Skip to content
Snippets Groups Projects
Commit 9e2ce82e authored by Antoine R. Dumont's avatar Antoine R. Dumont
Browse files

New upstream version 0.0.15

parents b97a6495 56c18ede
No related branches found
Tags debian/0.25.0-1_swh1
No related merge requests found
Metadata-Version: 1.0
Name: swh.indexer
Version: 0.0.14
Version: 0.0.15
Summary: Software Heritage Content Indexer
Home-page: https://forge.softwareheritage.org/diffusion/78/
Author: Software Heritage developers
......
Metadata-Version: 1.0
Name: swh.indexer
Version: 0.0.14
Version: 0.0.15
Summary: Software Heritage Content Indexer
Home-page: https://forge.softwareheritage.org/diffusion/78/
Author: Software Heritage developers
......
......@@ -10,12 +10,6 @@ import sys
from swh.core import utils, hashutil
from swh.scheduler.celery_backend.config import app
from . import tasks, TASK_NAMES # noqa
task_name = TASK_NAMES['orchestrator_all']
orchestrator_task = app.tasks[task_name]
def read_from_stdin():
for sha1 in sys.stdin:
......@@ -32,31 +26,44 @@ def gen_sha1(batch):
yield sha1s
def run_with_limit(limit, batch):
def run_with_limit(task, limit, batch):
count = 0
for sha1s in gen_sha1(batch):
count += len(sha1s)
print('%s sent - [%s, ...]' % (len(sha1s), sha1s[0]))
orchestrator_task.delay(sha1s)
task.delay(sha1s)
if count >= limit:
return
def run_no_limit(batch):
def run_no_limit(task, batch):
for sha1s in gen_sha1(batch):
print('%s sent - [%s, ...]' % (len(sha1s), sha1s[0]))
orchestrator_task.delay(sha1s)
task.delay(sha1s)
@click.command(help='Read sha1 from stdin and send them for indexing')
@click.option('--limit', default=None, help='Limit the number of data to read')
@click.option('--batch', default='10', help='Group data by batch')
def main(limit, batch):
@click.option('--task-name', default='orchestrator_all', help='')
def main(limit, batch, task_name):
batch = int(batch)
from . import tasks, TASK_NAMES # noqa
possible_tasks = TASK_NAMES.keys()
if task_name not in possible_tasks:
print('The task_name can only be one of %s' %
', '.join(possible_tasks))
return
task_name = TASK_NAMES['orchestrator_all']
task = app.tasks[task_name]
if limit:
run_with_limit(int(limit), batch)
run_with_limit(task, int(limit), batch)
else:
run_no_limit(batch)
run_no_limit(task, batch)
if __name__ == '__main__':
......
v0.0.14-0-g2c192c5
\ No newline at end of file
v0.0.15-0-g56c18ed
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment