From 4a9608f31c247b5b1284844609703f3ea4f95423 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <antoine.romain.dumont@gmail.com>
Date: Mon, 2 Dec 2019 15:49:38 +0100
Subject: [PATCH] lister/tasks: Standardize return statements
The following commit adapts the return statements from both lister and their
associated tasks. This standardizes on what other modules (e.g. both dvcs and
package loaders) do.
---
swh/lister/bitbucket/tasks.py | 5 +++--
swh/lister/cgit/lister.py | 4 ++++
swh/lister/cgit/tasks.py | 2 +-
swh/lister/core/indexing_lister.py | 3 +++
swh/lister/core/page_by_page_lister.py | 4 ++++
swh/lister/core/simple_lister.py | 5 +++++
swh/lister/cran/tasks.py | 2 +-
swh/lister/debian/lister.py | 12 +++++++-----
swh/lister/debian/tasks.py | 2 +-
swh/lister/github/tasks.py | 5 +++--
swh/lister/gitlab/tasks.py | 5 +++--
swh/lister/gnu/tasks.py | 4 ++--
swh/lister/npm/tasks.py | 4 ++--
swh/lister/phabricator/tasks.py | 4 ++--
swh/lister/pypi/tasks.py | 2 +-
15 files changed, 42 insertions(+), 21 deletions(-)
diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py
index 28b29d64..3b64de01 100644
--- a/swh/lister/bitbucket/tasks.py
+++ b/swh/lister/bitbucket/tasks.py
@@ -14,13 +14,13 @@ GROUP_SPLIT = 10000
def list_bitbucket_incremental(**lister_args):
'''Incremental update of the BitBucket forge'''
lister = BitBucketLister(**lister_args)
- lister.run(min_bound=lister.db_last_index(), max_bound=None)
+ return lister.run(min_bound=lister.db_last_index(), max_bound=None)
@shared_task(name=__name__ + '.RangeBitBucketLister')
def _range_bitbucket_lister(start, end, **lister_args):
lister = BitBucketLister(**lister_args)
- lister.run(min_bound=start, max_bound=end)
+ return lister.run(min_bound=start, max_bound=end)
@shared_task(name=__name__ + '.FullBitBucketRelister', bind=True)
@@ -44,6 +44,7 @@ def list_bitbucket_full(self, split=None, **lister_args):
promise.save() # so that we can restore the GroupResult in tests
except (NotImplementedError, AttributeError):
self.log.info('Unable to call save_group with current result backend.')
+ # FIXME: what to do in terms of return here?
return promise.id
diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py
index 37464c08..a8da6c6a 100644
--- a/swh/lister/cgit/lister.py
+++ b/swh/lister/cgit/lister.py
@@ -78,6 +78,7 @@ class CGitLister(ListerBase):
}
def run(self):
+ status = 'uneventful'
total = 0
for repos in grouper(self.get_repos(), 10):
models = list(filter(None, (self.build_model(repo)
@@ -87,6 +88,9 @@ class CGitLister(ListerBase):
self.db_session.commit()
total += len(injected_repos)
logger.debug('Scheduled %s tasks for %s', total, self.url)
+ status = 'eventful'
+
+ return {'status': status}
def get_repos(self):
"""Generate git 'project' URLs found on the current CGit server
diff --git a/swh/lister/cgit/tasks.py b/swh/lister/cgit/tasks.py
index 12952485..2d60e36d 100644
--- a/swh/lister/cgit/tasks.py
+++ b/swh/lister/cgit/tasks.py
@@ -10,7 +10,7 @@ from .lister import CGitLister
@shared_task(name=__name__ + '.CGitListerTask')
def list_cgit(**lister_args):
'''Lister task for CGit instances'''
- CGitLister(**lister_args).run()
+ return CGitLister(**lister_args).run()
@shared_task(name=__name__ + '.ping')
diff --git a/swh/lister/core/indexing_lister.py b/swh/lister/core/indexing_lister.py
index 48be968c..2e7f3005 100644
--- a/swh/lister/core/indexing_lister.py
+++ b/swh/lister/core/indexing_lister.py
@@ -213,6 +213,7 @@ class IndexingLister(ListerBase):
Returns:
nothing
"""
+ status = 'uneventful'
self.min_index = min_bound
self.max_index = max_bound
@@ -243,9 +244,11 @@ class IndexingLister(ListerBase):
logger.debug('Flushing updates at index %s', i)
self.db_session.commit()
self.db_session = self.mk_session()
+ status = 'eventful'
self.db_session.commit()
self.db_session = self.mk_session()
+ return {'status': status}
class IndexingHttpLister(ListerHttpTransport, IndexingLister):
diff --git a/swh/lister/core/page_by_page_lister.py b/swh/lister/core/page_by_page_lister.py
index 3d6d9c74..8bcce45c 100644
--- a/swh/lister/core/page_by_page_lister.py
+++ b/swh/lister/core/page_by_page_lister.py
@@ -110,6 +110,7 @@ class PageByPageLister(ListerBase):
nothing
"""
+ status = 'uneventful'
page = min_bound or 0
loop_count = 0
@@ -127,6 +128,7 @@ class PageByPageLister(ListerBase):
elif not injected_repos:
logging.info('Repositories already seen, stopping')
break
+ status = 'eventful'
next_page = self.get_next_target_from_response(response)
@@ -149,6 +151,8 @@ class PageByPageLister(ListerBase):
self.db_session.commit()
self.db_session = self.mk_session()
+ return {'status': status}
+
class PageByPageHttpLister(ListerHttpTransport, PageByPageLister):
"""Convenience class for ensuring right lookup and init order when
diff --git a/swh/lister/core/simple_lister.py b/swh/lister/core/simple_lister.py
index a355d9df..fa09ed88 100644
--- a/swh/lister/core/simple_lister.py
+++ b/swh/lister/core/simple_lister.py
@@ -89,3 +89,8 @@ class SimpleLister(ListerBase):
response, injected_repos = self.ingest_data(dump_not_used_identifier)
if not response and not injected_repos:
logging.info('No response from api server, stopping')
+ status = 'uneventful'
+ else:
+ status = 'eventful'
+
+ return {'status': status}
diff --git a/swh/lister/cran/tasks.py b/swh/lister/cran/tasks.py
index 6802619f..74eef746 100644
--- a/swh/lister/cran/tasks.py
+++ b/swh/lister/cran/tasks.py
@@ -10,7 +10,7 @@ from swh.lister.cran.lister import CRANLister
@shared_task(name=__name__ + '.CRANListerTask')
def list_cran(**lister_args):
'''Lister task for the CRAN registry'''
- CRANLister(**lister_args).run()
+ return CRANLister(**lister_args).run()
@shared_task(name=__name__ + '.ping')
diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py
index cc9ecd38..ef9853be 100644
--- a/swh/lister/debian/lister.py
+++ b/swh/lister/debian/lister.py
@@ -216,12 +216,14 @@ class DebianLister(ListerHttpTransport, ListerBase):
.one_or_none()
if not distribution:
- raise ValueError("Distribution %s is not registered" %
- self.distribution)
+ logger.error("Distribution %s is not registered" %
+ self.distribution)
+ return {'status': 'failed'}
if not distribution.type == 'deb':
- raise ValueError("Distribution %s is not a Debian derivative" %
- distribution)
+ logger.error("Distribution %s is not a Debian derivative" %
+ distribution)
+ return {'status': 'failed'}
date = self.date
@@ -250,4 +252,4 @@ class DebianLister(ListerHttpTransport, ListerBase):
self.db_session.commit()
- return True
+ return {'status': 'eventful'}
diff --git a/swh/lister/debian/tasks.py b/swh/lister/debian/tasks.py
index d542d04c..04d12977 100644
--- a/swh/lister/debian/tasks.py
+++ b/swh/lister/debian/tasks.py
@@ -10,7 +10,7 @@ from .lister import DebianLister
@shared_task(name=__name__ + '.DebianListerTask')
def list_debian_distribution(distribution, **lister_args):
'''List a Debian distribution'''
- DebianLister(distribution=distribution, **lister_args).run()
+ return DebianLister(distribution=distribution, **lister_args).run()
@shared_task(name=__name__ + '.ping')
diff --git a/swh/lister/github/tasks.py b/swh/lister/github/tasks.py
index 207ddf9d..1b9f37e7 100644
--- a/swh/lister/github/tasks.py
+++ b/swh/lister/github/tasks.py
@@ -15,13 +15,13 @@ GROUP_SPLIT = 10000
def list_github_incremental(**lister_args):
'Incremental update of GitHub'
lister = GitHubLister(**lister_args)
- lister.run(min_bound=lister.db_last_index(), max_bound=None)
+ return lister.run(min_bound=lister.db_last_index(), max_bound=None)
@shared_task(name=__name__ + '.RangeGitHubLister')
def _range_github_lister(start, end, **lister_args):
lister = GitHubLister(**lister_args)
- lister.run(min_bound=start, max_bound=end)
+ return lister.run(min_bound=start, max_bound=end)
@shared_task(name=__name__ + '.FullGitHubRelister', bind=True)
@@ -44,6 +44,7 @@ def list_github_full(self, split=None, **lister_args):
promise.save() # so that we can restore the GroupResult in tests
except (NotImplementedError, AttributeError):
self.log.info('Unable to call save_group with current result backend.')
+ # FIXME: what to do in terms of return here?
return promise.id
diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py
index bdb23cd9..e6a17552 100644
--- a/swh/lister/gitlab/tasks.py
+++ b/swh/lister/gitlab/tasks.py
@@ -20,13 +20,13 @@ def list_gitlab_incremental(**lister_args):
lister = GitLabLister(**lister_args)
total_pages = lister.get_pages_information()[1]
# stopping as soon as existing origins for that instance are detected
- lister.run(min_bound=1, max_bound=total_pages, check_existence=True)
+ return lister.run(min_bound=1, max_bound=total_pages, check_existence=True)
@shared_task(name=__name__ + '.RangeGitLabLister')
def _range_gitlab_lister(start, end, **lister_args):
lister = GitLabLister(**lister_args)
- lister.run(min_bound=start, max_bound=end)
+ return lister.run(min_bound=start, max_bound=end)
@shared_task(name=__name__ + '.FullGitLabRelister', bind=True)
@@ -43,6 +43,7 @@ def list_gitlab_full(self, **lister_args):
promise.save()
except (NotImplementedError, AttributeError):
self.log.info('Unable to call save_group with current result backend.')
+ # FIXME: what to do in terms of return here?
return promise.id
diff --git a/swh/lister/gnu/tasks.py b/swh/lister/gnu/tasks.py
index 1eb70612..edcde7e3 100644
--- a/swh/lister/gnu/tasks.py
+++ b/swh/lister/gnu/tasks.py
@@ -9,8 +9,8 @@ from .lister import GNULister
@shared_task(name=__name__ + '.GNUListerTask')
def list_gnu_full(**lister_args):
- 'List lister for the GNU source code archive'
- GNULister(**lister_args).run()
+ """List lister for the GNU source code archive"""
+ return GNULister(**lister_args).run()
@shared_task(name=__name__ + '.ping')
diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py
index 94b991b1..1e4a51ca 100644
--- a/swh/lister/npm/tasks.py
+++ b/swh/lister/npm/tasks.py
@@ -45,7 +45,7 @@ def list_npm_full(**lister_args):
'Full lister for the npm (javascript) registry'
lister = NpmLister(**lister_args)
with save_registry_state(lister):
- lister.run()
+ return lister.run()
@shared_task(name=__name__ + '.NpmIncrementalListerTask')
@@ -54,7 +54,7 @@ def list_npm_incremental(**lister_args):
lister = NpmIncrementalLister(**lister_args)
update_seq_start = get_last_update_seq(lister)
with save_registry_state(lister):
- lister.run(min_bound=update_seq_start)
+ return lister.run(min_bound=update_seq_start)
@shared_task(name=__name__ + '.ping')
diff --git a/swh/lister/phabricator/tasks.py b/swh/lister/phabricator/tasks.py
index 361feddb..614f4f2a 100644
--- a/swh/lister/phabricator/tasks.py
+++ b/swh/lister/phabricator/tasks.py
@@ -9,8 +9,8 @@ from swh.lister.phabricator.lister import PhabricatorLister
@shared_task(name=__name__ + '.FullPhabricatorLister')
def list_phabricator_full(**lister_args):
- 'Full update of a Phabricator instance'
- PhabricatorLister(**lister_args).run()
+ """Full update of a Phabricator instance"""
+ return PhabricatorLister(**lister_args).run()
@shared_task(name=__name__ + '.ping')
diff --git a/swh/lister/pypi/tasks.py b/swh/lister/pypi/tasks.py
index 2412f26c..b59e6b0f 100644
--- a/swh/lister/pypi/tasks.py
+++ b/swh/lister/pypi/tasks.py
@@ -10,7 +10,7 @@ from .lister import PyPILister
@shared_task(name=__name__ + '.PyPIListerTask')
def list_pypi(**lister_args):
'Full update of the PyPI (python) registry'
- PyPILister(**lister_args).run()
+ return PyPILister(**lister_args).run()
@shared_task(name=__name__ + '.ping')
--
GitLab