listers: Ensure run can be called without bounds arguments

Closes T2001

swh/lister/bitbucket/lister.py

@@ -5,7 +5,7 @@
 import logging
 import iso8601
 
-from datetime import datetime
+from datetime import datetime, timezone
 from urllib import parse
 
 from swh.lister.bitbucket.models import BitBucketModel
@@ -21,7 +21,7 @@ class BitBucketLister(IndexingHttpLister):
     LISTER_NAME = 'bitbucket'
     DEFAULT_URL = 'https://api.bitbucket.org/2.0'
     instance = 'bitbucket'
-    default_min_bound = datetime.utcfromtimestamp(0)
+    default_min_bound = datetime.fromtimestamp(0, timezone.utc)
 
     def __init__(self, url=None, override_config=None, per_page=100):
         super().__init__(url=url, override_config=override_config)

swh/lister/bitbucket/tests/test_bb_lister.py

@@ -35,6 +35,16 @@ class BitBucketListerTester(HttpListerTester, unittest.TestCase):
     entries_per_page = 10
     convert_type = staticmethod(convert_type)
 
+    def request_index(self, request):
+        """(Override) This is needed to emulate the listing bootstrap
+        when no min_bound is provided to run
+        """
+        m = self.test_re.search(request.path_url)
+        idx = convert_type(m.group(1))
+        if idx == self.Lister.default_min_bound:
+            idx = self.first_index
+        return idx
+
     @requests_mock.Mocker()
     def test_fetch_none_nodb(self, http_mocker):
         """Overridden because index is not an integer nor a string

swh/lister/core/tests/test_lister.py

@@ -208,6 +208,17 @@ class HttpListerTester(HttpListerTesterBase, abc.ABC):
         self.mock_scheduler(fl)
         return fl
 
+    @requests_mock.Mocker()
+    def test_fetch_no_bounds_yesdb(self, http_mocker):
+        fl = self.create_fl_with_db(http_mocker)
+
+        fl.run()
+
+        self.assertEqual(fl.db_last_index(), self.last_index)
+        ingested_repos = list(fl.db_query_range(self.first_index,
+                                                self.last_index))
+        self.assertEqual(len(ingested_repos), self.entries_per_page)
+
     @requests_mock.Mocker()
     def test_fetch_multiple_pages_yesdb(self, http_mocker):
 

swh/lister/github/lister.py

@@ -16,6 +16,7 @@ class GitHubLister(IndexingHttpLister):
     API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)')
     LISTER_NAME = 'github'
     instance = 'github'  # There is only 1 instance of such lister
+    default_min_bound = 0
 
     def get_model_from_repo(self, repo):
         return {

swh/lister/github/tests/test_gh_lister.py

@@ -16,7 +16,7 @@ class GitHubListerTester(HttpListerTester, unittest.TestCase):
     lister_subdir = 'github'
     good_api_response_file = 'api_response.json'
     bad_api_response_file = 'api_empty_response.json'
-    first_index = 26
+    first_index = 0
     last_index = 368
     entries_per_page = 100
     convert_type = int

swh/lister/phabricator/tests/test_lister.py

@@ -74,17 +74,6 @@ class PhabricatorListerTester(HttpListerTester, unittest.TestCase):
                 'https://svn.blender.org/svnroot/bf-blender/',
                 get_repo_url(repo['attachments']['uris']['uris']))
 
-    @requests_mock.Mocker()
-    def test_full_listing(self, http_mocker):
-        fl = self.create_fl_with_db(http_mocker)
-
-        fl.run()
-
-        self.assertEqual(fl.db_last_index(), self.last_index)
-        ingested_repos = list(fl.db_query_range(self.first_index,
-                                                self.last_index))
-        self.assertEqual(len(ingested_repos), self.entries_per_page)
-
     @requests_mock.Mocker()
     def test_scheduled_tasks(self, http_mocker):
         fl = self.create_fl_with_db(http_mocker)