diff --git a/conftest.py b/conftest.py
index ea09c62e54c521e4362a6048f462e75b3df25a72..b4775dc47073bc556fe5b46af9c1344a251224b2 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2022 The Software Heritage developers
+# Copyright (C) 2020-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -13,8 +13,6 @@ os.environ["LC_ALL"] = "C.UTF-8"
@pytest.fixture(autouse=True)
-def tenacity_wait(mocker):
- # Stops tenacity from blocking lister tests for 50x errors
- from swh.lister.pattern import Lister
-
- mocker.patch.object(Lister.http_request.retry, "sleep")
+def mock_sleep(mocker):
+ # Stops tenacity from blocking lister tests when retrying
+ return mocker.patch("time.sleep")
diff --git a/requirements.txt b/requirements.txt
index 6c03204c22421efd21c20c264287cd21cf4c2b32..e8bf14cc263f3d68cd248c94f3f8814306281618 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,6 +10,6 @@ python_debian
repomd
requests
setuptools
-tenacity >= 6.2
+tenacity >= 8.4.2
testing.postgresql
toml
diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py
index 7a78ae258c020d4ae06ca7c01ed46686291662ef..7882d31f30c1aaa69324c26cebd9fb17ee82fbac 100644
--- a/swh/lister/bitbucket/tests/test_lister.py
+++ b/swh/lister/bitbucket/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2023 The Software Heritage developers
+# Copyright (C) 2017-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -109,8 +109,6 @@ def test_bitbucket_lister_rate_limit_hit(
lister = BitbucketLister(scheduler=swh_scheduler, page_size=10)
- mocker.patch.object(lister.http_request.retry, "sleep")
-
stats = lister.run()
assert stats.pages == 2
@@ -200,8 +198,6 @@ def test_bitbucket_lister_buggy_page(
lister = BitbucketLister(scheduler=swh_scheduler, page_size=10)
- mocker.patch.object(lister.http_request.retry, "sleep")
-
stats = lister.run()
assert stats.pages == 2
diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py
index 44953bb12e3069167aab59c54ae7e3527a0add73..1be417c4747fce81228adf85c58abda53acfa32f 100644
--- a/swh/lister/cgit/tests/test_lister.py
+++ b/swh/lister/cgit/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2023 The Software Heritage developers
+# Copyright (C) 2019-2024 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -276,8 +276,6 @@ def test_lister_cgit_get_pages_with_pages_and_retry(
lister_cgit = CGitLister(swh_scheduler, url=url)
- mocker.patch.object(lister_cgit.http_request.retry, "sleep")
-
repos: List[List[str]] = list(lister_cgit.get_pages())
flattened_repos = sum(repos, [])
# we should have 16 repos (listed on 3 pages)
diff --git a/swh/lister/gitiles/tests/test_lister.py b/swh/lister/gitiles/tests/test_lister.py
index 2b1211766787f54e5c324595338cbc0ed42537c7..2fb20f50e7c4c45364cb2a83085e57bee3369f87 100644
--- a/swh/lister/gitiles/tests/test_lister.py
+++ b/swh/lister/gitiles/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2023 The Software Heritage developers
+# Copyright (C) 2023-2024 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -104,7 +104,5 @@ def test_lister_gitiles_get_pages_with_pages_and_retry(
lister_gitiles = GitilesLister(swh_scheduler, url=url)
- mocker.patch.object(lister_gitiles.http_request.retry, "sleep")
-
pages: List[str] = list(lister_gitiles.get_pages())
assert len(pages) == 7
diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py
index f8dd50442877574bca6cd4209bc7c90596f95c8d..4cd4b0623272ba476ea2037e717ef879152c3e6b 100644
--- a/swh/lister/gitlab/tests/test_lister.py
+++ b/swh/lister/gitlab/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2022 The Software Heritage developers
+# Copyright (C) 2017-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -212,7 +212,9 @@ def test_lister_gitlab_incremental(swh_scheduler, requests_mock, datadir):
assert listed_origin.last_update is not None
-def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker):
+def test_lister_gitlab_rate_limit(
+ swh_scheduler, requests_mock, datadir, mocker, mock_sleep
+):
"""Gitlab lister supports rate-limit"""
instance = "gite.lirmm.fr"
url = api_url(instance)
@@ -240,9 +242,6 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker)
additional_matcher=_match_request,
)
- # To avoid this test being too slow, we mock sleep within the retry behavior
- mock_sleep = mocker.patch.object(lister.get_page_result.retry, "sleep")
-
listed_result = lister.run()
expected_nb_origins = len(response1) + len(response2)
@@ -253,7 +252,7 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker)
@pytest.mark.parametrize("status_code", [502, 503, 520])
def test_lister_gitlab_http_errors(
- swh_scheduler, requests_mock, datadir, mocker, status_code
+ swh_scheduler, requests_mock, datadir, mocker, status_code, mock_sleep
):
"""Gitlab lister should retry requests when encountering HTTP 50x errors"""
instance = "gite.lirmm.fr"
@@ -281,9 +280,6 @@ def test_lister_gitlab_http_errors(
additional_matcher=_match_request,
)
- # To avoid this test being too slow, we mock sleep within the retry behavior
- mock_sleep = mocker.patch.object(lister.get_page_result.retry, "sleep")
-
listed_result = lister.run()
expected_nb_origins = len(response1) + len(response2)
diff --git a/swh/lister/gitweb/tests/test_lister.py b/swh/lister/gitweb/tests/test_lister.py
index 30f3dc916f717632904eb99f27c19364b85be88c..63534215c8f4f6c5ddb3d3c876b89960b752a7f3 100644
--- a/swh/lister/gitweb/tests/test_lister.py
+++ b/swh/lister/gitweb/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2023 The Software Heritage developers
+# Copyright (C) 2023-2024 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -105,8 +105,6 @@ def test_lister_gitweb_get_pages_with_pages_and_retry(
lister_gitweb = GitwebLister(swh_scheduler, url=url)
- mocker.patch.object(lister_gitweb.http_request.retry, "sleep")
-
pages: List[List[str]] = list(lister_gitweb.get_pages())
flattened_repos = sum(pages, [])
assert len(pages) == 1
diff --git a/swh/lister/golang/tests/test_lister.py b/swh/lister/golang/tests/test_lister.py
index 94cea57f214b7d0f28eb712669e96bfe82b8cca3..a6d24bf26181fc4f9a6ab17deba9f1cdd736a3c3 100644
--- a/swh/lister/golang/tests/test_lister.py
+++ b/swh/lister/golang/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2022-2023 The Software Heritage developers
+# Copyright (C) 2022-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -97,10 +97,7 @@ def _generate_responses(datadir, requests_mock):
requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses)
-def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir):
- # Exponential retries take a long time, so stub time.sleep
- mocked_sleep = mocker.patch.object(GolangLister.http_request.retry, "sleep")
-
+def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir, mock_sleep):
# first listing, should return one origin per package
lister = GolangLister(scheduler=swh_scheduler)
@@ -126,7 +123,7 @@ def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir):
# Test `time.sleep` is called with exponential retries
assert_sleep_calls(
- mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE]
+ mocker, mock_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE]
)
# doing it all again (without incremental) should give us the same result
diff --git a/swh/lister/launchpad/tests/test_lister.py b/swh/lister/launchpad/tests/test_lister.py
index d6081bfadd365079e2bfa41265374bc74638af7e..50e0b43c207db86673bdd24d44ea70f18c4da580 100644
--- a/swh/lister/launchpad/tests/test_lister.py
+++ b/swh/lister/launchpad/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2022 The Software Heritage developers
+# Copyright (C) 2020-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -234,8 +234,6 @@ def test_launchpad_lister_raise_during_listing(
swh_scheduler, mocker, launchpad_response1, launchpad_bzr_response
):
lister = LaunchpadLister(scheduler=swh_scheduler)
- # Exponential retries take a long time, so stub time.sleep
- mocker.patch.object(lister._page_request.retry, "sleep")
mock_getRepositories, mock_getBranches = _mock_launchpad(
mocker,
diff --git a/swh/lister/maven/tests/test_lister.py b/swh/lister/maven/tests/test_lister.py
index 81a57eeaedd9ac664c734399fdb4889c523dddfb..5f879de85eb5cb174d8513e3ad19778231da0b79 100644
--- a/swh/lister/maven/tests/test_lister.py
+++ b/swh/lister/maven/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2021-2022 The Software Heritage developers
+# Copyright (C) 2021-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -80,11 +80,6 @@ def network_requests_mock(requests_mock, requests_mock_datadir, maven_index_full
requests_mock.get(INDEX_URL, content=maven_index_full)
-@pytest.fixture(autouse=True)
-def retry_sleep_mock(mocker):
- mocker.patch.object(MavenLister.http_request.retry, "sleep")
-
-
def test_maven_full_listing(swh_scheduler):
"""Covers full listing of multiple pages, checking page results and listed
origins, statelessness."""
diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py
index 7c4fa9346a2aa1654470f153b994f331b7a3715c..ea27778f15f9aae1b45813ee6b7207ad514edfc0 100644
--- a/swh/lister/npm/tests/test_lister.py
+++ b/swh/lister/npm/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2022 The Software Heritage developers
+# Copyright (C) 2018-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -35,11 +35,6 @@ def npm_incremental_listing_page2(datadir):
return json.loads(Path(datadir, "npm_incremental_page2.json").read_text())
-@pytest.fixture(autouse=True)
-def retry_sleep_mock(mocker):
- mocker.patch.object(NpmLister.http_request.retry, "sleep")
-
-
def _check_listed_npm_packages(lister, packages, scheduler_origins):
for package in packages:
package_name = package["doc"]["name"]
diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py
index acea2b7af00fe4ad6f0525fd1446a6d07c01a861..3b10bc091c88408443b4cf19ab082e1a1ef04a33 100644
--- a/swh/lister/phabricator/tests/test_lister.py
+++ b/swh/lister/phabricator/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2022 The Software Heritage developers
+# Copyright (C) 2019-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -27,11 +27,6 @@ def phabricator_repositories_page2(datadir):
)
-@pytest.fixture(autouse=True)
-def retry_sleep_mock(mocker):
- mocker.patch.object(PhabricatorLister.http_request.retry, "sleep")
-
-
def test_get_repo_url(phabricator_repositories_page1):
repos = phabricator_repositories_page1["result"]["data"]
for repo in repos:
diff --git a/swh/lister/sourceforge/tests/test_lister.py b/swh/lister/sourceforge/tests/test_lister.py
index 4261ce5e7df2dd77b8cbf5b9d6cc8de774cd72d5..a7b767e7a29daf089e4bb004d974eb726b3d2fd4 100644
--- a/swh/lister/sourceforge/tests/test_lister.py
+++ b/swh/lister/sourceforge/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2021-2023 The Software Heritage developers
+# Copyright (C) 2021-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -367,12 +367,11 @@ def test_sourceforge_lister_incremental(swh_scheduler, requests_mock, datadir, m
_check_listed_origins(lister, swh_scheduler)
-def test_sourceforge_lister_retry(swh_scheduler, requests_mock, mocker, datadir):
+def test_sourceforge_lister_retry(
+ swh_scheduler, requests_mock, mocker, datadir, mock_sleep
+):
lister = SourceForgeLister(scheduler=swh_scheduler)
- # Exponential retries take a long time, so stub time.sleep
- mocked_sleep = mocker.patch.object(lister.http_request.retry, "sleep")
-
requests_mock.get(
MAIN_SITEMAP_URL,
[
@@ -431,18 +430,15 @@ def test_sourceforge_lister_retry(swh_scheduler, requests_mock, mocker, datadir)
_check_listed_origins(lister, swh_scheduler)
# Test `time.sleep` is called with exponential retries
- assert_sleep_calls(mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, 1])
+ assert_sleep_calls(mocker, mock_sleep, [1, WAIT_EXP_BASE, 1, 1])
@pytest.mark.parametrize("status_code", [500, 503, 504, 403, 404])
def test_sourceforge_lister_http_error(
- swh_scheduler, requests_mock, status_code, mocker
+ swh_scheduler, requests_mock, status_code, mocker, mock_sleep
):
lister = SourceForgeLister(scheduler=swh_scheduler)
- # Exponential retries take a long time, so stub time.sleep
- mocked_sleep = mocker.patch.object(lister.http_request.retry, "sleep")
-
requests_mock.get(MAIN_SITEMAP_URL, status_code=status_code)
with pytest.raises(HTTPError):
@@ -452,7 +448,7 @@ def test_sourceforge_lister_http_error(
if status_code >= 500:
exp_retries = [1.0, 10.0, 100.0, 1000.0]
- assert_sleep_calls(mocker, mocked_sleep, exp_retries)
+ assert_sleep_calls(mocker, mock_sleep, exp_retries)
@pytest.mark.parametrize("status_code", [500, 503, 504, 403, 404])
@@ -460,8 +456,6 @@ def test_sourceforge_lister_project_error(
datadir, swh_scheduler, requests_mock, status_code, mocker
):
lister = SourceForgeLister(scheduler=swh_scheduler)
- # Exponential retries take a long time, so stub time.sleep
- mocker.patch.object(lister.http_request.retry, "sleep")
requests_mock.get(
MAIN_SITEMAP_URL,
diff --git a/swh/lister/stagit/tests/test_lister.py b/swh/lister/stagit/tests/test_lister.py
index 15fbdcb12b6a0354fbafa6f77a87fd22cd23f143..dc810a0a83b874a05dd849679037d9ff5d9773e9 100644
--- a/swh/lister/stagit/tests/test_lister.py
+++ b/swh/lister/stagit/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2023 The Software Heritage developers
+# Copyright (C) 2023-2024 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -97,8 +97,6 @@ def test_lister_stagit_get_pages_with_pages_and_retry(
lister_stagit = StagitLister(swh_scheduler, url=url)
- mocker.patch.object(lister_stagit.http_request.retry, "sleep")
-
pages: List[List[str]] = list(lister_stagit.get_pages())
flattened_repos = sum(pages, [])
assert len(pages) == 1
diff --git a/swh/lister/tuleap/tests/test_lister.py b/swh/lister/tuleap/tests/test_lister.py
index 722e70810f4fc20fabbf0003eeb884243bd26e2e..019494100d26196a38c68ee2d5d706b91138ae3b 100644
--- a/swh/lister/tuleap/tests/test_lister.py
+++ b/swh/lister/tuleap/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2021-2022 The Software Heritage developers
+# Copyright (C) 2021-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -92,11 +92,6 @@ def tuleap_repo_3(datadir) -> Tuple[str, Dict[str, str], List[RepoPage], List[st
return text, headers, page_results, origin_urls
-@pytest.fixture(autouse=True)
-def retry_sleep_mock(mocker):
- mocker.patch.object(TuleapLister.http_request.retry, "sleep")
-
-
def check_listed_origins(lister_urls: List[str], scheduler_origins: List[ListedOrigin]):
"""Asserts that the two collections have the same origin URLs.