diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py index 3461ebe6d6a1a738dc6b5d77b8f9b029fabfce64..5c6287e0b63dc7ac8301a74096db93f46146301c 100644 --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -124,16 +124,18 @@ class BitbucketLister(Lister[BitbucketListerState, List[Dict[str, Any]]]): body = self.http_request(self.url, params=self.url_params).json() yield body["values"] except HTTPError as e: - if e.response is not None and e.response.status_code == 500: + if e.response is not None and e.response.status_code >= 500: logger.warning( - "URL %s is buggy (error 500), skip it and get next page.", + "URL %s is buggy (error %s), skip it and get next page.", e.response.url, + e.response.status_code, ) body = self.http_request( self.url, params={ "pagelen": self.url_params["pagelen"], "fields": "next", + "after": last_repo_cdate, }, ).json() diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py index 7882d31f30c1aaa69324c26cebd9fb17ee82fbac..9583c95168750669ab28898a1f3e097fc121c105 100644 --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -190,7 +190,8 @@ def test_bitbucket_lister_buggy_page( BitbucketLister.API_URL, [ {"json": bb_api_repositories_page1, "status_code": 200}, - *[{"json": None, "status_code": 500}] * MAX_NUMBER_ATTEMPTS, + *[{"json": None, "status_code": 500}] * (MAX_NUMBER_ATTEMPTS - 1), + {"json": None, "status_code": 504}, {"json": {"next": bb_api_repositories_page1["next"]}, "status_code": 200}, {"json": bb_api_repositories_page2, "status_code": 200}, ],