bitbucket: Fix request to get next page of buggy page

The bitbucket Web API to list repositories has buggy pages that
needs to be skipped to continue the listing.

Previously the request to get the next page when a buggy page
is detected was missing the after query parameter so the request
was always returning the second page of repositories listing
endpoint.

Also refine buggy page detection by considering all HTTP status
code >= 500.

swh/lister/bitbucket/lister.py

@@ -124,16 +124,18 @@ class BitbucketLister(Lister[BitbucketListerState, List[Dict[str, Any]]]):
                 body = self.http_request(self.url, params=self.url_params).json()
                 yield body["values"]
             except HTTPError as e:
-                if e.response is not None and e.response.status_code == 500:
+                if e.response is not None and e.response.status_code >= 500:
                     logger.warning(
-                        "URL %s is buggy (error 500), skip it and get next page.",
+                        "URL %s is buggy (error %s), skip it and get next page.",
                         e.response.url,
+                        e.response.status_code,
                     )
                     body = self.http_request(
                         self.url,
                         params={
                             "pagelen": self.url_params["pagelen"],
                             "fields": "next",
+                            "after": last_repo_cdate,
                         },
                     ).json()
 

swh/lister/bitbucket/tests/test_lister.py

@@ -190,7 +190,8 @@ def test_bitbucket_lister_buggy_page(
         BitbucketLister.API_URL,
         [
             {"json": bb_api_repositories_page1, "status_code": 200},
-            *[{"json": None, "status_code": 500}] * MAX_NUMBER_ATTEMPTS,
+            *[{"json": None, "status_code": 500}] * (MAX_NUMBER_ATTEMPTS - 1),
+            {"json": None, "status_code": 504},
             {"json": {"next": bb_api_repositories_page1["next"]}, "status_code": 200},
             {"json": bb_api_repositories_page2, "status_code": 200},
         ],