diff --git a/swh/lister/bioconductor/lister.py b/swh/lister/bioconductor/lister.py index 95895cdecea54d80eec6ae608cb03f3c0a1b3562..e050f76c87b549df7af548ec670a212b5fa46b30 100644 --- a/swh/lister/bioconductor/lister.py +++ b/swh/lister/bioconductor/lister.py @@ -127,6 +127,7 @@ class BioconductorLister(Lister[BioconductorListerState, BioconductorListerPage] packages_txt = self.http_request(url).text packages = self.parse_packages(packages_txt) except HTTPError as e: + assert e.response is not None logger.debug( "Skipping page since got %s response for %s", e.response.status_code, diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py index 00d8abfffc728e8e3fe8f398ab3c74ff4516a2ea..f129bc0dc3557bb7a422c6a6a975fd34b82826d5 100644 --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -125,7 +125,7 @@ class BitbucketLister(Lister[BitbucketListerState, List[Dict[str, Any]]]): body = self.http_request(self.url, params=self.url_params).json() yield body["values"] except HTTPError as e: - if e.response.status_code == 500: + if e.response is not None and e.response.status_code == 500: logger.warning( "URL %s is buggy (error 500), skip it and get next page.", e.response.url, diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py index 1ab17cb5013a91d6f5e0d77cbc3fc4275f1aa689..af3d70362c58fe20d6aab1ca3eec4d0110504ac7 100644 --- a/swh/lister/cgit/lister.py +++ b/swh/lister/cgit/lister.py @@ -160,6 +160,7 @@ class CGitLister(StatelessLister[Repositories]): try: bs = self._get_and_parse(repository_url) except HTTPError as e: + assert e.response is not None logger.warning( "Unexpected HTTP status code %s on %s", e.response.status_code, diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py index d2b1bd5229f5a1e39710e9e900793e7d8026d789..e7c3fb585a988138cc62cd4858adcab58cf168bc 100644 --- a/swh/lister/gitlab/lister.py +++ b/swh/lister/gitlab/lister.py @@ -56,6 +56,7 @@ def _if_rate_limited(retry_state) -> bool: exc = attempt.exception() return ( isinstance(exc, HTTPError) + and exc.response is not None and exc.response.status_code == codes.forbidden and int(exc.response.headers.get("RateLimit-Remaining", "0")) == 0 ) or is_retryable_exception(exc) diff --git a/swh/lister/gitweb/lister.py b/swh/lister/gitweb/lister.py index 86720c349e700a0985ee9f84ae7408e6593a9fa2..26521141fb21b9554db6b9aab0e66f0ccdf7586a 100644 --- a/swh/lister/gitweb/lister.py +++ b/swh/lister/gitweb/lister.py @@ -125,6 +125,7 @@ class GitwebLister(StatelessLister[Repositories]): try: bs = self._get_and_parse(repository_url) except HTTPError as e: + assert e.response is not None logger.warning( "Unexpected HTTP status code %s on %s", e.response.status_code, diff --git a/swh/lister/gogs/lister.py b/swh/lister/gogs/lister.py index b9db3d359ad86c16b5523fa35997abab99e60077..0d1547c59dd209e7496806b941775428cbf04c84 100644 --- a/swh/lister/gogs/lister.py +++ b/swh/lister/gogs/lister.py @@ -135,7 +135,8 @@ class GogsLister(Lister[GogsListerState, GogsListerPage]): response = self.http_request(url, params=params) except HTTPError as http_error: if ( - http_error.response.status_code == 500 + http_error.response is not None + and http_error.response.status_code == 500 ): # Temporary hack for skipping fatal repos (T4423) url_parts = urlparse(url) query: Dict[str, Any] = dict(parse_qsl(url_parts.query)) diff --git a/swh/lister/stagit/lister.py b/swh/lister/stagit/lister.py index b87ca66bf296d9e08a7bcead320fd30683961e0c..7a2187f7d0229ac0c84e4e336dd123f8a964d875 100644 --- a/swh/lister/stagit/lister.py +++ b/swh/lister/stagit/lister.py @@ -113,6 +113,7 @@ class StagitLister(StatelessLister[Repositories]): try: bs = self._get_and_parse(repository_url) except HTTPError as e: + assert e.response is not None logger.warning( "Unexpected HTTP status code %s on %s", e.response.status_code,