From 7344d264e7ad08148e6e187b525643fbdc30a50c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio=20=28Lunar=29?=
<lunar@softwareheritage.org>
Date: Wed, 18 Oct 2023 10:14:55 +0200
Subject: [PATCH] Ensure HTTPError.response is not None
The implementation of `HTTPError` in `requests` does not guarantee that
the `response` property will always be set. So we need to ensure it is
not `None` before looking for the return code, for example.
This also makes mypy checks pass again, as `types-request` was updated
in 2.31.0.9 to better match this particular aspect. See:
https://github.com/python/typeshed/pull/10875
---
swh/lister/bioconductor/lister.py | 1 +
swh/lister/bitbucket/lister.py | 2 +-
swh/lister/cgit/lister.py | 1 +
swh/lister/gitlab/lister.py | 1 +
swh/lister/gitweb/lister.py | 1 +
swh/lister/gogs/lister.py | 3 ++-
swh/lister/stagit/lister.py | 1 +
7 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/swh/lister/bioconductor/lister.py b/swh/lister/bioconductor/lister.py
index 95895cde..e050f76c 100644
--- a/swh/lister/bioconductor/lister.py
+++ b/swh/lister/bioconductor/lister.py
@@ -127,6 +127,7 @@ class BioconductorLister(Lister[BioconductorListerState, BioconductorListerPage]
packages_txt = self.http_request(url).text
packages = self.parse_packages(packages_txt)
except HTTPError as e:
+ assert e.response is not None
logger.debug(
"Skipping page since got %s response for %s",
e.response.status_code,
diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py
index 00d8abff..f129bc0d 100644
--- a/swh/lister/bitbucket/lister.py
+++ b/swh/lister/bitbucket/lister.py
@@ -125,7 +125,7 @@ class BitbucketLister(Lister[BitbucketListerState, List[Dict[str, Any]]]):
body = self.http_request(self.url, params=self.url_params).json()
yield body["values"]
except HTTPError as e:
- if e.response.status_code == 500:
+ if e.response is not None and e.response.status_code == 500:
logger.warning(
"URL %s is buggy (error 500), skip it and get next page.",
e.response.url,
diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py
index 1ab17cb5..af3d7036 100644
--- a/swh/lister/cgit/lister.py
+++ b/swh/lister/cgit/lister.py
@@ -160,6 +160,7 @@ class CGitLister(StatelessLister[Repositories]):
try:
bs = self._get_and_parse(repository_url)
except HTTPError as e:
+ assert e.response is not None
logger.warning(
"Unexpected HTTP status code %s on %s",
e.response.status_code,
diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py
index d2b1bd52..e7c3fb58 100644
--- a/swh/lister/gitlab/lister.py
+++ b/swh/lister/gitlab/lister.py
@@ -56,6 +56,7 @@ def _if_rate_limited(retry_state) -> bool:
exc = attempt.exception()
return (
isinstance(exc, HTTPError)
+ and exc.response is not None
and exc.response.status_code == codes.forbidden
and int(exc.response.headers.get("RateLimit-Remaining", "0")) == 0
) or is_retryable_exception(exc)
diff --git a/swh/lister/gitweb/lister.py b/swh/lister/gitweb/lister.py
index 86720c34..26521141 100644
--- a/swh/lister/gitweb/lister.py
+++ b/swh/lister/gitweb/lister.py
@@ -125,6 +125,7 @@ class GitwebLister(StatelessLister[Repositories]):
try:
bs = self._get_and_parse(repository_url)
except HTTPError as e:
+ assert e.response is not None
logger.warning(
"Unexpected HTTP status code %s on %s",
e.response.status_code,
diff --git a/swh/lister/gogs/lister.py b/swh/lister/gogs/lister.py
index b9db3d35..0d1547c5 100644
--- a/swh/lister/gogs/lister.py
+++ b/swh/lister/gogs/lister.py
@@ -135,7 +135,8 @@ class GogsLister(Lister[GogsListerState, GogsListerPage]):
response = self.http_request(url, params=params)
except HTTPError as http_error:
if (
- http_error.response.status_code == 500
+ http_error.response is not None
+ and http_error.response.status_code == 500
): # Temporary hack for skipping fatal repos (T4423)
url_parts = urlparse(url)
query: Dict[str, Any] = dict(parse_qsl(url_parts.query))
diff --git a/swh/lister/stagit/lister.py b/swh/lister/stagit/lister.py
index b87ca66b..7a2187f7 100644
--- a/swh/lister/stagit/lister.py
+++ b/swh/lister/stagit/lister.py
@@ -113,6 +113,7 @@ class StagitLister(StatelessLister[Repositories]):
try:
bs = self._get_and_parse(repository_url)
except HTTPError as e:
+ assert e.response is not None
logger.warning(
"Unexpected HTTP status code %s on %s",
e.response.status_code,
--
GitLab