pattern: introduce a requests_extra_kwargs parameter

This new parameter allows passing extra kwargs to calls to the requests
library, e.g. to set timeouts or TLS certificate verification.

swh/lister/gitlab/lister.py

@@ -156,7 +156,7 @@ class GitLabLister(Lister[GitLabListerState, PageResult]):
     )
     def get_page_result(self, url: str) -> PageResult:
         logger.debug("Fetching URL %s", url)
-        response = self.session.get(url)
+        response = self.session.get(url, **self.requests_kwargs())
         if response.status_code != 200:
             logger.warning(
                 "Unexpected HTTP status code %s on %s: %s",
@@ -175,7 +175,7 @@ class GitLabLister(Lister[GitLabListerState, PageResult]):
             while True:
                 next_id_after = id_after + self.per_page
                 url = url.replace(f"id_after={id_after}", f"id_after={next_id_after}")
-                response = self.session.get(url)
+                response = self.session.get(url, **self.requests_kwargs())
                 if response.status_code == 200:
                     break
                 else:

swh/lister/pattern.py

@@ -90,6 +90,7 @@ class Lister(Generic[StateType, PageType]):
       max_origins_per_page: the maximum number of origins processed per page
       enable_origins: whether the created origins should be enabled or not
       record_batch_size: maximum number of records to flush to the scheduler at once.
+      requests_extra_kwargs: extra keyword arguments to pass to :mod:`requests` calls.
 
     Generic types:
       - *StateType*: concrete lister type; should usually be a :class:`dataclass` for
@@ -113,6 +114,7 @@ class Lister(Generic[StateType, PageType]):
         enable_origins: bool = True,
         with_github_session: bool = False,
         record_batch_size: int = 1000,
+        requests_extra_kwargs: Dict[str, Any] = {},
     ):
         if not self.LISTER_NAME:
             raise ValueError("Must set the LISTER_NAME attribute on Lister classes")
@@ -168,6 +170,7 @@ class Lister(Generic[StateType, PageType]):
         self.max_origins_per_page = max_origins_per_page
         self.enable_origins = enable_origins
         self.record_batch_size = record_batch_size
+        self.requests_extra_kwargs = requests_extra_kwargs
 
     def build_url(self, instance: str) -> str:
         """Optionally build the forge url to list. When the url is not provided in the
@@ -188,11 +191,14 @@ class Lister(Generic[StateType, PageType]):
             raise ValueError("Instance should only be a net location.")
         return f"https://{instance}"
 
+    def requests_kwargs(self, **kwargs):
+        return {**self.requests_extra_kwargs, **kwargs}
+
     @http_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
     def http_request(self, url: str, method="GET", **kwargs) -> requests.Response:
         logger.debug("Fetching URL %s with params %s", url, kwargs.get("params"))
 
-        response = self.session.request(method, url, **kwargs)
+        response = self.session.request(method, url, **self.requests_kwargs(**kwargs))
         if response.status_code not in (200, 304):
             logger.warning(
                 "Unexpected HTTP status code %s on %s: %s",