gitlab: Support authentication

Related to T2987

swh/lister/gitlab/lister.py

@@ -5,6 +5,7 @@
 
 from dataclasses import asdict, dataclass
 import logging
+import random
 from typing import Any, Dict, Iterator, Optional, Tuple
 from urllib.parse import parse_qs, urlparse
 
@@ -101,10 +102,7 @@ class GitLabLister(Lister[GitLabListerState, PageResult]):
         if instance is None:
             instance = parse_url(url).host
         super().__init__(
-            scheduler=scheduler,
-            credentials=None,  # anonymous for now
-            url=url,
-            instance=instance,
+            scheduler=scheduler, url=url, instance=instance, credentials=credentials,
         )
         self.incremental = incremental
         self.last_page: Optional[str] = None
@@ -114,6 +112,15 @@ class GitLabLister(Lister[GitLabListerState, PageResult]):
             {"Accept": "application/json", "User-Agent": USER_AGENT}
         )
 
+        if len(self.credentials) > 0:
+            cred = random.choice(self.credentials)
+            logger.info(
+                "Using %s credentials from user %s", self.instance, cred["username"]
+            )
+            api_token = cred["password"]
+            if api_token:
+                self.session.headers["Authorization"] = f"Bearer {api_token}"
+
     def state_from_dict(self, d: Dict[str, Any]) -> GitLabListerState:
         return GitLabListerState(**d)
 

swh/lister/gitlab/tests/test_lister.py

@@ -198,6 +198,21 @@ def test_lister_gitlab_rate_limit(swh_scheduler, requests_mock, datadir, mocker)
     assert_sleep_calls(mocker, mock_sleep, [1, WAIT_EXP_BASE])
 
 
+def test_lister_gitlab_credentials(swh_scheduler):
+    """Gitlab lister supports credentials configuration
+
+    """
+    instance = "gitlab"
+    credentials = {
+        "gitlab": {instance: [{"username": "user", "password": "api-token"}]}
+    }
+    url = api_url(instance)
+    lister = GitLabLister(
+        scheduler=swh_scheduler, url=url, instance=instance, credentials=credentials
+    )
+    assert lister.session.headers["Authorization"] == "Bearer api-token"
+
+
 @pytest.mark.parametrize(
     "url,expected_result",
     [