From 6bb36697bea8288c9d84a77c4e0dc0fc82bf27c6 Mon Sep 17 00:00:00 2001
From: Antoine Eiche <lewo@abesis.fr>
Date: Wed, 25 Mar 2020 17:52:19 +0100
Subject: [PATCH] client: add `visits` method to get the list of an origin
visits
A test is added to ensure the pagination and the data typing are
working as expected.
---
swh/web/client/client.py | 46 +++++
swh/web/client/tests/api_data.py | 210 ++++++++++++++++++++
swh/web/client/tests/conftest.py | 10 +-
swh/web/client/tests/gen-api-data.sh | 2 +
swh/web/client/tests/test_web_api_client.py | 15 ++
5 files changed, 281 insertions(+), 2 deletions(-)
diff --git a/swh/web/client/client.py b/swh/web/client/client.py
index 9ea2200..a1a4b40 100644
--- a/swh/web/client/client.py
+++ b/swh/web/client/client.py
@@ -43,6 +43,8 @@ from .auth import AuthenticationError, OpenIDConnectSession, SWH_OIDC_SERVER_URL
PIDish = Union[PID, str]
+ORIGIN_VISIT = "origin_visit"
+
def _get_pid(pidish: PIDish) -> PID:
"""Parse string to PID if needed"""
@@ -104,6 +106,10 @@ def typify(data: Any, obj_type: str) -> Any:
)
elif obj_type == CONTENT:
pass # nothing to do for contents
+ elif obj_type == ORIGIN_VISIT:
+ data['date'] = to_date(data['date'])
+ if data['snapshot'] is not None:
+ data['snapshot'] = to_pid(SNAPSHOT, data['snapshot'])
else:
raise ValueError(f"invalid object type: {obj_type}")
@@ -330,6 +336,46 @@ class WebAPIClient:
else:
done = True
+ def visits(self,
+ origin: str,
+ per_page: Optional[int] = None,
+ last_visit: Optional[int] = None,
+ **req_args) -> Generator[Dict[str, Any], None, None]:
+ """List visits of an origin
+
+ Args:
+ origin: the URL of a software origin
+ per_page: the number of visits to list
+ last_visit: visit to start listing from
+ req_args: extra keyword arguments for requests.get()
+
+ Returns:
+ an iterator over visits of the origin
+
+ Raises:
+ requests.HTTPError: if HTTP request fails
+
+ """
+ done = False
+ r = None
+
+ params = []
+ if last_visit is not None:
+ params.append(("last_visit", last_visit))
+ if per_page is not None:
+ params.append(("per_page", per_page))
+
+ query = f'origin/{origin}/visits/'
+
+ while not done:
+ r = self._call(query, http_method='get', params=params, **req_args)
+ yield from [typify(v, ORIGIN_VISIT) for v in r.json()]
+ if 'next' in r.links and 'url' in r.links['next']:
+ params = []
+ query = r.links['next']['url']
+ else:
+ done = True
+
def content_exists(self, pid: PIDish, **req_args) -> bool:
"""Check if a content object exists in the archive
diff --git a/swh/web/client/tests/api_data.py b/swh/web/client/tests/api_data.py
index 34136b4..f804ee6 100644
--- a/swh/web/client/tests/api_data.py
+++ b/swh/web/client/tests/api_data.py
@@ -7511,4 +7511,214 @@ API_DATA = {
"next_branch": null
}
""", # NoQA: E501 # NoQA: E501
+ "origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=50&per_page=10": # NoQA: E501
+ r"""
+[
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 49,
+ "date": "2018-07-31T04:34:23.298931+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/49/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 48,
+ "date": "2018-07-27T10:12:23.069912+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/48/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 47,
+ "date": "2018-07-24T15:34:14.433021+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/47/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 46,
+ "date": "2018-07-23T22:07:33.025195+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/46/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 45,
+ "date": "2018-07-21T16:50:41.944584+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/45/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 44,
+ "date": "2018-07-21T12:17:24.044885+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/44/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 43,
+ "date": "2018-07-20T13:58:53.926748+00:00",
+ "type": "git",
+ "status": "full",
+ "snapshot": "c5d63cd27b09068b1f6913a90dfe44f1276091c2",
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/43/",
+ "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/c5d63cd27b09068b1f6913a90dfe44f1276091c2/"
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 42,
+ "date": "2018-07-20T00:38:20.356513+00:00",
+ "type": "git",
+ "status": "full",
+ "snapshot": "456550ea74af4e2eecaa406629efaaf0b9b5f976",
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/42/",
+ "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/456550ea74af4e2eecaa406629efaaf0b9b5f976/"
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 41,
+ "date": "2018-07-18T12:42:33.408489+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/41/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 40,
+ "date": "2018-07-17T16:40:42.302353+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/40/",
+ "snapshot_url": null
+ }
+]
+ """, # NoQA: E501
+ "origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=40&per_page=10": # NoQA: E501
+ r"""
+[
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 39,
+ "date": "2018-07-16T23:24:15.061318+00:00",
+ "type": "git",
+ "status": "full",
+ "snapshot": "1c2d51e9cda9958736394454ce8875e5913a1925",
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/39/",
+ "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/1c2d51e9cda9958736394454ce8875e5913a1925/"
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 38,
+ "date": "2018-07-16T10:43:14.651509+00:00",
+ "type": "git",
+ "status": "full",
+ "snapshot": "8fc8f205ba2d16a798b85467bd9f3cc6b938cc5c",
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/38/",
+ "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/8fc8f205ba2d16a798b85467bd9f3cc6b938cc5c/"
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 37,
+ "date": "2018-07-15T18:45:41.744499+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/37/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 36,
+ "date": "2018-07-15T10:41:05.001029+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/36/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 35,
+ "date": "2018-07-14T10:42:58.232545+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/35/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 34,
+ "date": "2018-07-14T05:31:45.565907+00:00",
+ "type": "git",
+ "status": "full",
+ "snapshot": "588fa42fefd27047474072da57ac2c83c3a481a7",
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/34/",
+ "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/588fa42fefd27047474072da57ac2c83c3a481a7/"
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 33,
+ "date": "2018-07-12T22:17:10.133596+00:00",
+ "type": "git",
+ "status": "full",
+ "snapshot": "9f9c7b2d52038b1d1fb6b1ff7387f0446c3f9da0",
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/33/",
+ "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/9f9c7b2d52038b1d1fb6b1ff7387f0446c3f9da0/"
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 32,
+ "date": "2018-07-12T14:45:54.291987+00:00",
+ "type": "git",
+ "status": "partial",
+ "snapshot": null,
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/32/",
+ "snapshot_url": null
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 31,
+ "date": "2018-07-11T18:30:43.843317+00:00",
+ "type": "git",
+ "status": "full",
+ "snapshot": "3d4c9c5c9a7de4d174b8a02d23bf4a55b2d3a911",
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/31/",
+ "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/3d4c9c5c9a7de4d174b8a02d23bf4a55b2d3a911/"
+ },
+ {
+ "origin": "https://github.com/NixOS/nixpkgs",
+ "visit": 30,
+ "date": "2018-07-10T07:37:24.991560+00:00",
+ "type": "git",
+ "status": "full",
+ "snapshot": "100de51846f317e6ab48da79d985cefa6fdefe42",
+ "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/30/",
+ "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/100de51846f317e6ab48da79d985cefa6fdefe42/"
+ }
+]
+ """, # NoQA: E501
}
diff --git a/swh/web/client/tests/conftest.py b/swh/web/client/tests/conftest.py
index 2cf4567..a0b9d49 100644
--- a/swh/web/client/tests/conftest.py
+++ b/swh/web/client/tests/conftest.py
@@ -11,14 +11,20 @@ from swh.web.client import WebAPIClient
@pytest.fixture
def web_api_mock(requests_mock):
+ # monkey patch URLs that require a special response headers
for api_call, data in API_DATA.items():
headers = {}
if api_call == "snapshot/cabcc7d7bf639bbe1cc3b41989e1806618dd5764/":
- # monkey patch the only URL that require a special response headers
- # (to make the client init and follow pagination)
+ # to make the client init and follow pagination
headers = {
"Link": f'<{API_URL}/{api_call}?branches_count=1000&branches_from=refs/tags/v3.0-rc7>; rel="next"' # NoQA: E501
}
+ elif api_call == "origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=50&per_page=10": # NoQA: E501
+ # to make the client follow pagination
+ headers = {
+ "Link":
+ f"<{API_URL}/origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=40&per_page=10>; rel=\"next\"" # NoQA: E501
+ }
requests_mock.get(f"{API_URL}/{api_call}", text=data, headers=headers)
return requests_mock
diff --git a/swh/web/client/tests/gen-api-data.sh b/swh/web/client/tests/gen-api-data.sh
index 9245f26..72656b9 100755
--- a/swh/web/client/tests/gen-api-data.sh
+++ b/swh/web/client/tests/gen-api-data.sh
@@ -18,6 +18,8 @@ urls="${urls} revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/"
urls="${urls} snapshot/6a3a2cf0b2b90ce7ae1cf0a221ed68035b686f5a/"
urls="${urls} snapshot/cabcc7d7bf639bbe1cc3b41989e1806618dd5764/"
urls="${urls} snapshot/cabcc7d7bf639bbe1cc3b41989e1806618dd5764/?branches_count=1000&branches_from=refs/tags/v3.0-rc7"
+urls="${urls} origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=50&per_page=10"
+urls="${urls} origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=40&per_page=10"
echo "# GENERATED FILE, DO NOT EDIT."
echo "# Run './gen-api-data.sh > api_data.py' instead."
diff --git a/swh/web/client/tests/test_web_api_client.py b/swh/web/client/tests/test_web_api_client.py
index d190ce8..acc3935 100644
--- a/swh/web/client/tests/test_web_api_client.py
+++ b/swh/web/client/tests/test_web_api_client.py
@@ -205,3 +205,18 @@ def test_authenticate_failure(web_api_client, web_api_mock):
web_api_client.authenticate(refresh_token)
assert e.match(repr(oidc_error_response))
+
+
+def test_get_visits(web_api_client, web_api_mock):
+ obj = web_api_client.visits('https://github.com/NixOS/nixpkgs',
+ last_visit=50,
+ per_page=10)
+ visits = [v for v in obj]
+ assert len(visits) == 20
+
+ timestamp = parse_date('2018-07-31 04:34:23.298931+00:00')
+ assert visits[0]['date'] == timestamp
+
+ assert visits[0]["snapshot"] is None
+ snapshot_pid = 'swh:1:snp:456550ea74af4e2eecaa406629efaaf0b9b5f976'
+ assert visits[7]["snapshot"] == parse_pid(snapshot_pid)
--
GitLab