From 5003e6588f0ce9ee1152eeb637f4275e62c51544 Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Fri, 23 Aug 2024 11:46:24 +0200
Subject: [PATCH] crates: Remove crates metadata as loader argument
Those extrinsic metadata can be directly fetched by the loader
through the crates Web API, plus it contains more metadata fields.
---
swh/lister/crates/lister.py | 10 -----
swh/lister/crates/tests/test_lister.py | 55 --------------------------
2 files changed, 65 deletions(-)
diff --git a/swh/lister/crates/lister.py b/swh/lister/crates/lister.py
index 73bf798b..fe8b56af 100644
--- a/swh/lister/crates/lister.py
+++ b/swh/lister/crates/lister.py
@@ -227,7 +227,6 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]):
last_update = crate_versions[0]["last_update"]
artifacts = []
- crates_metadata = []
for entry in crate_versions:
# Build an artifact entry following original-artifacts-json specification
@@ -243,14 +242,6 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]):
}
)
- crates_metadata.append(
- {
- "version": entry["version"],
- "yanked": entry["yanked"],
- "last_update": entry["last_update"],
- }
- )
-
yield ListedOrigin(
lister_id=self.lister_obj.id,
visit_type=self.VISIT_TYPE,
@@ -258,7 +249,6 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]):
last_update=iso8601.parse_date(last_update),
extra_loader_arguments={
"artifacts": artifacts,
- "crates_metadata": crates_metadata,
},
)
diff --git a/swh/lister/crates/tests/test_lister.py b/swh/lister/crates/tests/test_lister.py
index 1bcb52ef..646a4611 100644
--- a/swh/lister/crates/tests/test_lister.py
+++ b/swh/lister/crates/tests/test_lister.py
@@ -37,23 +37,6 @@ expected_origins = [
"url": "https://static.crates.io/crates/rand/rand-0.1.3-experimental.crate",
},
],
- "crates_metadata": [
- {
- "version": "0.1.1",
- "last_update": "2017-11-30 03:33:14.186028",
- "yanked": False,
- },
- {
- "version": "0.1.2",
- "last_update": "2017-11-30 03:14:27.545115",
- "yanked": False,
- },
- {
- "version": "0.1.3-experimental",
- "last_update": "2017-11-30 05:24:37.146115",
- "yanked": False,
- },
- ],
},
{
"url": "https://crates.io/crates/regex",
@@ -91,28 +74,6 @@ expected_origins = [
"url": "https://static.crates.io/crates/regex/regex-0.1.3.crate",
},
],
- "crates_metadata": [
- {
- "version": "0.1.0",
- "last_update": "2017-11-30 02:51:27.240551",
- "yanked": False,
- },
- {
- "version": "0.1.1",
- "last_update": "2017-11-30 03:03:20.143103",
- "yanked": False,
- },
- {
- "version": "0.1.2",
- "last_update": "2017-11-30 02:29:20.01125",
- "yanked": False,
- },
- {
- "version": "0.1.3",
- "last_update": "2017-11-30 02:26:59.236947",
- "yanked": False,
- },
- ],
},
{
"url": "https://crates.io/crates/regex-syntax",
@@ -126,13 +87,6 @@ expected_origins = [
"url": "https://static.crates.io/crates/regex-syntax/regex-syntax-0.1.0.crate", # noqa: B950
},
],
- "crates_metadata": [
- {
- "version": "0.1.0",
- "last_update": "2017-11-30 03:37:17.449539",
- "yanked": False,
- },
- ],
},
]
@@ -148,13 +102,6 @@ expected_origins_incremental = {
},
}
],
- "crates_metadata": [
- {
- "version": "1.0.12",
- "yanked": False,
- "last_update": "2022-08-15 13:52:11.642129",
- }
- ],
}
@@ -185,7 +132,6 @@ def test_crates_lister(datadir, tmp_path, swh_scheduler, requests_mock_datadir):
scheduled.visit_type,
scheduled.url,
scheduled.extra_loader_arguments["artifacts"],
- scheduled.extra_loader_arguments["crates_metadata"],
)
for scheduled in sorted(scheduler_origins, key=lambda scheduled: scheduled.url)
] == [
@@ -193,7 +139,6 @@ def test_crates_lister(datadir, tmp_path, swh_scheduler, requests_mock_datadir):
"crates",
expected["url"],
expected["artifacts"],
- expected["crates_metadata"],
)
for expected in sorted(expected_origins, key=lambda expected: expected["url"])
]
--
GitLab