diff --git a/swh/lister/puppet/__init__.py b/swh/lister/puppet/__init__.py
index e56cee6f69544591f48692aa1bebb5db0428db00..3e5e28d8dc542012dd8ae8278c1105b402925166 100644
--- a/swh/lister/puppet/__init__.py
+++ b/swh/lister/puppet/__init__.py
@@ -38,7 +38,7 @@ this pattern::
"https://forge.puppet.com/modules/{owner}/{pkgname}"
-For each origin `last_update`is set via the module "updated_at" value.
+For each origin `last_update` is set via the module "updated_at" value.
As the api also returns all existing versions for a package, we build an `artifacts`
dict in `extra_loader_arguments` with the archive tarball corresponding to each
existing versions.
@@ -47,20 +47,27 @@ Example for ``file_concat`` module located at
https://forge.puppet.com/modules/electrical/file_concat::
{
- "artifacts": {
- "1.0.0": {
- "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
- "version": "1.0.0",
- "filename": "electrical-file_concat-1.0.0.tar.gz",
- "last_update": "2015-04-09T12:03:13-07:00",
- },
- "1.0.1": {
+ "artifacts": [
+ {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", # noqa: B950
"version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz",
"last_update": "2015-04-17T01:03:46-07:00",
+ "checksums": {
+ "md5": "74901a89544134478c2dfde5efbb7f14",
+ "sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6", # noqa: B950
+ },
+ },
+ {
+ "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
+ "version": "1.0.0",
+ "filename": "electrical-file_concat-1.0.0.tar.gz",
+ "last_update": "2015-04-09T12:03:13-07:00",
+ "checksums": {
+ "length": 13289,
+ },
},
- }
+ ],
}
Running tests
diff --git a/swh/lister/puppet/lister.py b/swh/lister/puppet/lister.py
index 4982e9263dc704847366065529dd54d92105ac94..80ac3da26377f66db440e66593ccc76a4bbd0904 100644
--- a/swh/lister/puppet/lister.py
+++ b/swh/lister/puppet/lister.py
@@ -76,7 +76,7 @@ class PuppetLister(StatelessLister[PuppetListerPage]):
pkgname = entry["name"]
owner = entry["owner"]["slug"]
url = f"https://forge.puppet.com/modules/{owner}/{pkgname}"
- artifacts = {}
+ artifacts = []
for release in entry["releases"]:
# Build an artifact entry following original-artifacts-json specification
# https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950
@@ -92,15 +92,17 @@ class PuppetLister(StatelessLister[PuppetListerPage]):
# use file length as basic content check instead
checksums["length"] = release["file_size"]
- artifacts[release["version"]] = {
- "filename": release["file_uri"].split("/")[-1],
- "url": urljoin(self.BASE_URL, release["file_uri"]),
- "version": release["version"],
- "last_update": datetime.strptime(
- release["created_at"], dt_parse_pattern
- ).isoformat(),
- "checksums": checksums,
- }
+ artifacts.append(
+ {
+ "filename": release["file_uri"].split("/")[-1],
+ "url": urljoin(self.BASE_URL, release["file_uri"]),
+ "version": release["version"],
+ "last_update": datetime.strptime(
+ release["created_at"], dt_parse_pattern
+ ).isoformat(),
+ "checksums": checksums,
+ }
+ )
yield ListedOrigin(
lister_id=self.lister_obj.id,
diff --git a/swh/lister/puppet/tests/test_lister.py b/swh/lister/puppet/tests/test_lister.py
index 5dbfd899cf1fb2ac965739fc37a88795185ebd3f..80e5a633b23b314f6a66827f1d004c06185fa3a2 100644
--- a/swh/lister/puppet/tests/test_lister.py
+++ b/swh/lister/puppet/tests/test_lister.py
@@ -7,19 +7,11 @@ from swh.lister.puppet.lister import PuppetLister
# flake8: noqa: B950
-expected_origins = {
- "https://forge.puppet.com/modules/electrical/file_concat": {
- "artifacts": {
- "1.0.0": {
- "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz",
- "version": "1.0.0",
- "filename": "electrical-file_concat-1.0.0.tar.gz",
- "last_update": "2015-04-09T12:03:13-07:00",
- "checksums": {
- "length": 13289,
- },
- },
- "1.0.1": {
+expected_origins = [
+ {
+ "url": "https://forge.puppet.com/modules/electrical/file_concat",
+ "artifacts": [
+ {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz",
"version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz",
@@ -29,29 +21,21 @@ expected_origins = {
"sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6",
},
},
- }
- },
- "https://forge.puppet.com/modules/puppetlabs/puppetdb": {
- "artifacts": {
- "1.0.0": {
- "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz",
+ {
+ "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz",
"version": "1.0.0",
- "filename": "puppetlabs-puppetdb-1.0.0.tar.gz",
- "last_update": "2012-09-19T16:51:22-07:00",
- "checksums": {
- "length": 16336,
- },
- },
- "7.9.0": {
- "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz",
- "version": "7.9.0",
- "filename": "puppetlabs-puppetdb-7.9.0.tar.gz",
- "last_update": "2021-06-24T07:48:54-07:00",
+ "filename": "electrical-file_concat-1.0.0.tar.gz",
+ "last_update": "2015-04-09T12:03:13-07:00",
"checksums": {
- "length": 42773,
+ "length": 13289,
},
},
- "7.10.0": {
+ ],
+ },
+ {
+ "url": "https://forge.puppet.com/modules/puppetlabs/puppetdb",
+ "artifacts": [
+ {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.10.0.tar.gz",
"version": "7.10.0",
"filename": "puppetlabs-puppetdb-7.10.0.tar.gz",
@@ -61,20 +45,30 @@ expected_origins = {
"sha256": "49b1a542fbd2a1378c16cb04809e0f88bf4f3e45979532294fb1f03f56c97fbb",
},
},
- }
- },
- "https://forge.puppet.com/modules/saz/memcached": {
- "artifacts": {
- "1.0.0": {
- "url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz",
+ {
+ "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz",
+ "version": "7.9.0",
+ "filename": "puppetlabs-puppetdb-7.9.0.tar.gz",
+ "last_update": "2021-06-24T07:48:54-07:00",
+ "checksums": {
+ "length": 42773,
+ },
+ },
+ {
+ "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz",
"version": "1.0.0",
- "filename": "saz-memcached-1.0.0.tar.gz",
- "last_update": "2011-11-20T13:40:30-08:00",
+ "filename": "puppetlabs-puppetdb-1.0.0.tar.gz",
+ "last_update": "2012-09-19T16:51:22-07:00",
"checksums": {
- "length": 2472,
+ "length": 16336,
},
},
- "8.1.0": {
+ ],
+ },
+ {
+ "url": "https://forge.puppet.com/modules/saz/memcached",
+ "artifacts": [
+ {
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz",
"version": "8.1.0",
"filename": "saz-memcached-8.1.0.tar.gz",
@@ -84,9 +78,18 @@ expected_origins = {
"sha256": "883d6186e91c2c3fed13ae2009c3aa596657f6707b76f1f7efc6203c6e4ae986",
},
},
- }
+ {
+ "url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz",
+ "version": "1.0.0",
+ "filename": "saz-memcached-1.0.0.tar.gz",
+ "last_update": "2011-11-20T13:40:30-08:00",
+ "checksums": {
+ "length": 2472,
+ },
+ },
+ ],
},
-}
+]
def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler):
@@ -100,7 +103,18 @@ def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler):
assert len(scheduler_origins) == len(expected_origins)
- for origin in scheduler_origins:
- assert origin.visit_type == "puppet"
- assert origin.url in expected_origins
- assert origin.extra_loader_arguments == expected_origins[origin.url]
+ assert [
+ (
+ scheduled.visit_type,
+ scheduled.url,
+ scheduled.extra_loader_arguments["artifacts"],
+ )
+ for scheduled in sorted(scheduler_origins, key=lambda scheduled: scheduled.url)
+ ] == [
+ (
+ "puppet",
+ expected["url"],
+ expected["artifacts"],
+ )
+ for expected in sorted(expected_origins, key=lambda expected: expected["url"])
+ ]