Skip to content
Snippets Groups Projects
Commit 8355fee2 authored by Franck Bret's avatar Franck Bret
Browse files

Puppet: Switch artifacts from dict to list

parent ca4ab7f2
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
swh/lister/puppet/__init__.py
+ 17
− 10
View file @ 8355fee2
...@@ -38,7 +38,7 @@ this pattern:: ...@@ -38,7 +38,7 @@ this pattern::
"https://forge.puppet.com/modules/{owner}/{pkgname}" "https://forge.puppet.com/modules/{owner}/{pkgname}"
For each origin `last_update`is set via the module "updated_at" value. For each origin `last_update` is set via the module "updated_at" value.
As the api also returns all existing versions for a package, we build an `artifacts` As the api also returns all existing versions for a package, we build an `artifacts`
dict in `extra_loader_arguments` with the archive tarball corresponding to each dict in `extra_loader_arguments` with the archive tarball corresponding to each
existing versions. existing versions.
...@@ -47,20 +47,27 @@ Example for ``file_concat`` module located at ...@@ -47,20 +47,27 @@ Example for ``file_concat`` module located at
https://forge.puppet.com/modules/electrical/file_concat:: https://forge.puppet.com/modules/electrical/file_concat::
{ {
"artifacts": { "artifacts": [
"1.0.0": { {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
},
"1.0.1": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", # noqa: B950 "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", # noqa: B950
"version": "1.0.1", "version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz", "filename": "electrical-file_concat-1.0.1.tar.gz",
"last_update": "2015-04-17T01:03:46-07:00", "last_update": "2015-04-17T01:03:46-07:00",
"checksums": {
"md5": "74901a89544134478c2dfde5efbb7f14",
"sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6", # noqa: B950
},
},
{
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", # noqa: B950
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
"checksums": {
"length": 13289,
},
}, },
} ],
} }
Running tests Running tests
......
swh/lister/puppet/lister.py
+ 12
− 10
View file @ 8355fee2
...@@ -76,7 +76,7 @@ class PuppetLister(StatelessLister[PuppetListerPage]): ...@@ -76,7 +76,7 @@ class PuppetLister(StatelessLister[PuppetListerPage]):
pkgname = entry["name"] pkgname = entry["name"]
owner = entry["owner"]["slug"] owner = entry["owner"]["slug"]
url = f"https://forge.puppet.com/modules/{owner}/{pkgname}" url = f"https://forge.puppet.com/modules/{owner}/{pkgname}"
artifacts = {} artifacts = []
for release in entry["releases"]: for release in entry["releases"]:
# Build an artifact entry following original-artifacts-json specification # Build an artifact entry following original-artifacts-json specification
# https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950 # https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950
...@@ -92,15 +92,17 @@ class PuppetLister(StatelessLister[PuppetListerPage]): ...@@ -92,15 +92,17 @@ class PuppetLister(StatelessLister[PuppetListerPage]):
# use file length as basic content check instead # use file length as basic content check instead
checksums["length"] = release["file_size"] checksums["length"] = release["file_size"]
artifacts[release["version"]] = { artifacts.append(
"filename": release["file_uri"].split("/")[-1], {
"url": urljoin(self.BASE_URL, release["file_uri"]), "filename": release["file_uri"].split("/")[-1],
"version": release["version"], "url": urljoin(self.BASE_URL, release["file_uri"]),
"last_update": datetime.strptime( "version": release["version"],
release["created_at"], dt_parse_pattern "last_update": datetime.strptime(
).isoformat(), release["created_at"], dt_parse_pattern
"checksums": checksums, ).isoformat(),
} "checksums": checksums,
}
)
yield ListedOrigin( yield ListedOrigin(
lister_id=self.lister_obj.id, lister_id=self.lister_obj.id,
......
swh/lister/puppet/tests/test_lister.py
+ 62
− 48
View file @ 8355fee2
...@@ -7,19 +7,11 @@ from swh.lister.puppet.lister import PuppetLister ...@@ -7,19 +7,11 @@ from swh.lister.puppet.lister import PuppetLister
# flake8: noqa: B950 # flake8: noqa: B950
expected_origins = { expected_origins = [
"https://forge.puppet.com/modules/electrical/file_concat": { {
"artifacts": { "url": "https://forge.puppet.com/modules/electrical/file_concat",
"1.0.0": { "artifacts": [
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz", {
"version": "1.0.0",
"filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2015-04-09T12:03:13-07:00",
"checksums": {
"length": 13289,
},
},
"1.0.1": {
"url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz", "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.1.tar.gz",
"version": "1.0.1", "version": "1.0.1",
"filename": "electrical-file_concat-1.0.1.tar.gz", "filename": "electrical-file_concat-1.0.1.tar.gz",
...@@ -29,29 +21,21 @@ expected_origins = { ...@@ -29,29 +21,21 @@ expected_origins = {
"sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6", "sha256": "15e973613ea038d8a4f60bafe2d678f88f53f3624c02df3157c0043f4a400de6",
}, },
}, },
} {
}, "url": "https://forgeapi.puppet.com/v3/files/electrical-file_concat-1.0.0.tar.gz",
"https://forge.puppet.com/modules/puppetlabs/puppetdb": {
"artifacts": {
"1.0.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz",
"version": "1.0.0", "version": "1.0.0",
"filename": "puppetlabs-puppetdb-1.0.0.tar.gz", "filename": "electrical-file_concat-1.0.0.tar.gz",
"last_update": "2012-09-19T16:51:22-07:00", "last_update": "2015-04-09T12:03:13-07:00",
"checksums": {
"length": 16336,
},
},
"7.9.0": {
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz",
"version": "7.9.0",
"filename": "puppetlabs-puppetdb-7.9.0.tar.gz",
"last_update": "2021-06-24T07:48:54-07:00",
"checksums": { "checksums": {
"length": 42773, "length": 13289,
}, },
}, },
"7.10.0": { ],
},
{
"url": "https://forge.puppet.com/modules/puppetlabs/puppetdb",
"artifacts": [
{
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.10.0.tar.gz", "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.10.0.tar.gz",
"version": "7.10.0", "version": "7.10.0",
"filename": "puppetlabs-puppetdb-7.10.0.tar.gz", "filename": "puppetlabs-puppetdb-7.10.0.tar.gz",
...@@ -61,20 +45,30 @@ expected_origins = { ...@@ -61,20 +45,30 @@ expected_origins = {
"sha256": "49b1a542fbd2a1378c16cb04809e0f88bf4f3e45979532294fb1f03f56c97fbb", "sha256": "49b1a542fbd2a1378c16cb04809e0f88bf4f3e45979532294fb1f03f56c97fbb",
}, },
}, },
} {
}, "url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-7.9.0.tar.gz",
"https://forge.puppet.com/modules/saz/memcached": { "version": "7.9.0",
"artifacts": { "filename": "puppetlabs-puppetdb-7.9.0.tar.gz",
"1.0.0": { "last_update": "2021-06-24T07:48:54-07:00",
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz", "checksums": {
"length": 42773,
},
},
{
"url": "https://forgeapi.puppet.com/v3/files/puppetlabs-puppetdb-1.0.0.tar.gz",
"version": "1.0.0", "version": "1.0.0",
"filename": "saz-memcached-1.0.0.tar.gz", "filename": "puppetlabs-puppetdb-1.0.0.tar.gz",
"last_update": "2011-11-20T13:40:30-08:00", "last_update": "2012-09-19T16:51:22-07:00",
"checksums": { "checksums": {
"length": 2472, "length": 16336,
}, },
}, },
"8.1.0": { ],
},
{
"url": "https://forge.puppet.com/modules/saz/memcached",
"artifacts": [
{
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz", "url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz",
"version": "8.1.0", "version": "8.1.0",
"filename": "saz-memcached-8.1.0.tar.gz", "filename": "saz-memcached-8.1.0.tar.gz",
...@@ -84,9 +78,18 @@ expected_origins = { ...@@ -84,9 +78,18 @@ expected_origins = {
"sha256": "883d6186e91c2c3fed13ae2009c3aa596657f6707b76f1f7efc6203c6e4ae986", "sha256": "883d6186e91c2c3fed13ae2009c3aa596657f6707b76f1f7efc6203c6e4ae986",
}, },
}, },
} {
"url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz",
"version": "1.0.0",
"filename": "saz-memcached-1.0.0.tar.gz",
"last_update": "2011-11-20T13:40:30-08:00",
"checksums": {
"length": 2472,
},
},
],
}, },
} ]
def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler): def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler):
...@@ -100,7 +103,18 @@ def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler): ...@@ -100,7 +103,18 @@ def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler):
assert len(scheduler_origins) == len(expected_origins) assert len(scheduler_origins) == len(expected_origins)
for origin in scheduler_origins: assert [
assert origin.visit_type == "puppet" (
assert origin.url in expected_origins scheduled.visit_type,
assert origin.extra_loader_arguments == expected_origins[origin.url] scheduled.url,
scheduled.extra_loader_arguments["artifacts"],
)
for scheduled in sorted(scheduler_origins, key=lambda scheduled: scheduled.url)
] == [
(
"puppet",
expected["url"],
expected["artifacts"],
)
for expected in sorted(expected_origins, key=lambda expected: expected["url"])
]
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment