diff --git a/swh/lister/packagist/lister.py b/swh/lister/packagist/lister.py index 5f7a4a5334ef622298542ebb13ef1d41743234ef..26f529c30d0f6911d21f5608c989770af0173dd5 100644 --- a/swh/lister/packagist/lister.py +++ b/swh/lister/packagist/lister.py @@ -143,6 +143,7 @@ class PackagistLister(Lister[PackagistListerState, PackagistPageType]): metadata_url = package_url_format.format(package_name=package_name) metadata = self.api_request(metadata_url) packages = metadata.get("packages", {}) + format_json = metadata.get("minified") if not packages: # package metadata not updated since last listing return None @@ -150,7 +151,16 @@ class PackagistLister(Lister[PackagistListerState, PackagistPageType]): if package_info is None: # missing package metadata in response return None - return package_info.values() # could be an empty response though -> [] + logger.debug( + "package-name: %s, package-info: %s", package_name, package_info + ) + if format_json == "composer/2.0": # /p2/ output + # In that format, the package info is a list of dict for each package + # version + return package_info + else: + # Otherwise, /p/, /packages/ urls returns a dict output + return package_info.values() except requests.HTTPError: # error when getting package metadata (usually 404 when a package has # been removed), skip it and process next package diff --git a/swh/lister/packagist/tests/data/ljjackson_linnworks.json b/swh/lister/packagist/tests/data/ljjackson_linnworks.json index ba57a81bd75c37f54dac0188724c9e238acfff9f..41c9207f43bbb0b15e2dcd0446e41fb471637e00 100644 --- a/swh/lister/packagist/tests/data/ljjackson_linnworks.json +++ b/swh/lister/packagist/tests/data/ljjackson_linnworks.json @@ -1,7 +1,7 @@ { "packages": { - "ljjackson/linnworks": { - "0.1": { + "ljjackson/linnworks": [ + { "name": "ljjackson/linnworks", "description": "A PHP API Integration of Linnworks.", "keywords": [], @@ -9,11 +9,13 @@ "version": "0.1", "version_normalized": "0.1.0.0", "license": [], - "authors": [{ - "name": "Liam Jackson", - "homepage": "https://github.com/ljjackson", - "role": "Developer" - }], + "authors": [ + { + "name": "Liam Jackson", + "homepage": "https://github.com/ljjackson", + "role": "Developer" + } + ], "source": { "type": "git", "url": "https://github.com/ljjackson/linnworks.git", @@ -37,47 +39,13 @@ "guzzlehttp/guzzle": "^6.3", "ext-json": "*" }, - "uid": 2535139 - }, - "dev-master": { - "name": "ljjackson/linnworks", - "description": "A PHP API Integration of Linnworks.", - "keywords": [], - "homepage": "https://github.com/ljjackson", - "version": "dev-master", - "version_normalized": "9999999-dev", - "license": [], - "authors": [{ - "name": "Liam Jackson", - "homepage": "https://github.com/ljjackson", - "role": "Developer" - }], - "source": { - "type": "git", - "url": "https://github.com/ljjackson/linnworks.git", - "reference": "7c6b1209dc3bafad4284b130bda8450f3478ea26" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/ljjackson/linnworks/zipball/7c6b1209dc3bafad4284b130bda8450f3478ea26", - "reference": "7c6b1209dc3bafad4284b130bda8450f3478ea26", - "shasum": "" - }, - "type": "library", - "time": "2018-11-01T21:45:50+00:00", - "autoload": { - "psr-4": { - "LJJackson\\Linnworks\\": "src/" - } - }, - "require": { - "guzzlehttp/guzzle": "^6.3", - "ext-json": "*", - "php": "^7.1.3", - "nesbot/carbon": "*" - }, - "uid": 2517334 + "abandoned": true, + "support": { + "issues": "https://github.com/ljjackson/linnworks/issues", + "source": "https://github.com/ljjackson/linnworks/tree/0.1" + } } - } - } -} \ No newline at end of file + ] + }, + "minified": "composer/2.0" +} diff --git a/swh/lister/packagist/tests/test_lister.py b/swh/lister/packagist/tests/test_lister.py index fa8cdfcf2625b98eafc427ec65d7ac7dc4b216ec..5591314dedba772b77f4c6db0b542e2fee4d1bc6 100644 --- a/swh/lister/packagist/tests/test_lister.py +++ b/swh/lister/packagist/tests/test_lister.py @@ -66,7 +66,7 @@ def test_packagist_lister(swh_scheduler, requests_mock, datadir, requests_mock_d ( "https://github.com/ljjackson/linnworks.git", # API goes 404 "git", - datetime.datetime.fromisoformat("2018-11-01T21:45:50+00:00"), + datetime.datetime.fromisoformat("2018-10-22T19:52:25+00:00"), ), ( "https://github.com/spryker-eco/computop-api", # SSH URL in manifest