packagist: Fix json parsing which is different depending on page

swh/lister/packagist/lister.py

@@ -143,6 +143,7 @@ class PackagistLister(Lister[PackagistListerState, PackagistPageType]):
             metadata_url = package_url_format.format(package_name=package_name)
             metadata = self.api_request(metadata_url)
             packages = metadata.get("packages", {})
+            format_json = metadata.get("minified")
             if not packages:
                 # package metadata not updated since last listing
                 return None
@@ -150,7 +151,16 @@ class PackagistLister(Lister[PackagistListerState, PackagistPageType]):
             if package_info is None:
                 # missing package metadata in response
                 return None
-            return package_info.values()  # could be an empty response though -> []
+            logger.debug(
+                "package-name: %s, package-info: %s", package_name, package_info
+            )
+            if format_json == "composer/2.0":  # /p2/ output
+                # In that format, the package info is a list of dict for each package
+                # version
+                return package_info
+            else:
+                # Otherwise, /p/, /packages/ urls returns a dict output
+                return package_info.values()
         except requests.HTTPError:
             # error when getting package metadata (usually 404 when a package has
             # been removed), skip it and process next package

swh/lister/packagist/tests/data/ljjackson_linnworks.json

 {
   "packages": {
-    "ljjackson/linnworks": {
-      "0.1": {
+    "ljjackson/linnworks": [
+      {
         "name": "ljjackson/linnworks",
         "description": "A PHP API Integration of Linnworks.",
         "keywords": [],
@@ -9,11 +9,13 @@
         "version": "0.1",
         "version_normalized": "0.1.0.0",
         "license": [],
-        "authors": [{
-          "name": "Liam Jackson",
-          "homepage": "https://github.com/ljjackson",
-          "role": "Developer"
-        }],
+        "authors": [
+          {
+            "name": "Liam Jackson",
+            "homepage": "https://github.com/ljjackson",
+            "role": "Developer"
+          }
+        ],
         "source": {
           "type": "git",
           "url": "https://github.com/ljjackson/linnworks.git",
@@ -37,47 +39,13 @@
           "guzzlehttp/guzzle": "^6.3",
           "ext-json": "*"
         },
-        "uid": 2535139
-      },
-      "dev-master": {
-        "name": "ljjackson/linnworks",
-        "description": "A PHP API Integration of Linnworks.",
-        "keywords": [],
-        "homepage": "https://github.com/ljjackson",
-        "version": "dev-master",
-        "version_normalized": "9999999-dev",
-        "license": [],
-        "authors": [{
-          "name": "Liam Jackson",
-          "homepage": "https://github.com/ljjackson",
-          "role": "Developer"
-        }],
-        "source": {
-          "type": "git",
-          "url": "https://github.com/ljjackson/linnworks.git",
-          "reference": "7c6b1209dc3bafad4284b130bda8450f3478ea26"
-        },
-        "dist": {
-          "type": "zip",
-          "url": "https://api.github.com/repos/ljjackson/linnworks/zipball/7c6b1209dc3bafad4284b130bda8450f3478ea26",
-          "reference": "7c6b1209dc3bafad4284b130bda8450f3478ea26",
-          "shasum": ""
-        },
-        "type": "library",
-        "time": "2018-11-01T21:45:50+00:00",
-        "autoload": {
-          "psr-4": {
-            "LJJackson\\Linnworks\\": "src/"
-          }
-        },
-        "require": {
-          "guzzlehttp/guzzle": "^6.3",
-          "ext-json": "*",
-          "php": "^7.1.3",
-          "nesbot/carbon": "*"
-        },
-        "uid": 2517334
+        "abandoned": true,
+        "support": {
+          "issues": "https://github.com/ljjackson/linnworks/issues",
+          "source": "https://github.com/ljjackson/linnworks/tree/0.1"
+        }
       }
-    }
-  }
-}
\ No newline at end of file
+    ]
+  },
+  "minified": "composer/2.0"
+}

swh/lister/packagist/tests/test_lister.py

@@ -66,7 +66,7 @@ def test_packagist_lister(swh_scheduler, requests_mock, datadir, requests_mock_d
         (
             "https://github.com/ljjackson/linnworks.git",  # API goes 404
             "git",
-            datetime.datetime.fromisoformat("2018-11-01T21:45:50+00:00"),
+            datetime.datetime.fromisoformat("2018-10-22T19:52:25+00:00"),
         ),
         (
             "https://github.com/spryker-eco/computop-api",  # SSH URL in manifest