crates: Loader implements incremental mode

Add incremental support based on sha256 EXTID
Manage release date for each versions of a package
Adapt test dataset and add incremental test cases

Related T4104

docs/package-loader-specifications.rst

@@ -67,13 +67,13 @@ Here is an overview of the fields (+ internal version name + branch name) used b
      - metadata is intrinsic
    * - crates
      - ``p_info.​version``
-     - ``release_name(​version, filename) + "\n\n" + i_metadata.description + "\n"``
+     - ``release_name(​version)``
      - =version
-     - Synthetic release for Crate source package {p_info.name} version {p_info.version} {description}
+     - Synthetic release for Crate source package {p_info.name} version {p_info.version}
      - true
-     - from int metadata
-     - from ext metadata
-     - ``i_metadata`` for intrinsic metadata, ``e_metadata`` for extrinsic metadata
+     - from intrinsic metadata
+     - from extrinsic metadata
+     - ""
    * - debian
      - =``version``
      - ``release_name(​version)``

swh/loader/package/crates/loader.py

@@ -3,139 +3,35 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-from distutils.version import StrictVersion
+from datetime import datetime
 import json
 from pathlib import Path
+import string
 from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple
 from urllib.parse import urlparse
 
 import attr
+from packaging.version import parse as parse_version
 import toml
-from typing_extensions import TypedDict
 
-from swh.loader.package.loader import BasePackageInfo, PackageLoader
-from swh.loader.package.utils import cached_method, get_url_body, release_name
-from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone
+from swh.loader.package.loader import (
+    BasePackageInfo,
+    PackageLoader,
+    RawExtrinsicMetadataCore,
+)
+from swh.loader.package.utils import EMPTY_AUTHOR, release_name
+from swh.model.model import (
+    MetadataAuthority,
+    MetadataAuthorityType,
+    ObjectType,
+    Person,
+    Release,
+    Sha1Git,
+    TimestampWithTimezone,
+)
 from swh.storage.interface import StorageInterface
 
 
-class ExtrinsicPackageMetadata(TypedDict):
-    """Data structure for package extrinsic metadata pulled from http api endpoint.
-
-    We set only the keys we need according to what is available when querying
-    https://crates.io/api/v1/crates/<name>, where `name` is the name of the crate
-    package (see JSON response example at https://crates.io/api/v1/crates/hg-core).
-
-    Usage example:
-
-    .. code-block:: python
-
-        e_metadata = ExtrinsicPackageMetadata(**self.info())
-
-    """  # noqa
-
-    categories: List[Dict[Any, Any]]
-    """Related categories"""
-
-    crate: Dict[Any, Any]
-    """Crate project information"""
-
-    keywords: List[Any]
-    """Keywords"""
-
-    versions: List[Dict[Any, Any]]
-    """A list of released versions for a crate"""
-
-
-class ExtrinsicVersionPackageMetadata(TypedDict):
-    """Data structure for specific package version extrinsic metadata, pulled
-    from http api endpoint.
-
-    Similar to `ExtrinsicPackageMetadata` in its usage, but we flatten the data
-    related to a specific version.
-    """
-
-    crate: str
-    """The package name"""
-
-    crate_size: int
-    """The package size"""
-
-    created_at: str
-    """First released at"""
-
-    downloads: str
-    """Number of downloads"""
-
-    license: str
-    """Package license"""
-
-    num: str
-    """Package version"""
-
-    published_by: Dict[Any, Any]
-    """Publishers information"""
-
-    updated_at: str
-    """Last update"""
-
-    yanked: bool
-    """Is that version yanked? (yanked means release-level deprecation)"""
-
-
-class IntrinsicPackageMetadata(TypedDict):
-    """Data structure for specific package version intrinsic metadata.
-
-    Data is extracted from the crate package's .toml file. Then the data of the
-    'package' entry is flattened.
-
-    Cargo.toml file content example:
-
-    .. code-block:: toml
-
-        [package]
-        name = "hg-core"
-        version = "0.0.1"
-        authors = ["Georges Racinet <georges.racinet@octobus.net>"]
-        description = "Mercurial pure Rust core library, with no assumption on
-        Python bindings (FFI)"
-        homepage = "https://mercurial-scm.org"
-        license = "GPL-2.0-or-later"
-        repository = "https://www.mercurial-scm.org/repo/hg"
-
-        [lib]
-        name = "hg"
-        [dev-dependencies.rand]
-        version = "~0.6"
-
-        [dev-dependencies.rand_pcg]
-        version = "~0.1"
-
-    :param toml: toml object
-    """
-
-    name: str
-    """The package name"""
-
-    version: str
-    """Package version"""
-
-    authors: List[str]
-    """Authors"""
-
-    description: str
-    """Package and release description"""
-
-    homepage: str
-    """Homepage of the project"""
-
-    license: str
-    """Package license"""
-
-    repository: str
-    """Source code repository"""
-
-
 @attr.s
 class CratesPackageInfo(BasePackageInfo):
 
@@ -145,16 +41,20 @@ class CratesPackageInfo(BasePackageInfo):
     version = attr.ib(type=str)
     """Current version"""
 
-    e_metadata: Dict[str, Any] = attr.ib(factory=ExtrinsicPackageMetadata)
-    """Extrinsic package metadata, common to all versions"""
+    sha256 = attr.ib(type=str)
+    """Extid as sha256"""
 
-    e_metadata_version: Dict[str, Any] = attr.ib(
-        factory=ExtrinsicVersionPackageMetadata
-    )
-    """Extrinsic package metadata specific to a version"""
+    last_update = attr.ib(type=datetime)
+    """Last update as release date"""
 
-    i_metadata: Dict[str, Any] = attr.ib(factory=IntrinsicPackageMetadata)
-    """Intrinsic metadata of the current package version"""
+    yanked = attr.ib(type=bool)
+    """Whether the package is yanked or not"""
+
+    MANIFEST_FORMAT = string.Template(
+        "name $name\nshasum $sha256\nurl $url\nversion $version\nlast_update $last_update"
+    )
+    EXTID_TYPE = "crates-manifest-sha256"
+    EXTID_VERSION = 0
 
 
 def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]:
@@ -171,35 +71,6 @@ def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]:
     return toml.load(dir_path / "Cargo.toml")
 
 
-def extract_author(p_info: CratesPackageInfo) -> Person:
-    """Extract package author from intrinsic metadata and return it as a
-    `Person` model.
-
-    Args:
-        p_info: CratesPackageInfo that should contains i_metadata entries
-
-    Returns:
-        Only one author (Person) of the package. Currently limited by internal detail
-        of the swh stack (see T3887).
-    """
-    authors = p_info.i_metadata["authors"]
-    fullname = authors[0]  # TODO: here we have a list of author, see T3887
-    return Person.from_fullname(fullname.encode())
-
-
-def extract_description(p_info: CratesPackageInfo) -> str:
-    """Extract package description from intrinsic metadata and return it as a
-    string.
-
-    Args:
-        p_info: CratesPackageInfo that should contains i_metadata and entries
-
-    Returns:
-        Package description from metadata.
-    """
-    return p_info.i_metadata["description"]
-
-
 class CratesLoader(PackageLoader[CratesPackageInfo]):
     """Load Crates package origins into swh archive."""
 
@@ -210,6 +81,7 @@ class CratesLoader(PackageLoader[CratesPackageInfo]):
         storage: StorageInterface,
         url: str,
         artifacts: List[Dict[str, Any]],
+        crates_metadata: List[Dict[str, Any]],
         **kwargs,
     ):
         """Constructor
@@ -223,8 +95,8 @@ class CratesLoader(PackageLoader[CratesPackageInfo]):
                 A list of dict listing all existing released versions for a
                 package (Usually set with crates lister `extra_loader_arguments`).
                 Each line is a dict that should have an `url`
-                (where to download package specific version) and a `version` entry.
-
+                (where to download package specific version), a `version`, a
+                `filename` and a `checksums['sha256']` entry.
 
                 Example::
 
@@ -232,29 +104,34 @@ class CratesLoader(PackageLoader[CratesPackageInfo]):
                         {
                             "version": <version>,
                             "url": "https://static.crates.io/crates/<package_name>/<package_name>-<version>.crate",
+                            "filename": "<package_name>-<version>.crate",
+                            "checksums": {
+                                "sha256": "<sha256>",
+                            },
                         }
                     ]
+
+            crates_metadata:
+                Same as previously but for Crates metadata.
+                For now it only has one boolean key `yanked`.
+
+                Example::
+
+                    [
+                        {
+                            "version": "<version>",
+                            "yanked": <yanked>,
+                        },
+                    ]
         """  # noqa
         super().__init__(storage=storage, url=url, **kwargs)
         self.url = url
         self.artifacts: Dict[str, Dict] = {
             artifact["version"]: artifact for artifact in artifacts
         }
-
-    @cached_method
-    def _raw_info(self) -> bytes:
-        """Get crate metadata (fetched from http api endpoint set as self.url)
-
-        Returns:
-            Content response as bytes. Content response is a json document.
-        """
-        return get_url_body(self.url)
-
-    @cached_method
-    def info(self) -> Dict:
-        """Parse http api json response and return the crate metadata information
-        as a Dict."""
-        return json.loads(self._raw_info())
+        self.crates_metadata: Dict[str, Dict] = {
+            data["version"]: data for data in crates_metadata
+        }
 
     def get_versions(self) -> Sequence[str]:
         """Get all released versions of a crate
@@ -267,7 +144,7 @@ class CratesLoader(PackageLoader[CratesPackageInfo]):
                 ["0.1.1", "0.10.2"]
         """
         versions = list(self.artifacts.keys())
-        versions.sort(key=StrictVersion)
+        versions.sort(key=parse_version)
         return versions
 
     def get_default_version(self) -> str:
@@ -282,6 +159,12 @@ class CratesLoader(PackageLoader[CratesPackageInfo]):
         """
         return self.get_versions()[-1]
 
+    def get_metadata_authority(self):
+        return MetadataAuthority(
+            type=MetadataAuthorityType.FORGE,
+            url="https://crates.io/",
+        )
+
     def get_package_info(self, version: str) -> Iterator[Tuple[str, CratesPackageInfo]]:
         """Get release name and package information from version
 
@@ -291,62 +174,63 @@ class CratesLoader(PackageLoader[CratesPackageInfo]):
         Returns:
             Iterator of tuple (release_name, p_info)
         """
-        artifact = self.artifacts[version]
+        artifact = self.artifacts[version].copy()
         filename = artifact["filename"]
+        assert artifact["checksums"]["sha256"]
+        sha256 = artifact["checksums"]["sha256"]
         package_name = urlparse(self.url).path.split("/")[-1]
         url = artifact["url"]
 
-        # Get extrinsic metadata from http api
-        e_metadata = ExtrinsicPackageMetadata(**self.info())  # type: ignore[misc]
+        crate_metadata = self.crates_metadata[version].copy()
+        yanked = crate_metadata["yanked"]
+        last_update = datetime.fromisoformat(crate_metadata["last_update"])
 
-        # Extract crate info for current version (One .crate file for a given version)
-        (crate_version,) = [
-            crate for crate in e_metadata["versions"] if crate["num"] == version
-        ]
-        e_metadata_version = ExtrinsicVersionPackageMetadata(  # type: ignore[misc]
-            **crate_version
-        )
+        # Remove "version" from artifact to follow "original-artifacts-json" extrinsic
+        # metadata format specifications
+        # See https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#extrinsic-metadata-formats  # noqa: B950
+        del artifact["version"]
 
         p_info = CratesPackageInfo(
             name=package_name,
             filename=filename,
             url=url,
             version=version,
-            e_metadata=e_metadata,
-            e_metadata_version=e_metadata_version,
+            sha256=sha256,
+            checksums={"sha256": sha256},
+            yanked=yanked,
+            last_update=last_update,
+            directory_extrinsic_metadata=[
+                RawExtrinsicMetadataCore(
+                    format="crates-package-json",
+                    metadata=json.dumps([crate_metadata]).encode(),
+                ),
+            ],
         )
         yield release_name(version, filename), p_info
 
     def build_release(
         self, p_info: CratesPackageInfo, uncompressed_path: str, directory: Sha1Git
     ) -> Optional[Release]:
+
         # Extract intrinsic metadata from dir_path/Cargo.toml
-        name = p_info.name
-        version = p_info.version
-        dir_path = Path(uncompressed_path, f"{name}-{version}")
-        i_metadata_raw = extract_intrinsic_metadata(dir_path)
-        # Get only corresponding key of IntrinsicPackageMetadata
-        i_metadata_keys = [k for k in IntrinsicPackageMetadata.__annotations__.keys()]
-        # We use data only from "package" entry
-        i_metadata = {
-            k: v for k, v in i_metadata_raw["package"].items() if k in i_metadata_keys
-        }
-        p_info.i_metadata = IntrinsicPackageMetadata(**i_metadata)  # type: ignore[misc]
+        dir_path = Path(uncompressed_path, f"{p_info.name}-{p_info.version}")
+        i_metadata = extract_intrinsic_metadata(dir_path)
+
+        author = EMPTY_AUTHOR
+        authors = i_metadata.get("package", {}).get("authors")
+        if authors and isinstance(authors, list):
+            # TODO: here we have a list of author, see T3887
+            author = Person.from_fullname(authors[0].encode())
 
-        author = extract_author(p_info)
-        description = extract_description(p_info)
         message = (
             f"Synthetic release for Crate source package {p_info.name} "
-            f"version {p_info.version}\n\n"
-            f"{description}\n"
+            f"version {p_info.version}\n"
         )
-        # The only way to get a value for updated_at is through extrinsic metadata
-        updated_at = p_info.e_metadata_version.get("updated_at")
 
         return Release(
-            name=version.encode(),
+            name=p_info.version.encode(),
+            date=TimestampWithTimezone.from_datetime(p_info.last_update),
             author=author,
-            date=TimestampWithTimezone.from_iso8601(updated_at),
             message=message.encode(),
             target_type=ObjectType.DIRECTORY,
             target=directory,

swh/loader/package/crates/tests/data/expected.json

+[
+  {
+    "url": "https://crates.io/api/v1/crates/hg-core",
+    "artifacts": [
+      {
+        "version": "0.0.1",
+        "checksums": {
+          "sha256": "233409cca39eccab8075d3414884a2e9096aa3f7ceb4139685134c9f2561e734"
+        },
+        "filename": "hg-core-0.0.1.crate",
+        "url": "https://static.crates.io/crates/hg-core/hg-core-0.0.1.crate"
+      }
+    ],
+    "crates_metadata": [
+      {
+        "version": "0.0.1",
+        "yanked": false,
+        "last_update": "2019-04-16T18:48:11.404457+00:00"
+      }
+    ]
+  },
+  {
+    "url": "https://crates.io/api/v1/crates/micro-timer",
+    "artifacts": [
+      {
+        "version": "0.1.0",
+        "checksums": {
+          "sha256": "837c3955f0dfbd1a95f4388e06acb2f1a8030e28d251867e603b6be7b93da783"
+        },
+        "filename": "micro-timer-0.1.0.crate",
+        "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.0.crate"
+      },
+      {
+        "version": "0.1.1",
+        "checksums": {
+          "sha256": "bfb0e972e961d6bfd8d344ccf05c29ae39b7d817ced30568204495aee4bab519"
+        },
+        "filename": "micro-timer-0.1.1.crate",
+        "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.1.crate"
+      },
+      {
+        "version": "0.1.2",
+        "checksums": {
+          "sha256": "53251c2ab69c97447b4d6bfb327f7608ba0f7108e4aa3d4f559934d6cd088d11"
+        },
+        "filename": "micro-timer-0.1.2.crate",
+        "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.2.crate"
+      },
+      {
+        "version": "0.2.0",
+        "checksums": {
+          "sha256": "d5a39e92178f6f0c8acb1111258327eab2447e2ab574e5868e2770ed28940d70"
+        },
+        "filename": "micro-timer-0.2.0.crate",
+        "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.2.0.crate"
+      },
+      {
+        "version": "0.2.1",
+        "checksums": {
+          "sha256": "7889cfebb345bd706ecb93e7b67e7e6af1d68ce16e3f0c109bf8b7bcb79e55af"
+        },
+        "filename": "micro-timer-0.2.1.crate",
+        "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.2.1.crate"
+      },
+      {
+        "version": "0.3.0",
+        "checksums": {
+          "sha256": "d92b84c26f807891fd492ceecc11ca2bdaf73583665275080ead1c127ed861a4"
+        },
+        "filename": "micro-timer-0.3.0.crate",
+        "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.3.0.crate"
+      },
+      {
+        "version": "0.3.1",
+        "checksums": {
+          "sha256": "740acae0cfa83cf78207aab16ba4a41dce7101bca774c63eac8c41fd5ce382a5"
+        },
+        "filename": "micro-timer-0.3.1.crate",
+        "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.3.1.crate"
+      },
+      {
+        "version": "0.4.0",
+        "checksums": {
+          "sha256": "475222c2def55b7166390398a0c033ceb9a1cdfe008ff6ebd22773fb2029a60f"
+        },
+        "filename": "micro-timer-0.4.0.crate",
+        "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.4.0.crate"
+      }
+    ],
+    "crates_metadata": [
+      {
+        "version": "0.1.0",
+        "yanked": false,
+        "last_update": "2020-02-27T14:31:49.131258+00:00"
+      },
+      {
+        "version": "0.1.1",
+        "yanked": false,
+        "last_update": "2020-02-27T15:17:53.486346+00:00"
+      },
+      {
+        "version": "0.1.2",
+        "yanked": false,
+        "last_update": "2020-02-27T23:35:41.872176+00:00"
+      },
+      {
+        "version": "0.2.0",
+        "yanked": false,
+        "last_update": "2020-03-23T10:57:04.418462+00:00"
+      },
+      {
+        "version": "0.2.1",
+        "yanked": false,
+        "last_update": "2020-03-23T11:22:26.288804+00:00"
+      },
+      {
+        "version": "0.3.0",
+        "yanked": false,
+        "last_update": "2020-06-02T11:38:33.047581+00:00"
+      },
+      {
+        "version": "0.3.1",
+        "yanked": false,
+        "last_update": "2020-06-22T16:40:06.754009+00:00"
+      },
+      {
+        "version": "0.4.0",
+        "yanked": false,
+        "last_update": "2020-09-28T13:40:49.593030+00:00"
+      }
+    ]
+  }
+]

swh/loader/package/crates/tests/data/fake_crates.sh

 #!/usr/bin/env bash
 
-# Script to generate fake crates files and fake http api response.
+# Script to generate fake crates files.
 
 set -euo pipefail
 
@@ -45,7 +45,7 @@ echo -e '''[package]
 edition = "2018"
 name = "micro-timer"
 version = "0.1.0"
-authors = ["Raphaël Gomès <rgomes@octobus.net>"]
+# authors = ["Raphaël Gomès <rgomes@octobus.net>"]  # commented for testing empty authors
 description = "Dumb tiny logging timer"
 homepage = "https://heptapod.octobus.net/Alphare/micro-timer"
 readme = "README.md"
@@ -243,17 +243,6 @@ cp micro-timer-0.3.0.crate ../../https_static.crates.io/crates_micro-timer_micro
 cp micro-timer-0.3.1.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.3.1.crate
 cp micro-timer-0.4.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.4.0.crate
 
-# Creates some http file response for test purposes.
-mkdir ../../https_crates.io
-
-# hg-core, https://crates.io/api/v1/crates/hg-core
-echo -e '''{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2019-04-16T18:48:11.404457+00:00","description":"Mercurial pure Rust core library, with no assumption on Python bindings (FFI)","documentation":null,"downloads":442,"exact_match":false,"homepage":"https://mercurial-scm.org","id":"hg-core","keywords":[],"links":{"owner_team":"/api/v1/crates/hg-core/owner_team","owner_user":"/api/v1/crates/hg-core/owner_user","owners":"/api/v1/crates/hg-core/owners","reverse_dependencies":"/api/v1/crates/hg-core/reverse_dependencies","version_downloads":"/api/v1/crates/hg-core/downloads","versions":null},"max_stable_version":"0.0.1","max_version":"0.0.1","name":"hg-core","newest_version":"0.0.1","recent_downloads":40,"repository":"https://www.mercurial-scm.org/repo/hg","updated_at":"2019-04-16T18:48:11.404457+00:00","versions":[145309]},"keywords":[],"versions":[{"audit_actions":[],"crate":"hg-core","crate_size":21344,"created_at":"2019-04-16T18:48:11.404457+00:00","dl_path":"/api/v1/crates/hg-core/0.0.1/download","downloads":442,"features":{},"id":145309,"license":"GPL-2.0-or-later","links":{"authors":"/api/v1/crates/hg-core/0.0.1/authors","dependencies":"/api/v1/crates/hg-core/0.0.1/dependencies","version_downloads":"/api/v1/crates/hg-core/0.0.1/downloads"},"num":"0.0.1","published_by":{"avatar":"https://avatars0.githubusercontent.com/u/474220?v=4","id":45544,"login":"gracinet","name":"Georges Racinet","url":"https://github.com/gracinet"},"readme_path":"/api/v1/crates/hg-core/0.0.1/readme","updated_at":"2019-04-16T18:48:11.404457+00:00","yanked":false}]}
-''' > ../../https_crates.io/api_v1_crates_hg-core
-
-# micro-timer, https://crates.io/api/v1/crates/micro-timer
-echo -e '''{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2020-02-27T14:31:49.131258+00:00","description":"Dumb tiny logging timer","documentation":null,"downloads":44245,"exact_match":false,"homepage":"https://foss.heptapod.net/octobus/rust/micro-timer","id":"micro-timer","keywords":[],"links":{"owner_team":"/api/v1/crates/micro-timer/owner_team","owner_user":"/api/v1/crates/micro-timer/owner_user","owners":"/api/v1/crates/micro-timer/owners","reverse_dependencies":"/api/v1/crates/micro-timer/reverse_dependencies","version_downloads":"/api/v1/crates/micro-timer/downloads","versions":null},"max_stable_version":"0.4.0","max_version":"0.4.0","name":"micro-timer","newest_version":"0.4.0","recent_downloads":3910,"repository":"https://foss.heptapod.net/octobus/rust/micro-timer","updated_at":"2020-09-28T13:40:49.593030+00:00","versions":[288167,254896,248120,223660,223652,216405,216156,216139]},"keywords":[],"versions":[{"audit_actions":[{"action":"publish","time":"2020-09-28T13:40:49.593030+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3513,"created_at":"2020-09-28T13:40:49.593030+00:00","dl_path":"/api/v1/crates/micro-timer/0.4.0/download","downloads":337,"features":{},"id":288167,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.4.0/authors","dependencies":"/api/v1/crates/micro-timer/0.4.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.4.0/downloads"},"num":"0.4.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.4.0/readme","updated_at":"2020-09-28T13:40:49.593030+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-22T16:40:06.754009+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3357,"created_at":"2020-06-22T16:40:06.754009+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.1/download","downloads":37853,"features":{},"id":254896,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.1/authors","dependencies":"/api/v1/crates/micro-timer/0.3.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.1/downloads"},"num":"0.3.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.1/readme","updated_at":"2020-06-22T16:40:06.754009+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-02T11:38:33.047581+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3306,"created_at":"2020-06-02T11:38:33.047581+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.0/download","downloads":4163,"features":{},"id":248120,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.0/authors","dependencies":"/api/v1/crates/micro-timer/0.3.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.0/downloads"},"num":"0.3.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.0/readme","updated_at":"2020-06-02T11:38:33.047581+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T11:22:26.288804+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2937,"created_at":"2020-03-23T11:22:26.288804+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.1/download","downloads":1301,"features":{},"id":223660,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.1/authors","dependencies":"/api/v1/crates/micro-timer/0.2.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.1/downloads"},"num":"0.2.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.1/readme","updated_at":"2020-03-23T11:22:26.288804+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T10:57:04.418462+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2941,"created_at":"2020-03-23T10:57:04.418462+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.0/download","downloads":104,"features":{},"id":223652,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.0/authors","dependencies":"/api/v1/crates/micro-timer/0.2.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.0/downloads"},"num":"0.2.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.0/readme","updated_at":"2020-03-23T10:57:04.418462+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T23:35:41.872176+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":4927,"created_at":"2020-02-27T23:35:41.872176+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.2/download","downloads":258,"features":{},"id":216405,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.2/authors","dependencies":"/api/v1/crates/micro-timer/0.1.2/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.2/downloads"},"num":"0.1.2","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.2/readme","updated_at":"2020-02-27T23:35:41.872176+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T15:17:53.486346+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2916,"created_at":"2020-02-27T15:17:53.486346+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.1/download","downloads":111,"features":{},"id":216156,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.1/authors","dependencies":"/api/v1/crates/micro-timer/0.1.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.1/downloads"},"num":"0.1.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.1/readme","updated_at":"2020-02-27T15:17:53.486346+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T14:31:49.131258+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2902,"created_at":"2020-02-27T14:31:49.131258+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.0/download","downloads":118,"features":{},"id":216139,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.0/authors","dependencies":"/api/v1/crates/micro-timer/0.1.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.0/downloads"},"num":"0.1.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.0/readme","updated_at":"2020-02-27T14:31:49.131258+00:00","yanked":false}]}
-''' > ../../https_crates.io/api_v1_crates_micro-timer
-
 # Clean up removing tmp_dir
 cd ../../
 rm -r tmp_dir/

swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core

-{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2019-04-16T18:48:11.404457+00:00","description":"Mercurial pure Rust core library, with no assumption on Python bindings (FFI)","documentation":null,"downloads":442,"exact_match":false,"homepage":"https://mercurial-scm.org","id":"hg-core","keywords":[],"links":{"owner_team":"/api/v1/crates/hg-core/owner_team","owner_user":"/api/v1/crates/hg-core/owner_user","owners":"/api/v1/crates/hg-core/owners","reverse_dependencies":"/api/v1/crates/hg-core/reverse_dependencies","version_downloads":"/api/v1/crates/hg-core/downloads","versions":null},"max_stable_version":"0.0.1","max_version":"0.0.1","name":"hg-core","newest_version":"0.0.1","recent_downloads":40,"repository":"https://www.mercurial-scm.org/repo/hg","updated_at":"2019-04-16T18:48:11.404457+00:00","versions":[145309]},"keywords":[],"versions":[{"audit_actions":[],"crate":"hg-core","crate_size":21344,"created_at":"2019-04-16T18:48:11.404457+00:00","dl_path":"/api/v1/crates/hg-core/0.0.1/download","downloads":442,"features":{},"id":145309,"license":"GPL-2.0-or-later","links":{"authors":"/api/v1/crates/hg-core/0.0.1/authors","dependencies":"/api/v1/crates/hg-core/0.0.1/dependencies","version_downloads":"/api/v1/crates/hg-core/0.0.1/downloads"},"num":"0.0.1","published_by":{"avatar":"https://avatars0.githubusercontent.com/u/474220?v=4","id":45544,"login":"gracinet","name":"Georges Racinet","url":"https://github.com/gracinet"},"readme_path":"/api/v1/crates/hg-core/0.0.1/readme","updated_at":"2019-04-16T18:48:11.404457+00:00","yanked":false}]}
-

swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_micro-timer

-{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2020-02-27T14:31:49.131258+00:00","description":"Dumb tiny logging timer","documentation":null,"downloads":44245,"exact_match":false,"homepage":"https://foss.heptapod.net/octobus/rust/micro-timer","id":"micro-timer","keywords":[],"links":{"owner_team":"/api/v1/crates/micro-timer/owner_team","owner_user":"/api/v1/crates/micro-timer/owner_user","owners":"/api/v1/crates/micro-timer/owners","reverse_dependencies":"/api/v1/crates/micro-timer/reverse_dependencies","version_downloads":"/api/v1/crates/micro-timer/downloads","versions":null},"max_stable_version":"0.4.0","max_version":"0.4.0","name":"micro-timer","newest_version":"0.4.0","recent_downloads":3910,"repository":"https://foss.heptapod.net/octobus/rust/micro-timer","updated_at":"2020-09-28T13:40:49.593030+00:00","versions":[288167,254896,248120,223660,223652,216405,216156,216139]},"keywords":[],"versions":[{"audit_actions":[{"action":"publish","time":"2020-09-28T13:40:49.593030+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3513,"created_at":"2020-09-28T13:40:49.593030+00:00","dl_path":"/api/v1/crates/micro-timer/0.4.0/download","downloads":337,"features":{},"id":288167,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.4.0/authors","dependencies":"/api/v1/crates/micro-timer/0.4.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.4.0/downloads"},"num":"0.4.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.4.0/readme","updated_at":"2020-09-28T13:40:49.593030+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-22T16:40:06.754009+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3357,"created_at":"2020-06-22T16:40:06.754009+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.1/download","downloads":37853,"features":{},"id":254896,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.1/authors","dependencies":"/api/v1/crates/micro-timer/0.3.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.1/downloads"},"num":"0.3.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.1/readme","updated_at":"2020-06-22T16:40:06.754009+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-02T11:38:33.047581+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3306,"created_at":"2020-06-02T11:38:33.047581+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.0/download","downloads":4163,"features":{},"id":248120,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.0/authors","dependencies":"/api/v1/crates/micro-timer/0.3.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.0/downloads"},"num":"0.3.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.0/readme","updated_at":"2020-06-02T11:38:33.047581+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T11:22:26.288804+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2937,"created_at":"2020-03-23T11:22:26.288804+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.1/download","downloads":1301,"features":{},"id":223660,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.1/authors","dependencies":"/api/v1/crates/micro-timer/0.2.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.1/downloads"},"num":"0.2.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.1/readme","updated_at":"2020-03-23T11:22:26.288804+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T10:57:04.418462+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2941,"created_at":"2020-03-23T10:57:04.418462+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.0/download","downloads":104,"features":{},"id":223652,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.0/authors","dependencies":"/api/v1/crates/micro-timer/0.2.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.0/downloads"},"num":"0.2.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.0/readme","updated_at":"2020-03-23T10:57:04.418462+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T23:35:41.872176+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":4927,"created_at":"2020-02-27T23:35:41.872176+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.2/download","downloads":258,"features":{},"id":216405,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.2/authors","dependencies":"/api/v1/crates/micro-timer/0.1.2/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.2/downloads"},"num":"0.1.2","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.2/readme","updated_at":"2020-02-27T23:35:41.872176+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T15:17:53.486346+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2916,"created_at":"2020-02-27T15:17:53.486346+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.1/download","downloads":111,"features":{},"id":216156,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.1/authors","dependencies":"/api/v1/crates/micro-timer/0.1.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.1/downloads"},"num":"0.1.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.1/readme","updated_at":"2020-02-27T15:17:53.486346+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T14:31:49.131258+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2902,"created_at":"2020-02-27T14:31:49.131258+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.0/download","downloads":118,"features":{},"id":216139,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.0/authors","dependencies":"/api/v1/crates/micro-timer/0.1.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.0/downloads"},"num":"0.1.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.0/readme","updated_at":"2020-02-27T14:31:49.131258+00:00","yanked":false}]}
-

swh/loader/package/crates/tests/test_tasks.py

@@ -24,7 +24,23 @@ def crates_listed_origin(crates_lister):
         url="some-url/api/v1/crates/some-package",
         visit_type="crates",
         extra_loader_arguments={
-            "artifacts": [{"version": "0.0.1", "url": "some-package-0.0.1.crate"}],
+            "artifacts": [
+                {
+                    "version": "0.0.1",
+                    "filename": "some-package-0.0.1.crate",
+                    "url": "https://somewhere/some-package-0.0.1.crate",
+                    "checksums": {
+                        "sha256": "5de32cb59a062672560d6f0842c4aa7714727457b9fe2daf8987d995a176a405",  # noqa: B950
+                    },
+                },
+            ],
+            "crates_metadata": [
+                {
+                    "version": "0.0.1",
+                    "yanked": True,
+                    "last_update": "1970-01-01T00:00:00.000000+00:00",
+                },
+            ],
         },
     )