opam: Write package definitions to the extrinsic metadata storage

swh/loader/package/opam/loader.py

@@ -10,9 +10,20 @@ from typing import Iterator, List, Optional, Tuple
 
 import attr
 
-from swh.loader.package.loader import BasePackageInfo, PackageLoader
+from swh.loader.package.loader import (
+    BasePackageInfo,
+    PackageLoader,
+    RawExtrinsicMetadataCore,
+)
 from swh.loader.package.utils import cached_method
-from swh.model.model import Person, Revision, RevisionType, Sha1Git
+from swh.model.model import (
+    MetadataAuthority,
+    MetadataAuthorityType,
+    Person,
+    Revision,
+    RevisionType,
+    Sha1Git,
+)
 from swh.storage.interface import StorageInterface
 
 
@@ -103,6 +114,9 @@ class OpamLoader(PackageLoader[OpamPackageInfo]):
     def get_package_file(self, version: str) -> str:
         return f"{self.get_package_dir()}/{self.get_package_name(version)}/opam"
 
+    def get_metadata_authority(self):
+        return MetadataAuthority(type=MetadataAuthorityType.FORGE, url=self.opam_url)
+
     @cached_method
     def _compute_versions(self) -> List[str]:
         """Compute the versions using opam internals
@@ -171,24 +185,24 @@ class OpamLoader(PackageLoader[OpamPackageInfo]):
         """Return the most recent version of the package as default."""
         return self._compute_versions()[-1]
 
-    def get_enclosed_single_line_field(self, field, version) -> Optional[str]:
+    def _opam_show_args(self, version: str):
         package_file = self.get_package_file(version)
-        result = opam_read(
-            [
-                "opam",
-                "show",
-                "--color",
-                "never",
-                "--safe",
-                "--normalise",
-                "--root",
-                self.opam_root,
-                "--file",
-                package_file,
-                "--field",
-                field,
-            ]
-        )
+
+        return [
+            "opam",
+            "show",
+            "--color",
+            "never",
+            "--safe",
+            "--normalise",
+            "--root",
+            self.opam_root,
+            "--file",
+            package_file,
+        ]
+
+    def get_enclosed_single_line_field(self, field, version) -> Optional[str]:
+        result = opam_read(self._opam_show_args(version) + ["--field", field])
 
         # Sanitize the result if any (remove trailing \n and enclosing ")
         return result.strip().strip('"') if result else None
@@ -210,8 +224,21 @@ class OpamLoader(PackageLoader[OpamPackageInfo]):
         fullname = b"" if maintainer_field is None else str.encode(maintainer_field)
         committer = Person(fullname=fullname, name=None, email=None)
 
+        with Popen(self._opam_show_args(version) + ["--raw"], stdout=PIPE) as proc:
+            assert proc.stdout is not None
+            metadata = proc.stdout.read()
+
         yield self.get_package_name(version), OpamPackageInfo(
-            url=url, filename=None, author=author, committer=committer, version=version
+            url=url,
+            filename=None,
+            author=author,
+            committer=committer,
+            version=version,
+            directory_extrinsic_metadata=[
+                RawExtrinsicMetadataCore(
+                    metadata=metadata, format="opam-package-definition",
+                )
+            ],
         )
 
     def build_revision(

swh/loader/package/opam/tests/test_opam.py

@@ -3,11 +3,65 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-
+from swh.loader.package import __version__
+from swh.loader.package.loader import RawExtrinsicMetadataCore
 from swh.loader.package.opam.loader import OpamLoader, OpamPackageInfo
 from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats
 from swh.model.hashutil import hash_to_bytes
-from swh.model.model import Person, Snapshot, SnapshotBranch, TargetType
+from swh.model.model import (
+    MetadataAuthority,
+    MetadataAuthorityType,
+    MetadataFetcher,
+    Person,
+    RawExtrinsicMetadata,
+    Snapshot,
+    SnapshotBranch,
+    TargetType,
+)
+from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID, ObjectType
+from swh.storage.interface import PagedResult
+
+OCB_METADATA = b"""\
+opam-version: "2.0"
+name: "ocb"
+version: "0.1"
+synopsis: "SVG badge generator"
+description:
+  "An OCaml library for SVG badge generation. There\'s also a command-line tool provided."
+maintainer: "OCamlPro <contact@ocamlpro.com>"
+authors: "OCamlPro <contact@ocamlpro.com>"
+license: "ISC"
+homepage: "https://ocamlpro.github.io/ocb/"
+doc: "https://ocamlpro.github.io/ocb/api/"
+bug-reports: "https://github.com/OCamlPro/ocb/issues"
+depends: [
+  "ocaml" {>= "4.05"}
+  "dune" {>= "2.0"}
+  "odoc" {with-doc}
+]
+build: [
+  ["dune" "subst"] {dev}
+  [
+    "dune"
+    "build"
+    "-p"
+    name
+    "-j"
+    jobs
+    "@install"
+    "@runtest" {with-test}
+    "@doc" {with-doc}
+  ]
+]
+dev-repo: "git+https://github.com/OCamlPro/ocb.git"
+url {
+  src: "https://github.com/OCamlPro/ocb/archive/0.1.tar.gz"
+  checksum: [
+    "sha256=aa27684fbda1b8036ae7e3c87de33a98a9cd2662bcc91c8447e00e41476b6a46"
+    "sha512=1260344f184dd8c8074b0439dbcc8a5d59550a654c249cd61913d4c150c664f37b76195ddca38f7f6646d08bddb320ceb8d420508450b4f09a233cd5c22e6b9b"
+  ]
+}
+"""  # noqa
 
 
 def test_opam_loader_no_opam_repository_fails(swh_storage, tmpdir, datadir):
@@ -181,7 +235,6 @@ def test_opam_revision(tmpdir, requests_mock_datadir, swh_storage, datadir):
     expected_package_info = OpamPackageInfo(
         url="https://github.com/OCamlPro/ocb/archive/0.1.tar.gz",
         filename=None,
-        directory_extrinsic_metadata=[],
         author=Person(
             fullname=b"OCamlPro <contact@ocamlpro.com>", name=None, email=None
         ),
@@ -189,6 +242,11 @@ def test_opam_revision(tmpdir, requests_mock_datadir, swh_storage, datadir):
             fullname=b"OCamlPro <contact@ocamlpro.com>", name=None, email=None
         ),
         version="0.1",
+        directory_extrinsic_metadata=[
+            RawExtrinsicMetadataCore(
+                metadata=OCB_METADATA, format="opam-package-definition",
+            )
+        ],
     )
 
     assert branch_name == expected_branch_name
@@ -202,3 +260,58 @@ def test_opam_revision(tmpdir, requests_mock_datadir, swh_storage, datadir):
 
     assert revision.author == expected_package_info.author
     assert revision.committer == expected_package_info.committer
+
+
+def test_opam_metadata(tmpdir, requests_mock_datadir, swh_storage, datadir):
+    opam_url = f"file://{datadir}/fake_opam_repo"
+    opam_root = tmpdir
+    opam_instance = "loadertest"
+
+    opam_package = "ocb"
+    url = f"opam+{opam_url}/packages/{opam_package}"
+
+    loader = OpamLoader(
+        swh_storage,
+        url,
+        opam_root,
+        opam_instance,
+        opam_url,
+        opam_package,
+        initialize_opam_root=True,
+    )
+
+    actual_load_status = loader.load()
+
+    assert actual_load_status["status"] == "eventful"
+
+    expected_revision_id = b"o\xad\x7f=\x07\xbb\xaah\xdbI(\xb0'\x10z\xfc\xff\x06x\x1b"
+
+    revision = swh_storage.revision_get([expected_revision_id])[0]
+    assert revision is not None
+
+    revision_swhid = CoreSWHID(
+        object_type=ObjectType.REVISION, object_id=expected_revision_id
+    )
+    directory_swhid = ExtendedSWHID(
+        object_type=ExtendedObjectType.DIRECTORY, object_id=revision.directory
+    )
+    metadata_authority = MetadataAuthority(
+        type=MetadataAuthorityType.FORGE, url=opam_url,
+    )
+    expected_metadata = [
+        RawExtrinsicMetadata(
+            target=directory_swhid,
+            authority=metadata_authority,
+            fetcher=MetadataFetcher(
+                name="swh.loader.package.opam.loader.OpamLoader", version=__version__,
+            ),
+            discovery_date=loader.visit_date,
+            format="opam-package-definition",
+            metadata=OCB_METADATA,
+            origin=url,
+            revision=revision_swhid,
+        )
+    ]
+    assert swh_storage.raw_extrinsic_metadata_get(
+        directory_swhid, metadata_authority,
+    ) == PagedResult(next_page_token=None, results=expected_metadata,)