Golang module loader

This uses the Golang proxy since that's what `go get` uses, and since it
probably offer better performance than most direct sources.

docs/package-loader-specifications.rst

@@ -77,6 +77,15 @@ Here is an overview of the fields (+ internal version name + branch name) used b
        ``i_version`` is the intrinsic version (eg. ``0.7.2-3``) while ``version``
        contains the debian suite name (eg. ``stretch/contrib/0.7.2-3``) and is
        passed as arg
+   * - golang
+     - ``p_info.​version``
+     - ``release_name(version)``
+     - =version
+     - Synthetic release for Golang source package {p_info.name} version {p_info.version}
+     - true
+     - ""
+     - from ext metadata
+     - Golang offers basically no metadata outside of version and timestamp
    * - deposit
      - HEAD
      - only HEAD

setup.py

@@ -62,6 +62,7 @@ setup(
         loader.crates=swh.loader.package.crates:register
         loader.debian=swh.loader.package.debian:register
         loader.deposit=swh.loader.package.deposit:register
+        loader.golang=swh.loader.package.golang:register
         loader.nixguix=swh.loader.package.nixguix:register
         loader.npm=swh.loader.package.npm:register
         loader.opam=swh.loader.package.opam:register

swh/loader/package/golang/__init__.py

+# Copyright (C) 2022  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+from typing import Any, Mapping
+
+
+def register() -> Mapping[str, Any]:
+    """Register the current worker module's definition"""
+    from .loader import GolangLoader
+
+    return {
+        "task_modules": [f"{__name__}.tasks"],
+        "loader": GolangLoader,
+    }

swh/loader/package/golang/loader.py

+# Copyright (C) 2022  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import json
+import logging
+from typing import Iterator, Optional, Sequence, Tuple
+
+import attr
+
+from swh.loader.package.loader import BasePackageInfo, PackageLoader
+from swh.loader.package.utils import EMPTY_AUTHOR, api_info, release_name
+from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone
+from swh.storage.interface import StorageInterface
+
+logger = logging.getLogger(__name__)
+
+
+@attr.s
+class GolangPackageInfo(BasePackageInfo):
+    name = attr.ib(type=str)
+    timestamp = attr.ib(type=Optional[TimestampWithTimezone])
+
+
+class GolangLoader(PackageLoader[GolangPackageInfo]):
+    """Load Golang module zip file into SWH archive."""
+
+    visit_type = "golang"
+    GOLANG_PKG_DEV_URL = "https://pkg.go.dev"
+    GOLANG_PROXY_URL = "https://proxy.golang.org"
+
+    def __init__(
+        self,
+        storage: StorageInterface,
+        url: str,
+        max_content_size: Optional[int] = None,
+        **kwargs,
+    ):
+        super().__init__(storage, url, max_content_size=max_content_size, **kwargs)
+        # The lister saves human-usable URLs, so we translate them to proxy URLs
+        # for use in the loader.
+        # This URL format is detailed in https://go.dev/ref/mod#goproxy-protocol
+        assert url.startswith(
+            self.GOLANG_PKG_DEV_URL
+        ), "Go package URL (%s) not from %s" % (url, self.GOLANG_PKG_DEV_URL)
+        self.name = url[len(self.GOLANG_PKG_DEV_URL) + 1 :]
+        self.url = url.replace(self.GOLANG_PKG_DEV_URL, self.GOLANG_PROXY_URL)
+
+    def get_versions(self) -> Sequence[str]:
+        return api_info(f"{self.url}/@v/list").decode().splitlines()
+
+    def get_default_version(self) -> str:
+        latest = api_info(f"{self.url}/@latest")
+        return json.loads(latest)["Version"]
+
+    def _raw_info(self, version: str) -> dict:
+        url = f"{self.url}/@v/{version}.info"
+        return json.loads(api_info(url))
+
+    def get_package_info(self, version: str) -> Iterator[Tuple[str, GolangPackageInfo]]:
+        # Encode the name because creating nested folders can become problematic
+        encoded_name = self.name.replace("/", "__")
+        filename = f"{encoded_name}-{version}.zip"
+        timestamp = TimestampWithTimezone.from_iso8601(self._raw_info(version)["Time"])
+        p_info = GolangPackageInfo(
+            url=f"{self.url}/@v/{version}.zip",
+            filename=filename,
+            version=version,
+            timestamp=timestamp,
+            name=self.name,
+        )
+        yield release_name(version), p_info
+
+    def build_release(
+        self, p_info: GolangPackageInfo, uncompressed_path: str, directory: Sha1Git
+    ) -> Optional[Release]:
+        msg = (
+            f"Synthetic release for Golang source package {p_info.name} "
+            f"version {p_info.version}\n"
+        )
+
+        return Release(
+            name=p_info.version.encode(),
+            message=msg.encode(),
+            date=p_info.timestamp,
+            author=EMPTY_AUTHOR,  # Go modules offer very little metadata
+            target_type=ObjectType.DIRECTORY,
+            target=directory,
+            synthetic=True,
+        )

swh/loader/package/golang/tasks.py

+# Copyright (C) 2022  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from celery import shared_task
+
+from swh.loader.package.golang.loader import GolangLoader
+
+
+@shared_task(name=__name__ + ".LoadGolang")
+def load_golang(**kwargs):
+    """Load Golang module"""
+    loader = GolangLoader.from_configfile(**kwargs)
+    return loader.load()

swh/loader/package/golang/tests/data/https_proxy.golang.org/example.com_basic-go-module_@latest

+{"Version":"v0.1.3","Time":"2022-03-15T13:54:34Z"}

swh/loader/package/golang/tests/data/https_proxy.golang.org/example.com_basic-go-module_@v_list

+v0.1.3

swh/loader/package/golang/tests/data/https_proxy.golang.org/example.com_basic-go-module_@v_v0.1.3.info

+{"Version":"v0.1.3","Time":"2022-03-17T15:42:55Z"}

swh/loader/package/golang/tests/test_golang.py

+# Copyright (C) 2022  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.loader.package.golang.loader import GolangLoader
+
+
+def test_golang_loader_first_visit(swh_storage, requests_mock_datadir):
+    url = "https://pkg.go.dev/example.com/basic-go-module"
+    loader = GolangLoader(swh_storage, url)
+
+    assert loader.load()["status"] == "eventful"

swh/loader/package/golang/tests/test_tasks.py

+# Copyright (C) 2022  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+def test_tasks_golang_loader(
+    mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config
+):
+    mock_load = mocker.patch("swh.loader.package.golang.loader.GolangLoader.load")
+    mock_load.return_value = {"status": "eventful"}
+
+    res = swh_scheduler_celery_app.send_task(
+        "swh.loader.package.golang.tasks.LoadGolang",
+        kwargs={"url": "https://pkg.go.dev/golang.org/whatever/package"},
+    )
+    assert res
+    res.wait()
+    assert res.successful()
+    assert mock_load.called
+    assert res.result == {"status": "eventful"}