From e5e8dce2ce36ad7018f99b151507b2a0d527dd28 Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Fri, 10 Jan 2025 15:36:11 +0100
Subject: [PATCH] golang: Handle errors when fetching default version
If an error occurs when attempting to fetch and parse latest version
info, return the latest version in versions list instead.
---
swh/loader/package/golang/loader.py | 18 ++++++++++++++----
swh/loader/package/golang/tests/test_golang.py | 18 ++++++++++++++++++
2 files changed, 32 insertions(+), 4 deletions(-)
diff --git a/swh/loader/package/golang/loader.py b/swh/loader/package/golang/loader.py
index c1890da1..7c77a78c 100644
--- a/swh/loader/package/golang/loader.py
+++ b/swh/loader/package/golang/loader.py
@@ -6,9 +6,11 @@
import json
import logging
import re
-from typing import Iterator, Optional, Sequence, Tuple
+from typing import Iterator, List, Optional, Sequence, Tuple
import attr
+from looseversion import LooseVersion2
+from requests import HTTPError
from swh.loader.core.utils import (
EMPTY_AUTHOR,
@@ -16,6 +18,7 @@ from swh.loader.core.utils import (
get_url_body,
release_name,
)
+from swh.loader.exception import NotFound
from swh.loader.package.loader import BasePackageInfo, PackageLoader
from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone
from swh.storage.interface import StorageInterface
@@ -58,8 +61,12 @@ class GolangLoader(PackageLoader[GolangPackageInfo]):
self.url = url.replace(self.GOLANG_PKG_DEV_URL, self.GOLANG_PROXY_URL)
self.url = _uppercase_encode(self.url)
+ @cached_method
+ def _get_versions(self) -> List[str]:
+ return get_url_body(f"{self.url}/@v/list").decode().splitlines()
+
def get_versions(self) -> Sequence[str]:
- versions = get_url_body(f"{self.url}/@v/list").decode().splitlines()
+ versions = self._get_versions()
# some go packages only have a development version not listed by the endpoint above,
# so ensure to return it or it will be missed by the golang loader
default_version = self.get_default_version()
@@ -69,8 +76,11 @@ class GolangLoader(PackageLoader[GolangPackageInfo]):
@cached_method
def get_default_version(self) -> str:
- latest = get_url_body(f"{self.url}/@latest")
- return json.loads(latest)["Version"]
+ try:
+ latest = get_url_body(f"{self.url}/@latest")
+ return json.loads(latest)["Version"]
+ except (NotFound, HTTPError, json.JSONDecodeError):
+ return max(self._get_versions(), key=LooseVersion2)
def _raw_info(self, version: str) -> dict:
url = f"{self.url}/@v/{_uppercase_encode(version)}.info"
diff --git a/swh/loader/package/golang/tests/test_golang.py b/swh/loader/package/golang/tests/test_golang.py
index e19864a9..a72b77df 100644
--- a/swh/loader/package/golang/tests/test_golang.py
+++ b/swh/loader/package/golang/tests/test_golang.py
@@ -34,3 +34,21 @@ def test_golang_loader_package_with_dev_version_only(
loader = GolangLoader(swh_storage, url)
assert loader.load()["status"] == "eventful"
+
+
+def test_golang_latest_version_not_found(
+ swh_storage, requests_mock_datadir, requests_mock
+):
+ url = "https://pkg.go.dev/github.com/adam-hanna/arrayOperations"
+ requests_mock.get(
+ "https://proxy.golang.org/github.com/adam-hanna/array!operations/@latest",
+ status_code=404,
+ )
+ loader = GolangLoader(swh_storage, url)
+
+ assert loader.load()["status"] == "eventful"
+ assert set(loader.last_snapshot().branches) == {
+ b"releases/v1.0.1",
+ b"releases/v1.0.1-RC1",
+ b"HEAD",
+ }
--
GitLab