Skip to content
Snippets Groups Projects
Commit 9e263a0f authored by Jenkins for Software Heritage's avatar Jenkins for Software Heritage
Browse files

New upstream version 2.4.1

parents 0171bd5c b3579ed0
No related branches found
No related tags found
No related merge requests found
Metadata-Version: 2.1
Name: swh.loader.core
Version: 2.3.0
Version: 2.4.1
Summary: Software Heritage Base Loader
Home-page: https://forge.softwareheritage.org/diffusion/DLDBASE
Author: Software Heritage developers
......
Metadata-Version: 2.1
Name: swh.loader.core
Version: 2.3.0
Version: 2.4.1
Summary: Software Heritage Base Loader
Home-page: https://forge.softwareheritage.org/diffusion/DLDBASE
Author: Software Heritage developers
......
......@@ -50,6 +50,7 @@ swh/loader/core/tests/test_loader.py
swh/loader/core/tests/test_utils.py
swh/loader/package/__init__.py
swh/loader/package/loader.py
swh/loader/package/py.typed
swh/loader/package/utils.py
swh/loader/package/archive/__init__.py
swh/loader/package/archive/loader.py
......@@ -207,6 +208,7 @@ swh/loader/package/tests/test_loader_metadata.py
swh/loader/package/tests/test_utils.py
swh/loader/tests/__init__.py
swh/loader/tests/conftest.py
swh/loader/tests/py.typed
swh/loader/tests/test_cli.py
swh/loader/tests/test_init.py
swh/loader/tests/data/0805nexter-1.1.0.tar.gz
\ No newline at end of file
......@@ -3,7 +3,6 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import email.utils
import logging
from os import path
import re
......@@ -265,20 +264,8 @@ def uid_to_person(uid: str) -> Dict[str, str]:
- fullname: the actual uid input
"""
logger.debug("uid: %s", uid)
ret = {
"name": "",
"email": "",
"fullname": uid,
}
name, mail = email.utils.parseaddr(uid)
if name and email:
ret["name"] = name
ret["email"] = mail
else:
ret["name"] = uid
return ret
person = Person.from_fullname(uid.encode("utf-8"))
return {k: v.decode("utf-8") for k, v in person.to_dict().items() if v is not None}
def prepare_person(person: Mapping[str, str]) -> Person:
......
......@@ -326,6 +326,20 @@ class PackageLoader(BaseLoader, Generic[TPackageInfo]):
for extid_target in extid_targets
if extid_target.object_type == ObjectType.RELEASE
}
# Exclude missing targets
missing_releases = {
CoreSWHID(object_type=ObjectType.RELEASE, object_id=id_)
for id_ in self.storage.release_missing(
[swhid.object_id for swhid in release_extid_targets]
)
}
if missing_releases:
logger.error(
"Found ExtIDs pointing to missing releases: %s", missing_releases
)
release_extid_targets -= missing_releases
extid_target2 = self.select_extid_target(p_info, release_extid_targets)
if extid_target2:
return extid_target2
......@@ -750,7 +764,8 @@ class PackageLoader(BaseLoader, Generic[TPackageInfo]):
status_visit = "partial"
status_load = "failed"
self._load_extids(new_extids)
if status_load != "failed":
self._load_extids(new_extids)
return self.finalize_visit(
snapshot=snapshot,
......
......@@ -35,7 +35,7 @@ from swh.storage.interface import StorageInterface
logger = logging.getLogger(__name__)
EMPTY_PERSON = Person(fullname=b"", name=None, email=None)
EMPTY_PERSON = Person.from_fullname(b"")
EXTID_TYPE = "npm-archive-sha1"
......
......@@ -212,8 +212,8 @@ def test_npm_extract_npm_package_author(datadir):
}"""
)
assert extract_npm_package_author(package_json_no_authors) == Person(
fullname=b"", name=None, email=None
assert extract_npm_package_author(package_json_no_authors) == Person.from_fullname(
b""
)
......
......@@ -217,11 +217,11 @@ class OpamLoader(PackageLoader[OpamPackageInfo]):
authors_field = self.get_enclosed_single_line_field("authors:", version)
fullname = b"" if authors_field is None else str.encode(authors_field)
author = Person(fullname=fullname, name=None, email=None)
author = Person.from_fullname(fullname)
maintainer_field = self.get_enclosed_single_line_field("maintainer:", version)
fullname = b"" if maintainer_field is None else str.encode(maintainer_field)
committer = Person(fullname=fullname, name=None, email=None)
committer = Person.from_fullname(fullname)
with Popen(self._opam_show_args(version) + ["--raw"], stdout=PIPE) as proc:
assert proc.stdout is not None
......
......@@ -140,9 +140,7 @@ def test_opam_loader_one_version(tmpdir, requests_mock_datadir, datadir, swh_sto
target=hash_to_bytes("00412ee5bc601deb462e55addd1004715116785e"),
target_type=ModelObjectType.DIRECTORY,
synthetic=True,
author=Person(
fullname=b"OCamlPro <contact@ocamlpro.com>", name=None, email=None
),
author=Person.from_fullname(b"OCamlPro <contact@ocamlpro.com>"),
date=None,
id=release_id,
)
......@@ -248,12 +246,8 @@ def test_opam_release(tmpdir, requests_mock_datadir, swh_storage, datadir):
expected_package_info = OpamPackageInfo(
url="https://github.com/OCamlPro/ocb/archive/0.1.tar.gz",
filename=None,
author=Person(
fullname=b"OCamlPro <contact@ocamlpro.com>", name=None, email=None
),
committer=Person(
fullname=b"OCamlPro <contact@ocamlpro.com>", name=None, email=None
),
author=Person.from_fullname(b"OCamlPro <contact@ocamlpro.com>"),
committer=Person.from_fullname(b"OCamlPro <contact@ocamlpro.com>"),
version="0.1",
directory_extrinsic_metadata=[
RawExtrinsicMetadataCore(
......
# Marker file for PEP 561.
......@@ -93,17 +93,31 @@ def test_loader_origin_visit_failure(swh_storage):
def test_resolve_object_from_extids() -> None:
loader = PackageLoader(None, None) # type: ignore
storage = get_storage("memory")
target = b"\x01" * 20
rel1 = Release(
name=b"aaaa",
message=b"aaaa",
target=target,
target_type=ModelObjectType.DIRECTORY,
synthetic=False,
)
rel2 = Release(
name=b"bbbb",
message=b"bbbb",
target=target,
target_type=ModelObjectType.DIRECTORY,
synthetic=False,
)
storage.release_add([rel1, rel2])
loader = PackageLoader(storage, "http://example.org/") # type: ignore
p_info = Mock(wraps=BasePackageInfo(None, None, None)) # type: ignore
# The PackageInfo does not support extids
p_info.extid.return_value = None
known_extids = {
("extid-type", 0, b"extid-of-aaaa"): [
CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"a" * 20),
]
}
known_extids = {("extid-type", 0, b"extid-of-aaaa"): [rel1.swhid()]}
whitelist = {b"unused"}
assert loader.resolve_object_from_extids(known_extids, p_info, whitelist) is None
......@@ -118,23 +132,52 @@ def test_resolve_object_from_extids() -> None:
# Some known extid, and the PackageInfo is one of them (ie. cache hit),
# and the target release was in the previous snapshot
whitelist = {b"a" * 20}
assert loader.resolve_object_from_extids(
known_extids, p_info, whitelist
) == CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"a" * 20)
whitelist = {rel1.id}
assert (
loader.resolve_object_from_extids(known_extids, p_info, whitelist)
== rel1.swhid()
)
# Same as before, but there is more than one extid, and only one is an allowed
# release
whitelist = {b"a" * 20}
known_extids = {
("extid-type", 0, b"extid-of-aaaa"): [
CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"b" * 20),
CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"a" * 20),
]
}
assert loader.resolve_object_from_extids(
known_extids, p_info, whitelist
) == CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"a" * 20)
whitelist = {rel1.id}
known_extids = {("extid-type", 0, b"extid-of-aaaa"): [rel2.swhid(), rel1.swhid()]}
assert (
loader.resolve_object_from_extids(known_extids, p_info, whitelist)
== rel1.swhid()
)
def test_resolve_object_from_extids_missing_target() -> None:
storage = get_storage("memory")
target = b"\x01" * 20
rel = Release(
name=b"aaaa",
message=b"aaaa",
target=target,
target_type=ModelObjectType.DIRECTORY,
synthetic=False,
)
loader = PackageLoader(storage, "http://example.org/") # type: ignore
p_info = Mock(wraps=BasePackageInfo(None, None, None)) # type: ignore
known_extids = {("extid-type", 0, b"extid-of-aaaa"): [rel.swhid()]}
p_info.extid.return_value = ("extid-type", 0, b"extid-of-aaaa")
whitelist = {rel.id}
# Targeted release is missing from the storage
assert loader.resolve_object_from_extids(known_extids, p_info, whitelist) is None
storage.release_add([rel])
# Targeted release now exists
assert (
loader.resolve_object_from_extids(known_extids, p_info, whitelist)
== rel.swhid()
)
def test_load_get_known_extids() -> None:
......@@ -160,13 +203,26 @@ def test_load_extids() -> None:
the new ExtIDs"""
storage = get_storage("memory")
origin = "http://example.org"
rel1_swhid = CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"a" * 20)
rel2_swhid = CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"b" * 20)
rel3_swhid = CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"c" * 20)
rel4_swhid = CoreSWHID(object_type=ObjectType.RELEASE, object_id=b"d" * 20)
dir_swhid = CoreSWHID(object_type=ObjectType.DIRECTORY, object_id=b"e" * 20)
rels = [
Release(
name=f"v{i}.0".encode(),
message=b"blah\n",
target=dir_swhid.object_id,
target_type=ModelObjectType.DIRECTORY,
synthetic=True,
)
for i in (1, 2, 3, 4)
]
storage.release_add(rels[0:3])
origin = "http://example.org"
rel1_swhid = rels[0].swhid()
rel2_swhid = rels[1].swhid()
rel3_swhid = rels[2].swhid()
rel4_swhid = rels[3].swhid()
# Results of a previous load
storage.extid_add(
[
......
......@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
DOWNLOAD_HASHES = set(["sha1", "sha256", "length"])
EMPTY_AUTHOR = Person(fullname=b"", name=None, email=None,)
EMPTY_AUTHOR = Person.from_fullname(b"")
def api_info(url: str, **extra_params) -> bytes:
......
# Marker file for PEP 561.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment