setup.py

-#!/usr/bin/env python3
-# Copyright (C) 2015-2020  The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
-
-from io import open
-from os import path
-
-from setuptools import find_packages, setup
-
-here = path.abspath(path.dirname(__file__))
-
-# Get the long description from the README file
-with open(path.join(here, "README.md"), encoding="utf-8") as f:
-    long_description = f.read()
-
-
-def parse_requirements(name=None):
-    if name:
-        reqf = "requirements-%s.txt" % name
-    else:
-        reqf = "requirements.txt"
-
-    requirements = []
-    if not path.exists(reqf):
-        return requirements
-
-    with open(reqf) as f:
-        for line in f.readlines():
-            line = line.strip()
-            if not line or line.startswith("#"):
-                continue
-            requirements.append(line)
-    return requirements
-
-
-blake2_requirements = ['pyblake2;python_version<"3.6"']
-
-setup(
-    name="swh.model",
-    description="Software Heritage data model",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    python_requires=">=3.7",
-    author="Software Heritage developers",
-    author_email="swh-devel@inria.fr",
-    url="https://forge.softwareheritage.org/diffusion/DMOD/",
-    packages=find_packages(),
-    setup_requires=["setuptools-scm"],
-    use_scm_version=True,
-    install_requires=(
-        parse_requirements() + parse_requirements("swh") + blake2_requirements
-    ),
-    extras_require={
-        "cli": parse_requirements("cli"),
-        "testing-minimal": parse_requirements("test"),
-        "testing": parse_requirements("test") + parse_requirements("cli"),
-    },
-    include_package_data=True,
-    entry_points="""
-        [console_scripts]
-        swh-identify=swh.model.cli:identify
-        [swh.cli.subcommands]
-        identify=swh.model.cli
-    """,
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-        "Operating System :: OS Independent",
-        "Development Status :: 5 - Production/Stable",
-    ],
-    project_urls={
-        "Bug Reports": "https://forge.softwareheritage.org/maniphest",
-        "Funding": "https://www.softwareheritage.org/donate",
-        "Source": "https://forge.softwareheritage.org/source/swh-model",
-        "Documentation": "https://docs.softwareheritage.org/devel/swh-model/",
-    },
-)

swh/__init__.py

-from pkgutil import extend_path
-from typing import List
-
-__path__: List[str] = extend_path(__path__, __name__)

swh/model/cli.py

@@ -5,7 +5,7 @@
 
 import os
 import sys
-from typing import Dict, Iterable, Optional
+from typing import Callable, Dict, Iterable, Optional
 
 # WARNING: do not import unnecessary things here to keep cli startup time under
 # control
@@ -20,13 +20,15 @@ except ImportError:
     exit(1)
 
 try:
-    from swh.core.cli import swh as swh_cli_group
+    import swh.core.cli
+
+    cli_command = swh.core.cli.swh.command
 except ImportError:
     # stub so that swh-identify can be used when swh-core isn't installed
-    swh_cli_group = click  # type: ignore
+    cli_command = click.command
 
 from swh.model.from_disk import Directory
-from swh.model.identifiers import CoreSWHID, ObjectType
+from swh.model.swhids import CoreSWHID
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 
@@ -42,7 +44,7 @@ _DULWICH_TYPES = {
 
 class CoreSWHIDParamType(click.ParamType):
     """Click argument that accepts a core SWHID and returns them as
-    :class:`swh.model.identifiers.CoreSWHID` instances """
+    :class:`swh.model.swhids.CoreSWHID` instances"""
 
     name = "SWHID"
 
@@ -69,35 +71,35 @@ def swhid_of_file_content(data) -> CoreSWHID:
     return object.swhid()
 
 
-def model_of_dir(path: bytes, exclude_patterns: Iterable[bytes] = None) -> Directory:
-    from swh.model.from_disk import accept_all_directories, ignore_directories_patterns
+def model_of_dir(
+    path: bytes,
+    exclude_patterns: Optional[Iterable[bytes]] = None,
+    update_info: Optional[Callable[[int], None]] = None,
+) -> Directory:
+    from swh.model.from_disk import accept_all_paths, ignore_directories_patterns
 
-    dir_filter = (
+    path_filter = (
         ignore_directories_patterns(path, exclude_patterns)
         if exclude_patterns
-        else accept_all_directories
+        else accept_all_paths
     )
 
-    return Directory.from_disk(path=path, dir_filter=dir_filter)
+    return Directory.from_disk(
+        path=path, path_filter=path_filter, progress_callback=update_info
+    )
 
 
-def swhid_of_dir(path: bytes, exclude_patterns: Iterable[bytes] = None) -> CoreSWHID:
+def swhid_of_dir(
+    path: bytes, exclude_patterns: Optional[Iterable[bytes]] = None
+) -> CoreSWHID:
     obj = model_of_dir(path, exclude_patterns)
     return obj.swhid()
 
 
 def swhid_of_origin(url):
-    from swh.model.hashutil import hash_to_bytes
-    from swh.model.identifiers import (
-        ExtendedObjectType,
-        ExtendedSWHID,
-        origin_identifier,
-    )
+    from swh.model.model import Origin
 
-    return ExtendedSWHID(
-        object_type=ExtendedObjectType.ORIGIN,
-        object_id=hash_to_bytes(origin_identifier({"url": url})),
-    )
+    return Origin(url).swhid()
 
 
 def swhid_of_git_repo(path) -> CoreSWHID:
@@ -110,7 +112,7 @@ def swhid_of_git_repo(path) -> CoreSWHID:
         )
 
     from swh.model import hashutil
-    from swh.model.identifiers import snapshot_identifier
+    from swh.model.model import Snapshot
 
     repo = dulwich.repo.Repo(path)
 
@@ -133,10 +135,7 @@ def swhid_of_git_repo(path) -> CoreSWHID:
 
     snapshot = {"branches": branches}
 
-    return CoreSWHID(
-        object_type=ObjectType.SNAPSHOT,
-        object_id=hashutil.hash_to_bytes(snapshot_identifier(snapshot)),
-    )
+    return Snapshot.from_dict(snapshot).swhid()
 
 
 def identify_object(
@@ -181,7 +180,7 @@ def identify_object(
     return swhid
 
 
-@swh_cli_group.command(context_settings=CONTEXT_SETTINGS)
+@cli_command(context_settings=CONTEXT_SETTINGS)
 @click.option(
     "--dereference/--no-dereference",
     "follow_symlinks",
@@ -220,7 +219,10 @@ def identify_object(
     help="reference identifier to be compared with computed one",
 )
 @click.option(
-    "-r", "--recursive", is_flag=True, help="compute SWHID recursively",
+    "-r",
+    "--recursive",
+    is_flag=True,
+    help="compute SWHID recursively",
 )
 @click.argument("objects", nargs=-1, required=True)
 def identify(
@@ -237,30 +239,26 @@ def identify(
 
     For more details about SWHIDs see:
 
-    \b
     https://docs.softwareheritage.org/devel/swh-model/persistent-identifiers.html
 
     Tip: you can pass "-" to identify the content of standard input.
 
-    \b
     Examples::
 
-    \b
       $ swh identify fork.c kmod.c sched/deadline.c
       swh:1:cnt:2e391c754ae730bd2d8520c2ab497c403220c6e3    fork.c
       swh:1:cnt:0277d1216f80ae1adeed84a686ed34c9b2931fc2    kmod.c
       swh:1:cnt:57b939c81bce5d06fa587df8915f05affbe22b82    sched/deadline.c
 
-    \b
       $ swh identify --no-filename /usr/src/linux/kernel/
       swh:1:dir:f9f858a48d663b3809c9e2f336412717496202ab
 
-    \b
       $ git clone --mirror https://forge.softwareheritage.org/source/helloworld.git
+
       $ swh identify --type snapshot helloworld.git/
-      swh:1:snp:510aa88bdc517345d258c1fc2babcd0e1f905e93	helloworld.git
+      swh:1:snp:510aa88bdc517345d258c1fc2babcd0e1f905e93    helloworld.git
 
-    """  # NoQA  # overlong lines in shell examples are fine
+    """
     from functools import partial
     import logging
 
@@ -311,7 +309,7 @@ def identify(
                 click.echo("SWHID mismatch: %s != %s" % (verify, swhid))
                 sys.exit(1)
         else:
-            for (obj, swhid) in results:
+            for obj, swhid in results:
                 msg = swhid
                 if show_filename:
                     msg = "%s\t%s" % (swhid, os.fsdecode(obj))

swh/model/collections.py

-# Copyright (C) 2020 The Software Heritage developers
+# Copyright (C) 2020-2023 The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+from __future__ import annotations
+
+"""Utility data structures."""
+
 from collections.abc import Mapping
+import copy
 from typing import Dict, Generic, Iterable, Optional, Tuple, TypeVar, Union
 
 KT = TypeVar("KT")
@@ -11,36 +16,40 @@ VT = TypeVar("VT")
 
 
 class ImmutableDict(Mapping, Generic[KT, VT]):
-    data: Tuple[Tuple[KT, VT], ...]
+    """A frozen dictionary.
+
+    This class behaves like a dictionary, but internally stores objects in a tuple,
+    so it is both immutable and hashable."""
+
+    _data: Dict[KT, VT]
 
     def __init__(
         self,
-        data: Union[
-            Iterable[Tuple[KT, VT]], "ImmutableDict[KT, VT]", Dict[KT, VT]
-        ] = {},
+        data: Union[Iterable[Tuple[KT, VT]], ImmutableDict[KT, VT], Dict[KT, VT]] = {},
     ):
         if isinstance(data, dict):
-            self.data = tuple(item for item in data.items())
+            self._data = data
         elif isinstance(data, ImmutableDict):
-            self.data = data.data
+            self._data = data._data
         else:
-            self.data = tuple(data)
+            self._data = {k: v for k, v in data}
+
+    @property
+    def data(self):
+        return tuple(self._data.items())
 
     def __repr__(self):
         return f"ImmutableDict({dict(self.data)!r})"
 
     def __getitem__(self, key):
-        for (k, v) in self.data:
-            if k == key:
-                return v
-        raise KeyError(key)
+        return self._data[key]
 
     def __iter__(self):
-        for (k, v) in self.data:
+        for k, v in self.data:
             yield k
 
     def __len__(self):
-        return len(self.data)
+        return len(self._data)
 
     def items(self):
         yield from self.data
@@ -48,15 +57,9 @@ class ImmutableDict(Mapping, Generic[KT, VT]):
     def __hash__(self):
         return hash(tuple(sorted(self.data)))
 
-    def copy_pop(self, popped_key) -> Tuple[Optional[VT], "ImmutableDict[KT, VT]"]:
+    def copy_pop(self, popped_key) -> Tuple[Optional[VT], ImmutableDict[KT, VT]]:
         """Returns a copy of this ImmutableDict without the given key,
         as well as the value associated to the key."""
-        popped_value = None
-        new_items = []
-        for (key, value) in self.data:
-            if key == popped_key:
-                popped_value = value
-            else:
-                new_items.append((key, value))
-
+        new_items = copy.deepcopy(self._data)
+        popped_value: Optional[VT] = new_items.pop(popped_key, None)
         return (popped_value, ImmutableDict(new_items))

swh/model/discovery.py

+# Copyright (C) 2022 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+"""Primitives for finding unknown content efficiently."""
+
+from __future__ import annotations
+
+from collections import namedtuple
+import itertools
+import logging
+import random
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    NamedTuple,
+    Optional,
+    Set,
+    Union,
+)
+
+from typing_extensions import Protocol, runtime_checkable
+
+from .from_disk import model
+from .model import Sha1Git
+
+logger = logging.getLogger(__name__)
+
+# Maximum amount when sampling from the undecided set of directory entries
+SAMPLE_SIZE = 1000
+
+# Sets of sha1 of contents, skipped contents and directories respectively
+Sample: NamedTuple = namedtuple(
+    "Sample", ["contents", "skipped_contents", "directories"]
+)
+
+
+@runtime_checkable
+class ArchiveDiscoveryInterface(Protocol):
+    """Interface used in discovery code to abstract over ways of connecting to
+    the SWH archive (direct storage, web API, etc.) for all methods needed by
+    discovery algorithms."""
+
+    contents: List[model.Content]
+    skipped_contents: List[model.SkippedContent]
+    directories: List[model.Directory]
+
+    def __init__(
+        self,
+        contents: List[model.Content],
+        skipped_contents: List[model.SkippedContent],
+        directories: List[model.Directory],
+    ) -> None:
+        self.contents = contents
+        self.skipped_contents = skipped_contents
+        self.directories = directories
+
+    def content_missing(self, contents: List[Sha1Git]) -> Iterable[Sha1Git]:
+        """List content missing from the archive by sha1"""
+
+    def skipped_content_missing(
+        self, skipped_contents: List[Sha1Git]
+    ) -> Iterable[Sha1Git]:
+        """List skipped content missing from the archive by sha1"""
+
+    def directory_missing(self, directories: List[Sha1Git]) -> Iterable[Sha1Git]:
+        """List directories missing from the archive by sha1"""
+
+
+class BaseDiscoveryGraph:
+    """Creates the base structures and methods needed for discovery algorithms.
+    Subclasses should override ``get_sample`` to affect how the discovery is made.
+
+    The `update_info_callback` is an optional argument that will get called for
+    each new piece of information we get. The callback arguments are `(content,
+    known)`.
+    - content: the relevant model.Content object,
+    - known: a boolean, True if the file is known to the archive False otherwise.
+    """
+
+    def __init__(
+        self,
+        contents,
+        skipped_contents,
+        directories,
+        update_info_callback: Optional[Callable[[Any, bool], None]] = None,
+    ):
+        self._all_contents: Mapping[
+            Sha1Git, Union[model.Content, model.SkippedContent]
+        ] = {}
+        self._undecided_directories: Set[Sha1Git] = set()
+        self._children: Mapping[Sha1Git, Set[Sha1Git]] = {}
+        self._parents: Mapping[model.DirectoryEntry, Set[Any]] = {}
+        self.undecided: Set[Sha1Git] = set()
+
+        for content in itertools.chain(contents, skipped_contents):
+            self.undecided.add(content.sha1_git)
+            self._all_contents[content.sha1_git] = content
+
+        for directory in directories:
+            self.undecided.add(directory.id)
+            self._undecided_directories.add(directory.id)
+            self._children[directory.id] = {c.target for c in directory.entries}
+            for child in directory.entries:
+                self._parents.setdefault(child.target, set()).add(directory.id)
+
+        self.undecided |= self._undecided_directories
+        self.known: Set[Sha1Git] = set()
+        self.unknown: Set[Sha1Git] = set()
+        self._update_info_callback = update_info_callback
+        self._sha1_to_obj = {}
+        for content in itertools.chain(contents, skipped_contents):
+            self._sha1_to_obj[content.sha1_git] = content
+        for directory in directories:
+            self._sha1_to_obj[directory.id] = directory
+
+    def mark_known(self, entries: Iterable[Sha1Git]):
+        """Mark ``entries`` and those they imply as known in the SWH archive"""
+        self._mark_entries(entries, self._children, self.known)
+
+    def mark_unknown(self, entries: Iterable[Sha1Git]):
+        """Mark ``entries`` and those they imply as unknown in the SWH archive"""
+        self._mark_entries(entries, self._parents, self.unknown)
+
+    def _mark_entries(
+        self,
+        entries: Iterable[Sha1Git],
+        transitive_mapping: Mapping[Any, Any],
+        target_set: Set[Any],
+    ):
+        """Use Merkle graph properties to mark a directory entry as known or unknown.
+
+        If an entry is known, then all of its descendants are known. If it's
+        unknown, then all of its ancestors are unknown.
+
+        - ``entries``: directory entries to mark along with their ancestors/descendants
+          where applicable.
+        - ``transitive_mapping``: mapping from an entry to the next entries to mark
+          in the hierarchy, if any.
+        - ``target_set``: set where marked entries will be added.
+
+        """
+        callback = self._update_info_callback
+        to_process = set(entries)
+        while to_process:
+            current = to_process.pop()
+            target_set.add(current)
+            new = current in self.undecided
+            self.undecided.discard(current)
+            self._undecided_directories.discard(current)
+            next_entries = transitive_mapping.get(current, set()) & self.undecided
+            to_process.update(next_entries)
+            if new and callback is not None:
+                obj = self._sha1_to_obj[current]
+                callback(obj, current in self.known)
+
+    def get_sample(
+        self,
+    ) -> Sample:
+        """Return a three-tuple of samples from the undecided sets of contents,
+        skipped contents and directories respectively.
+        These samples will be queried against the storage which will tell us
+        which are known."""
+        raise NotImplementedError()
+
+    def do_query(self, archive: ArchiveDiscoveryInterface, sample: Sample) -> None:
+        """Given a three-tuple of samples, ask the archive which are known or
+        unknown and mark them as such."""
+
+        methods = (
+            archive.content_missing,
+            archive.skipped_content_missing,
+            archive.directory_missing,
+        )
+
+        for sample_per_type, method in zip(sample, methods):
+            if not sample_per_type:
+                continue
+            known = set(sample_per_type)
+            unknown = set(method(list(sample_per_type)))
+            known -= unknown
+
+            self.mark_known(known)
+            self.mark_unknown(unknown)
+
+
+class RandomDirSamplingDiscoveryGraph(BaseDiscoveryGraph):
+    """Use a random sampling using only directories.
+
+    This allows us to find a statistically good spread of entries in the graph
+    with a smaller population than using all types of entries. When there are
+    no more directories, only contents or skipped contents are undecided if any
+    are left: we send them directly to the storage since they should be few and
+    their structure flat."""
+
+    def get_sample(self) -> Sample:
+        if self._undecided_directories:
+            if len(self._undecided_directories) <= SAMPLE_SIZE:
+                return Sample(
+                    contents=set(),
+                    skipped_contents=set(),
+                    directories=set(self._undecided_directories),
+                )
+            sample = random.sample(tuple(self._undecided_directories), SAMPLE_SIZE)
+            directories = {o for o in sample}
+            return Sample(
+                contents=set(), skipped_contents=set(), directories=directories
+            )
+
+        contents = set()
+        skipped_contents = set()
+
+        for sha1 in self.undecided:
+            obj = self._all_contents[sha1]
+            obj_type = obj.object_type
+            if obj_type == model.Content.object_type:
+                contents.add(sha1)
+            elif obj_type == model.SkippedContent.object_type:
+                skipped_contents.add(sha1)
+            else:
+                raise TypeError(f"Unexpected object type {obj_type}")
+
+        return Sample(
+            contents=contents, skipped_contents=skipped_contents, directories=set()
+        )
+
+
+def filter_known_objects(
+    archive: ArchiveDiscoveryInterface,
+    update_info_callback: Optional[Callable[[Any, bool], None]] = None,
+):
+    """Filter ``archive``'s ``contents``, ``skipped_contents`` and ``directories``
+    to only return those that are unknown to the SWH archive using a discovery
+    algorithm.
+
+    The `update_info_callback` is an optional argument that will get called for
+    each new piece of information we get. The callback arguments are `(content,
+    known)`.
+    - content: the relevant model.Content object,
+    - known: a boolean, True if the file is known to the archive False otherwise.
+    """
+    contents = archive.contents
+    skipped_contents = archive.skipped_contents
+    directories = archive.directories
+
+    contents_count = len(contents)
+    skipped_contents_count = len(skipped_contents)
+    directories_count = len(directories)
+
+    graph = RandomDirSamplingDiscoveryGraph(
+        contents,
+        skipped_contents,
+        directories,
+        update_info_callback=update_info_callback,
+    )
+
+    while graph.undecided:
+        sample = graph.get_sample()
+        graph.do_query(archive, sample)
+
+    contents = [c for c in contents if c.sha1_git in graph.unknown]
+    skipped_contents = [c for c in skipped_contents if c.sha1_git in graph.unknown]
+    directories = [c for c in directories if c.id in graph.unknown]
+
+    logger.debug(
+        "Filtered out %d contents, %d skipped contents and %d directories",
+        contents_count - len(contents),
+        skipped_contents_count - len(skipped_contents),
+        directories_count - len(directories),
+    )
+
+    return (contents, skipped_contents, directories)

swh/model/fields/compound.py

@@ -27,7 +27,10 @@ def validate_against_schema(model, schema, value):
     if not isinstance(value, dict):
         raise ValidationError(
             "Unexpected type %(type)s for %(model)s, expected dict",
-            params={"model": model, "type": value.__class__.__name__,},
+            params={
+                "model": model,
+                "type": value.__class__.__name__,
+            },
             code="model-unexpected-type",
         )
 

swh/model/fields/hashes.py

@@ -96,7 +96,9 @@ def validate_hash(value, hash_type):
 
     raise ValidationError(
         "Unexpected type %(type)s for hash, expected str or bytes",
-        params={"type": value.__class__.__name__,},
+        params={
+            "type": value.__class__.__name__,
+        },
         code="unexpected-hash-value-type",
     )
 

swh/model/fields/simple.py

@@ -18,7 +18,10 @@ def validate_type(value, type):
             typestr = type.__name__
         raise ValidationError(
             "Unexpected type %(type)s, expected %(expected_type)s",
-            params={"type": value.__class__.__name__, "expected_type": typestr,},
+            params={
+                "type": value.__class__.__name__,
+                "expected_type": typestr,
+            },
             code="unexpected-type",
         )
 

swh/model/hashutil.py

@@ -56,9 +56,11 @@ import functools
 import hashlib
 from io import BytesIO
 import os
-from typing import Callable, Dict, Optional
+from typing import Callable, Dict, Optional, Union
 
-ALGORITHMS = set(["sha1", "sha256", "sha1_git", "blake2s256", "blake2b512"])
+ALGORITHMS = set(
+    ["sha1", "sha256", "sha1_git", "blake2s256", "blake2b512", "md5", "sha512"]
+)
 """Hashing algorithms supported by this module"""
 
 DEFAULT_ALGORITHMS = set(["sha1", "sha256", "sha1_git", "blake2s256"])
@@ -70,7 +72,7 @@ Subset of :const:`ALGORITHMS`.
 HASH_BLOCK_SIZE = 32768
 """Block size for streaming hash computations made in this module"""
 
-_blake2_hash_cache = {}  # type: Dict[str, Callable]
+_blake2_hash_cache: Dict[str, Callable] = {}
 
 
 class MultiHash:
@@ -160,9 +162,7 @@ class MultiHash:
 
 
 def _new_blake2_hash(algo):
-    """Return a function that initializes a blake2 hash.
-
-    """
+    """Return a function that initializes a blake2 hash."""
     if algo in _blake2_hash_cache:
         return _blake2_hash_cache[algo]()
 
@@ -183,20 +183,8 @@ def _new_blake2_hash(algo):
                 "Digest size for algorithm %s must be a multiple of 8" % algo
             )
 
-    if lalgo in hashlib.algorithms_available:
-        # Handle the case where OpenSSL ships the given algorithm
-        # (e.g. Python 3.5 on Debian 9 stretch)
-        _blake2_hash_cache[algo] = lambda: hashlib.new(lalgo)
-    else:
-        # Try using the built-in implementation for Python 3.6+
-        if blake_family in hashlib.algorithms_available:
-            blake2 = getattr(hashlib, blake_family)
-        else:
-            import pyblake2
-
-            blake2 = getattr(pyblake2, blake_family)
-
-        _blake2_hash_cache[algo] = lambda: blake2(digest_size=digest_size)
+    blake2 = getattr(hashlib, blake_family)
+    _blake2_hash_cache[algo] = lambda: blake2(digest_size=digest_size)
 
     return _blake2_hash_cache[algo]()
 
@@ -307,7 +295,7 @@ def hash_git_data(data, git_type, base_algo="sha1"):
 
 
 @functools.lru_cache()
-def hash_to_hex(hash):
+def hash_to_hex(hash: Union[str, bytes]) -> str:
     """Converts a hash (in hex or bytes form) to its hexadecimal ascii form
 
     Args:
@@ -323,7 +311,7 @@ def hash_to_hex(hash):
 
 
 @functools.lru_cache()
-def hash_to_bytehex(hash):
+def hash_to_bytehex(hash: bytes) -> bytes:
     """Converts a hash to its hexadecimal bytes representation
 
     Args:
@@ -336,7 +324,7 @@ def hash_to_bytehex(hash):
 
 
 @functools.lru_cache()
-def hash_to_bytes(hash):
+def hash_to_bytes(hash: Union[str, bytes]) -> bytes:
     """Converts a hash (in hex or bytes form) to its raw bytes form
 
     Args:
@@ -352,7 +340,7 @@ def hash_to_bytes(hash):
 
 
 @functools.lru_cache()
-def bytehex_to_hash(hex):
+def bytehex_to_hash(hex: bytes) -> bytes:
     """Converts a hexadecimal bytes representation of a hash to that hash
 
     Args:

swh/model/merkle.py

-# Copyright (C) 2017-2020 The Software Heritage developers
+# Copyright (C) 2017-2022 The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
 """Merkle tree data structure"""
 
-import abc
-from collections.abc import Mapping
-from typing import Dict, Iterator, List, Set
-
-
-def deep_update(left, right):
-    """Recursively update the left mapping with deeply nested values from the right
-    mapping.
-
-    This function is useful to merge the results of several calls to
-    :func:`MerkleNode.collect`.
-
-    Arguments:
-      left: a mapping (modified by the update operation)
-      right: a mapping
-
-    Returns:
-      the left mapping, updated with nested values from the right mapping
-
-    Example:
-        >>> a = {
-        ...     'key1': {
-        ...         'key2': {
-        ...              'key3': 'value1/2/3',
-        ...         },
-        ...     },
-        ... }
-        >>> deep_update(a, {
-        ...     'key1': {
-        ...         'key2': {
-        ...              'key4': 'value1/2/4',
-        ...         },
-        ...     },
-        ... }) == {
-        ...     'key1': {
-        ...         'key2': {
-        ...             'key3': 'value1/2/3',
-        ...             'key4': 'value1/2/4',
-        ...         },
-        ...     },
-        ... }
-        True
-        >>> deep_update(a, {
-        ...     'key1': {
-        ...         'key2': {
-        ...              'key3': 'newvalue1/2/3',
-        ...         },
-        ...     },
-        ... }) == {
-        ...     'key1': {
-        ...         'key2': {
-        ...             'key3': 'newvalue1/2/3',
-        ...             'key4': 'value1/2/4',
-        ...         },
-        ...     },
-        ... }
-        True
+from __future__ import annotations
 
-    """
-    for key, rvalue in right.items():
-        if isinstance(rvalue, Mapping):
-            new_lvalue = deep_update(left.get(key, {}), rvalue)
-            left[key] = new_lvalue
-        else:
-            left[key] = rvalue
-    return left
+import abc
+from typing import Any, Dict, Iterator, List, Set
 
 
 class MerkleNode(dict, metaclass=abc.ABCMeta):
@@ -141,7 +79,7 @@ class MerkleNode(dict, metaclass=abc.ABCMeta):
         for parent in self.parents:
             parent.invalidate_hash()
 
-    def update_hash(self, *, force=False):
+    def update_hash(self, *, force=False) -> Any:
         """Recursively compute the hash of the current node.
 
         Args:
@@ -161,14 +99,17 @@ class MerkleNode(dict, metaclass=abc.ABCMeta):
         return self.__hash
 
     @property
-    def hash(self):
+    def hash(self) -> Any:
         """The hash of the current node, as calculated by
         :func:`compute_hash`.
         """
         return self.update_hash()
 
+    def __hash__(self):
+        return hash(self.hash)
+
     @abc.abstractmethod
-    def compute_hash(self):
+    def compute_hash(self) -> Any:
         """Compute the hash of the current node.
 
         The hash should depend on the data of the node, as well as on hashes
@@ -223,47 +164,24 @@ class MerkleNode(dict, metaclass=abc.ABCMeta):
         """
         return self.data
 
-    def collect_node(self, **kwargs):
-        """Collect the data for the current node, for use by :func:`collect`.
-
-        Arguments:
-          kwargs: passed as-is to :func:`get_data`.
-
-        Returns:
-          A :class:`dict` compatible with :func:`collect`.
-        """
+    def collect_node(self) -> Set[MerkleNode]:
+        """Collect the current node if it has not been yet, for use by :func:`collect`."""
         if not self.collected:
             self.collected = True
-            return {self.object_type: {self.hash: self.get_data(**kwargs)}}
+            return {self}
         else:
-            return {}
+            return set()
 
-    def collect(self, **kwargs):
-        """Collect the data for all nodes in the subtree rooted at `self`.
-
-        The data is deduplicated by type and by hash.
-
-        Arguments:
-          kwargs: passed as-is to :func:`get_data`.
+    def collect(self) -> Set[MerkleNode]:
+        """Collect the added and modified nodes in the subtree rooted at `self`
+        since the last collect operation.
 
         Returns:
-           A :class:`dict` with the following structure::
-
-             {
-               'typeA': {
-                 node1.hash: node1.get_data(),
-                 node2.hash: node2.get_data(),
-               },
-               'typeB': {
-                 node3.hash: node3.get_data(),
-                 ...
-               },
-               ...
-             }
+           A :class:`set` of collected nodes
         """
-        ret = self.collect_node(**kwargs)
+        ret = self.collect_node()
         for child in self.values():
-            deep_update(ret, child.collect(**kwargs))
+            ret.update(child.collect())
 
         return ret
 
@@ -277,18 +195,20 @@ class MerkleNode(dict, metaclass=abc.ABCMeta):
         for child in self.values():
             child.reset_collect()
 
-    def iter_tree(self) -> Iterator["MerkleNode"]:
-        """Yields all children nodes, recursively. Common nodes are
-        deduplicated.
+    def iter_tree(self, dedup=True) -> Iterator[MerkleNode]:
+        """Yields all children nodes, recursively. Common nodes are deduplicated
+        by default (deduplication can be turned off setting the given argument
+        'dedup' to False).
         """
-        yield from self._iter_tree(set())
+        yield from self._iter_tree(seen=set(), dedup=dedup)
 
-    def _iter_tree(self, seen: Set[bytes]) -> Iterator["MerkleNode"]:
+    def _iter_tree(self, seen: Set[bytes], dedup) -> Iterator[MerkleNode]:
         if self.hash not in seen:
-            seen.add(self.hash)
+            if dedup:
+                seen.add(self.hash)
             yield self
             for child in self.values():
-                yield from child._iter_tree(seen=seen)
+                yield from child._iter_tree(seen=seen, dedup=dedup)
 
 
 class MerkleLeaf(MerkleNode):
@@ -297,7 +217,7 @@ class MerkleLeaf(MerkleNode):
     A Merkle leaf is simply a Merkle node with children disabled.
     """
 
-    __slots__ = []  # type: List[str]
+    __slots__: List[str] = []
 
     def __setitem__(self, name, child):
         raise ValueError("%s is a leaf" % self.__class__.__name__)

swh/model/swhids.py

+# Copyright (C) 2015-2021  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+"""
+Classes to represent :ref:`SWH persistend IDentifiers <persistent-identifiers>`.
+
+:class:`CoreSWHID` represents a SWHID with no qualifier, and :class:`QualifiedSWHID`
+represents a SWHID that may have qualifiers.
+:class:`ExtendedSWHID` extends the definition of SWHID to other object types,
+and is used internally in Software Heritage; it does not support qualifiers.
+"""
+
+from __future__ import annotations
+
+import enum
+import re
+from typing import Any, Dict, Generic, Optional, Tuple, Type, TypeVar, Union
+import urllib.parse
+
+import attr
+from attrs_strict import type_validator
+
+from .exceptions import ValidationError
+from .hashutil import hash_to_bytes, hash_to_hex
+
+
+class ObjectType(enum.Enum):
+    """Possible object types of a QualifiedSWHID or CoreSWHID.
+
+    The values of each variant is what is used in the SWHID's string representation."""
+
+    SNAPSHOT = "snp"
+    REVISION = "rev"
+    RELEASE = "rel"
+    DIRECTORY = "dir"
+    CONTENT = "cnt"
+
+
+class ExtendedObjectType(enum.Enum):
+    """Possible object types of an ExtendedSWHID.
+
+    The variants are a superset of :class:`ObjectType`'s"""
+
+    SNAPSHOT = "snp"
+    REVISION = "rev"
+    RELEASE = "rel"
+    DIRECTORY = "dir"
+    CONTENT = "cnt"
+    ORIGIN = "ori"
+    RAW_EXTRINSIC_METADATA = "emd"
+
+
+SWHID_NAMESPACE = "swh"
+SWHID_VERSION = 1
+SWHID_TYPES = ["snp", "rel", "rev", "dir", "cnt"]
+EXTENDED_SWHID_TYPES = SWHID_TYPES + ["ori", "emd"]
+SWHID_SEP = ":"
+SWHID_CTXT_SEP = ";"
+SWHID_QUALIFIERS = {"origin", "anchor", "visit", "path", "lines"}
+
+SWHID_RE_RAW = (
+    f"(?P<namespace>{SWHID_NAMESPACE})"
+    f"{SWHID_SEP}(?P<scheme_version>{SWHID_VERSION})"
+    f"{SWHID_SEP}(?P<object_type>{'|'.join(EXTENDED_SWHID_TYPES)})"
+    f"{SWHID_SEP}(?P<object_id>[0-9a-f]{{40}})"
+    f"({SWHID_CTXT_SEP}(?P<qualifiers>\\S+))?"
+)
+SWHID_RE = re.compile(SWHID_RE_RAW)
+
+
+# type of the "object_type" attribute of the SWHID class; either
+# ObjectType or ExtendedObjectType
+_TObjectType = TypeVar("_TObjectType", ObjectType, ExtendedObjectType)
+
+# the SWHID class itself (this is used so that X.from_string() can return X
+# for all X subclass of _BaseSWHID)
+_TSWHID = TypeVar("_TSWHID", bound="_BaseSWHID")
+
+
+@attr.s(frozen=True, kw_only=True, repr=False)
+class _BaseSWHID(Generic[_TObjectType]):
+    """Common base class for CoreSWHID, QualifiedSWHID, and ExtendedSWHID.
+
+    This is an "abstract" class and should not be instantiated directly;
+    it only exists to deduplicate code between these three SWHID classes."""
+
+    namespace = attr.ib(type=str, default=SWHID_NAMESPACE)
+    """the namespace of the identifier, defaults to ``swh``"""
+
+    scheme_version = attr.ib(type=int, default=SWHID_VERSION)
+    """the scheme version of the identifier, defaults to 1"""
+
+    # overridden by subclasses
+    object_type: _TObjectType
+    """the type of object the identifier points to"""
+
+    object_id = attr.ib(type=bytes, validator=type_validator())
+    """object's identifier"""
+
+    @namespace.validator
+    def check_namespace(self, attribute, value):
+        if value != SWHID_NAMESPACE:
+            raise ValidationError(
+                "Invalid SWHID: invalid namespace: %(namespace)s",
+                params={"namespace": value},
+            )
+
+    @scheme_version.validator
+    def check_scheme_version(self, attribute, value):
+        if value != SWHID_VERSION:
+            raise ValidationError(
+                "Invalid SWHID: invalid version: %(version)s", params={"version": value}
+            )
+
+    @object_id.validator
+    def check_object_id(self, attribute, value):
+        if len(value) != 20:
+            raise ValidationError(
+                "Invalid SWHID: invalid checksum: %(object_id)s",
+                params={"object_id": hash_to_hex(value)},
+            )
+
+    def __str__(self) -> str:
+        return self._format_core_swhid()
+
+    def _format_core_swhid(self) -> str:
+        return SWHID_SEP.join(
+            [
+                self.namespace,
+                str(self.scheme_version),
+                self.object_type.value,
+                hash_to_hex(self.object_id),
+            ]
+        )
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}.from_string('{self}')"
+
+    @classmethod
+    def from_string(cls: Type[_TSWHID], s: str) -> _TSWHID:
+        parts = _parse_swhid(s)
+        if parts.pop("qualifiers"):
+            raise ValidationError(f"{cls.__name__} does not support qualifiers.")
+        try:
+            return cls(**parts)
+        except ValueError as e:
+            raise ValidationError(
+                "ValueError: %(args)s", params={"args": e.args}
+            ) from None
+
+
+@attr.s(frozen=True, kw_only=True, repr=False)
+class CoreSWHID(_BaseSWHID[ObjectType]):
+    """
+    Dataclass holding the relevant info associated to a SoftWare Heritage
+    persistent IDentifier (SWHID).
+
+    Unlike `QualifiedSWHID`, it is restricted to core SWHIDs, ie. SWHIDs
+    with no qualifiers.
+
+    Raises:
+        swh.model.exceptions.ValidationError: In case of invalid object type or id
+
+    To get the raw SWHID string from an instance of this class,
+    use the :func:`str` function:
+
+    >>> swhid = CoreSWHID(
+    ...     object_type=ObjectType.CONTENT,
+    ...     object_id=bytes.fromhex('8ff44f081d43176474b267de5451f2c2e88089d0'),
+    ... )
+    >>> str(swhid)
+    'swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0'
+
+    And vice-versa with :meth:`CoreSWHID.from_string`:
+
+    >>> swhid == CoreSWHID.from_string(
+    ...     "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0"
+    ... )
+    True
+    """
+
+    object_type = attr.ib(
+        type=ObjectType, validator=type_validator(), converter=ObjectType
+    )
+    """the type of object the identifier points to"""
+
+    def to_extended(self) -> ExtendedSWHID:
+        """Converts this CoreSWHID into an ExtendedSWHID.
+
+        As ExtendedSWHID is a superset of CoreSWHID, this is lossless."""
+        return ExtendedSWHID(
+            namespace=self.namespace,
+            scheme_version=self.scheme_version,
+            object_type=ExtendedObjectType(self.object_type.value),
+            object_id=self.object_id,
+        )
+
+    def to_qualified(self) -> QualifiedSWHID:
+        """Converts this CoreSWHID into a QualifiedSWHID.
+
+        As QualifiedSWHID is a superset of CoreSWHID, this is lossless."""
+        return QualifiedSWHID(
+            namespace=self.namespace,
+            scheme_version=self.scheme_version,
+            object_type=self.object_type,
+            object_id=self.object_id,
+        )
+
+
+def _parse_core_swhid(swhid: Union[str, CoreSWHID, None]) -> Optional[CoreSWHID]:
+    if swhid is None or isinstance(swhid, CoreSWHID):
+        return swhid
+    else:
+        return CoreSWHID.from_string(swhid)
+
+
+def _parse_lines_qualifier(
+    lines: Union[str, Tuple[int, Optional[int]], None],
+) -> Optional[Tuple[int, Optional[int]]]:
+    try:
+        if lines is None or isinstance(lines, tuple):
+            return lines
+        elif "-" in lines:
+            (from_, to) = lines.split("-", 2)
+            return (int(from_), int(to))
+        else:
+            return (int(lines), None)
+    except ValueError:
+        raise ValidationError(
+            "Invalid format for the lines qualifier: %(lines)s", params={"lines": lines}
+        )
+
+
+def _parse_path_qualifier(path: Union[str, bytes, None]) -> Optional[bytes]:
+    if path is None or isinstance(path, bytes):
+        return path
+    else:
+        return urllib.parse.unquote_to_bytes(path)
+
+
+@attr.s(frozen=True, kw_only=True, repr=False)
+class QualifiedSWHID(_BaseSWHID[ObjectType]):
+    """
+    Dataclass holding the relevant info associated to a SoftWare Heritage
+    persistent IDentifier (SWHID)
+
+    Raises:
+        swh.model.exceptions.ValidationError: In case of invalid object type or id
+
+    To get the raw SWHID string from an instance of this class,
+    use the :func:`str` function:
+
+    >>> swhid = QualifiedSWHID(
+    ...     object_type=ObjectType.CONTENT,
+    ...     object_id=bytes.fromhex('8ff44f081d43176474b267de5451f2c2e88089d0'),
+    ...     lines=(5, 10),
+    ... )
+    >>> str(swhid)
+    'swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0;lines=5-10'
+
+    And vice-versa with :meth:`QualifiedSWHID.from_string`:
+
+    >>> swhid == QualifiedSWHID.from_string(
+    ...     "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0;lines=5-10"
+    ... )
+    True
+    """
+
+    object_type = attr.ib(
+        type=ObjectType, validator=type_validator(), converter=ObjectType
+    )
+    """the type of object the identifier points to"""
+
+    # qualifiers:
+
+    origin = attr.ib(type=Optional[str], default=None, validator=type_validator())
+    """the software origin where an object has been found or observed in the wild,
+    as an URI"""
+
+    visit = attr.ib(type=Optional[CoreSWHID], default=None, converter=_parse_core_swhid)
+    """the core identifier of a snapshot corresponding to a specific visit
+    of a repository containing the designated object"""
+
+    anchor = attr.ib(
+        type=Optional[CoreSWHID],
+        default=None,
+        validator=type_validator(),
+        converter=_parse_core_swhid,
+    )
+    """a designated node in the Merkle DAG relative to which a path to the object
+    is specified, as the core identifier of a directory, a revision, a release,
+    or a snapshot"""
+
+    path = attr.ib(
+        type=Optional[bytes],
+        default=None,
+        validator=type_validator(),
+        converter=_parse_path_qualifier,
+    )
+    """the absolute file path, from the root directory associated to the anchor node,
+    to the object; when the anchor denotes a directory or a revision, and almost always
+    when it’s a release, the root directory is uniquely determined;
+    when the anchor denotes a snapshot, the root directory is the one pointed to by HEAD
+    (possibly indirectly), and undefined if such a reference is missing"""
+
+    Lines = Tuple[int, Optional[int]]
+    lines = attr.ib(
+        type=Optional[Lines],
+        default=None,
+        validator=type_validator(),
+        converter=_parse_lines_qualifier,
+    )
+    """lines: line number(s) of interest, usually within a content object"""
+
+    @visit.validator
+    def check_visit(self, attribute, value):
+        if value and value.object_type != ObjectType.SNAPSHOT:
+            raise ValidationError(
+                "The 'visit' qualifier must be a 'snp' SWHID, not '%(type)s'",
+                params={"type": value.object_type.value},
+            )
+
+    @anchor.validator
+    def check_anchor(self, attribute, value):
+        if value and value.object_type not in (
+            ObjectType.DIRECTORY,
+            ObjectType.REVISION,
+            ObjectType.RELEASE,
+            ObjectType.SNAPSHOT,
+        ):
+            raise ValidationError(
+                "The 'visit' qualifier must be a 'dir', 'rev', 'rel', or 'snp' SWHID, "
+                "not '%s(type)s'",
+                params={"type": value.object_type.value},
+            )
+
+    def to_dict(self) -> Dict[str, Optional[str | bytes | CoreSWHID | Lines]]:
+        """Returns a dictionary version of this QSWHID for json serialization"""
+        return {
+            "swhid": self._format_core_swhid(),
+            "origin": self.origin,
+            "visit": self.visit,
+            "anchor": self.anchor,
+            "path": self.path,
+            "lines": self.lines,
+        }
+
+    def qualifiers(self) -> Dict[str, str]:
+        """Returns URL-escaped qualifiers of this SWHID, for use in serialization"""
+        origin = self.origin
+        if origin:
+            unescaped_origin = origin
+            origin = origin.replace("%", "%25")
+            origin = origin.replace(";", "%3B")
+            assert (
+                urllib.parse.unquote(origin) == unescaped_origin
+            ), "Escaping ';' in the origin qualifier corrupted the origin URL."
+
+        d: Dict[str, Optional[str]] = {
+            "origin": origin,
+            "visit": str(self.visit) if self.visit else None,
+            "anchor": str(self.anchor) if self.anchor else None,
+            "path": (
+                urllib.parse.quote_from_bytes(self.path)
+                if self.path is not None
+                else None
+            ),
+            "lines": (
+                "-".join(str(line) for line in self.lines if line is not None)
+                if self.lines
+                else None
+            ),
+        }
+        return {k: v for (k, v) in d.items() if v is not None}
+
+    def __str__(self) -> str:
+        swhid = self._format_core_swhid()
+        qualifiers = self.qualifiers()
+        if qualifiers:
+            for k, v in qualifiers.items():
+                swhid += "%s%s=%s" % (SWHID_CTXT_SEP, k, v)
+        return swhid
+
+    def __repr__(self) -> str:
+        return super().__repr__()
+
+    @classmethod
+    def from_string(cls, s: str) -> QualifiedSWHID:
+        parts = _parse_swhid(s)
+        qualifiers = parts.pop("qualifiers")
+        invalid_qualifiers = set(qualifiers) - SWHID_QUALIFIERS
+        if invalid_qualifiers:
+            raise ValidationError(
+                "Invalid qualifier(s): %(qualifiers)s",
+                params={"qualifiers": ", ".join(invalid_qualifiers)},
+            )
+        if "origin" in qualifiers:
+            qualifiers["origin"] = urllib.parse.unquote(qualifiers["origin"])
+
+        try:
+            return QualifiedSWHID(**parts, **qualifiers)
+        except ValueError as e:
+            raise ValidationError(
+                "ValueError: %(args)s", params={"args": e.args}
+            ) from None
+
+
+@attr.s(frozen=True, kw_only=True, repr=False)
+class ExtendedSWHID(_BaseSWHID[ExtendedObjectType]):
+    """
+    Dataclass holding the relevant info associated to a SoftWare Heritage
+    persistent IDentifier (SWHID).
+
+    It extends  `CoreSWHID`, by allowing non-standard object types; and should
+    only be used internally to Software Heritage.
+
+    Raises:
+        swh.model.exceptions.ValidationError: In case of invalid object type or id
+
+    To get the raw SWHID string from an instance of this class,
+    use the :func:`str` function:
+
+    >>> swhid = ExtendedSWHID(
+    ...     object_type=ExtendedObjectType.CONTENT,
+    ...     object_id=bytes.fromhex('8ff44f081d43176474b267de5451f2c2e88089d0'),
+    ... )
+    >>> str(swhid)
+    'swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0'
+
+    And vice-versa with :meth:`CoreSWHID.from_string`:
+
+    >>> swhid == ExtendedSWHID.from_string(
+    ...     "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0"
+    ... )
+    True
+    """
+
+    object_type = attr.ib(
+        type=ExtendedObjectType,
+        validator=type_validator(),
+        converter=ExtendedObjectType,
+    )
+    """the type of object the identifier points to"""
+
+
+def _parse_swhid(swhid: str) -> Dict[str, Any]:
+    """Parse a Software Heritage identifier (SWHID) from string (see:
+    :ref:`persistent-identifiers`.)
+
+    This is for internal use; use :meth:`CoreSWHID.from_string`,
+    :meth:`QualifiedSWHID.from_string`, or :meth:`ExtendedSWHID.from_string` instead,
+    as they perform validation and build a dataclass.
+
+    Args:
+        swhid (str): A persistent identifier
+
+    Raises:
+        swh.model.exceptions.ValidationError: if passed string is not a valid SWHID
+
+    """
+    m = SWHID_RE.fullmatch(swhid)
+    if not m:
+        raise ValidationError(
+            "Invalid SWHID: invalid syntax: %(swhid)s", params={"swhid": swhid}
+        )
+    parts: Dict[str, Any] = m.groupdict()
+
+    qualifiers_raw = parts["qualifiers"]
+    parts["qualifiers"] = {}
+    if qualifiers_raw:
+        for qualifier in qualifiers_raw.split(SWHID_CTXT_SEP):
+            try:
+                k, v = qualifier.split("=", maxsplit=1)
+                parts["qualifiers"][k] = v
+            except ValueError:
+                raise ValidationError(
+                    "Invalid SWHID: invalid qualifier: %(qualifier)s",
+                    params={"qualifier": qualifier},
+                )
+
+    parts["scheme_version"] = int(parts["scheme_version"])
+    parts["object_id"] = hash_to_bytes(parts["object_id"])
+    return parts

swh/model/tests/fields/test_compound.py

@@ -157,7 +157,9 @@ class ValidateCompound(unittest.TestCase):
     def test_validate_whole_schema_shortcut_previous_error(self):
         with self.assertRaises(ValidationError) as cm:
             compound.validate_against_schema(
-                self.test_model, self.test_schema_shortcut, self.test_value_missing,
+                self.test_model,
+                self.test_schema_shortcut,
+                self.test_value_missing,
             )
 
         exc = cm.exception
@@ -167,7 +169,9 @@ class ValidateCompound(unittest.TestCase):
     def test_validate_whole_schema(self):
         with self.assertRaises(ValidationError) as cm:
             compound.validate_against_schema(
-                self.test_model, self.test_schema_shortcut, self.test_value,
+                self.test_model,
+                self.test_schema_shortcut,
+                self.test_value,
             )
 
         # The exception should be of the form:

swh/model/tests/swh_model_data.py

@@ -4,12 +4,11 @@
 # See top-level LICENSE file for more information
 
 import datetime
-from typing import Dict, Sequence
+from typing import Dict, List, Sequence, cast
 
 import attr
 
 from swh.model.hashutil import MultiHash, hash_to_bytes
-from swh.model.identifiers import ExtendedSWHID
 from swh.model.model import (
     BaseModel,
     Content,
@@ -19,6 +18,7 @@ from swh.model.model import (
     MetadataAuthority,
     MetadataAuthorityType,
     MetadataFetcher,
+    ModelObjectType,
     ObjectType,
     Origin,
     OriginVisit,
@@ -31,14 +31,15 @@ from swh.model.model import (
     SkippedContent,
     Snapshot,
     SnapshotBranch,
-    TargetType,
+    SnapshotTargetType,
     Timestamp,
     TimestampWithTimezone,
 )
+from swh.model.swhids import ExtendedSWHID
 
 UTC = datetime.timezone.utc
 
-CONTENTS = [
+CONTENTS: List[Content] = [
     Content(
         length=4,
         data=f"foo{i}".encode(),
@@ -56,7 +57,7 @@ CONTENTS = [
     for i in range(10)
 ]
 
-SKIPPED_CONTENTS = [
+SKIPPED_CONTENTS: List[SkippedContent] = [
     SkippedContent(
         length=4,
         status="absent",
@@ -84,25 +85,29 @@ duplicate_content2 = attr.evolve(duplicate_content1, sha1_git=bytes(sha1_array))
 DUPLICATE_CONTENTS = [duplicate_content1, duplicate_content2]
 
 
-COMMITTERS = [
+COMMITTERS: List[Person] = [
     Person(fullname=b"foo", name=b"foo", email=b""),
     Person(fullname=b"bar", name=b"bar", email=b""),
 ]
 
-DATES = [
+DATES: List[TimestampWithTimezone] = [
     TimestampWithTimezone(
-        timestamp=Timestamp(seconds=1234567891, microseconds=0,),
-        offset=120,
-        negative_utc=False,
+        timestamp=Timestamp(
+            seconds=1234567891,
+            microseconds=0,
+        ),
+        offset_bytes=b"+0200",
     ),
     TimestampWithTimezone(
-        timestamp=Timestamp(seconds=1234567892, microseconds=0,),
-        offset=120,
-        negative_utc=False,
+        timestamp=Timestamp(
+            seconds=1234567892,
+            microseconds=0,
+        ),
+        offset_bytes=b"+0200",
     ),
 ]
 
-REVISIONS = [
+REVISIONS: List[Revision] = [
     Revision(
         id=hash_to_bytes("66c7c1cd9673275037140f2abff7b7b11fc9439c"),
         message=b"hello",
@@ -133,21 +138,39 @@ REVISIONS = [
         parents=(),
         extra_headers=((b"foo", b"bar"),),
     ),
+    Revision(
+        id=hash_to_bytes("51580d63b8dcc0ec73e74994e66896858542840a"),
+        message=b"hello",
+        date=DATES[0],
+        committer=COMMITTERS[0],
+        author=COMMITTERS[0],
+        committer_date=DATES[0],
+        type=RevisionType.GIT,
+        directory=b"\x01" * 20,
+        synthetic=False,
+        metadata=None,
+        parents=(hash_to_bytes("9b918dd063cec85c2bc63cc7f167e29f5894dcbc"),),
+        raw_manifest=(
+            b"commit 207\x00"
+            b"tree 0101010101010101010101010101010101010101\n"
+            b"parent 9B918DD063CEC85C2BC63CC7F167E29F5894DCBC"  # upper-cased
+            b"nauthor foo 1234567891 +0200\n"
+            b"committer foo 1234567891 +0200"
+            b"\n\nhello"
+        ),
+    ),
 ]
 
-EXTIDS = [
-    ExtID(extid_type="git256", extid=b"\x03" * 32, target=REVISIONS[0].swhid(),),
-    ExtID(extid_type="hg", extid=b"\x04" * 20, target=REVISIONS[1].swhid(),),
-]
-
-RELEASES = [
+RELEASES: List[Release] = [
     Release(
         id=hash_to_bytes("8059dc4e17fcd0e51ca3bcd6b80f4577d281fd08"),
         name=b"v0.0.1",
         date=TimestampWithTimezone(
-            timestamp=Timestamp(seconds=1234567890, microseconds=0,),
-            offset=120,
-            negative_utc=False,
+            timestamp=Timestamp(
+                seconds=1234567890,
+                microseconds=0,
+            ),
+            offset_bytes=b"+0200",
         ),
         author=COMMITTERS[0],
         target_type=ObjectType.REVISION,
@@ -165,14 +188,42 @@ RELEASES = [
         message=b"bar",
         synthetic=False,
     ),
+    Release(
+        id=hash_to_bytes("1cdd1e87234b6f066d0855a3b5b567638a55d583"),
+        name=b"v0.0.1",
+        date=TimestampWithTimezone(
+            timestamp=Timestamp(
+                seconds=1234567890,
+                microseconds=0,
+            ),
+            offset_bytes=b"+0200",
+        ),
+        author=COMMITTERS[0],
+        target_type=ObjectType.REVISION,
+        target=b"\x04" * 20,
+        message=b"foo",
+        synthetic=False,
+        raw_manifest=(
+            b"tag 102\x00"
+            b"object 0404040404040404040404040404040404040404\n"
+            b"type commit\n"
+            b"tag v0.0.1\n"
+            b"tagger foo 1234567890 +200"  # missing leading 0 for timezone
+            b"\n\nfoo"
+        ),
+    ),
 ]
 
-ORIGINS = [
-    Origin(url="https://somewhere.org/den/fox",),
-    Origin(url="https://overtherainbow.org/fox/den",),
+ORIGINS: List[Origin] = [
+    Origin(
+        url="https://somewhere.org/den/fox",
+    ),
+    Origin(
+        url="https://overtherainbow.org/fox/den",
+    ),
 ]
 
-ORIGIN_VISITS = [
+ORIGIN_VISITS: List[OriginVisit] = [
     OriginVisit(
         origin=ORIGINS[0].url,
         date=datetime.datetime(2013, 5, 7, 4, 20, 39, 369271, tzinfo=UTC),
@@ -209,7 +260,7 @@ ORIGIN_VISITS = [
 # visit dates counterpart. Otherwise, we are hitting storage-wise the "on conflict"
 # ignore policy (because origin-visit-add creates an origin-visit-status with the same
 # parameters from the origin-visit {origin, visit, date}...
-ORIGIN_VISIT_STATUSES = [
+ORIGIN_VISIT_STATUSES: List[OriginVisitStatus] = [
     OriginVisitStatus(
         origin=ORIGINS[0].url,
         date=datetime.datetime(2013, 5, 7, 4, 20, 39, 432222, tzinfo=UTC),
@@ -258,7 +309,7 @@ ORIGIN_VISIT_STATUSES = [
 ]
 
 
-DIRECTORIES = [
+DIRECTORIES: List[Directory] = [
     Directory(id=hash_to_bytes("4b825dc642cb6eb9a060e54bf8d69288fbee4904"), entries=()),
     Directory(
         id=hash_to_bytes("87b339104f7dc2a8163dec988445e3987995545f"),
@@ -276,19 +327,38 @@ DIRECTORIES = [
                 target=hash_to_bytes("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
             ),
             DirectoryEntry(
-                name=b"subprepo1", perms=0o160000, type="rev", target=REVISIONS[1].id,
+                name=b"subprepo1",
+                perms=0o160000,
+                type="rev",
+                target=REVISIONS[1].id,
             ),
         ),
     ),
+    Directory(
+        id=hash_to_bytes("d135a91ac82a754e7f4bdeff8d56ef06d921eb7d"),
+        entries=(
+            DirectoryEntry(
+                name=b"file1.ext",
+                perms=0o644,
+                type="file",
+                target=b"\x11" * 20,
+            ),
+        ),
+        raw_manifest=(
+            b"tree 34\x00"
+            + b"00644 file1.ext\x00"  # added two leading zeros
+            + b"\x11" * 20
+        ),
+    ),
 ]
 
 
-SNAPSHOTS = [
+SNAPSHOTS: List[Snapshot] = [
     Snapshot(
         id=hash_to_bytes("9e78d7105c5e0f886487511e2a92377b4ee4c32a"),
         branches={
             b"master": SnapshotBranch(
-                target_type=TargetType.REVISION, target=REVISIONS[0].id
+                target_type=SnapshotTargetType.REVISION, target=REVISIONS[0].id
             )
         },
     ),
@@ -296,19 +366,21 @@ SNAPSHOTS = [
         id=hash_to_bytes("0e7f84ede9a254f2cd55649ad5240783f557e65f"),
         branches={
             b"target/revision": SnapshotBranch(
-                target_type=TargetType.REVISION, target=REVISIONS[0].id,
+                target_type=SnapshotTargetType.REVISION,
+                target=REVISIONS[0].id,
             ),
             b"target/alias": SnapshotBranch(
-                target_type=TargetType.ALIAS, target=b"target/revision"
+                target_type=SnapshotTargetType.ALIAS, target=b"target/revision"
             ),
             b"target/directory": SnapshotBranch(
-                target_type=TargetType.DIRECTORY, target=DIRECTORIES[0].id,
+                target_type=SnapshotTargetType.DIRECTORY,
+                target=DIRECTORIES[0].id,
             ),
             b"target/release": SnapshotBranch(
-                target_type=TargetType.RELEASE, target=RELEASES[0].id
+                target_type=SnapshotTargetType.RELEASE, target=RELEASES[0].id
             ),
             b"target/snapshot": SnapshotBranch(
-                target_type=TargetType.SNAPSHOT,
+                target_type=SnapshotTargetType.SNAPSHOT,
                 target=hash_to_bytes("9e78d7105c5e0f886487511e2a92377b4ee4c32a"),
             ),
         },
@@ -316,17 +388,23 @@ SNAPSHOTS = [
 ]
 
 
-METADATA_AUTHORITIES = [
+METADATA_AUTHORITIES: List[MetadataAuthority] = [
     MetadataAuthority(
-        type=MetadataAuthorityType.FORGE, url="http://example.org/", metadata={},
+        type=MetadataAuthorityType.FORGE,
+        url="http://example.org/",
+        metadata={},
     ),
 ]
 
-METADATA_FETCHERS = [
-    MetadataFetcher(name="test-fetcher", version="1.0.0", metadata={},)
+METADATA_FETCHERS: List[MetadataFetcher] = [
+    MetadataFetcher(
+        name="test-fetcher",
+        version="1.0.0",
+        metadata={},
+    )
 ]
 
-RAW_EXTRINSIC_METADATA = [
+RAW_EXTRINSIC_METADATA: List[RawExtrinsicMetadata] = [
     RawExtrinsicMetadata(
         target=Origin("http://example.org/foo.git").swhid(),
         discovery_date=datetime.datetime(2020, 7, 30, 17, 8, 20, tzinfo=UTC),
@@ -345,22 +423,54 @@ RAW_EXTRINSIC_METADATA = [
     ),
 ]
 
+EXTIDS: List[ExtID] = [
+    ExtID(
+        extid_type="git256",
+        extid=b"\x03" * 32,
+        target=REVISIONS[0].swhid(),
+    ),
+    ExtID(
+        extid_type="hg",
+        extid=b"\x04" * 20,
+        target=REVISIONS[1].swhid(),
+    ),
+    ExtID(
+        extid_type="hg-nodeid",
+        extid=b"\x05" * 20,
+        target=REVISIONS[1].swhid(),
+        extid_version=1,
+    ),
+    ExtID(
+        extid_type="tarball-sha256",
+        extid=b"\x03" * 32,
+        target=DIRECTORIES[0].swhid(),
+        payload_type="disarchive",
+        payload=CONTENTS[0].sha1_git,
+    ),
+]
 
-TEST_OBJECTS: Dict[str, Sequence[BaseModel]] = {
-    "content": CONTENTS,
-    "directory": DIRECTORIES,
-    "extid": EXTIDS,
-    "metadata_authority": METADATA_AUTHORITIES,
-    "metadata_fetcher": METADATA_FETCHERS,
-    "origin": ORIGINS,
-    "origin_visit": ORIGIN_VISITS,
-    "origin_visit_status": ORIGIN_VISIT_STATUSES,
-    "raw_extrinsic_metadata": RAW_EXTRINSIC_METADATA,
-    "release": RELEASES,
-    "revision": REVISIONS,
-    "snapshot": SNAPSHOTS,
-    "skipped_content": SKIPPED_CONTENTS,
-}
+TEST_OBJECTS: Dict[ModelObjectType, Sequence[BaseModel]] = {}
+# generate this mapping with code to avoid error
+for objects in [
+    CONTENTS,
+    DIRECTORIES,
+    EXTIDS,
+    METADATA_AUTHORITIES,
+    METADATA_FETCHERS,
+    ORIGINS,
+    ORIGIN_VISITS,
+    ORIGIN_VISIT_STATUSES,
+    RAW_EXTRINSIC_METADATA,
+    RELEASES,
+    REVISIONS,
+    SNAPSHOTS,
+    SKIPPED_CONTENTS,
+]:
+    objects = cast(List[BaseModel], objects)
+    object_type = objects[0].object_type
+    assert all(object_type == o.object_type for o in objects)
+    assert object_type not in TEST_OBJECTS
+    TEST_OBJECTS[object_type] = objects
 
 SAMPLE_FOLDER_SWHIDS = [
     "swh:1:dir:e8b0f1466af8608c8a3fb9879db172b887e80759",

swh/model/tests/test_cli.py

@@ -19,7 +19,6 @@ from swh.model.tests.swh_model_data import SAMPLE_FOLDER_SWHIDS
 from swh.model.tests.test_from_disk import DataMixin
 
 
-@pytest.mark.fs
 class TestIdentify(DataMixin, unittest.TestCase):
     def setUp(self):
         super().setUp()
@@ -78,7 +77,9 @@ class TestIdentify(DataMixin, unittest.TestCase):
         with unittest.mock.patch.dict(sys.modules, {"dulwich": None}):
             with tempfile.TemporaryDirectory(prefix="swh.model.cli") as d:
                 result = self.runner.invoke(
-                    cli.identify, ["--type", "snapshot", d], catch_exceptions=False,
+                    cli.identify,
+                    ["--type", "snapshot", d],
+                    catch_exceptions=False,
                 )
 
         assert result.exit_code == 1
@@ -94,7 +95,8 @@ class TestIdentify(DataMixin, unittest.TestCase):
         """identify symlink --- both itself and target"""
         regular = os.path.join(self.tmpdir_name, b"foo.txt")
         link = os.path.join(self.tmpdir_name, b"bar.txt")
-        open(regular, "w").write("foo\n")
+        with open(regular, "w") as f:
+            f.write("foo\n")
         os.symlink(os.path.basename(regular), link)
 
         result = self.runner.invoke(cli.identify, [link])

swh/model/tests/test_discovery.py

+# Copyright (C) 2023 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from dataclasses import dataclass
+from typing import Iterable, List
+
+from swh.model import discovery, model
+from swh.model.hashutil import hash_to_bytes
+from swh.model.model import Sha1Git
+from swh.model.tests.test_identifiers import directory_example
+
+pytest_plugins = ["aiohttp.pytest_plugin"]
+
+UNKNOWN_HASH = hash_to_bytes("17140cb6109f1e3296dc52e2b2cd29bcb40e86be")
+KNOWN_CONTENT_HASH = hash_to_bytes("e8e4106de42e2d5d5efab6a9422b9a8677c993c8")
+KNOWN_DIRECTORY_HASH = hash_to_bytes("d7ed3d2c31d608823be58b1cbe57605310615231")
+KNOWN_DIRECTORY_HASH_2 = hash_to_bytes("c76724e9a0be4b60f4bf0cb48b261df8eda94b1d")
+
+
+@dataclass
+class FakeArchive:
+    contents: List[model.Content]
+    skipped_contents: List[model.SkippedContent]
+    directories: List[model.Directory]
+
+    def content_missing(self, contents: List[Sha1Git]) -> Iterable[Sha1Git]:
+        return []
+
+    def skipped_content_missing(
+        self, skipped_contents: List[Sha1Git]
+    ) -> Iterable[Sha1Git]:
+        """List skipped content missing from the archive by sha1"""
+        return []
+
+    def directory_missing(self, directories: List[Sha1Git]) -> Iterable[Sha1Git]:
+        """List directories missing from the archive by sha1"""
+        return []
+
+
+def test_filter_known_objects(monkeypatch):
+    # Test with smaller sample sizes to actually trigger the random sampling
+    monkeypatch.setattr(discovery, "SAMPLE_SIZE", 1)
+
+    base_directory = model.Directory.from_dict(directory_example)
+
+    # Hardcoding another hash is enough since it's all that's being checked
+    directory_data = directory_example.copy()
+    directory_data["id"] = KNOWN_DIRECTORY_HASH_2
+    other_directory = model.Directory.from_dict(directory_data)
+    archive = FakeArchive(
+        contents=[model.Content.from_data(b"blabla")],
+        skipped_contents=[model.SkippedContent.from_data(b"blabla2", reason="reason")],
+        directories=[
+            base_directory,
+            other_directory,
+        ],
+    )
+    assert archive.contents[0].sha1_git == KNOWN_CONTENT_HASH
+    assert archive.directories[0].id == KNOWN_DIRECTORY_HASH
+    assert archive.directories[1].id == KNOWN_DIRECTORY_HASH_2
+    (contents, skipped_contents, directories) = discovery.filter_known_objects(archive)
+    assert len(contents) == 0
+    assert len(skipped_contents) == 0
+    assert len(directories) == 0