diff --git a/PKG-INFO b/PKG-INFO
index 479c4cc885d38a38b955a19d00297184c6004e43..82b5632091f301de09de09fea6a742bf00ec2f8f 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: swh.model
-Version: 0.4.0
+Version: 0.5.0
Summary: Software Heritage data model
Home-page: https://forge.softwareheritage.org/diffusion/DMOD/
Author: Software Heritage developers
diff --git a/swh.model.egg-info/PKG-INFO b/swh.model.egg-info/PKG-INFO
index 479c4cc885d38a38b955a19d00297184c6004e43..82b5632091f301de09de09fea6a742bf00ec2f8f 100644
--- a/swh.model.egg-info/PKG-INFO
+++ b/swh.model.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: swh.model
-Version: 0.4.0
+Version: 0.5.0
Summary: Software Heritage data model
Home-page: https://forge.softwareheritage.org/diffusion/DMOD/
Author: Software Heritage developers
diff --git a/swh.model.egg-info/SOURCES.txt b/swh.model.egg-info/SOURCES.txt
index 74c28607d46eeefa9f5c9f724229ccfddf41df7b..763a95cb9133c6375cceb9709b9ae918df62c525 100644
--- a/swh.model.egg-info/SOURCES.txt
+++ b/swh.model.egg-info/SOURCES.txt
@@ -17,6 +17,7 @@ swh.model.egg-info/requires.txt
swh.model.egg-info/top_level.txt
swh/model/__init__.py
swh/model/cli.py
+swh/model/collections.py
swh/model/exceptions.py
swh/model/from_disk.py
swh/model/hashutil.py
@@ -25,6 +26,7 @@ swh/model/identifiers.py
swh/model/merkle.py
swh/model/model.py
swh/model/py.typed
+swh/model/test_identifiers.py
swh/model/toposort.py
swh/model/validators.py
swh/model/fields/__init__.py
@@ -35,6 +37,7 @@ swh/model/tests/__init__.py
swh/model/tests/generate_testdata.py
swh/model/tests/generate_testdata_from_disk.py
swh/model/tests/test_cli.py
+swh/model/tests/test_collections.py
swh/model/tests/test_from_disk.py
swh/model/tests/test_generate_testdata.py
swh/model/tests/test_hashutil.py
diff --git a/swh/model/collections.py b/swh/model/collections.py
new file mode 100644
index 0000000000000000000000000000000000000000..38d28a329de2e02c5077da83a20001a4b38513c7
--- /dev/null
+++ b/swh/model/collections.py
@@ -0,0 +1,59 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from collections.abc import Mapping
+from typing import Dict, Generic, Iterable, Optional, Tuple, TypeVar, Union
+
+KT = TypeVar("KT")
+VT = TypeVar("VT")
+
+
+class ImmutableDict(Mapping, Generic[KT, VT]):
+ data: Tuple[Tuple[KT, VT], ...]
+
+ def __init__(
+ self,
+ data: Union[
+ Iterable[Tuple[KT, VT]], "ImmutableDict[KT, VT]", Dict[KT, VT]
+ ] = {},
+ ):
+ if isinstance(data, dict):
+ self.data = tuple(item for item in data.items())
+ elif isinstance(data, ImmutableDict):
+ self.data = data.data
+ else:
+ self.data = tuple(data)
+
+ def __getitem__(self, key):
+ for (k, v) in self.data:
+ if k == key:
+ return v
+ raise KeyError(key)
+
+ def __iter__(self):
+ for (k, v) in self.data:
+ yield k
+
+ def __len__(self):
+ return len(self.data)
+
+ def items(self):
+ yield from self.data
+
+ def __hash__(self):
+ return hash(tuple(sorted(self.data)))
+
+ def copy_pop(self, popped_key) -> Tuple[Optional[VT], "ImmutableDict[KT, VT]"]:
+ """Returns a copy of this ImmutableDict without the given key,
+ as well as the value associated to the key."""
+ popped_value = None
+ new_items = []
+ for (key, value) in self.data:
+ if key == popped_key:
+ popped_value = value
+ else:
+ new_items.append((key, value))
+
+ return (popped_value, ImmutableDict(new_items))
diff --git a/swh/model/identifiers.py b/swh/model/identifiers.py
index 3c0a46ebc5e4d8ca2faed299aee68adc8b0a92a8..e1cf0dfe3b80fe40f0df23beb3c493098f82c2fd 100644
--- a/swh/model/identifiers.py
+++ b/swh/model/identifiers.py
@@ -8,10 +8,12 @@ import datetime
import hashlib
from functools import lru_cache
-from typing import Any, Dict, NamedTuple
+from typing import Any, Dict, Union
+import attr
from deprecated import deprecated
+from .collections import ImmutableDict
from .exceptions import ValidationError
from .fields.hashes import validate_sha1
from .hashutil import hash_git_data, hash_to_hex, MultiHash
@@ -649,19 +651,8 @@ _object_type_map = {
}
-_SWHID = NamedTuple(
- "SWHID",
- [
- ("namespace", str),
- ("scheme_version", int),
- ("object_type", str),
- ("object_id", str),
- ("metadata", Dict[str, Any]),
- ],
-)
-
-
-class SWHID(_SWHID):
+@attr.s(frozen=True)
+class SWHID:
"""
Named tuple holding the relevant info associated to a SoftWare Heritage
persistent IDentifier (SWHID)
@@ -699,39 +690,41 @@ class SWHID(_SWHID):
# 'swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0'
"""
- __slots__ = ()
-
- def __new__(
- cls,
- namespace: str = SWHID_NAMESPACE,
- scheme_version: int = SWHID_VERSION,
- object_type: str = "",
- object_id: str = "",
- metadata: Dict[str, Any] = {},
- ):
- o = _object_type_map.get(object_type)
- if not o:
- raise ValidationError(
- "Wrong input: Supported types are %s" % (list(_object_type_map.keys()))
- )
- if namespace != SWHID_NAMESPACE:
+ namespace = attr.ib(type=str, default="swh")
+ scheme_version = attr.ib(type=int, default=1)
+ object_type = attr.ib(type=str, default="")
+ object_id = attr.ib(type=str, converter=hash_to_hex, default="") # type: ignore
+ metadata = attr.ib(
+ type=ImmutableDict[str, Any], converter=ImmutableDict, default=ImmutableDict()
+ )
+
+ @namespace.validator
+ def check_namespace(self, attribute, value):
+ if value != SWHID_NAMESPACE:
raise ValidationError(
"Wrong format: only supported namespace is '%s'" % SWHID_NAMESPACE
)
- if scheme_version != SWHID_VERSION:
+
+ @scheme_version.validator
+ def check_scheme_version(self, attribute, value):
+ if value != SWHID_VERSION:
raise ValidationError(
"Wrong format: only supported version is %d" % SWHID_VERSION
)
- # internal swh representation resolution
- if isinstance(object_id, dict):
- object_id = object_id[o["key_id"]]
+ @object_type.validator
+ def check_object_type(self, attribute, value):
+ if value not in _object_type_map:
+ raise ValidationError(
+ "Wrong input: Supported types are %s" % (list(_object_type_map.keys()))
+ )
- validate_sha1(object_id) # can raise if invalid hash
- object_id = hash_to_hex(object_id)
- return super().__new__(
- cls, namespace, scheme_version, object_type, object_id, metadata
- )
+ @object_id.validator
+ def check_object_id(self, attribute, value):
+ validate_sha1(value) # can raise if invalid hash
+
+ def to_dict(self) -> Dict[str, Any]:
+ return attr.asdict(self)
def __str__(self) -> str:
o = _object_type_map.get(self.object_type)
@@ -756,15 +749,14 @@ class PersistentId(SWHID):
"""
- def __new__(cls, *args, **kwargs):
- return super(cls, PersistentId).__new__(cls, *args, **kwargs)
+ pass
def swhid(
object_type: str,
- object_id: str,
+ object_id: Union[str, Dict[str, Any]],
scheme_version: int = 1,
- metadata: Dict[str, Any] = {},
+ metadata: Union[ImmutableDict[str, Any], Dict[str, Any]] = ImmutableDict(),
) -> str:
"""Compute :ref:`persistent-identifiers`
@@ -782,11 +774,14 @@ def swhid(
the SWHID of the object
"""
+ if isinstance(object_id, dict):
+ o = _object_type_map[object_type]
+ object_id = object_id[o["key_id"]]
swhid = SWHID(
scheme_version=scheme_version,
object_type=object_type,
object_id=object_id,
- metadata=metadata,
+ metadata=metadata, # type: ignore # mypy can't properly unify types
)
return str(swhid)
@@ -848,7 +843,13 @@ def parse_swhid(swhid: str) -> SWHID:
except Exception:
msg = "Contextual data is badly formatted, form key=val expected"
raise ValidationError(msg)
- return SWHID(_ns, int(_version), _type, _id, _metadata)
+ return SWHID(
+ _ns,
+ int(_version),
+ _type,
+ _id,
+ _metadata, # type: ignore # mypy can't properly unify types
+ )
@deprecated("Use swh.model.identifiers.parse_swhid instead")
@@ -858,4 +859,4 @@ def parse_persistent_identifier(persistent_id: str) -> PersistentId:
.. deprecated:: 0.3.8
Use :func:`swh.model.identifiers.parse_swhid` instead
"""
- return PersistentId(**parse_swhid(persistent_id)._asdict())
+ return PersistentId(**parse_swhid(persistent_id).to_dict())
diff --git a/swh/model/model.py b/swh/model/model.py
index c4f185f5075a8b68171aa30080d2fa01e5271c5d..66649015cc73cb80bd8ac7b98081333179ea27f9 100644
--- a/swh/model/model.py
+++ b/swh/model/model.py
@@ -6,10 +6,9 @@
import datetime
from abc import ABCMeta, abstractmethod
-from copy import deepcopy
from enum import Enum
from hashlib import sha256
-from typing import Dict, Iterable, Optional, Tuple, TypeVar, Union
+from typing import Any, Dict, Iterable, Optional, Tuple, TypeVar, Union
from typing_extensions import Final
import attr
@@ -17,14 +16,17 @@ from attrs_strict import type_validator
import dateutil.parser
import iso8601
+from .collections import ImmutableDict
+from .hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, MultiHash
from .identifiers import (
normalize_timestamp,
directory_identifier,
revision_identifier,
release_identifier,
snapshot_identifier,
+ SWHID,
+ parse_swhid,
)
-from .hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, MultiHash
class MissingData(Exception):
@@ -40,13 +42,28 @@ SHA1_SIZE = 20
Sha1Git = bytes
+KT = TypeVar("KT")
+VT = TypeVar("VT")
+
+
+def freeze_optional_dict(
+ d: Union[None, Dict[KT, VT], ImmutableDict[KT, VT]] # type: ignore
+) -> Optional[ImmutableDict[KT, VT]]:
+ if isinstance(d, dict):
+ return ImmutableDict(d)
+ else:
+ return d
+
+
def dictify(value):
"Helper function used by BaseModel.to_dict()"
if isinstance(value, BaseModel):
return value.to_dict()
+ elif isinstance(value, SWHID):
+ return str(value)
elif isinstance(value, Enum):
return value.value
- elif isinstance(value, dict):
+ elif isinstance(value, (dict, ImmutableDict)):
return {k: dictify(v) for k, v in value.items()}
elif isinstance(value, tuple):
return tuple(dictify(v) for v in value)
@@ -276,7 +293,10 @@ class OriginVisitStatus(BaseModel):
)
snapshot = attr.ib(type=Optional[Sha1Git], validator=type_validator())
metadata = attr.ib(
- type=Optional[Dict[str, object]], validator=type_validator(), default=None
+ type=Optional[ImmutableDict[str, object]],
+ validator=type_validator(),
+ converter=freeze_optional_dict,
+ default=None,
)
@@ -331,7 +351,9 @@ class Snapshot(BaseModel, HashableObject):
object_type: Final = "snapshot"
branches = attr.ib(
- type=Dict[bytes, Optional[SnapshotBranch]], validator=type_validator()
+ type=ImmutableDict[bytes, Optional[SnapshotBranch]],
+ validator=type_validator(),
+ converter=freeze_optional_dict,
)
id = attr.ib(type=Sha1Git, validator=type_validator(), default=b"")
@@ -343,10 +365,10 @@ class Snapshot(BaseModel, HashableObject):
def from_dict(cls, d):
d = d.copy()
return cls(
- branches={
- name: SnapshotBranch.from_dict(branch) if branch else None
+ branches=ImmutableDict(
+ (name, SnapshotBranch.from_dict(branch) if branch else None)
for (name, branch) in d.pop("branches").items()
- },
+ ),
**d,
)
@@ -365,7 +387,10 @@ class Release(BaseModel, HashableObject):
type=Optional[TimestampWithTimezone], validator=type_validator(), default=None
)
metadata = attr.ib(
- type=Optional[Dict[str, object]], validator=type_validator(), default=None
+ type=Optional[ImmutableDict[str, object]],
+ validator=type_validator(),
+ converter=freeze_optional_dict,
+ default=None,
)
id = attr.ib(type=Sha1Git, validator=type_validator(), default=b"")
@@ -430,7 +455,10 @@ class Revision(BaseModel, HashableObject):
directory = attr.ib(type=Sha1Git, validator=type_validator())
synthetic = attr.ib(type=bool, validator=type_validator())
metadata = attr.ib(
- type=Optional[Dict[str, object]], validator=type_validator(), default=None
+ type=Optional[ImmutableDict[str, object]],
+ validator=type_validator(),
+ converter=freeze_optional_dict,
+ default=None,
)
parents = attr.ib(type=Tuple[Sha1Git, ...], validator=type_validator(), default=())
id = attr.ib(type=Sha1Git, validator=type_validator(), default=b"")
@@ -446,12 +474,11 @@ class Revision(BaseModel, HashableObject):
# ensure metadata is a deep copy of whatever was given, and if needed
# extract extra_headers from there
if self.metadata:
- metadata = deepcopy(self.metadata)
+ metadata = self.metadata
if not self.extra_headers and "extra_headers" in metadata:
+ (extra_headers, metadata) = metadata.copy_pop("extra_headers")
object.__setattr__(
- self,
- "extra_headers",
- tuplify_extra_headers(metadata.pop("extra_headers")),
+ self, "extra_headers", tuplify_extra_headers(extra_headers),
)
attr.validate(self)
object.__setattr__(self, "metadata", metadata)
@@ -696,3 +723,264 @@ class SkippedContent(BaseContent):
if d2.pop("data", None) is not None:
raise ValueError('SkippedContent has no "data" attribute %r' % d)
return super().from_dict(d2, use_subclass=False)
+
+
+class MetadataAuthorityType(Enum):
+ DEPOSIT = "deposit"
+ FORGE = "forge"
+ REGISTRY = "registry"
+
+
+@attr.s(frozen=True)
+class MetadataAuthority(BaseModel):
+ """Represents an entity that provides metadata about an origin or
+ software artifact."""
+
+ type = attr.ib(type=MetadataAuthorityType, validator=type_validator())
+ url = attr.ib(type=str, validator=type_validator())
+ metadata = attr.ib(
+ type=Optional[ImmutableDict[str, Any]],
+ default=None,
+ validator=type_validator(),
+ converter=freeze_optional_dict,
+ )
+
+ def to_dict(self):
+ d = super().to_dict()
+ if d["metadata"] is None:
+ del d["metadata"]
+ return d
+
+ @classmethod
+ def from_dict(cls, d):
+ d["type"] = MetadataAuthorityType(d["type"])
+ return super().from_dict(d)
+
+
+@attr.s(frozen=True)
+class MetadataFetcher(BaseModel):
+ """Represents a software component used to fetch metadata from a metadata
+ authority, and ingest them into the Software Heritage archive."""
+
+ name = attr.ib(type=str, validator=type_validator())
+ version = attr.ib(type=str, validator=type_validator())
+ metadata = attr.ib(
+ type=Optional[ImmutableDict[str, Any]],
+ default=None,
+ validator=type_validator(),
+ converter=freeze_optional_dict,
+ )
+
+ def to_dict(self):
+ d = super().to_dict()
+ if d["metadata"] is None:
+ del d["metadata"]
+ return d
+
+
+class MetadataTargetType(Enum):
+ """The type of object extrinsic metadata refer to."""
+
+ CONTENT = "content"
+ DIRECTORY = "directory"
+ REVISION = "revision"
+ RELEASE = "release"
+ SNAPSHOT = "snapshot"
+ ORIGIN = "origin"
+
+
+@attr.s(frozen=True)
+class RawExtrinsicMetadata(BaseModel):
+ # target object
+ type = attr.ib(type=MetadataTargetType, validator=type_validator())
+ id = attr.ib(type=Union[str, SWHID], validator=type_validator())
+ """URL if type=MetadataTargetType.ORIGIN, else core SWHID"""
+
+ # source
+ discovery_date = attr.ib(type=datetime.datetime, validator=type_validator())
+ authority = attr.ib(type=MetadataAuthority, validator=type_validator())
+ fetcher = attr.ib(type=MetadataFetcher, validator=type_validator())
+
+ # the metadata itself
+ format = attr.ib(type=str, validator=type_validator())
+ metadata = attr.ib(type=bytes, validator=type_validator())
+
+ # context
+ origin = attr.ib(type=Optional[str], default=None, validator=type_validator())
+ visit = attr.ib(type=Optional[int], default=None, validator=type_validator())
+ snapshot = attr.ib(type=Optional[SWHID], default=None, validator=type_validator())
+ release = attr.ib(type=Optional[SWHID], default=None, validator=type_validator())
+ revision = attr.ib(type=Optional[SWHID], default=None, validator=type_validator())
+ path = attr.ib(type=Optional[bytes], default=None, validator=type_validator())
+ directory = attr.ib(type=Optional[SWHID], default=None, validator=type_validator())
+
+ @id.validator
+ def check_id(self, attribute, value):
+ if self.type == MetadataTargetType.ORIGIN:
+ if isinstance(value, SWHID) or value.startswith("swh:"):
+ raise ValueError(
+ "Got SWHID as id for origin metadata (expected an URL)."
+ )
+ else:
+ self._check_pid(self.type.value, value)
+
+ @origin.validator
+ def check_origin(self, attribute, value):
+ if value is None:
+ return
+
+ if self.type not in (
+ MetadataTargetType.SNAPSHOT,
+ MetadataTargetType.RELEASE,
+ MetadataTargetType.REVISION,
+ MetadataTargetType.DIRECTORY,
+ MetadataTargetType.CONTENT,
+ ):
+ raise ValueError(
+ f"Unexpected 'origin' context for {self.type.value} object: {value}"
+ )
+
+ if value.startswith("swh:"):
+ # Technically this is valid; but:
+ # 1. SWHIDs are URIs, not URLs
+ # 2. if a SWHID gets here, it's very likely to be a mistake
+ # (and we can remove this check if it turns out there is a
+ # legitimate use for it).
+ raise ValueError(f"SWHID used as context origin URL: {value}")
+
+ @visit.validator
+ def check_visit(self, attribute, value):
+ if value is None:
+ return
+
+ if self.type not in (
+ MetadataTargetType.SNAPSHOT,
+ MetadataTargetType.RELEASE,
+ MetadataTargetType.REVISION,
+ MetadataTargetType.DIRECTORY,
+ MetadataTargetType.CONTENT,
+ ):
+ raise ValueError(
+ f"Unexpected 'visit' context for {self.type.value} object: {value}"
+ )
+
+ if self.origin is None:
+ raise ValueError("'origin' context must be set if 'visit' is.")
+
+ if value <= 0:
+ raise ValueError("Nonpositive visit id")
+
+ @snapshot.validator
+ def check_snapshot(self, attribute, value):
+ if value is None:
+ return
+
+ if self.type not in (
+ MetadataTargetType.RELEASE,
+ MetadataTargetType.REVISION,
+ MetadataTargetType.DIRECTORY,
+ MetadataTargetType.CONTENT,
+ ):
+ raise ValueError(
+ f"Unexpected 'snapshot' context for {self.type.value} object: {value}"
+ )
+
+ self._check_pid("snapshot", value)
+
+ @release.validator
+ def check_release(self, attribute, value):
+ if value is None:
+ return
+
+ if self.type not in (
+ MetadataTargetType.REVISION,
+ MetadataTargetType.DIRECTORY,
+ MetadataTargetType.CONTENT,
+ ):
+ raise ValueError(
+ f"Unexpected 'release' context for {self.type.value} object: {value}"
+ )
+
+ self._check_pid("release", value)
+
+ @revision.validator
+ def check_revision(self, attribute, value):
+ if value is None:
+ return
+
+ if self.type not in (MetadataTargetType.DIRECTORY, MetadataTargetType.CONTENT,):
+ raise ValueError(
+ f"Unexpected 'revision' context for {self.type.value} object: {value}"
+ )
+
+ self._check_pid("revision", value)
+
+ @path.validator
+ def check_path(self, attribute, value):
+ if value is None:
+ return
+
+ if self.type not in (MetadataTargetType.DIRECTORY, MetadataTargetType.CONTENT,):
+ raise ValueError(
+ f"Unexpected 'path' context for {self.type.value} object: {value}"
+ )
+
+ @directory.validator
+ def check_directory(self, attribute, value):
+ if value is None:
+ return
+
+ if self.type not in (MetadataTargetType.CONTENT,):
+ raise ValueError(
+ f"Unexpected 'directory' context for {self.type.value} object: {value}"
+ )
+
+ self._check_pid("directory", value)
+
+ def _check_pid(self, expected_object_type, pid):
+ if isinstance(pid, str):
+ raise ValueError(f"Expected SWHID, got a string: {pid}")
+
+ if pid.object_type != expected_object_type:
+ raise ValueError(
+ f"Expected SWHID type '{expected_object_type}', "
+ f"got '{pid.object_type}' in {pid}"
+ )
+
+ if pid.metadata:
+ raise ValueError(f"Expected core SWHID, but got: {pid}")
+
+ def to_dict(self):
+ d = super().to_dict()
+ context_keys = (
+ "origin",
+ "visit",
+ "snapshot",
+ "release",
+ "revision",
+ "directory",
+ "path",
+ )
+ for context_key in context_keys:
+ if d[context_key] is None:
+ del d[context_key]
+ return d
+
+ @classmethod
+ def from_dict(cls, d):
+ d = {
+ **d,
+ "type": MetadataTargetType(d["type"]),
+ "authority": MetadataAuthority.from_dict(d["authority"]),
+ "fetcher": MetadataFetcher.from_dict(d["fetcher"]),
+ }
+
+ if d["type"] != MetadataTargetType.ORIGIN:
+ d["id"] = parse_swhid(d["id"])
+
+ swhid_keys = ("snapshot", "release", "revision", "directory")
+ for swhid_key in swhid_keys:
+ if d.get(swhid_key):
+ d[swhid_key] = parse_swhid(d[swhid_key])
+
+ return super().from_dict(d)
diff --git a/swh/model/test_identifiers.py b/swh/model/test_identifiers.py
new file mode 100644
index 0000000000000000000000000000000000000000..412e00a0a6b77d9de1a8f73acb6bc31181f3ad3a
--- /dev/null
+++ b/swh/model/test_identifiers.py
@@ -0,0 +1,72 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.model.identifiers import SWHID
+
+
+def test_swhid_hash():
+ object_id = "94a9ed024d3859793618152ea559a168bbcbb5e2"
+
+ assert hash(SWHID(object_type="directory", object_id=object_id)) == hash(
+ SWHID(object_type="directory", object_id=object_id)
+ )
+
+ assert hash(
+ SWHID(
+ object_type="directory",
+ object_id=object_id,
+ metadata={"foo": "bar", "baz": "qux"},
+ )
+ ) == hash(
+ SWHID(
+ object_type="directory",
+ object_id=object_id,
+ metadata={"foo": "bar", "baz": "qux"},
+ )
+ )
+
+ # Different order of the dictionary, so the underlying order of the tuple in
+ # ImmutableDict is different.
+ assert hash(
+ SWHID(
+ object_type="directory",
+ object_id=object_id,
+ metadata={"foo": "bar", "baz": "qux"},
+ )
+ ) == hash(
+ SWHID(
+ object_type="directory",
+ object_id=object_id,
+ metadata={"baz": "qux", "foo": "bar"},
+ )
+ )
+
+
+def test_swhid_eq():
+ object_id = "94a9ed024d3859793618152ea559a168bbcbb5e2"
+
+ assert SWHID(object_type="directory", object_id=object_id) == SWHID(
+ object_type="directory", object_id=object_id
+ )
+
+ assert SWHID(
+ object_type="directory",
+ object_id=object_id,
+ metadata={"foo": "bar", "baz": "qux"},
+ ) == SWHID(
+ object_type="directory",
+ object_id=object_id,
+ metadata={"foo": "bar", "baz": "qux"},
+ )
+
+ assert SWHID(
+ object_type="directory",
+ object_id=object_id,
+ metadata={"foo": "bar", "baz": "qux"},
+ ) == SWHID(
+ object_type="directory",
+ object_id=object_id,
+ metadata={"baz": "qux", "foo": "bar"},
+ )
diff --git a/swh/model/tests/test_collections.py b/swh/model/tests/test_collections.py
new file mode 100644
index 0000000000000000000000000000000000000000..d096009d82d975d10d1a7241cb27e8189b8c7d45
--- /dev/null
+++ b/swh/model/tests/test_collections.py
@@ -0,0 +1,86 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+
+from swh.model.collections import ImmutableDict
+
+
+def test_immutabledict_empty():
+ d = ImmutableDict()
+
+ assert d == {}
+ assert d != {"foo": "bar"}
+
+ assert list(d) == []
+ assert list(d.items()) == []
+
+
+def test_immutabledict_one_item():
+ d = ImmutableDict({"foo": "bar"})
+
+ assert d == {"foo": "bar"}
+ assert d != {}
+
+ assert d["foo"] == "bar"
+ with pytest.raises(KeyError, match="bar"):
+ d["bar"]
+
+ assert list(d) == ["foo"]
+ assert list(d.items()) == [("foo", "bar")]
+
+
+def test_immutabledict_from_iterable():
+ d1 = ImmutableDict()
+ d2 = ImmutableDict({"foo": "bar"})
+
+ assert ImmutableDict([]) == d1
+ assert ImmutableDict([("foo", "bar")]) == d2
+
+
+def test_immutabledict_from_immutabledict():
+ d1 = ImmutableDict()
+ d2 = ImmutableDict({"foo": "bar"})
+
+ assert ImmutableDict(d1) == d1
+ assert ImmutableDict(d2) == d2
+
+
+def test_immutabledict_immutable():
+ d = ImmutableDict({"foo": "bar"})
+
+ with pytest.raises(TypeError, match="item assignment"):
+ d["bar"] = "baz"
+
+ with pytest.raises(TypeError, match="item deletion"):
+ del d["foo"]
+
+
+def test_immutabledict_copy_pop():
+ d = ImmutableDict({"foo": "bar", "baz": "qux"})
+
+ assert d.copy_pop("foo") == ("bar", ImmutableDict({"baz": "qux"}))
+
+ assert d.copy_pop("not a key") == (None, d)
+
+
+def test_hash():
+ assert hash(ImmutableDict()) == hash(ImmutableDict({}))
+ assert hash(ImmutableDict({"foo": "bar"})) == hash(ImmutableDict({"foo": "bar"}))
+ assert hash(ImmutableDict({"foo": "bar", "baz": "qux"})) == hash(
+ ImmutableDict({"foo": "bar", "baz": "qux"})
+ )
+ assert hash(ImmutableDict({"foo": "bar", "baz": "qux"})) == hash(
+ ImmutableDict({"baz": "qux", "foo": "bar"})
+ )
+
+
+def test_equality_order():
+ assert ImmutableDict({"foo": "bar", "baz": "qux"}) == ImmutableDict(
+ {"foo": "bar", "baz": "qux"}
+ )
+ assert ImmutableDict({"foo": "bar", "baz": "qux"}) == ImmutableDict(
+ {"baz": "qux", "foo": "bar"}
+ )
diff --git a/swh/model/tests/test_identifiers.py b/swh/model/tests/test_identifiers.py
index c03b9eff0e9ebebaac06994be8e94ee1e4431736..5acbd2d38c19466bda2c14b16595ac9d3d41cc0e 100644
--- a/swh/model/tests/test_identifiers.py
+++ b/swh/model/tests/test_identifiers.py
@@ -915,6 +915,16 @@ class SnapshotIdentifier(unittest.TestCase):
)
actual_result = identifiers.parse_swhid(swhid)
self.assertEqual(actual_result, expected_result)
+ self.assertEqual(
+ expected_result.to_dict(),
+ {
+ "namespace": "swh",
+ "scheme_version": _version,
+ "object_type": _type,
+ "object_id": _hash,
+ "metadata": _metadata,
+ },
+ )
def test_parse_swhid_parsing_error(self):
for swhid in [
diff --git a/swh/model/tests/test_model.py b/swh/model/tests/test_model.py
index edfc829b74e394eacf9b5c6e027eaee07605aac2..b6aa82cb6d081221499ea9e3ad639f28b167c7b8 100644
--- a/swh/model/tests/test_model.py
+++ b/swh/model/tests/test_model.py
@@ -25,6 +25,11 @@ from swh.model.model import (
TimestampWithTimezone,
MissingData,
Person,
+ RawExtrinsicMetadata,
+ MetadataTargetType,
+ MetadataAuthority,
+ MetadataAuthorityType,
+ MetadataFetcher,
)
from swh.model.hashutil import hash_to_bytes, MultiHash
import swh.model.hypothesis_strategies as strategies
@@ -33,6 +38,8 @@ from swh.model.identifiers import (
revision_identifier,
release_identifier,
snapshot_identifier,
+ parse_swhid,
+ SWHID,
)
from swh.model.tests.test_identifiers import (
directory_example,
@@ -678,3 +685,435 @@ def test_object_type_is_final():
check_final(subcls)
check_final(BaseModel)
+
+
+_metadata_authority = MetadataAuthority(
+ type=MetadataAuthorityType.FORGE, url="https://forge.softwareheritage.org",
+)
+_metadata_fetcher = MetadataFetcher(name="test-fetcher", version="0.0.1",)
+_content_swhid = parse_swhid("swh:1:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2")
+_origin_url = "https://forge.softwareheritage.org/source/swh-model.git"
+_common_metadata_fields = dict(
+ discovery_date=datetime.datetime.now(),
+ authority=_metadata_authority,
+ fetcher=_metadata_fetcher,
+ format="json",
+ metadata=b'{"foo": "bar"}',
+)
+
+
+def test_metadata_valid():
+ """Checks valid RawExtrinsicMetadata objects don't raise an error."""
+
+ # Simplest case
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN, id=_origin_url, **_common_metadata_fields
+ )
+
+ # Object with an SWHID
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT, id=_content_swhid, **_common_metadata_fields
+ )
+
+
+def test_metadata_to_dict():
+ """Checks valid RawExtrinsicMetadata objects don't raise an error."""
+
+ common_fields = {
+ "authority": {"type": "forge", "url": "https://forge.softwareheritage.org",},
+ "fetcher": {"name": "test-fetcher", "version": "0.0.1",},
+ "discovery_date": _common_metadata_fields["discovery_date"],
+ "format": "json",
+ "metadata": b'{"foo": "bar"}',
+ }
+
+ m = RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN, id=_origin_url, **_common_metadata_fields
+ )
+ assert m.to_dict() == {
+ "type": "origin",
+ "id": _origin_url,
+ **common_fields,
+ }
+ assert RawExtrinsicMetadata.from_dict(m.to_dict()) == m
+
+ m = RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT, id=_content_swhid, **_common_metadata_fields
+ )
+ assert m.to_dict() == {
+ "type": "content",
+ "id": "swh:1:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2",
+ **common_fields,
+ }
+ assert RawExtrinsicMetadata.from_dict(m.to_dict()) == m
+
+
+def test_metadata_invalid_id():
+ """Checks various invalid values for the 'id' field."""
+
+ # SWHID for an origin
+ with pytest.raises(ValueError, match="expected an URL"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN, id=_content_swhid, **_common_metadata_fields
+ )
+
+ # SWHID for an origin (even when passed as string)
+ with pytest.raises(ValueError, match="expected an URL"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN,
+ id="swh:1:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2",
+ **_common_metadata_fields,
+ )
+
+ # URL for a non-origin
+ with pytest.raises(ValueError, match="Expected SWHID, got a string"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT, id=_origin_url, **_common_metadata_fields
+ )
+
+ # SWHID passed as string instead of SWHID
+ with pytest.raises(ValueError, match="Expected SWHID, got a string"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id="swh:1:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2",
+ **_common_metadata_fields,
+ )
+
+ # Object type does not match the SWHID
+ with pytest.raises(
+ ValueError, match="Expected SWHID type 'revision', got 'content'"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.REVISION,
+ id=_content_swhid,
+ **_common_metadata_fields,
+ )
+
+ # Non-core SWHID
+ with pytest.raises(ValueError, match="Expected core SWHID"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=SWHID(
+ object_type="content",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ metadata={"foo": "bar"},
+ ),
+ **_common_metadata_fields,
+ )
+
+
+def test_metadata_validate_context_origin():
+ """Checks validation of RawExtrinsicMetadata.origin."""
+
+ # Origins can't have an 'origin' context
+ with pytest.raises(
+ ValueError, match="Unexpected 'origin' context for origin object"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN,
+ id=_origin_url,
+ origin=_origin_url,
+ **_common_metadata_fields,
+ )
+
+ # but all other types can
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ origin=_origin_url,
+ **_common_metadata_fields,
+ )
+
+ # SWHIDs aren't valid origin URLs
+ with pytest.raises(ValueError, match="SWHID used as context origin URL"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ origin="swh:1:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2",
+ **_common_metadata_fields,
+ )
+
+
+def test_metadata_validate_context_visit():
+ """Checks validation of RawExtrinsicMetadata.visit."""
+
+ # Origins can't have a 'visit' context
+ with pytest.raises(
+ ValueError, match="Unexpected 'visit' context for origin object"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN,
+ id=_origin_url,
+ visit=42,
+ **_common_metadata_fields,
+ )
+
+ # but all other types can
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ origin=_origin_url,
+ visit=42,
+ **_common_metadata_fields,
+ )
+
+ # Missing 'origin'
+ with pytest.raises(ValueError, match="'origin' context must be set if 'visit' is"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ visit=42,
+ **_common_metadata_fields,
+ )
+
+ # visit id must be positive
+ with pytest.raises(ValueError, match="Nonpositive visit id"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ origin=_origin_url,
+ visit=-42,
+ **_common_metadata_fields,
+ )
+
+
+def test_metadata_validate_context_snapshot():
+ """Checks validation of RawExtrinsicMetadata.snapshot."""
+
+ # Origins can't have a 'snapshot' context
+ with pytest.raises(
+ ValueError, match="Unexpected 'snapshot' context for origin object"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN,
+ id=_origin_url,
+ snapshot=SWHID(
+ object_type="snapshot",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
+
+ # but content can
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ snapshot=SWHID(
+ object_type="snapshot", object_id="94a9ed024d3859793618152ea559a168bbcbb5e2"
+ ),
+ **_common_metadata_fields,
+ )
+
+ # Non-core SWHID
+ with pytest.raises(ValueError, match="Expected core SWHID"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ snapshot=SWHID(
+ object_type="snapshot",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ metadata={"foo": "bar"},
+ ),
+ **_common_metadata_fields,
+ )
+
+ # SWHID type doesn't match the expected type of this context key
+ with pytest.raises(
+ ValueError, match="Expected SWHID type 'snapshot', got 'content'"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ snapshot=SWHID(
+ object_type="content",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
+
+
+def test_metadata_validate_context_release():
+ """Checks validation of RawExtrinsicMetadata.release."""
+
+ # Origins can't have a 'release' context
+ with pytest.raises(
+ ValueError, match="Unexpected 'release' context for origin object"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN,
+ id=_origin_url,
+ release=SWHID(
+ object_type="release",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
+
+ # but content can
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ release=SWHID(
+ object_type="release", object_id="94a9ed024d3859793618152ea559a168bbcbb5e2"
+ ),
+ **_common_metadata_fields,
+ )
+
+ # Non-core SWHID
+ with pytest.raises(ValueError, match="Expected core SWHID"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ release=SWHID(
+ object_type="release",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ metadata={"foo": "bar"},
+ ),
+ **_common_metadata_fields,
+ )
+
+ # SWHID type doesn't match the expected type of this context key
+ with pytest.raises(
+ ValueError, match="Expected SWHID type 'release', got 'content'"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ release=SWHID(
+ object_type="content",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
+
+
+def test_metadata_validate_context_revision():
+ """Checks validation of RawExtrinsicMetadata.revision."""
+
+ # Origins can't have a 'revision' context
+ with pytest.raises(
+ ValueError, match="Unexpected 'revision' context for origin object"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN,
+ id=_origin_url,
+ revision=SWHID(
+ object_type="revision",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
+
+ # but content can
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ revision=SWHID(
+ object_type="revision", object_id="94a9ed024d3859793618152ea559a168bbcbb5e2"
+ ),
+ **_common_metadata_fields,
+ )
+
+ # Non-core SWHID
+ with pytest.raises(ValueError, match="Expected core SWHID"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ revision=SWHID(
+ object_type="revision",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ metadata={"foo": "bar"},
+ ),
+ **_common_metadata_fields,
+ )
+
+ # SWHID type doesn't match the expected type of this context key
+ with pytest.raises(
+ ValueError, match="Expected SWHID type 'revision', got 'content'"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ revision=SWHID(
+ object_type="content",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
+
+
+def test_metadata_validate_context_path():
+ """Checks validation of RawExtrinsicMetadata.path."""
+
+ # Origins can't have a 'path' context
+ with pytest.raises(ValueError, match="Unexpected 'path' context for origin object"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN,
+ id=_origin_url,
+ path=b"/foo/bar",
+ **_common_metadata_fields,
+ )
+
+ # but content can
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ path=b"/foo/bar",
+ **_common_metadata_fields,
+ )
+
+
+def test_metadata_validate_context_directory():
+ """Checks validation of RawExtrinsicMetadata.directory."""
+
+ # Origins can't have a 'directory' context
+ with pytest.raises(
+ ValueError, match="Unexpected 'directory' context for origin object"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.ORIGIN,
+ id=_origin_url,
+ directory=SWHID(
+ object_type="directory",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
+
+ # but content can
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ directory=SWHID(
+ object_type="directory",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
+
+ # Non-core SWHID
+ with pytest.raises(ValueError, match="Expected core SWHID"):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ directory=SWHID(
+ object_type="directory",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ metadata={"foo": "bar"},
+ ),
+ **_common_metadata_fields,
+ )
+
+ # SWHID type doesn't match the expected type of this context key
+ with pytest.raises(
+ ValueError, match="Expected SWHID type 'directory', got 'content'"
+ ):
+ RawExtrinsicMetadata(
+ type=MetadataTargetType.CONTENT,
+ id=_content_swhid,
+ directory=SWHID(
+ object_type="content",
+ object_id="94a9ed024d3859793618152ea559a168bbcbb5e2",
+ ),
+ **_common_metadata_fields,
+ )
diff --git a/version.txt b/version.txt
index 43fb728930ad36e25ed0b993c35a86c66a3e2145..c513ada33a46f85b32bfcc470bafa961cc1c0306 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-v0.4.0-0-ga7d9aca
\ No newline at end of file
+v0.5.0-0-g0547a51
\ No newline at end of file