diff --git a/PKG-INFO b/PKG-INFO
index 4dc3739e0b2526788499fd3b3280e82f01054480..479c4cc885d38a38b955a19d00297184c6004e43 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: swh.model
-Version: 0.3.8
+Version: 0.4.0
Summary: Software Heritage data model
Home-page: https://forge.softwareheritage.org/diffusion/DMOD/
Author: Software Heritage developers
diff --git a/swh.model.egg-info/PKG-INFO b/swh.model.egg-info/PKG-INFO
index 4dc3739e0b2526788499fd3b3280e82f01054480..479c4cc885d38a38b955a19d00297184c6004e43 100644
--- a/swh.model.egg-info/PKG-INFO
+++ b/swh.model.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: swh.model
-Version: 0.3.8
+Version: 0.4.0
Summary: Software Heritage data model
Home-page: https://forge.softwareheritage.org/diffusion/DMOD/
Author: Software Heritage developers
diff --git a/swh/model/hypothesis_strategies.py b/swh/model/hypothesis_strategies.py
index 821343bb006783de3d57eb09ef1c175d511c3e5e..21e922eed40cfc95db664bce1acc4ce0d344ef35 100644
--- a/swh/model/hypothesis_strategies.py
+++ b/swh/model/hypothesis_strategies.py
@@ -18,6 +18,7 @@ from hypothesis.strategies import (
from_regex,
integers,
just,
+ lists,
none,
one_of,
sampled_from,
@@ -229,6 +230,12 @@ def releases():
revision_metadata = metadata_dicts
+def extra_headers():
+ return lists(
+ tuples(binary(min_size=0, max_size=50), binary(min_size=0, max_size=500))
+ ).map(tuple)
+
+
def revisions_d():
return builds(
dict,
@@ -242,6 +249,7 @@ def revisions_d():
directory=sha1_git(),
type=sampled_from([x.value for x in RevisionType]),
metadata=optional(revision_metadata()),
+ extra_headers=extra_headers(),
)
# TODO: metadata['extra_headers'] can have binary keys and values
diff --git a/swh/model/identifiers.py b/swh/model/identifiers.py
index de2082dafc189c82d50a0055a2da9d131c5be087..3c0a46ebc5e4d8ca2faed299aee68adc8b0a92a8 100644
--- a/swh/model/identifiers.py
+++ b/swh/model/identifiers.py
@@ -426,7 +426,7 @@ def revision_identifier(revision):
- author_date
- committer
- committer_date
- - metadata -> extra_headers
+ - extra_headers or metadata -> extra_headers
- message
A revision's identifier is the 'git'-checksum of a commit manifest
@@ -486,22 +486,13 @@ def revision_identifier(revision):
)
# Handle extra headers
- metadata = revision.get("metadata")
- if not metadata:
- metadata = {}
+ metadata = revision.get("metadata") or {}
+ extra_headers = revision.get("extra_headers", ())
+ if not extra_headers and "extra_headers" in metadata:
+ extra_headers = metadata["extra_headers"]
- for key, value in metadata.get("extra_headers", []):
-
- # Integer values: decimal representation
- if isinstance(value, int):
- value = str(value).encode("utf-8")
-
- # Unicode string values: utf-8 encoding
- if isinstance(value, str):
- value = value.encode("utf-8")
-
- # encode the key to utf-8
- components.extend([key.encode("utf-8"), b" ", escape_newlines(value), b"\n"])
+ for key, value in extra_headers:
+ components.extend([key, b" ", escape_newlines(value), b"\n"])
if revision["message"] is not None:
components.extend([b"\n", revision["message"]])
diff --git a/swh/model/model.py b/swh/model/model.py
index f585051965d6d13881274b45cc3ac84986a5a4d1..c4f185f5075a8b68171aa30080d2fa01e5271c5d 100644
--- a/swh/model/model.py
+++ b/swh/model/model.py
@@ -6,9 +6,10 @@
import datetime
from abc import ABCMeta, abstractmethod
+from copy import deepcopy
from enum import Enum
from hashlib import sha256
-from typing import Dict, Optional, Tuple, TypeVar, Union
+from typing import Dict, Iterable, Optional, Tuple, TypeVar, Union
from typing_extensions import Final
import attr
@@ -410,6 +411,10 @@ class RevisionType(Enum):
MERCURIAL = "hg"
+def tuplify_extra_headers(value: Iterable) -> Tuple:
+ return tuple((k, v) for k, v in value)
+
+
@attr.s(frozen=True)
class Revision(BaseModel, HashableObject):
object_type: Final = "revision"
@@ -429,6 +434,27 @@ class Revision(BaseModel, HashableObject):
)
parents = attr.ib(type=Tuple[Sha1Git, ...], validator=type_validator(), default=())
id = attr.ib(type=Sha1Git, validator=type_validator(), default=b"")
+ extra_headers = attr.ib(
+ type=Tuple[Tuple[bytes, bytes], ...], # but it makes mypy sad
+ validator=type_validator(),
+ converter=tuplify_extra_headers, # type: ignore
+ default=(),
+ )
+
+ def __attrs_post_init__(self):
+ super().__attrs_post_init__()
+ # ensure metadata is a deep copy of whatever was given, and if needed
+ # extract extra_headers from there
+ if self.metadata:
+ metadata = deepcopy(self.metadata)
+ if not self.extra_headers and "extra_headers" in metadata:
+ object.__setattr__(
+ self,
+ "extra_headers",
+ tuplify_extra_headers(metadata.pop("extra_headers")),
+ )
+ attr.validate(self)
+ object.__setattr__(self, "metadata", metadata)
@staticmethod
def compute_hash(object_dict):
diff --git a/swh/model/tests/test_identifiers.py b/swh/model/tests/test_identifiers.py
index 6edb26c387e3c5cd190e6cfb7b70e92a01f8d1b5..c03b9eff0e9ebebaac06994be8e94ee1e4431736 100644
--- a/swh/model/tests/test_identifiers.py
+++ b/swh/model/tests/test_identifiers.py
@@ -370,12 +370,10 @@ dg1KdHOa34shrKDaOVzW
2015, 7, 12, 15, 10, 30, tzinfo=linus_tz
),
"message": b"Linux 4.2-rc2\n",
- "metadata": {
- "extra_headers": [
- ["svn-repo-uuid", "046f1af7-66c2-d61b-5410-ce57b7db7bff"],
- ["svn-revision", 10],
- ]
- },
+ "extra_headers": (
+ (b"svn-repo-uuid", b"046f1af7-66c2-d61b-5410-ce57b7db7bff"),
+ (b"svn-revision", b"10"),
+ ),
}
self.revision_with_gpgsig = {
@@ -393,7 +391,7 @@ dg1KdHOa34shrKDaOVzW
"date": {"timestamp": 1428538899, "offset": 480,},
"committer": {"name": b"Jiang Xin", "email": b"worldhello.net@gmail.com",},
"committer_date": {"timestamp": 1428538899, "offset": 480,},
- "metadata": {"extra_headers": [["gpgsig", gpgsig],],},
+ "extra_headers": ((b"gpgsig", gpgsig),),
"message": b"""Merge branch 'master' of git://github.com/alexhenrie/git-po
* 'master' of git://github.com/alexhenrie/git-po:
@@ -450,12 +448,10 @@ dg1KdHOa34shrKDaOVzW
2015, 7, 12, 15, 10, 30, tzinfo=linus_tz
),
"message": b"Linux 4.2-rc2\n",
- "metadata": {
- "extra_headers": [
- ["svn-repo-uuid", "046f1af7-66c2-d61b-5410-ce57b7db7bff"],
- ["svn-revision", 10],
- ]
- },
+ "extra_headers": (
+ (b"svn-repo-uuid", b"046f1af7-66c2-d61b-5410-ce57b7db7bff"),
+ (b"svn-revision", b"10"),
+ ),
}
def test_revision_identifier(self):
diff --git a/swh/model/tests/test_model.py b/swh/model/tests/test_model.py
index df9492440c71e45a865b97dce75c71fa25637e80..edfc829b74e394eacf9b5c6e027eaee07605aac2 100644
--- a/swh/model/tests/test_model.py
+++ b/swh/model/tests/test_model.py
@@ -415,6 +415,194 @@ def test_skipped_content_origin_is_str(skipped_content_d):
SkippedContent.from_dict(skipped_content_d)
+# Revision
+
+
+def test_revision_extra_headers_no_headers():
+ rev_dict = revision_example.copy()
+ rev_dict.pop("id")
+ rev = Revision.from_dict(rev_dict)
+ rev_dict = attr.asdict(rev, recurse=False)
+
+ rev_model = Revision(**rev_dict)
+ assert rev_model.metadata is None
+ assert rev_model.extra_headers == ()
+
+ rev_dict["metadata"] = {
+ "something": "somewhere",
+ "some other thing": "stranger",
+ }
+ rev_model = Revision(**rev_dict)
+ assert rev_model.metadata == rev_dict["metadata"]
+ assert rev_model.extra_headers == ()
+
+
+def test_revision_extra_headers_with_headers():
+ rev_dict = revision_example.copy()
+ rev_dict.pop("id")
+ rev = Revision.from_dict(rev_dict)
+ rev_dict = attr.asdict(rev, recurse=False)
+ rev_dict["metadata"] = {
+ "something": "somewhere",
+ "some other thing": "stranger",
+ }
+ extra_headers = (
+ (b"header1", b"value1"),
+ (b"header2", b"42"),
+ (b"header3", b"should I?\u0000"),
+ (b"header1", b"again"),
+ )
+
+ rev_dict["extra_headers"] = extra_headers
+ rev_model = Revision(**rev_dict)
+ assert "extra_headers" not in rev_model.metadata
+ assert rev_model.extra_headers == extra_headers
+
+
+def test_revision_extra_headers_in_metadata():
+ rev_dict = revision_example.copy()
+ rev_dict.pop("id")
+ rev = Revision.from_dict(rev_dict)
+ rev_dict = attr.asdict(rev, recurse=False)
+ rev_dict["metadata"] = {
+ "something": "somewhere",
+ "some other thing": "stranger",
+ }
+
+ extra_headers = (
+ (b"header1", b"value1"),
+ (b"header2", b"42"),
+ (b"header3", b"should I?\u0000"),
+ (b"header1", b"again"),
+ )
+
+ # check the bw-compat init hook does the job
+ # ie. extra_headers are given in the metadata field
+ rev_dict["metadata"]["extra_headers"] = extra_headers
+ rev_model = Revision(**rev_dict)
+ assert "extra_headers" not in rev_model.metadata
+ assert rev_model.extra_headers == extra_headers
+
+
+def test_revision_extra_headers_as_lists():
+ rev_dict = revision_example.copy()
+ rev_dict.pop("id")
+ rev = Revision.from_dict(rev_dict)
+ rev_dict = attr.asdict(rev, recurse=False)
+ rev_dict["metadata"] = {}
+
+ extra_headers = (
+ (b"header1", b"value1"),
+ (b"header2", b"42"),
+ (b"header3", b"should I?\u0000"),
+ (b"header1", b"again"),
+ )
+
+ # check Revision.extra_headers tuplify does the job
+ rev_dict["extra_headers"] = [list(x) for x in extra_headers]
+ rev_model = Revision(**rev_dict)
+ assert "extra_headers" not in rev_model.metadata
+ assert rev_model.extra_headers == extra_headers
+
+
+def test_revision_extra_headers_type_error():
+ rev_dict = revision_example.copy()
+ rev_dict.pop("id")
+ rev = Revision.from_dict(rev_dict)
+ orig_rev_dict = attr.asdict(rev, recurse=False)
+ orig_rev_dict["metadata"] = {
+ "something": "somewhere",
+ "some other thing": "stranger",
+ }
+ extra_headers = (
+ ("header1", b"value1"),
+ (b"header2", 42),
+ ("header1", "again"),
+ )
+ # check headers one at a time
+ # if given as extra_header
+ for extra_header in extra_headers:
+ rev_dict = copy.deepcopy(orig_rev_dict)
+ rev_dict["extra_headers"] = (extra_header,)
+ with pytest.raises(AttributeTypeError):
+ Revision(**rev_dict)
+ # if given as metadata
+ for extra_header in extra_headers:
+ rev_dict = copy.deepcopy(orig_rev_dict)
+ rev_dict["metadata"]["extra_headers"] = (extra_header,)
+ with pytest.raises(AttributeTypeError):
+ Revision(**rev_dict)
+
+
+def test_revision_extra_headers_from_dict():
+ rev_dict = revision_example.copy()
+ rev_dict.pop("id")
+ rev_model = Revision.from_dict(rev_dict)
+ assert rev_model.metadata is None
+ assert rev_model.extra_headers == ()
+
+ rev_dict["metadata"] = {
+ "something": "somewhere",
+ "some other thing": "stranger",
+ }
+ rev_model = Revision.from_dict(rev_dict)
+ assert rev_model.metadata == rev_dict["metadata"]
+ assert rev_model.extra_headers == ()
+
+ extra_headers = (
+ (b"header1", b"value1"),
+ (b"header2", b"42"),
+ (b"header3", b"should I?\nmaybe\x00\xff"),
+ (b"header1", b"again"),
+ )
+ rev_dict["extra_headers"] = extra_headers
+ rev_model = Revision.from_dict(rev_dict)
+ assert "extra_headers" not in rev_model.metadata
+ assert rev_model.extra_headers == extra_headers
+
+
+def test_revision_extra_headers_in_metadata_from_dict():
+ rev_dict = revision_example.copy()
+ rev_dict.pop("id")
+
+ rev_dict["metadata"] = {
+ "something": "somewhere",
+ "some other thing": "stranger",
+ }
+ extra_headers = (
+ (b"header1", b"value1"),
+ (b"header2", b"42"),
+ (b"header3", b"should I?\nmaybe\x00\xff"),
+ (b"header1", b"again"),
+ )
+ # check the bw-compat init hook does the job
+ rev_dict["metadata"]["extra_headers"] = extra_headers
+ rev_model = Revision.from_dict(rev_dict)
+ assert "extra_headers" not in rev_model.metadata
+ assert rev_model.extra_headers == extra_headers
+
+
+def test_revision_extra_headers_as_lists_from_dict():
+ rev_dict = revision_example.copy()
+ rev_dict.pop("id")
+ rev_model = Revision.from_dict(rev_dict)
+ rev_dict["metadata"] = {
+ "something": "somewhere",
+ "some other thing": "stranger",
+ }
+ extra_headers = (
+ (b"header1", b"value1"),
+ (b"header2", b"42"),
+ (b"header3", b"should I?\nmaybe\x00\xff"),
+ (b"header1", b"again"),
+ )
+ # check Revision.extra_headers converter does the job
+ rev_dict["extra_headers"] = [list(x) for x in extra_headers]
+ rev_model = Revision.from_dict(rev_dict)
+ assert "extra_headers" not in rev_model.metadata
+ assert rev_model.extra_headers == extra_headers
+
+
# ID computation
diff --git a/version.txt b/version.txt
index c836ae1c21d9944f2be970821987bd7e8acf211d..43fb728930ad36e25ed0b993c35a86c66a3e2145 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-v0.3.8-0-g1ff0516
\ No newline at end of file
+v0.4.0-0-ga7d9aca
\ No newline at end of file