swh/model/tests/fields/test_compound.py

@@ -157,7 +157,9 @@ class ValidateCompound(unittest.TestCase):
     def test_validate_whole_schema_shortcut_previous_error(self):
         with self.assertRaises(ValidationError) as cm:
             compound.validate_against_schema(
-                self.test_model, self.test_schema_shortcut, self.test_value_missing,
+                self.test_model,
+                self.test_schema_shortcut,
+                self.test_value_missing,
             )
 
         exc = cm.exception
@@ -167,7 +169,9 @@ class ValidateCompound(unittest.TestCase):
     def test_validate_whole_schema(self):
         with self.assertRaises(ValidationError) as cm:
             compound.validate_against_schema(
-                self.test_model, self.test_schema_shortcut, self.test_value,
+                self.test_model,
+                self.test_schema_shortcut,
+                self.test_value,
             )
 
         # The exception should be of the form:

swh/model/tests/generate_testdata.py

@@ -4,12 +4,12 @@
 # See top-level LICENSE file for more information
 
 from datetime import datetime
-from pytz import all_timezones, timezone
 from random import choice, randint, random, shuffle
-from typing import List, Dict
+from typing import Dict, List
 
-from swh.model.hashutil import MultiHash
+from pytz import all_timezones, timezone
 
+from swh.model.hashutil import MultiHash
 
 PROTOCOLS = ["git", "http", "https", "deb", "svn", "mock"]
 DOMAINS = ["example.com", "some.long.host.name", "xn--n28h.tld"]

swh/model/tests/generate_testdata_from_disk.py

@@ -7,7 +7,7 @@ from operator import itemgetter
 import os
 import sys
 
-from swh.model.from_disk import Directory, DentryPerms
+from swh.model.from_disk import DentryPerms, Directory
 from swh.model.hashutil import ALGORITHMS, hash_to_hex
 
 

swh/model/tests/test_discovery.py

+# Copyright (C) 2023 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from dataclasses import dataclass
+from typing import Iterable, List
+
+from swh.model import discovery, model
+from swh.model.hashutil import hash_to_bytes
+from swh.model.model import Sha1Git
+from swh.model.tests.test_identifiers import directory_example
+
+pytest_plugins = ["aiohttp.pytest_plugin"]
+
+UNKNOWN_HASH = hash_to_bytes("17140cb6109f1e3296dc52e2b2cd29bcb40e86be")
+KNOWN_CONTENT_HASH = hash_to_bytes("e8e4106de42e2d5d5efab6a9422b9a8677c993c8")
+KNOWN_DIRECTORY_HASH = hash_to_bytes("d7ed3d2c31d608823be58b1cbe57605310615231")
+KNOWN_DIRECTORY_HASH_2 = hash_to_bytes("c76724e9a0be4b60f4bf0cb48b261df8eda94b1d")
+
+
+@dataclass
+class FakeArchive:
+    contents: List[model.Content]
+    skipped_contents: List[model.SkippedContent]
+    directories: List[model.Directory]
+
+    def content_missing(self, contents: List[Sha1Git]) -> Iterable[Sha1Git]:
+        return []
+
+    def skipped_content_missing(
+        self, skipped_contents: List[Sha1Git]
+    ) -> Iterable[Sha1Git]:
+        """List skipped content missing from the archive by sha1"""
+        return []
+
+    def directory_missing(self, directories: List[Sha1Git]) -> Iterable[Sha1Git]:
+        """List directories missing from the archive by sha1"""
+        return []
+
+
+def test_filter_known_objects(monkeypatch):
+    # Test with smaller sample sizes to actually trigger the random sampling
+    monkeypatch.setattr(discovery, "SAMPLE_SIZE", 1)
+
+    base_directory = model.Directory.from_dict(directory_example)
+
+    # Hardcoding another hash is enough since it's all that's being checked
+    directory_data = directory_example.copy()
+    directory_data["id"] = KNOWN_DIRECTORY_HASH_2
+    other_directory = model.Directory.from_dict(directory_data)
+    archive = FakeArchive(
+        contents=[model.Content.from_data(b"blabla")],
+        skipped_contents=[model.SkippedContent.from_data(b"blabla2", reason="reason")],
+        directories=[
+            base_directory,
+            other_directory,
+        ],
+    )
+    assert archive.contents[0].sha1_git == KNOWN_CONTENT_HASH
+    assert archive.directories[0].id == KNOWN_DIRECTORY_HASH
+    assert archive.directories[1].id == KNOWN_DIRECTORY_HASH_2
+    (contents, skipped_contents, directories) = discovery.filter_known_objects(archive)
+    assert len(contents) == 0
+    assert len(skipped_contents) == 0
+    assert len(directories) == 0

swh/model/tests/test_generate_testdata.py

@@ -3,9 +3,9 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-from .generate_testdata import gen_contents, gen_origins, ORIGINS
+from swh.model.model import BaseContent, Origin
 
-from swh.model.model import Origin, BaseContent
+from .generate_testdata import ORIGINS, gen_contents, gen_origins
 
 
 def test_gen_origins_empty():

swh/model/validators.py

@@ -3,8 +3,8 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-from .exceptions import ValidationError, NON_FIELD_ERRORS
 from . import fields
+from .exceptions import NON_FIELD_ERRORS, ValidationError
 from .hashutil import MultiHash, hash_to_bytes