swh/model/tests/fields/test_compound.py

@@ -157,7 +157,9 @@ class ValidateCompound(unittest.TestCase):
     def test_validate_whole_schema_shortcut_previous_error(self):
         with self.assertRaises(ValidationError) as cm:
             compound.validate_against_schema(
-                self.test_model, self.test_schema_shortcut, self.test_value_missing,
+                self.test_model,
+                self.test_schema_shortcut,
+                self.test_value_missing,
             )
 
         exc = cm.exception
@@ -167,7 +169,9 @@ class ValidateCompound(unittest.TestCase):
     def test_validate_whole_schema(self):
         with self.assertRaises(ValidationError) as cm:
             compound.validate_against_schema(
-                self.test_model, self.test_schema_shortcut, self.test_value,
+                self.test_model,
+                self.test_schema_shortcut,
+                self.test_value,
             )
 
         # The exception should be of the form:

swh/model/tests/generate_testdata.py

@@ -4,12 +4,12 @@
 # See top-level LICENSE file for more information
 
 from datetime import datetime
-from pytz import all_timezones, timezone
 from random import choice, randint, random, shuffle
-from typing import List, Dict
+from typing import Dict, List
 
-from swh.model.hashutil import MultiHash
+from pytz import all_timezones, timezone
 
+from swh.model.hashutil import MultiHash
 
 PROTOCOLS = ["git", "http", "https", "deb", "svn", "mock"]
 DOMAINS = ["example.com", "some.long.host.name", "xn--n28h.tld"]

swh/model/tests/generate_testdata_from_disk.py

@@ -7,7 +7,7 @@ from operator import itemgetter
 import os
 import sys
 
-from swh.model.from_disk import Directory, DentryPerms
+from swh.model.from_disk import DentryPerms, Directory
 from swh.model.hashutil import ALGORITHMS, hash_to_hex
 
 

swh/model/tests/test_cli.py

@@ -4,27 +4,29 @@
 # See top-level LICENSE file for more information
 
 import os
+import sys
 import tarfile
 import tempfile
 import unittest
+import unittest.mock
 
 from click.testing import CliRunner
 import pytest
 
 from swh.model import cli
 from swh.model.hashutil import hash_to_hex
+from swh.model.tests.swh_model_data import SAMPLE_FOLDER_SWHIDS
 from swh.model.tests.test_from_disk import DataMixin
 
 
-@pytest.mark.fs
 class TestIdentify(DataMixin, unittest.TestCase):
     def setUp(self):
         super().setUp()
         self.runner = CliRunner()
 
-    def assertPidOK(self, result, pid):
-        self.assertEqual(result.exit_code, 0)
-        self.assertEqual(result.output.split()[0], pid)
+    def assertSWHID(self, result, swhid):
+        self.assertEqual(result.exit_code, 0, result.output)
+        self.assertEqual(result.output.split()[0], swhid)
 
     def test_no_args(self):
         result = self.runner.invoke(cli.identify)
@@ -36,22 +38,23 @@ class TestIdentify(DataMixin, unittest.TestCase):
         for filename, content in self.contents.items():
             path = os.path.join(self.tmpdir_name, filename)
             result = self.runner.invoke(cli.identify, ["--type", "content", path])
-            self.assertPidOK(result, "swh:1:cnt:" + hash_to_hex(content["sha1_git"]))
+            self.assertSWHID(result, "swh:1:cnt:" + hash_to_hex(content["sha1_git"]))
 
     def test_content_id_from_stdin(self):
         """identify file content"""
         self.make_contents(self.tmpdir_name)
         for _, content in self.contents.items():
             result = self.runner.invoke(cli.identify, ["-"], input=content["data"])
-            self.assertPidOK(result, "swh:1:cnt:" + hash_to_hex(content["sha1_git"]))
+            self.assertSWHID(result, "swh:1:cnt:" + hash_to_hex(content["sha1_git"]))
 
     def test_directory_id(self):
         """identify an entire directory"""
         self.make_from_tarball(self.tmpdir_name)
         path = os.path.join(self.tmpdir_name, b"sample-folder")
         result = self.runner.invoke(cli.identify, ["--type", "directory", path])
-        self.assertPidOK(result, "swh:1:dir:e8b0f1466af8608c8a3fb9879db172b887e80759")
+        self.assertSWHID(result, "swh:1:dir:e8b0f1466af8608c8a3fb9879db172b887e80759")
 
+    @pytest.mark.requires_optional_deps
     def test_snapshot_id(self):
         """identify a snapshot"""
         tarball = os.path.join(
@@ -64,28 +67,43 @@ class TestIdentify(DataMixin, unittest.TestCase):
                 result = self.runner.invoke(
                     cli.identify, ["--type", "snapshot", repo_dir]
                 )
-                self.assertPidOK(
+                self.assertSWHID(
                     result, "swh:1:snp:abc888898124270905a0ef3c67e872ce08e7e0c1"
                 )
 
+    def test_snapshot_without_dulwich(self):
+        """checks swh-identify returns a 'nice' message instead of a traceback
+        when dulwich is not installed"""
+        with unittest.mock.patch.dict(sys.modules, {"dulwich": None}):
+            with tempfile.TemporaryDirectory(prefix="swh.model.cli") as d:
+                result = self.runner.invoke(
+                    cli.identify,
+                    ["--type", "snapshot", d],
+                    catch_exceptions=False,
+                )
+
+        assert result.exit_code == 1
+        assert "'swh.model[cli]'" in result.output
+
     def test_origin_id(self):
         """identify an origin URL"""
         url = "https://github.com/torvalds/linux"
         result = self.runner.invoke(cli.identify, ["--type", "origin", url])
-        self.assertPidOK(result, "swh:1:ori:b63a575fe3faab7692c9f38fb09d4bb45651bb0f")
+        self.assertSWHID(result, "swh:1:ori:b63a575fe3faab7692c9f38fb09d4bb45651bb0f")
 
     def test_symlink(self):
         """identify symlink --- both itself and target"""
         regular = os.path.join(self.tmpdir_name, b"foo.txt")
         link = os.path.join(self.tmpdir_name, b"bar.txt")
-        open(regular, "w").write("foo\n")
+        with open(regular, "w") as f:
+            f.write("foo\n")
         os.symlink(os.path.basename(regular), link)
 
         result = self.runner.invoke(cli.identify, [link])
-        self.assertPidOK(result, "swh:1:cnt:257cc5642cb1a054f08cc83f2d943e56fd3ebe99")
+        self.assertSWHID(result, "swh:1:cnt:257cc5642cb1a054f08cc83f2d943e56fd3ebe99")
 
         result = self.runner.invoke(cli.identify, ["--no-dereference", link])
-        self.assertPidOK(result, "swh:1:cnt:996f1789ff67c0e3f69ef5933a55d54c5d0e9954")
+        self.assertSWHID(result, "swh:1:cnt:996f1789ff67c0e3f69ef5933a55d54c5d0e9954")
 
     def test_show_filename(self):
         """filename is shown by default"""
@@ -108,7 +126,7 @@ class TestIdentify(DataMixin, unittest.TestCase):
             result = self.runner.invoke(
                 cli.identify, ["--type", "content", "--no-filename", path]
             )
-            self.assertPidOK(result, "swh:1:cnt:" + hash_to_hex(content["sha1_git"]))
+            self.assertSWHID(result, "swh:1:cnt:" + hash_to_hex(content["sha1_git"]))
 
     def test_auto_content(self):
         """automatic object type detection: content"""
@@ -127,7 +145,7 @@ class TestIdentify(DataMixin, unittest.TestCase):
     def test_auto_origin(self):
         """automatic object type detection: origin"""
         result = self.runner.invoke(cli.identify, ["https://github.com/torvalds/linux"])
-        self.assertEqual(result.exit_code, 0)
+        self.assertEqual(result.exit_code, 0, result.output)
         self.assertRegex(result.output, r"^swh:\d+:ori:")
 
     def test_verify_content(self):
@@ -139,10 +157,57 @@ class TestIdentify(DataMixin, unittest.TestCase):
             # match
             path = os.path.join(self.tmpdir_name, filename)
             result = self.runner.invoke(cli.identify, ["--verify", expected_id, path])
-            self.assertEqual(result.exit_code, 0)
+            self.assertEqual(result.exit_code, 0, result.output)
 
             # mismatch
             with open(path, "a") as f:
                 f.write("trailing garbage to make verification fail")
             result = self.runner.invoke(cli.identify, ["--verify", expected_id, path])
             self.assertEqual(result.exit_code, 1)
+
+    def test_exclude(self):
+        """exclude patterns"""
+        self.make_from_tarball(self.tmpdir_name)
+        path = os.path.join(self.tmpdir_name, b"sample-folder")
+
+        excluded_dir = os.path.join(path, b"excluded_dir\x96")
+        os.mkdir(excluded_dir)
+        with open(os.path.join(excluded_dir, b"some_file"), "w") as f:
+            f.write("content")
+
+        result = self.runner.invoke(
+            cli.identify, ["--type", "directory", "--exclude", "excluded_*", path]
+        )
+
+        self.assertSWHID(result, "swh:1:dir:e8b0f1466af8608c8a3fb9879db172b887e80759")
+
+    def test_recursive_directory(self):
+        self.make_from_tarball(self.tmpdir_name)
+        path = os.path.join(self.tmpdir_name, b"sample-folder")
+        result = self.runner.invoke(cli.identify, ["--recursive", path])
+        self.assertEqual(result.exit_code, 0, result.output)
+
+        result = result.output.split()
+        result_swhids = []
+        # get all SWHID from the result
+        for i in range(0, len(result)):
+            if i % 2 == 0:
+                result_swhids.append(result[i])
+
+        assert len(result_swhids) == len(SAMPLE_FOLDER_SWHIDS)
+        for swhid in SAMPLE_FOLDER_SWHIDS:
+            assert swhid in result_swhids
+
+    def test_recursive_directory_no_filename(self):
+        self.make_from_tarball(self.tmpdir_name)
+        path = os.path.join(self.tmpdir_name, b"sample-folder")
+        result = self.runner.invoke(
+            cli.identify, ["--recursive", "--no-filename", path]
+        )
+        self.assertEqual(result.exit_code, 0, result.output)
+
+        result_swhids = result.output.split()
+
+        assert len(result_swhids) == len(SAMPLE_FOLDER_SWHIDS)
+        for swhid in SAMPLE_FOLDER_SWHIDS:
+            assert swhid in result_swhids

swh/model/validators.py

@@ -3,8 +3,8 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-from .exceptions import ValidationError, NON_FIELD_ERRORS
 from . import fields
+from .exceptions import NON_FIELD_ERRORS, ValidationError
 from .hashutil import MultiHash, hash_to_bytes