(Re-)add tests for the loader.

When doing !11 (closed), I misunderstood how batching in the loader worked, so I ended up making these tests useless.

This makes them do their job again, and tests more cases, such as the threshold being reached.

It also makes the cascading more explicit by having more compact and documented tests.

---

Migrated from D770 (view on Phabricator)

swh/loader/core/tests/test_loader.py

@@ -6,15 +6,63 @@
 import datetime
 
 from swh.model.hashutil import hash_to_bytes
-from swh.storage.in_memory import Storage
+
+from swh.loader.core.loader import SWHLoader
 
 from . import BaseLoaderTest
 
 
+class DummyLoader(SWHLoader):
+    def cleanup(self):
+        pass
+
+    def prepare(self):
+        pass
+
+    def fetch_data(self):
+        pass
+
+    def store_data(self):
+        pass
+
+    def prepare_origin_visit(self):
+        origin = self.storage.origin_get(
+            self._test_prepare_origin_visit_data['origin'])
+        self.origin = origin
+        self.origin_id = origin['id']
+        self.origin_url = origin['url']
+        self.visit_date = datetime.datetime.utcnow()
+        self.storage.origin_visit_add(origin['id'], self.visit_date)
+
+    def parse_config_file(self, *args, **kwargs):
+        return {
+            'storage': {
+                'cls': 'memory',
+                'args': {
+                }
+            },
+
+            'send_contents': True,
+            'send_directories': True,
+            'send_revisions': True,
+            'send_releases': True,
+            'send_snapshot': True,
+
+            'content_packet_size': 2,
+            'content_packet_size_bytes': 8,
+            'directory_packet_size': 2,
+            'revision_packet_size': 2,
+            'release_packet_size': 2,
+
+            'content_size_limit': 10000,
+        }
+
+
 class DummyBaseLoaderTest(BaseLoaderTest):
     def setUp(self):
+        self.loader = DummyLoader(logging_class='dummyloader')
         # do not call voluntarily super().setUp()
-        self.storage = Storage()
+        self.storage = self.loader.storage
         contents = [
             {
                 'id': '34973274ccef6ab4dfaaf86599792fa9c3fe4689',
@@ -24,6 +72,7 @@ class DummyBaseLoaderTest(BaseLoaderTest):
                 'blake2s256': b'qux1',
                 'status': 'visible',
                 'data': b'data1',
+                'length': 5,
             },
             {
                 'id': '61c2b3a30496d329e21af70dd2d7e097046d07b7',
@@ -33,6 +82,7 @@ class DummyBaseLoaderTest(BaseLoaderTest):
                 'blake2s256': b'qux2',
                 'status': 'visible',
                 'data': b'data2',
+                'length': 5,
             },
         ]
         self.expected_contents = [content['id'] for content in contents]
@@ -52,12 +102,20 @@ class DummyBaseLoaderTest(BaseLoaderTest):
                 'id': b'rev1',
                 'date': None,
             },
+            {
+                'id': b'rev2',
+                'date': None,
+            },
         ]
         self.in_releases = [
             {
                 'id': b'rel1',
                 'date': None,
             },
+            {
+                'id': b'rel2',
+                'date': None,
+            },
         ]
         self.in_origins = [
             {
@@ -70,15 +128,19 @@ class DummyBaseLoaderTest(BaseLoaderTest):
             'branches': {},
         }
 
+        self.storage.origin_add(self.in_origins)
+
+        # used by prepare_origin_visit() when it gets called
+        self.loader._test_prepare_origin_visit_data = {
+            'origin': self.in_origins[0],
+        }
+
     def tearDown(self):
         # do not call voluntarily super().tearDown()
         pass
 
 
-class LoadTest1(DummyBaseLoaderTest):
-    def setUp(self):
-        super().setUp()
-
+class CoreLoaderTest(DummyBaseLoaderTest):
     def test_stateful_loader(self):
         """Stateful loader accumulates in place the sent data
 
@@ -86,14 +148,27 @@ class LoadTest1(DummyBaseLoaderTest):
         another story.
 
         """
-        self.storage.directory_add(self.in_directories)
-        self.storage.revision_add(self.in_revisions)
-        self.storage.release_add(self.in_releases)
+        self.loader.load()  # initialize the loader
+
+        self.loader.maybe_load_contents(self.in_contents[0:1])
+        self.loader.maybe_load_directories(self.in_directories[0:1])
+        self.loader.maybe_load_revisions(self.in_revisions[0:1])
+        self.loader.maybe_load_releases(self.in_releases[0:1])
 
         self.assertCountContents(0)
+        self.assertCountDirectories(0)
+        self.assertCountRevisions(0)
+        self.assertCountReleases(0)
+
+        self.loader.maybe_load_contents(self.in_contents[1:])
+        self.loader.maybe_load_directories(self.in_directories[1:])
+        self.loader.maybe_load_revisions(self.in_revisions[1:])
+        self.loader.maybe_load_releases(self.in_releases[1:])
+
+        self.assertCountContents(len(self.in_contents))
         self.assertCountDirectories(len(self.in_directories))
         self.assertCountRevisions(len(self.in_revisions))
-        self.assertCountSnapshots(0)
+        self.assertCountReleases(len(self.in_releases))
 
     def test_stateless_loader(self):
         """Stateless loader accumulates in place the sent data as well
@@ -102,50 +177,80 @@ class LoadTest1(DummyBaseLoaderTest):
         another story.
 
         """
-        (origin,) = self.storage.origin_add(self.in_origins)
-        visit = self.storage.origin_visit_add(
-            origin['id'], datetime.datetime.utcnow())
-        self.storage.content_add(self.in_contents)
-        self.storage.snapshot_add(origin, visit['visit'], self.in_snapshot)
+        self.loader.load()  # initialize the loader
+
+        self.loader.send_contents(self.in_contents[0:1])
+        self.loader.send_directories(self.in_directories[0:1])
+        self.loader.send_revisions(self.in_revisions[0:1])
+        self.loader.send_releases(self.in_releases[0:1])
+
+        self.assertCountContents(1)
+        self.assertCountDirectories(1)
+        self.assertCountRevisions(1)
+        self.assertCountReleases(1)
+
+        self.loader.send_contents(self.in_contents[1:])
+        self.loader.send_directories(self.in_directories[1:])
+        self.loader.send_revisions(self.in_revisions[1:])
+        self.loader.send_releases(self.in_releases[1:])
 
         self.assertCountContents(len(self.in_contents))
-        self.assertCountDirectories(0)
-        self.assertCountRevisions(0)
-        self.assertCountReleases(0)
-        self.assertCountSnapshots(1)
+        self.assertCountDirectories(len(self.in_directories))
+        self.assertCountRevisions(len(self.in_revisions))
+        self.assertCountReleases(len(self.in_releases))
 
+    def test_directory_cascade(self):
+        """Checks that sending a directory triggers sending contents"""
+        self.loader.load()  # initialize the loader
 
-class LoadTestContent(DummyBaseLoaderTest):
-    def test_load_contents(self):
-        """Loading contents should be ok
+        self.loader.maybe_load_contents(self.in_contents[0:1])
+        self.loader.maybe_load_directories(self.in_directories)
 
-        """
-        self.storage.content_add(self.in_contents)
-        self.assertCountContents(len(self.expected_contents))
-        self.assertContentsOk(self.expected_contents)
+        self.assertCountContents(1)
+        self.assertCountDirectories(len(self.in_directories))
 
-    def test_failing(self):
-        """Comparing wrong snapshot should fail.
+    def test_revision_cascade(self):
+        """Checks that sending a revision triggers sending contents and
+        directories."""
 
-        """
-        self.storage.content_add(self.in_contents)
-        with self.assertRaises(AssertionError):
-            self.assertContentsOk([])
+        self.loader.load()  # initialize the loader
 
+        self.loader.maybe_load_contents(self.in_contents[0:1])
+        self.loader.maybe_load_directories(self.in_directories[0:1])
+        self.loader.maybe_load_revisions(self.in_revisions)
 
-class LoadTestDirectory(DummyBaseLoaderTest):
-    def test_send_batch_directories(self):
-        """Loading directories should be ok
+        self.assertCountContents(1)
+        self.assertCountDirectories(1)
+        self.assertCountRevisions(len(self.in_revisions))
 
-        """
-        self.storage.directory_add(self.in_directories)
-        self.assertCountDirectories(len(self.in_directories))
-        self.assertDirectoriesOk(self.in_directories)
+    def test_release_cascade(self):
+        """Checks that sending a release triggers sending revisions,
+        contents, and directories."""
+        self.loader.load()  # initialize the loader
 
-    def test_failing(self):
-        """Comparing wrong snapshot should fail.
+        self.loader.maybe_load_contents(self.in_contents[0:1])
+        self.loader.maybe_load_directories(self.in_directories[0:1])
+        self.loader.maybe_load_revisions(self.in_revisions[0:1])
+        self.loader.maybe_load_releases(self.in_releases)
 
-        """
-        self.storage.directory_add(self.in_directories)
-        with self.assertRaises(AssertionError):
-            self.assertDirectoriesOk([])
+        self.assertCountContents(1)
+        self.assertCountDirectories(1)
+        self.assertCountRevisions(1)
+        self.assertCountReleases(len(self.in_releases))
+
+    def test_snapshot_cascade(self):
+        """Checks that sending a snapshot triggers sending releases,
+        revisions, contents, and directories."""
+        self.loader.load()  # initialize the loader
+
+        self.loader.maybe_load_contents(self.in_contents[0:1])
+        self.loader.maybe_load_directories(self.in_directories[0:1])
+        self.loader.maybe_load_revisions(self.in_revisions[0:1])
+        self.loader.maybe_load_releases(self.in_releases[0:1])
+        self.loader.maybe_load_snapshot(self.in_snapshot)
+
+        self.assertCountContents(1)
+        self.assertCountDirectories(1)
+        self.assertCountRevisions(1)
+        self.assertCountReleases(1)
+        self.assertCountSnapshots(1)