Skip to content
Snippets Groups Projects
Commit f86bc3d1 authored by vlorentz's avatar vlorentz
Browse files

Use the in-mem storage instead of mocks to test the git loader. 

Summary: Depends on D682.

Reviewers: #reviewers, ardumont

Reviewed By: #reviewers, ardumont

Subscribers: anlambert, ardumont, swh-public-ci

Maniphest Tasks: T1307

Differential Revision: https://forge.softwareheritage.org/D684
parents d5df3bfc c725b7c5
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
swh/loader/git/tests/__init__.py
+ 21
− 0
View file @ f86bc3d1
TEST_LOADER_CONFIG = {
'storage': {
'cls': 'memory',
'args': {
}
},
'send_contents': True,
'send_directories': True,
'send_revisions': True,
'send_releases': True,
'send_snapshot': True,
'content_size_limit': 100 * 1024 * 1024,
'content_packet_size': 10,
'content_packet_size_bytes': 100 * 1024 * 1024,
'directory_packet_size': 10,
'revision_packet_size': 10,
'release_packet_size': 10,
'save_data': False,
}
swh/loader/git/tests/test_loader.py
+ 27
− 114
View file @ f86bc3d1
......@@ -9,26 +9,15 @@ import tempfile
import subprocess
from swh.loader.git.loader import GitLoader, GitLoaderFromArchive
from swh.loader.core.tests import BaseLoaderTest, LoaderNoStorage
from swh.model.hashutil import hash_to_bytes, hash_to_bytehex
from swh.loader.core.tests import BaseLoaderTest
from . import TEST_LOADER_CONFIG
class MockStorage0:
"""The storage's state before anything is added."""
def snapshot_get_latest(self, origin_id):
return None
def content_missing(self, contents, key_hash='sha1'):
return [c[key_hash] for c in contents]
def directory_missing(self, directories):
return directories
def revision_missing(self, revisions):
return revisions
def object_find_by_sha1_git(self, ids):
return {}
class GitLoaderFromArchive(GitLoaderFromArchive):
def project_name_from_archive(self, archive_path):
# We don't want the project name to be 'resources'.
return 'testrepo'
CONTENT1 = {
......@@ -70,8 +59,6 @@ SNAPSHOT1 = {
},
}
SUBDIR_HASH = hash_to_bytes('d53f143d5f3aadb278aad60c4e9a17945a2d68de')
# directory hashes obtained with:
# gco b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a
# swh-hashtree --ignore '.git' --path .
......@@ -105,91 +92,6 @@ REVISIONS1 = {
}
class MockStorage1:
"""The storage's state after the first snapshot is loaded."""
def snapshot_get_latest(self, origin_id):
# The following line reencodes SNAPSHOT1 from the format expected
# by assertSnapshotOk to the one that Storage.snapshot_get_latest
# returns.
return {
'id': hash_to_bytes(SNAPSHOT1['id']),
'branches': {
branch_name.encode(): {
'target': hash_to_bytes(branch['target']),
'target_type': branch['target_type']}
for (branch_name, branch)
in SNAPSHOT1['branches'].items()}}
def content_missing(self, contents, key_hash='sha1'):
return map(hash_to_bytes,
{c[key_hash] for c in contents} - CONTENT1)
def directory_missing(self, directories):
assert all(isinstance(d, bytes) for d in directories)
return (set(directories) -
set(map(hash_to_bytes, REVISIONS1)) -
{hash_to_bytes(SUBDIR_HASH)})
def revision_missing(self, revisions):
assert all(isinstance(r, bytes) for r in revisions)
return list(set(revisions) - set(map(hash_to_bytes, REVISIONS1)))
def object_find_by_sha1_git(self, ids):
res = {}
for id_ in ids:
found = []
decoded_id = hash_to_bytehex(id_)
if decoded_id in REVISIONS1:
found.append({
'sha1_git': id_,
'type': 'revision',
'id': id_,
'object_id': 42,
})
elif decoded_id in REVISIONS1.values():
found.append({
'sha1_git': id_,
'type': 'directory',
'id': id_,
'object_id': 42,
})
elif decoded_id == SUBDIR_HASH:
found.append({
'sha1_git': id_,
'type': 'directory',
'id': id_,
'object_id': 42,
})
elif decoded_id in CONTENT1:
found.append({
'sha1_git': id_,
'type': 'content',
'id': id_,
'object_id': 42,
})
res[id_] = found
return res
class LoaderNoStorageMixin(LoaderNoStorage):
def __init__(self):
super().__init__()
self.origin_id = 1
self.visit = 1
self.storage = MockStorage0()
class GitLoaderNoStorage(LoaderNoStorageMixin, GitLoader):
pass
class GitLoaderFromArchiveNoStorage(LoaderNoStorageMixin,
GitLoaderFromArchive):
def project_name_from_archive(self, archive_path):
# We don't want the project name to be 'resources'.
return 'testrepo'
class BaseGitLoaderTest(BaseLoaderTest):
def setUp(self, archive_name, uncompress_archive, filename='testrepo'):
super().setUp(archive_name=archive_name, filename=filename,
......@@ -198,6 +100,11 @@ class BaseGitLoaderTest(BaseLoaderTest):
uncompress_archive=uncompress_archive)
class TestGitLoader(GitLoader):
def parse_config_file(self, *args, **kwargs):
return TEST_LOADER_CONFIG
class BaseDirGitLoaderTest(BaseGitLoaderTest):
"""Mixin base loader test to prepare the git
repository to uncompress, load and test the results.
......@@ -207,7 +114,8 @@ class BaseDirGitLoaderTest(BaseGitLoaderTest):
"""
def setUp(self):
super().setUp('testrepo.tgz', True)
self.loader = GitLoaderNoStorage()
self.loader = TestGitLoader()
self.storage = self.loader.storage
def load(self):
return self.loader.load(
......@@ -216,6 +124,11 @@ class BaseDirGitLoaderTest(BaseGitLoaderTest):
directory=self.destination_path)
class TestGitLoaderFromArchive(GitLoaderFromArchive):
def parse_config_file(self, *args, **kwargs):
return TEST_LOADER_CONFIG
class BaseZipGitLoaderTest(BaseGitLoaderTest):
"""Mixin base loader test to prepare the git
repository to uncompress, load and test the results.
......@@ -226,7 +139,8 @@ class BaseZipGitLoaderTest(BaseGitLoaderTest):
def setUp(self):
super().setUp('testrepo.tgz', True)
self._setup_zip()
self.loader = GitLoaderFromArchiveNoStorage()
self.loader = TestGitLoaderFromArchive()
self.storage = self.loader.storage
def _setup_zip(self):
self._zip_file = tempfile.NamedTemporaryFile('ab', suffix='.zip')
......@@ -264,15 +178,15 @@ class GitLoaderTests:
res = self.load()
self.assertEqual(res['status'], 'eventful', res)
self.assertCountContents(4) # two README, file1, file2
self.assertContentsContain(CONTENT1)
self.assertCountDirectories(7)
self.assertCountReleases(0) # FIXME: why not 2?
self.assertCountRevisions(7)
self.assertCountSnapshots(1)
self.assertRevisionsOk(REVISIONS1)
self.assertRevisionsContain(REVISIONS1)
self.assertSnapshotOk(SNAPSHOT1)
self.assertSnapshotEqual(SNAPSHOT1)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')
......@@ -283,7 +197,6 @@ class GitLoaderTests:
res = self.load()
self.assertEqual(res['status'], 'eventful')
self.loader.storage = MockStorage1()
res = self.load()
self.assertEqual(res['status'], 'uneventful')
self.assertCountSnapshots(1)
......@@ -335,10 +248,10 @@ class DirGitLoaderTest(BaseDirGitLoaderTest, GitLoaderTests):
self.assertCountRevisions(7 + 1)
self.assertCountSnapshots(1 + 1)
self.assertRevisionsOk(revisions)
self.assertRevisionsContain(revisions)
# TODO: how to check the snapshot id?
# self.assertSnapshotOk(SNAPSHOT1)
# self.assertSnapshotEqual(SNAPSHOT1)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')
......@@ -359,10 +272,10 @@ class DirGitLoaderTest(BaseDirGitLoaderTest, GitLoaderTests):
self.assertCountRevisions(7 + 2)
self.assertCountSnapshots(1 + 1 + 1)
self.assertRevisionsOk(revisions)
self.assertRevisionsContain(revisions)
# TODO: how to check the snapshot id?
# self.assertSnapshotOk(SNAPSHOT1)
# self.assertSnapshotEqual(SNAPSHOT1)
self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')
......
swh/loader/git/tests/test_updater.py
+ 9
− 6
View file @ f86bc3d1
from swh.loader.git.updater import BulkUpdater
from swh.loader.git.tests.test_loader import (
DirGitLoaderTest, LoaderNoStorageMixin)
from swh.loader.git.tests.test_loader import DirGitLoaderTest
class BulkUpdaterNoStorage(LoaderNoStorageMixin, BulkUpdater):
"""Subclass of BulkUpdater that uses a mock storage."""
pass
class TestBulkUpdater(BulkUpdater):
def parse_config_file(self, *args, **kwargs):
return {
**super().parse_config_file(*args, **kwargs),
'storage': {'cls': 'memory', 'args': {}}
}
class BulkUpdaterTest(DirGitLoaderTest):
"""Same tests as for the GitLoader, but running on BulkUpdater."""
def setUp(self):
super().setUp()
self.loader = BulkUpdaterNoStorage()
self.loader = TestBulkUpdater()
self.storage = self.loader.storage
def load(self):
return self.loader.load(
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment