From b7b7927ec97b61c4943a6b6f1eb17a72ca19dae1 Mon Sep 17 00:00:00 2001
From: David Douard <david.douard@sdfa3.org>
Date: Tue, 31 May 2022 18:24:03 +0200
Subject: [PATCH] use an in-memory storage for test_cookers and test_to_disk
to reduce test execution time a bit.
Keep test_git_bare_cooker using a postgresql backend, since it depends
on it (support for displayname).
---
swh/vault/tests/conftest.py | 24 +------
swh/vault/tests/test_cookers.py | 91 +++++++++++--------------
swh/vault/tests/test_git_bare_cooker.py | 23 ++++++-
3 files changed, 65 insertions(+), 73 deletions(-)
diff --git a/swh/vault/tests/conftest.py b/swh/vault/tests/conftest.py
index 5c7cbf2..e91753f 100644
--- a/swh/vault/tests/conftest.py
+++ b/swh/vault/tests/conftest.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 The Software Heritage developers
+# Copyright (C) 2020-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -12,7 +12,6 @@ import pytest
from pytest_postgresql import factories
from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact
-from swh.storage.postgresql.db import Db as StorageDb
from swh.vault import get_vault
from swh.vault.backend import VaultBackend
@@ -35,13 +34,6 @@ if pytest_v < pkg_resources.extern.packaging.version.parse("3.9"):
yield pathlib.Path(tmpdir)
-storage_postgresql_proc = factories.postgresql_proc(
- dbname="storage",
- load=[
- partial(initialize_database_for_module, "storage", StorageDb.current_version)
- ],
-)
-
vault_postgresql_proc = factories.postgresql_proc(
dbname="vault",
load=[
@@ -50,25 +42,15 @@ vault_postgresql_proc = factories.postgresql_proc(
)
postgres_vault = postgresql_fact("vault_postgresql_proc")
-postgres_storage = postgresql_fact(
- "storage_postgresql_proc",
- no_db_drop=True, # keep the db for performance reasons
-)
@pytest.fixture
-def swh_vault_config(postgres_vault, postgres_storage, tmp_path) -> Dict[str, Any]:
+def swh_vault_config(postgres_vault, tmp_path) -> Dict[str, Any]:
tmp_path = str(tmp_path)
return {
"db": postgres_vault.dsn,
"storage": {
- "cls": "postgresql",
- "db": postgres_storage.dsn,
- "objstorage": {
- "cls": "pathslicing",
- "root": tmp_path,
- "slicing": "0:1/1:5",
- },
+ "cls": "memory",
},
"cache": {
"cls": "pathslicing",
diff --git a/swh/vault/tests/test_cookers.py b/swh/vault/tests/test_cookers.py
index 2d1a368..48f53cc 100644
--- a/swh/vault/tests/test_cookers.py
+++ b/swh/vault/tests/test_cookers.py
@@ -17,6 +17,7 @@ import tempfile
import unittest
import unittest.mock
+import attrs
import dulwich.fastexport
import dulwich.index
import dulwich.objects
@@ -31,6 +32,7 @@ from swh.model.model import (
Release,
Revision,
RevisionType,
+ SkippedContent,
Snapshot,
SnapshotBranch,
TargetType,
@@ -437,32 +439,25 @@ class TestDirectoryCooker:
obj_id = hashutil.hash_to_bytes(obj_id_hex)
swhid = CoreSWHID(object_type=ObjectType.DIRECTORY, object_id=obj_id)
- # FIXME: storage.content_update() should be changed to allow things
- # like that
- with loader.storage.get_db().transaction() as cur:
- cur.execute(
- """update content set status = 'visible'
- where sha1 = %s""",
- (id_1,),
- )
- cur.execute(
- """update content set status = 'hidden'
- where sha1 = %s""",
- (id_2,),
- )
-
- cur.execute(
- """
- insert into skipped_content
- (sha1, sha1_git, sha256, blake2s256, length, reason)
- select sha1, sha1_git, sha256, blake2s256, length, 'no reason'
- from content
- where sha1 = %s
- """,
- (id_3,),
- )
-
- cur.execute("delete from content where sha1 = %s", (id_3,))
+ # alter the content of the storage
+ # 1/ make file 2 an hidden file object
+ loader.storage._allow_overwrite = True
+ cnt2 = attrs.evolve(
+ loader.storage.content_get([id_2])[0], status="hidden", data=file_2
+ )
+ loader.storage.content_add([cnt2])
+ assert loader.storage.content_get([id_2])[0].status == "hidden"
+
+ # 2/ make file 3 an skipped file object
+ cnt3 = loader.storage.content_get([id_3])[0].to_dict()
+ cnt3["status"] = "absent"
+ cnt3["reason"] = "no reason"
+ sk_cnt3 = SkippedContent.from_dict(cnt3)
+ loader.storage.skipped_content_add([sk_cnt3])
+ # dirty dirty dirty... let's pretend it is the equivalent of writing sql
+ # queries in the postgresql backend
+ for hashkey in loader.storage._cql_runner._content_indexes:
+ loader.storage._cql_runner._content_indexes[hashkey].pop(cnt3[hashkey])
with cook_extract_directory(loader.storage, swhid) as p:
assert (p / "file").read_bytes() == b"test1"
@@ -828,32 +823,26 @@ class RepoFixtures:
loader = git_loader(str(rp))
loader.load()
- # FIXME: storage.content_update() should be changed to allow things
- # like that
- with loader.storage.get_db().transaction() as cur:
- cur.execute(
- """update content set status = 'visible'
- where sha1 = %s""",
- (id_1,),
- )
- cur.execute(
- """update content set status = 'hidden'
- where sha1 = %s""",
- (id_2,),
- )
-
- cur.execute(
- """
- insert into skipped_content
- (sha1, sha1_git, sha256, blake2s256, length, reason)
- select sha1, sha1_git, sha256, blake2s256, length, 'no reason'
- from content
- where sha1 = %s
- """,
- (id_3,),
- )
+ # alter the content of the storage
+ # 1/ make file 2 an hidden file object
+ loader.storage._allow_overwrite = True
+ cnt2 = attrs.evolve(
+ loader.storage.content_get([id_2])[0], status="hidden", data=file_2
+ )
+ loader.storage.content_add([cnt2])
+ assert loader.storage.content_get([id_2])[0].status == "hidden"
+
+ # 2/ make file 3 an skipped file object
+ cnt3 = loader.storage.content_get([id_3])[0].to_dict()
+ cnt3["status"] = "absent"
+ cnt3["reason"] = "no reason"
+ sk_cnt3 = SkippedContent.from_dict(cnt3)
+ loader.storage.skipped_content_add([sk_cnt3])
+ # dirty dirty dirty... let's pretend it is the equivalent of writing sql
+ # queries in the postgresql backend
+ for hashkey in loader.storage._cql_runner._content_indexes:
+ loader.storage._cql_runner._content_indexes[hashkey].pop(cnt3[hashkey])
- cur.execute("delete from content where sha1 = %s", (id_3,))
return (loader, swhid)
def check_revision_filtered_objects(self, ert, p, swhid):
diff --git a/swh/vault/tests/test_git_bare_cooker.py b/swh/vault/tests/test_git_bare_cooker.py
index f3c27e9..138dd4d 100644
--- a/swh/vault/tests/test_git_bare_cooker.py
+++ b/swh/vault/tests/test_git_bare_cooker.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 The Software Heritage developers
+# Copyright (C) 2021-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -11,6 +11,7 @@ run on the bare cooker.
import datetime
import enum
+from functools import partial
import io
import subprocess
import tarfile
@@ -21,7 +22,9 @@ import attr
import dulwich.repo
import pytest
from pytest import param
+from pytest_postgresql import factories
+from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact
from swh.model.from_disk import DentryPerms
from swh.model.model import (
Content,
@@ -38,9 +41,27 @@ from swh.model.model import (
Timestamp,
TimestampWithTimezone,
)
+from swh.storage import get_storage
+from swh.storage.postgresql.db import Db as StorageBackend
from swh.vault.cookers.git_bare import GitBareCooker
from swh.vault.in_memory_backend import InMemoryVaultBackend
+storage_postgresql_proc = factories.postgresql_proc(
+ dbname="storage",
+ load=[
+ partial(
+ initialize_database_for_module, "storage", StorageBackend.current_version
+ )
+ ],
+)
+
+storage_postgresql = postgresql_fact("storage_postgresql_proc", no_db_drop=True)
+
+
+@pytest.fixture
+def swh_storage(storage_postgresql):
+ return get_storage("local", db=storage_postgresql.dsn, objstorage={"cls": "memory"})
+
class RootObjects(enum.Enum):
REVISION = enum.auto()
--
GitLab