Skip to content
Snippets Groups Projects
Commit b7b7927e authored by David Douard's avatar David Douard
Browse files

use an in-memory storage for test_cookers and test_to_disk 

to reduce test execution time a bit.
Keep test_git_bare_cooker using a postgresql backend, since it depends
on it (support for displayname).
parent 922a1ea0
No related branches found
No related tags found
1 merge request!150use an in-memory storage for tests
Hide whitespace changes
Inline Side-by-side
swh/vault/tests/conftest.py
+ 3
− 21
View file @ b7b7927e
# Copyright (C) 2020 The Software Heritage developers
# Copyright (C) 2020-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
......@@ -12,7 +12,6 @@ import pytest
from pytest_postgresql import factories
from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact
from swh.storage.postgresql.db import Db as StorageDb
from swh.vault import get_vault
from swh.vault.backend import VaultBackend
......@@ -35,13 +34,6 @@ if pytest_v < pkg_resources.extern.packaging.version.parse("3.9"):
yield pathlib.Path(tmpdir)
storage_postgresql_proc = factories.postgresql_proc(
dbname="storage",
load=[
partial(initialize_database_for_module, "storage", StorageDb.current_version)
],
)
vault_postgresql_proc = factories.postgresql_proc(
dbname="vault",
load=[
......@@ -50,25 +42,15 @@ vault_postgresql_proc = factories.postgresql_proc(
)
postgres_vault = postgresql_fact("vault_postgresql_proc")
postgres_storage = postgresql_fact(
"storage_postgresql_proc",
no_db_drop=True, # keep the db for performance reasons
)
@pytest.fixture
def swh_vault_config(postgres_vault, postgres_storage, tmp_path) -> Dict[str, Any]:
def swh_vault_config(postgres_vault, tmp_path) -> Dict[str, Any]:
tmp_path = str(tmp_path)
return {
"db": postgres_vault.dsn,
"storage": {
"cls": "postgresql",
"db": postgres_storage.dsn,
"objstorage": {
"cls": "pathslicing",
"root": tmp_path,
"slicing": "0:1/1:5",
},
"cls": "memory",
},
"cache": {
"cls": "pathslicing",
......
swh/vault/tests/test_cookers.py
+ 40
− 51
View file @ b7b7927e
......@@ -17,6 +17,7 @@ import tempfile
import unittest
import unittest.mock
import attrs
import dulwich.fastexport
import dulwich.index
import dulwich.objects
......@@ -31,6 +32,7 @@ from swh.model.model import (
Release,
Revision,
RevisionType,
SkippedContent,
Snapshot,
SnapshotBranch,
TargetType,
......@@ -437,32 +439,25 @@ class TestDirectoryCooker:
obj_id = hashutil.hash_to_bytes(obj_id_hex)
swhid = CoreSWHID(object_type=ObjectType.DIRECTORY, object_id=obj_id)
# FIXME: storage.content_update() should be changed to allow things
# like that
with loader.storage.get_db().transaction() as cur:
cur.execute(
"""update content set status = 'visible'
where sha1 = %s""",
(id_1,),
)
cur.execute(
"""update content set status = 'hidden'
where sha1 = %s""",
(id_2,),
)
cur.execute(
"""
insert into skipped_content
(sha1, sha1_git, sha256, blake2s256, length, reason)
select sha1, sha1_git, sha256, blake2s256, length, 'no reason'
from content
where sha1 = %s
""",
(id_3,),
)
cur.execute("delete from content where sha1 = %s", (id_3,))
# alter the content of the storage
# 1/ make file 2 an hidden file object
loader.storage._allow_overwrite = True
cnt2 = attrs.evolve(
loader.storage.content_get([id_2])[0], status="hidden", data=file_2
)
loader.storage.content_add([cnt2])
assert loader.storage.content_get([id_2])[0].status == "hidden"
# 2/ make file 3 an skipped file object
cnt3 = loader.storage.content_get([id_3])[0].to_dict()
cnt3["status"] = "absent"
cnt3["reason"] = "no reason"
sk_cnt3 = SkippedContent.from_dict(cnt3)
loader.storage.skipped_content_add([sk_cnt3])
# dirty dirty dirty... let's pretend it is the equivalent of writing sql
# queries in the postgresql backend
for hashkey in loader.storage._cql_runner._content_indexes:
loader.storage._cql_runner._content_indexes[hashkey].pop(cnt3[hashkey])
with cook_extract_directory(loader.storage, swhid) as p:
assert (p / "file").read_bytes() == b"test1"
......@@ -828,32 +823,26 @@ class RepoFixtures:
loader = git_loader(str(rp))
loader.load()
# FIXME: storage.content_update() should be changed to allow things
# like that
with loader.storage.get_db().transaction() as cur:
cur.execute(
"""update content set status = 'visible'
where sha1 = %s""",
(id_1,),
)
cur.execute(
"""update content set status = 'hidden'
where sha1 = %s""",
(id_2,),
)
cur.execute(
"""
insert into skipped_content
(sha1, sha1_git, sha256, blake2s256, length, reason)
select sha1, sha1_git, sha256, blake2s256, length, 'no reason'
from content
where sha1 = %s
""",
(id_3,),
)
# alter the content of the storage
# 1/ make file 2 an hidden file object
loader.storage._allow_overwrite = True
cnt2 = attrs.evolve(
loader.storage.content_get([id_2])[0], status="hidden", data=file_2
)
loader.storage.content_add([cnt2])
assert loader.storage.content_get([id_2])[0].status == "hidden"
# 2/ make file 3 an skipped file object
cnt3 = loader.storage.content_get([id_3])[0].to_dict()
cnt3["status"] = "absent"
cnt3["reason"] = "no reason"
sk_cnt3 = SkippedContent.from_dict(cnt3)
loader.storage.skipped_content_add([sk_cnt3])
# dirty dirty dirty... let's pretend it is the equivalent of writing sql
# queries in the postgresql backend
for hashkey in loader.storage._cql_runner._content_indexes:
loader.storage._cql_runner._content_indexes[hashkey].pop(cnt3[hashkey])
cur.execute("delete from content where sha1 = %s", (id_3,))
return (loader, swhid)
def check_revision_filtered_objects(self, ert, p, swhid):
......
swh/vault/tests/test_git_bare_cooker.py
+ 22
− 1
View file @ b7b7927e
# Copyright (C) 2021 The Software Heritage developers
# Copyright (C) 2021-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
......@@ -11,6 +11,7 @@ run on the bare cooker.
import datetime
import enum
from functools import partial
import io
import subprocess
import tarfile
......@@ -21,7 +22,9 @@ import attr
import dulwich.repo
import pytest
from pytest import param
from pytest_postgresql import factories
from swh.core.db.pytest_plugin import initialize_database_for_module, postgresql_fact
from swh.model.from_disk import DentryPerms
from swh.model.model import (
Content,
......@@ -38,9 +41,27 @@ from swh.model.model import (
Timestamp,
TimestampWithTimezone,
)
from swh.storage import get_storage
from swh.storage.postgresql.db import Db as StorageBackend
from swh.vault.cookers.git_bare import GitBareCooker
from swh.vault.in_memory_backend import InMemoryVaultBackend
storage_postgresql_proc = factories.postgresql_proc(
dbname="storage",
load=[
partial(
initialize_database_for_module, "storage", StorageBackend.current_version
)
],
)
storage_postgresql = postgresql_fact("storage_postgresql_proc", no_db_drop=True)
@pytest.fixture
def swh_storage(storage_postgresql):
return get_storage("local", db=storage_postgresql.dsn, objstorage={"cls": "memory"})
class RootObjects(enum.Enum):
REVISION = enum.auto()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment