diff --git a/pyproject.toml b/pyproject.toml index 5fa5406b8176e69369d1e1a29bfc796298f520d4..3738e71c1f59d860af794418573f5a336c0af6f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,15 +25,14 @@ dependencies = {file = "requirements.txt"} [tool.setuptools.dynamic.optional-dependencies] testing_core = {file = "requirements-test.txt"} logging = {file = ["requirements-logging.txt"]} -db = {file = ["requirements-db.txt", "requirements-db-pytestplugin.txt"]} +db = {file = ["requirements-db.txt"]} http = {file = "requirements-http.txt"} # kitchen sink, please do not use testing = {file = [ "requirements-test.txt", "requirements-logging.txt", "requirements-http.txt", - "requirements-db.txt", - "requirements-db-pytestplugin.txt"]} + "requirements-db.txt"]} [project.entry-points.console_scripts] "swh" = "swh.core.cli:main" diff --git a/requirements-db-pytestplugin.txt b/requirements-db-pytestplugin.txt deleted file mode 100644 index 815feddca4cdbb8b579ca61edc908e412b3f72a4..0000000000000000000000000000000000000000 --- a/requirements-db-pytestplugin.txt +++ /dev/null @@ -1,2 +0,0 @@ -# requirements for swh.core.db.pytest_plugin -pytest-postgresql >=3, < 4.0.0 # version 4.0 depends on psycopg 3. https://github.com/ClearcodeHQ/pytest-postgresql/blob/main/CHANGES.rst#400 diff --git a/requirements-test.txt b/requirements-test.txt index 71e33f9b1428642f00edff99c056a53f0e1b3720..299bd006a1e64cad1f1124dea35c017c257e1d26 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,7 @@ hypothesis >= 3.11.0 pytest pytest-mock +pytest-postgresql > 5 pytz requests-mock types-deprecated diff --git a/swh/core/db/pytest_plugin.py b/swh/core/db/pytest_plugin.py deleted file mode 100644 index 23b0609c8d9044addab06817abb8fa753441e26e..0000000000000000000000000000000000000000 --- a/swh/core/db/pytest_plugin.py +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright (C) 2020 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import glob -from importlib import import_module -import logging -import subprocess -from typing import Callable, Iterable, Iterator, List, Optional, Sequence, Set, Union -import warnings - -from _pytest.fixtures import FixtureRequest -from deprecated import deprecated -import psycopg2 -import pytest -from pytest_postgresql.compat import check_for_psycopg2, connection -from pytest_postgresql.executor import PostgreSQLExecutor -from pytest_postgresql.executor_noop import NoopExecutor -from pytest_postgresql.janitor import DatabaseJanitor - -from swh.core.db.db_utils import initialize_database_for_module -from swh.core.utils import basename_sortkey - -# to keep mypy happy regardless pytest-postgresql version -try: - _pytest_pgsql_get_config_module = import_module("pytest_postgresql.config") -except ImportError: - # pytest_postgresql < 3.0.0 - _pytest_pgsql_get_config_module = import_module("pytest_postgresql.factories") - -_pytest_postgresql_get_config = getattr(_pytest_pgsql_get_config_module, "get_config") - - -logger = logging.getLogger(__name__) - -initialize_database_for_module = deprecated( - version="2.10", - reason="Use swh.core.db.db_utils.initialize_database_for_module instead.", -)(initialize_database_for_module) - -warnings.warn( - "This pytest plugin is deprecated, it should not be used any more.", - category=DeprecationWarning, -) - - -class SWHDatabaseJanitor(DatabaseJanitor): - """SWH database janitor implementation with a a different setup/teardown policy than - than the stock one. Instead of dropping, creating and initializing the database for - each test, it creates and initializes the db once, then truncates the tables (and - sequences) in between tests. - - This is needed to have acceptable test performances. - - """ - - def __init__( - self, - user: str, - host: str, - port: int, - dbname: str, - version: Union[str, float], - password: Optional[str] = None, - isolation_level: Optional[int] = None, - connection_timeout: int = 60, - dump_files: Optional[Union[str, Sequence[str]]] = None, - no_truncate_tables: Set[str] = set(), - no_db_drop: bool = False, - ) -> None: - super().__init__(user, host, port, dbname, version) - # do no truncate the following tables - self.no_truncate_tables = set(no_truncate_tables) - self.no_db_drop = no_db_drop - self.dump_files = dump_files - - def psql_exec(self, fname: str) -> None: - conninfo = ( - f"host={self.host} user={self.user} port={self.port} dbname={self.dbname}" - ) - - subprocess.check_call( - [ - "psql", - "--quiet", - "--no-psqlrc", - "-v", - "ON_ERROR_STOP=1", - "-d", - conninfo, - "-f", - fname, - ] - ) - - def db_reset(self) -> None: - """Truncate tables (all but self.no_truncate_tables set) and sequences""" - with psycopg2.connect( - dbname=self.dbname, - user=self.user, - host=self.host, - port=self.port, - ) as cnx: - with cnx.cursor() as cur: - cur.execute( - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = %s", - ("public",), - ) - all_tables = set(table for (table,) in cur.fetchall()) - tables_to_truncate = all_tables - self.no_truncate_tables - - for table in tables_to_truncate: - cur.execute("TRUNCATE TABLE %s CASCADE" % table) - - cur.execute( - "SELECT sequence_name FROM information_schema.sequences " - "WHERE sequence_schema = %s", - ("public",), - ) - seqs = set(seq for (seq,) in cur.fetchall()) - for seq in seqs: - cur.execute("ALTER SEQUENCE %s RESTART;" % seq) - cnx.commit() - - def _db_exists(self, cur, dbname): - cur.execute( - "SELECT EXISTS " - "(SELECT datname FROM pg_catalog.pg_database WHERE datname= %s);", - (dbname,), - ) - row = cur.fetchone() - return (row is not None) and row[0] - - def init(self) -> None: - """Create database in postgresql out of a template it if it exists, bare - creation otherwise.""" - template_name = f"{self.dbname}_tmpl" - logger.debug("Initialize DB %s", self.dbname) - with self.cursor() as cur: - tmpl_exists = self._db_exists(cur, template_name) - db_exists = self._db_exists(cur, self.dbname) - if not db_exists: - if tmpl_exists: - logger.debug( - "Create %s from template %s", self.dbname, template_name - ) - cur.execute( - f'CREATE DATABASE "{self.dbname}" TEMPLATE "{template_name}";' - ) - else: - logger.debug("Create %s from scratch", self.dbname) - cur.execute(f'CREATE DATABASE "{self.dbname}";') - if self.dump_files: - logger.warning( - "Using dump_files on the postgresql_fact fixture " - "is deprecated. See swh.core documentation for more " - "details." - ) - for dump_file in gen_dump_files(self.dump_files): - logger.info(f"Loading {dump_file}") - self.psql_exec(dump_file) - else: - logger.debug("Reset %s", self.dbname) - self.db_reset() - - def drop(self) -> None: - """Drop database in postgresql.""" - if self.no_db_drop: - with self.cursor() as cur: - self._terminate_connection(cur, self.dbname) - else: - super().drop() - - -# the postgres_fact factory fixture below is mostly a copy of the code -# from pytest-postgresql. We need a custom version here to be able to -# specify our version of the DBJanitor we use. -@deprecated(version="2.10", reason="Use stock pytest_postgresql factory instead") -def postgresql_fact( - process_fixture_name: str, - dbname: Optional[str] = None, - load: Optional[Sequence[Union[Callable, str]]] = None, - isolation_level: Optional[int] = None, - modname: Optional[str] = None, - dump_files: Optional[Union[str, List[str]]] = None, - no_truncate_tables: Set[str] = {"dbversion"}, - no_db_drop: bool = False, -) -> Callable[[FixtureRequest], Iterator[connection]]: - """ - Return connection fixture factory for PostgreSQL. - - :param process_fixture_name: name of the process fixture - :param dbname: database name - :param load: SQL, function or function import paths to automatically load - into our test database - :param isolation_level: optional postgresql isolation level - defaults to server's default - :param modname: (swh) module name for which the database is created - :dump_files: (deprecated, use load instead) list of sql script files to - execute after the database has been created - :no_truncate_tables: list of table not to truncate between tests (only used - when no_db_drop is True) - :no_db_drop: if True, keep the database between tests; in which case, the - database is reset (see SWHDatabaseJanitor.db_reset()) by truncating - most of the tables. Note that this makes de facto tests (potentially) - interdependent, use with extra caution. - :returns: function which makes a connection to postgresql - """ - - @pytest.fixture - def postgresql_factory(request: FixtureRequest) -> Iterator[connection]: - """ - Fixture factory for PostgreSQL. - - :param request: fixture request object - :returns: postgresql client - """ - check_for_psycopg2() - proc_fixture: Union[PostgreSQLExecutor, NoopExecutor] = request.getfixturevalue( - process_fixture_name - ) - - pg_host = proc_fixture.host - pg_port = proc_fixture.port - pg_user = proc_fixture.user - pg_password = proc_fixture.password - pg_options = proc_fixture.options - pg_db = dbname or proc_fixture.dbname - pg_load = load or [] - assert pg_db is not None - - with SWHDatabaseJanitor( - pg_user, - pg_host, - pg_port, - pg_db, - proc_fixture.version, - pg_password, - isolation_level=isolation_level, - dump_files=dump_files, - no_truncate_tables=no_truncate_tables, - no_db_drop=no_db_drop, - ) as janitor: - db_connection: connection = psycopg2.connect( - dbname=pg_db, - user=pg_user, - password=pg_password, - host=pg_host, - port=pg_port, - options=pg_options, - ) - for load_element in pg_load: - janitor.load(load_element) - try: - yield db_connection - finally: - db_connection.close() - - return postgresql_factory - - -def gen_dump_files(dump_files: Union[str, Iterable[str]]) -> Iterator[str]: - """Generate files potentially resolving glob patterns if any""" - if isinstance(dump_files, str): - dump_files = [dump_files] - for dump_file in dump_files: - if glob.has_magic(dump_file): - # if the dump_file is a glob pattern one, resolve it - yield from ( - fname for fname in sorted(glob.glob(dump_file), key=basename_sortkey) - ) - else: - # otherwise, just return the filename - yield dump_file diff --git a/swh/core/db/tests/pytest_plugin/__init__.py b/swh/core/db/tests/pytest_plugin/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/swh/core/db/tests/pytest_plugin/data/0-schema.sql b/swh/core/db/tests/pytest_plugin/data/0-schema.sql deleted file mode 100644 index e6b008bd63cb024f661ec78a5d27ac36e87bacab..0000000000000000000000000000000000000000 --- a/swh/core/db/tests/pytest_plugin/data/0-schema.sql +++ /dev/null @@ -1,19 +0,0 @@ --- schema version table which won't get truncated -create table dbversion ( - version int primary key, - release timestamptz, - description text -); - --- a people table which won't get truncated -create table people ( - fullname text not null -); - --- a fun table which will get truncated for each test -create table fun ( - time timestamptz not null -); - --- one sequence to check for reset as well -create sequence serial; diff --git a/swh/core/db/tests/pytest_plugin/data/1-data.sql b/swh/core/db/tests/pytest_plugin/data/1-data.sql deleted file mode 100644 index 8680263b09539b3c8203e7e48102445b3f516b50..0000000000000000000000000000000000000000 --- a/swh/core/db/tests/pytest_plugin/data/1-data.sql +++ /dev/null @@ -1,15 +0,0 @@ --- insert some values in dbversion -insert into dbversion(version, release, description) values (1, '2016-02-22 15:56:28.358587+00', 'Work In Progress'); -insert into dbversion(version, release, description) values (2, '2016-02-24 18:05:54.887217+00', 'Work In Progress'); -insert into dbversion(version, release, description) values (3, '2016-10-21 14:10:18.629763+00', 'Work In Progress'); -insert into dbversion(version, release, description) values (4, '2017-08-08 19:01:11.723113+00', 'Work In Progress'); -insert into dbversion(version, release, description) values (7, '2018-03-30 12:58:39.256679+00', 'Work In Progress'); - -insert into fun(time) values ('2020-10-19 09:00:00.666999+00'); -insert into fun(time) values ('2020-10-18 09:00:00.666999+00'); -insert into fun(time) values ('2020-10-17 09:00:00.666999+00'); - -select nextval('serial'); - -insert into people(fullname) values ('dudess'); -insert into people(fullname) values ('dude'); diff --git a/swh/core/db/tests/pytest_plugin/test_pytest_plugin.py b/swh/core/db/tests/pytest_plugin/test_pytest_plugin.py deleted file mode 100644 index ba8c1b335263a3af83f0b9e386965e36dc14b097..0000000000000000000000000000000000000000 --- a/swh/core/db/tests/pytest_plugin/test_pytest_plugin.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright (C) 2020 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import glob -import os - -from pytest_postgresql import factories - -from swh.core.db import BaseDb -from swh.core.db.pytest_plugin import gen_dump_files, postgresql_fact - -SQL_DIR = os.path.join(os.path.dirname(__file__), "data") - -test_postgresql_proc = factories.postgresql_proc( - dbname="fun", - load=sorted(glob.glob(f"{SQL_DIR}/*.sql")), # type: ignore[arg-type] - # type ignored because load is typed as Optional[List[...]] instead of an - # Optional[Sequence[...]] in pytest_postgresql<4 -) - -# db with special policy for tables dbversion and people -postgres_fun = postgresql_fact( - "test_postgresql_proc", - no_db_drop=True, - no_truncate_tables={"dbversion", "people"}, -) - -postgres_fun2 = postgresql_fact( - "test_postgresql_proc", - dbname="fun2", - load=sorted(glob.glob(f"{SQL_DIR}/*.sql")), - no_truncate_tables={"dbversion", "people"}, - no_db_drop=True, -) - - -def test_smoke_test_fun_db_is_up(postgres_fun): - """This ensures the db is created and configured according to its dumps files.""" - with BaseDb.connect(postgres_fun.dsn).cursor() as cur: - cur.execute("select count(*) from dbversion") - nb_rows = cur.fetchone()[0] - assert nb_rows == 5 - - cur.execute("select count(*) from fun") - nb_rows = cur.fetchone()[0] - assert nb_rows == 3 - - cur.execute("select count(*) from people") - nb_rows = cur.fetchone()[0] - assert nb_rows == 2 - - # in data, we requested a value already so it starts at 2 - cur.execute("select nextval('serial')") - val = cur.fetchone()[0] - assert val == 2 - - -def test_smoke_test_fun2_db_is_up(postgres_fun2): - """This ensures the db is created and configured according to its dumps files.""" - with BaseDb.connect(postgres_fun2.dsn).cursor() as cur: - cur.execute("select count(*) from dbversion") - nb_rows = cur.fetchone()[0] - assert nb_rows == 5 - - cur.execute("select count(*) from fun") - nb_rows = cur.fetchone()[0] - assert nb_rows == 3 - - cur.execute("select count(*) from people") - nb_rows = cur.fetchone()[0] - assert nb_rows == 2 - - # in data, we requested a value already so it starts at 2 - cur.execute("select nextval('serial')") - val = cur.fetchone()[0] - assert val == 2 - - -def test_smoke_test_fun_db_is_still_up_and_got_reset(postgres_fun): - """This ensures that within another tests, the 'fun' db is still up, created (and not - configured again). This time, most of the data has been reset: - - except for tables 'dbversion' and 'people' which were left as is - - the other tables from the schema (here only "fun") got truncated - - the sequences got truncated as well - - """ - with BaseDb.connect(postgres_fun.dsn).cursor() as cur: - # db version is excluded from the truncate - cur.execute("select count(*) from dbversion") - nb_rows = cur.fetchone()[0] - assert nb_rows == 5 - - # people is also allowed not to be truncated - cur.execute("select count(*) from people") - nb_rows = cur.fetchone()[0] - assert nb_rows == 2 - - # table and sequence are reset - cur.execute("select count(*) from fun") - nb_rows = cur.fetchone()[0] - assert nb_rows == 0 - - cur.execute("select nextval('serial')") - val = cur.fetchone()[0] - assert val == 1 - - -# db with no special policy for tables truncation, all tables are reset -postgres_people = postgresql_fact( - "postgresql_proc", - dbname="people", - dump_files=f"{SQL_DIR}/*.sql", - no_truncate_tables=set(), - no_db_drop=True, -) - - -def test_gen_dump_files(): - files = [os.path.basename(fn) for fn in gen_dump_files(f"{SQL_DIR}/*.sql")] - assert files == ["0-schema.sql", "1-data.sql"] - - -def test_smoke_test_people_db_up(postgres_people): - """'people' db is up and configured""" - with BaseDb.connect(postgres_people.dsn).cursor() as cur: - cur.execute("select count(*) from dbversion") - nb_rows = cur.fetchone()[0] - assert nb_rows == 5 - - cur.execute("select count(*) from people") - nb_rows = cur.fetchone()[0] - assert nb_rows == 2 - - cur.execute("select count(*) from fun") - nb_rows = cur.fetchone()[0] - assert nb_rows == 3 - - cur.execute("select nextval('serial')") - val = cur.fetchone()[0] - assert val == 2 - - -def test_smoke_test_people_db_up_and_reset(postgres_people): - """'people' db is up and got reset on every tables and sequences""" - with BaseDb.connect(postgres_people.dsn).cursor() as cur: - # tables are truncated after the first round - cur.execute("select count(*) from dbversion") - nb_rows = cur.fetchone()[0] - assert nb_rows == 0 - - # tables are truncated after the first round - cur.execute("select count(*) from people") - nb_rows = cur.fetchone()[0] - assert nb_rows == 0 - - # table and sequence are reset - cur.execute("select count(*) from fun") - nb_rows = cur.fetchone()[0] - assert nb_rows == 0 - - cur.execute("select nextval('serial')") - val = cur.fetchone()[0] - assert val == 1 - - -# db with no initialization step, an empty db -postgres_no_init = postgresql_fact("postgresql_proc", dbname="something") - - -def test_smoke_test_db_no_init(postgres_no_init): - """We can connect to the db nonetheless""" - with BaseDb.connect(postgres_no_init.dsn).cursor() as cur: - cur.execute("select now()") - data = cur.fetchone()[0] - assert data is not None diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py index 8590427fa19b7a1071487ea7bcd9a0069fce6e3f..ef305258054c58834b9f4ce66dad4c681251d846 100644 --- a/swh/core/db/tests/test_cli.py +++ b/swh/core/db/tests/test_cli.py @@ -3,7 +3,6 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import copy import os import traceback @@ -59,24 +58,19 @@ def swh_db_cli(cli_runner, monkeypatch, postgresql): the cli to run appropriately (when not specifying the --dbname flag) """ - db_params = postgresql.get_dsn_parameters() - monkeypatch.setenv("PGHOST", db_params["host"]) - monkeypatch.setenv("PGUSER", db_params["user"]) - monkeypatch.setenv("PGPORT", db_params["port"]) + monkeypatch.setenv("PGHOST", postgresql.info.host) + monkeypatch.setenv("PGUSER", postgresql.info.user) + monkeypatch.setenv("PGPORT", postgresql.info.port) - return cli_runner, db_params + return cli_runner, postgresql.info def craft_conninfo(test_db, dbname=None) -> str: """Craft conninfo string out of the test_db object. This also allows to override the dbname.""" - db_params = test_db.get_dsn_parameters() - if dbname: - params = copy.deepcopy(db_params) - params["dbname"] = dbname - else: - params = db_params - return "postgresql://{user}@{host}:{port}/{dbname}".format(**params) + db_params = test_db.info + dbname = dbname if dbname else db_params.dbname + return f"postgresql://{db_params.user}@{db_params.host}:{db_params.port}/{dbname}" def test_cli_swh_db_create_and_init_db(cli_runner, postgresql, mock_import_swhmodule): @@ -145,7 +139,7 @@ def test_cli_swh_db_initialization_works_with_flags( assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) - with BaseDb.connect(postgresql.dsn).cursor() as cur: + with BaseDb.connect(postgresql.info.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 @@ -158,18 +152,18 @@ def test_cli_swh_db_initialization_with_env( module_name = "test.cli" # it's mocked here cli_runner, db_params = swh_db_cli result = cli_runner.invoke( - swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] + swhdb, ["init-admin", module_name, "--dbname", db_params.dbname] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( - swhdb, ["init", module_name, "--dbname", db_params["dbname"]] + swhdb, ["init", module_name, "--dbname", db_params.dbname] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) - with BaseDb.connect(postgresql.dsn).cursor() as cur: + with BaseDb.connect(postgresql.info.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 @@ -183,28 +177,28 @@ def test_cli_swh_db_initialization_idempotent( cli_runner, db_params = swh_db_cli result = cli_runner.invoke( - swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] + swhdb, ["init-admin", module_name, "--dbname", db_params.dbname] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( - swhdb, ["init", module_name, "--dbname", db_params["dbname"]] + swhdb, ["init", module_name, "--dbname", db_params.dbname] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( - swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] + swhdb, ["init-admin", module_name, "--dbname", db_params.dbname] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( - swhdb, ["init", module_name, "--dbname", db_params["dbname"]] + swhdb, ["init", module_name, "--dbname", db_params.dbname] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) - with BaseDb.connect(postgresql.dsn).cursor() as cur: + with BaseDb.connect(postgresql.info.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 diff --git a/swh/core/db/tests/test_db.py b/swh/core/db/tests/test_db.py index c0779436435a4aea77d308b69888783fe18bc61b..44932e80df9042febf07554d8160ac1fbb7a52ca 100644 --- a/swh/core/db/tests/test_db.py +++ b/swh/core/db/tests/test_db.py @@ -245,7 +245,7 @@ test_db = factories.postgresql("postgresql_proc", dbname="test-db2") @pytest.fixture def db_with_data(test_db, request): """Fixture to initialize a db with some data out of the "INIT_SQL above""" - db = BaseDb.connect(test_db.dsn) + db = BaseDb.connect(test_db.info.dsn) with db.cursor() as cur: psycopg2.extras.register_default_jsonb(cur) cur.execute(INIT_SQL)