From 7bf33cb4b513a80c3a2b94c079d73177fefc99cb Mon Sep 17 00:00:00 2001
From: David Douard <david.douard@sdfa3.org>
Date: Tue, 9 Jul 2019 16:05:18 +0200
Subject: [PATCH] cli: add a new 'db' cli group with an 'init' subcommand
that allows to initialize the database for a given swh package with db
connection credentials/dsn read from a standard swh config file.
Also add a couple of tests (only tests click group and command exist).
Closes T1896.
---
setup.py | 1 +
swh/core/cli/db.py | 117 ++++++++++++++++++++++++++++------
swh/core/db/tests/test_cli.py | 49 ++++++++++++++
3 files changed, 148 insertions(+), 19 deletions(-)
mode change 100755 => 100644 setup.py
create mode 100644 swh/core/db/tests/test_cli.py
diff --git a/setup.py b/setup.py
old mode 100755
new mode 100644
index dd9de8ac..66235fe8
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,7 @@ setup(
swh=swh.core.cli:main
swh-db-init=swh.core.cli.db:db_init
[swh.cli.subcommands]
+ db=swh.core.cli.db:db
db-init=swh.core.cli.db:db_init
''',
classifiers=[
diff --git a/swh/core/cli/db.py b/swh/core/cli/db.py
index 67fa9a39..cee02345 100755
--- a/swh/core/cli/db.py
+++ b/swh/core/cli/db.py
@@ -4,17 +4,95 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import glob
import logging
+from os import path
+import subprocess
import warnings
+
warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t
import click
from swh.core.cli import CONTEXT_SETTINGS
+from swh.core.config import read as config_read
logger = logging.getLogger(__name__)
+@click.group(name="db", context_settings=CONTEXT_SETTINGS)
+@click.option("--config-file", "-C", default=None,
+ type=click.Path(exists=True, dir_okay=False),
+ help="Configuration file.")
+@click.pass_context
+def db(ctx, config_file):
+ """Software Heritage database generic tools.
+ """
+ ctx.ensure_object(dict)
+ cfg = config_read(config_file)
+ ctx.obj["config"] = cfg
+
+
+@db.command(name="init", context_settings=CONTEXT_SETTINGS)
+@click.pass_context
+def init(ctx):
+ """Initialize the database for every Software Heritage module found in the
+ configuration file. For every configuration section in the config file
+ that:
+
+ 1. has the name of an existing swh package,
+ 2. has credentials for a local db access,
+
+ it will run the initialization scripts from the swh package against the
+ given database.
+
+ Example for the config file::
+
+ \b
+ storage:
+ cls: local
+ args:
+ db: postgresql:///?service=swh-storage
+ objstorage:
+ cls: remote
+ args:
+ url: http://swh-objstorage:5003/
+
+ the command:
+
+ swh db -C /path/to/config.yml init
+
+ will initialize the database for the `storage` section using initialization
+ scripts from the `swh.storage` package.
+ """
+
+ for modname, cfg in ctx.obj["config"].items():
+ if cfg.get("cls") == "local" and cfg.get("args"):
+ try:
+ sqlfiles = get_sql_for_package(modname)
+ except click.BadParameter:
+ logger.info(
+ "Failed to load/find sql initialization files for %s",
+ modname)
+
+ if sqlfiles:
+ conninfo = cfg["args"]["db"]
+ for sqlfile in sqlfiles:
+ subprocess.call_call(
+ [
+ "psql",
+ "--quiet",
+ "--no-psqlrc",
+ "-v",
+ "ON_ERROR_STOP=1",
+ "-d",
+ conninfo,
+ "-f",
+ sqlfile,
+ ]
+ )
+
+
@click.command(context_settings=CONTEXT_SETTINGS)
@click.argument('module', nargs=-1, required=True)
@click.option('--db-name', '-d', help='Database name.',
@@ -41,10 +119,6 @@ def db_init(module, db_name, create_db):
"""
# put import statements here so we can keep startup time of the main swh
# command as short as possible
- from os import path
- import glob
- from importlib import import_module
- from swh.core.utils import numfile_sortkey as sortkey
from swh.core.db.tests.db_testing import (
pg_createdb, pg_restore, DB_DUMP_TYPES,
swh_db_version
@@ -54,21 +128,7 @@ def db_init(module, db_name, create_db):
dump_files = []
for modname in module:
- if not modname.startswith('swh.'):
- modname = 'swh.{}'.format(modname)
- try:
- m = import_module(modname)
- except ImportError:
- raise click.BadParameter(
- 'Unable to load module {}'.format(modname))
-
- sqldir = path.join(path.dirname(m.__file__), 'sql')
- if not path.isdir(sqldir):
- raise click.BadParameter(
- 'Module {} does not provide a db schema '
- '(no sql/ dir)'.format(modname))
- dump_files.extend(sorted(glob.glob(path.join(sqldir, '*.sql')),
- key=sortkey))
+ dump_files.extend(get_sql_for_package(modname))
if create_db:
# Create the db (or fail silently if already existing)
@@ -89,3 +149,22 @@ def db_init(module, db_name, create_db):
click.secho('DONE database is {} version {}'.format(db_name, db_version),
fg='green', bold=True)
+
+
+def get_sql_for_package(modname):
+ from importlib import import_module
+ from swh.core.utils import numfile_sortkey as sortkey
+
+ if not modname.startswith("swh."):
+ modname = "swh.{}".format(modname)
+ try:
+ m = import_module(modname)
+ except ImportError:
+ raise click.BadParameter("Unable to load module {}".format(modname))
+
+ sqldir = path.join(path.dirname(m.__file__), "sql")
+ if not path.isdir(sqldir):
+ raise click.BadParameter(
+ "Module {} does not provide a db schema "
+ "(no sql/ dir)".format(modname))
+ return list(sorted(glob.glob(path.join(sqldir, "*.sql")), key=sortkey))
diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py
new file mode 100644
index 00000000..4a07fdc6
--- /dev/null
+++ b/swh/core/db/tests/test_cli.py
@@ -0,0 +1,49 @@
+#
+
+from click.testing import CliRunner
+
+from swh.core.cli import swh as swhmain
+from swh.core.cli.db import db as swhdb
+
+
+help_msg = '''Usage: swh [OPTIONS] COMMAND [ARGS]...
+
+ Command line interface for Software Heritage.
+
+Options:
+ -l, --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL]
+ Log level (default to INFO)
+ -h, --help Show this message and exit.
+
+Commands:
+ db Software Heritage database generic tools.
+'''
+
+
+def test_swh_help():
+ swhmain.add_command(swhdb)
+ runner = CliRunner()
+ result = runner.invoke(swhmain, ['-h'])
+ assert result.exit_code == 0
+ assert result.output == help_msg
+
+
+help_db_msg = '''Usage: swh db [OPTIONS] COMMAND [ARGS]...
+
+ Software Heritage database generic tools.
+
+Options:
+ -C, --config-file FILE Configuration file.
+ -h, --help Show this message and exit.
+
+Commands:
+ init Initialize the database for every Software Heritage module found in...
+'''
+
+
+def test_swh_db_help():
+ swhmain.add_command(swhdb)
+ runner = CliRunner()
+ result = runner.invoke(swhmain, ['db', '-h'])
+ assert result.exit_code == 0
+ assert result.output == help_db_msg
--
GitLab