cli: Add support for loading a logging configuration file

This should allow us to stop playing whack-a-mole with logging configuration
between Celery workers (where the logging configuration is hard-coded in our
initialization routines) and CLI tools (where we only have the log-level knob).

swh/core/cli/__init__.py

@@ -4,9 +4,11 @@
 # See top-level LICENSE file for more information
 
 import logging
+import logging.config
 
 import click
 import pkg_resources
+import yaml
 
 LOG_LEVEL_NAMES = ['NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
 
@@ -45,21 +47,43 @@ class AliasedGroup(click.Group):
         self.format_commands(ctx, formatter)
 
 
-@click.group(context_settings=CONTEXT_SETTINGS, cls=AliasedGroup)
-@click.option('--log-level', '-l', default='INFO',
+@click.group(
+    context_settings=CONTEXT_SETTINGS, cls=AliasedGroup,
+    option_notes='''\
+If both options are present, --log-level will override the root logger
+configuration set in --log-config.
+
+The --log-config YAML must conform to the logging.config.dictConfig schema
+documented at https://docs.python.org/3/library/logging.config.html.
+'''
+)
+@click.option('--log-level', '-l', default=None,
               type=click.Choice(LOG_LEVEL_NAMES),
               help="Log level (defaults to INFO).")
+@click.option('--log-config', default=None,
+              type=click.File('r'),
+              help="Python yaml logging configuration file.")
 @click.pass_context
-def swh(ctx, log_level):
+def swh(ctx, log_level, log_config):
     """Command line interface for Software Heritage.
     """
-    log_level = logging.getLevelName(log_level)
-    logging.root.setLevel(log_level)
+    if log_level is None and log_config is None:
+        log_level = 'INFO'
+
+    if log_config:
+        logging.config.dictConfig(yaml.safe_load(log_config.read()))
+
+    if log_level:
+        log_level = logging.getLevelName(log_level)
+        logging.root.setLevel(log_level)
+
     ctx.ensure_object(dict)
     ctx.obj['log_level'] = log_level
 
 
 def main():
+    # Even though swh() sets up logging, we need an earlier basic logging setup
+    # for the next few logging statements
     logging.basicConfig()
     # load plugins that define cli sub commands
     for entry_point in pkg_resources.iter_entry_points('swh.cli.subcommands'):

swh/core/db/tests/test_cli.py

@@ -13,8 +13,16 @@ help_msg = '''Usage: swh [OPTIONS] COMMAND [ARGS]...
 Options:
   -l, --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL]
                                   Log level (defaults to INFO).
+  --log-config FILENAME           Python yaml logging configuration file.
   -h, --help                      Show this message and exit.
 
+Notes:
+  If both options are present, --log-level will override the root logger
+  configuration set in --log-config.
+
+  The --log-config YAML must conform to the logging.config.dictConfig schema
+  documented at https://docs.python.org/3/library/logging.config.html.
+
 Commands:
   db  Software Heritage database generic tools.
 '''

swh/core/tests/test_cli.py

 #
 
 import logging
+import textwrap
 
 import click
 from click.testing import CliRunner
+import pytest
 
 from swh.core.cli import swh as swhmain
 
@@ -15,7 +17,15 @@ help_msg = '''Usage: swh [OPTIONS] COMMAND [ARGS]...
 Options:
   -l, --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL]
                                   Log level (defaults to INFO).
+  --log-config FILENAME           Python yaml logging configuration file.
   -h, --help                      Show this message and exit.
+
+Notes:
+  If both options are present, --log-level will override the root logger
+  configuration set in --log-config.
+
+  The --log-config YAML must conform to the logging.config.dictConfig schema
+  documented at https://docs.python.org/3/library/logging.config.html.
 '''
 
 
@@ -82,6 +92,84 @@ def test_loglevel_debug(caplog):
     assert result.output.strip() == '''Hello SWH!'''
 
 
+@pytest.fixture
+def log_config_path(tmp_path):
+    log_config = textwrap.dedent('''\
+    ---
+    version: 1
+    formatters:
+      formatter:
+        format: 'custom format:%(name)s:%(levelname)s:%(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        stream: ext://sys.stdout
+        formatter: formatter
+        level: DEBUG
+    root:
+      level: DEBUG
+      handlers:
+        - console
+    loggers:
+      dontshowdebug:
+        level: INFO
+    ''')
+
+    (tmp_path / 'log_config.yml').write_text(log_config)
+
+    yield str(tmp_path / 'log_config.yml')
+
+
+def test_log_config(caplog, log_config_path):
+    @swhmain.command(name='test')
+    @click.pass_context
+    def swhtest(ctx):
+        logging.debug('Root log debug')
+        logging.info('Root log info')
+        logging.getLogger('dontshowdebug').debug('Not shown')
+        logging.getLogger('dontshowdebug').info('Shown')
+
+    runner = CliRunner()
+    result = runner.invoke(
+        swhmain, [
+            '--log-config', log_config_path,
+            'test',
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert result.output.strip() == '\n'.join([
+        'custom format:root:DEBUG:Root log debug',
+        'custom format:root:INFO:Root log info',
+        'custom format:dontshowdebug:INFO:Shown',
+    ])
+
+
+def test_log_config_log_level_interaction(caplog, log_config_path):
+    @swhmain.command(name='test')
+    @click.pass_context
+    def swhtest(ctx):
+        logging.debug('Root log debug')
+        logging.info('Root log info')
+        logging.getLogger('dontshowdebug').debug('Not shown')
+        logging.getLogger('dontshowdebug').info('Shown')
+
+    runner = CliRunner()
+    result = runner.invoke(
+        swhmain, [
+            '--log-config', log_config_path,
+            '--log-level', 'INFO',
+            'test',
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert result.output.strip() == '\n'.join([
+        'custom format:root:INFO:Root log info',
+        'custom format:dontshowdebug:INFO:Shown',
+    ])
+
+
 def test_aliased_command():
     @swhmain.command(name='canonical-test')
     @click.pass_context