From b3313ebc8c760d514f9bdf7680ae3e5848cb14d8 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <antoine.romain.dumont@gmail.com>
Date: Fri, 20 Jan 2023 12:10:02 +0100
Subject: [PATCH] Add new module to allow common logging configuration from
swh.core
This module only depends on core python library.
That will allow to simplify deployment logging configuration per infrastructure.
Refs. swh/infra/sysadm-environment#4524
---
swh/core/cli/__init__.py | 29 ++----------
swh/core/logging.py | 55 +++++++++++++++++++++++
swh/core/tests/data/logging-config.yaml | 35 +++++++++++++++
swh/core/tests/test_logging.py | 59 +++++++++++++++++++++++++
4 files changed, 153 insertions(+), 25 deletions(-)
create mode 100644 swh/core/logging.py
create mode 100644 swh/core/tests/data/logging-config.yaml
create mode 100644 swh/core/tests/test_logging.py
diff --git a/swh/core/cli/__init__.py b/swh/core/cli/__init__.py
index c354f4d6..d9cf5e86 100644
--- a/swh/core/cli/__init__.py
+++ b/swh/core/cli/__init__.py
@@ -1,16 +1,16 @@
-# Copyright (C) 2019 The Software Heritage developers
+# Copyright (C) 2019-2023 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import logging
-import logging.config
-from typing import Optional
import warnings
import click
import pkg_resources
+from swh.core.logging import logging_configure
+
LOG_LEVEL_NAMES = ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
@@ -125,8 +125,6 @@ def swh(ctx, log_levels, log_config, sentry_dsn, sentry_debug):
"""Command line interface for Software Heritage."""
import signal
- import yaml
-
from ..sentry import init_sentry
signal.signal(signal.SIGTERM, clean_exit_on_signal)
@@ -134,26 +132,7 @@ def swh(ctx, log_levels, log_config, sentry_dsn, sentry_debug):
init_sentry(sentry_dsn=sentry_dsn, debug=sentry_debug)
- set_default_loglevel: Optional[str] = None
-
- if log_config:
- logging.config.dictConfig(yaml.safe_load(log_config.read()))
- set_default_loglevel = logging.root.getEffectiveLevel()
-
- if not log_levels:
- log_levels = []
-
- for module, log_level in log_levels:
- logger = logging.getLogger(module)
- log_level = logging.getLevelName(log_level)
- logger.setLevel(log_level)
-
- if module is None:
- set_default_loglevel = log_level
-
- if not set_default_loglevel:
- logging.root.setLevel("INFO")
- set_default_loglevel = "INFO"
+ set_default_loglevel = logging_configure(log_levels, log_config)
ctx.ensure_object(dict)
ctx.obj["log_level"] = set_default_loglevel
diff --git a/swh/core/logging.py b/swh/core/logging.py
new file mode 100644
index 00000000..334d958f
--- /dev/null
+++ b/swh/core/logging.py
@@ -0,0 +1,55 @@
+# Copyright (C) 2023 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+"""Logging module providing common swh logging configuration. This only depends on
+python core library.
+
+"""
+
+import logging
+from logging.config import dictConfig
+from typing import Any, List, Optional, Tuple
+
+from yaml import safe_load
+
+
+def logging_configure(
+ log_levels: List[Tuple[str, str]] = [], log_config: Optional[Any] = None
+) -> str:
+ """A default configuration function to unify swh module logger configuration.
+
+ The log_config YAML file must conform to the logging.config.dictConfig schema
+ documented at https://docs.python.org/3/library/logging.config.html.
+
+ Returns:
+ The actual root logger log level name defined.
+
+ """
+ set_default_loglevel: Optional[str] = None
+
+ if log_config:
+ with open(log_config, "r") as f:
+ config_dict = safe_load(f.read())
+ # Configure logging using a dictionary config
+ dictConfig(config_dict)
+ effective_level = logging.root.getEffectiveLevel()
+ set_default_loglevel = logging.getLevelName(effective_level)
+
+ if not log_levels:
+ log_levels = []
+
+ for module, log_level in log_levels:
+ logger = logging.getLogger(module)
+ log_level = logging.getLevelName(log_level)
+ logger.setLevel(log_level)
+
+ if module is None:
+ set_default_loglevel = log_level
+
+ if not set_default_loglevel:
+ logging.root.setLevel("INFO")
+ set_default_loglevel = "INFO"
+
+ return set_default_loglevel
diff --git a/swh/core/tests/data/logging-config.yaml b/swh/core/tests/data/logging-config.yaml
new file mode 100644
index 00000000..20d2a308
--- /dev/null
+++ b/swh/core/tests/data/logging-config.yaml
@@ -0,0 +1,35 @@
+---
+version: 1
+
+handlers:
+ console:
+ class: logging.StreamHandler
+ formatter: task
+ stream: ext://sys.stdout
+ systemd:
+ class: swh.core.logger.JournalHandler
+ formatter: task
+
+formatters:
+ task:
+ (): celery.app.log.TaskFormatter
+ fmt: "[%(asctime)s: %(levelname)s/%(processName)s] %(task_name)s[%(task_id)s]: %(message)s"
+ use_color: false
+
+loggers:
+ celery:
+ level: INFO
+ amqp:
+ level: WARNING
+ urllib3:
+ level: WARNING
+ azure.core.pipeline.policies.http_logging_policy:
+ level: WARNING
+ swh: {}
+ celery.task: {}
+
+root:
+ level: DEBUG
+ handlers:
+ - console
+ - systemd
diff --git a/swh/core/tests/test_logging.py b/swh/core/tests/test_logging.py
new file mode 100644
index 00000000..4fcfc854
--- /dev/null
+++ b/swh/core/tests/test_logging.py
@@ -0,0 +1,59 @@
+# Copyright (C) 2023 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import logging
+import os
+
+from yaml import safe_load
+
+from swh.core.logging import logging_configure
+
+
+def test_logging_configure_default():
+ """Logging should be configured to INFO by default"""
+ root_log_level = logging_configure()
+
+ assert root_log_level == "INFO"
+
+
+def test_logging_configure_with_override():
+ """Logging module should be configured according to log_levels provided."""
+ log_levels = [
+ ("swh.core.tests", "DEBUG"),
+ ("swh.core", "CRITICAL"),
+ ("swh.loader.core.tests", "ERROR"),
+ ("swh.loader.core", "WARNING"),
+ ]
+
+ for module, log_level in log_levels:
+ logger = logging.getLogger(module)
+ assert logger.getEffectiveLevel() != logging.getLevelName(log_level)
+
+ # Set it up
+ root_log_level = logging_configure(log_levels)
+ assert root_log_level == "INFO"
+
+ for module, log_level in log_levels:
+ logger = logging.getLogger(module)
+ assert logger.getEffectiveLevel() == logging.getLevelName(log_level)
+
+
+def test_logging_configure_from_yaml(datadir):
+ """Logging should be configurable from yaml configuration file."""
+ logging_config = os.path.join(datadir, "logging-config.yaml")
+ root_log_level = logging_configure([], logging_config)
+
+ with open(logging_config, "r") as f:
+ config = safe_load(f.read())
+
+ for module, logger_config in config["loggers"].items():
+ if not logger_config:
+ continue
+ log_level = logger_config["level"]
+
+ logger = logging.getLogger(module)
+ assert logger.getEffectiveLevel() == logging.getLevelName(log_level)
+
+ assert root_log_level == config["root"]["level"]
--
GitLab