From 8f0849a7c369fd01149d63bf15909680ce2844ab Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Mon, 16 Jan 2023 15:22:12 +0100
Subject: [PATCH] Allow logging configuration from configuration yaml file
This will allow proper logging configuration for the services which are currently
running in the dynamic infrastructure. Their logs are current written in the wrong
elasticsearch indices.
Ref. swh/infra/sysadm-environment#4524
---
requirements-swh.txt | 2 +-
swh/scheduler/celery_backend/config.py | 37 +++++++++++++++-----------
2 files changed, 23 insertions(+), 16 deletions(-)
diff --git a/requirements-swh.txt b/requirements-swh.txt
index cd766120..ac4a616a 100644
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1,2 +1,2 @@
-swh.core[db,http] >= 2.9
+swh.core[db,http] >= 2.19
swh.storage >= 0.11.1
diff --git a/swh/scheduler/celery_backend/config.py b/swh/scheduler/celery_backend/config.py
index 6b8d8e5c..eb23f1fd 100644
--- a/swh/scheduler/celery_backend/config.py
+++ b/swh/scheduler/celery_backend/config.py
@@ -79,10 +79,12 @@ def setup_log_handler(
We use the command-line loglevel for tasks only, as we never
really care about the debug messages from celery.
"""
+ from swh.core.logging import logging_configure
+
if loglevel is None:
loglevel = logging.DEBUG
if isinstance(loglevel, str):
- loglevel = logging._nameToLevel[loglevel]
+ loglevel = logging.getLevelName(loglevel)
formatter = logging.Formatter(format)
@@ -118,23 +120,28 @@ def setup_log_handler(
systemd_journal.setFormatter(formatter)
root_logger.addHandler(systemd_journal)
- logging.getLogger("celery").setLevel(logging.INFO)
+ # Retrieve a possible log_config path to a yaml log configuration file
+ log_config_path = kwargs.get("log_config")
+
+ logging_configure(
+ [
+ ("celery", logging.INFO)
+ # Silence amqp heartbeat_tick messages
+ ("amqp", loglevel),
+ # Silence useless "Starting new HTTP connection" messages
+ ("urllib3", logging.WARNING),
+ # Completely disable azure logspam
+ ("azure.core.pipeline.policies.http_logging_policy", logging.WARNING),
+ ("swh", loglevel),
+ # get_task_logger makes the swh tasks loggers children of celery.task
+ ("celery.task", loglevel),
+ ],
+ log_config_path,
+ )
- # Silence amqp heartbeat_tick messages
+ # extra step for amqp
logger = logging.getLogger("amqp")
logger.addFilter(lambda record: not record.msg.startswith("heartbeat_tick"))
- logger.setLevel(loglevel)
-
- # Silence useless "Starting new HTTP connection" messages
- logging.getLogger("urllib3").setLevel(logging.WARNING)
-
- # Completely disable azure logspam
- azure_logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy")
- azure_logger.setLevel(logging.WARNING)
-
- logging.getLogger("swh").setLevel(loglevel)
- # get_task_logger makes the swh tasks loggers children of celery.task
- logging.getLogger("celery.task").setLevel(loglevel)
return loglevel
--
GitLab