From edfd0b3324e5e1993c8da3b3cc76abd547365af9 Mon Sep 17 00:00:00 2001
From: David Douard <david.douard@sdfa3.org>
Date: Mon, 28 Jan 2019 10:16:46 +0100
Subject: [PATCH] Add a simple merge_configs function in config.py
that recursively merge 2 (config) dictionaries.
This is meant to be used to simplify a bit the config system of several
systems of the swh platform (related to T1410 and T826).
---
swh/core/config.py | 73 +++++++++++++++++++++++++++++
swh/core/tests/test_config.py | 87 +++++++++++++++++++++++++++++++++++
2 files changed, 160 insertions(+)
diff --git a/swh/core/config.py b/swh/core/config.py
index d53aaa5a..ac0ac4d4 100644
--- a/swh/core/config.py
+++ b/swh/core/config.py
@@ -7,6 +7,8 @@ import configparser
import logging
import os
import yaml
+from itertools import chain
+from copy import deepcopy
logger = logging.getLogger(__name__)
@@ -178,6 +180,77 @@ def merge_default_configs(base_config, *other_configs):
return full_config
+def merge_configs(base, other):
+ """Merge two config dictionaries
+
+ This does merge config dicts recursively, with the rules, for every value
+ of the dicts (with 'val' not being a dict):
+
+ - None + type -> type
+ - type + None -> None
+ - dict + dict -> dict (merged)
+ - val + dict -> TypeError
+ - dict + val -> TypeError
+ - val + val -> val (other)
+
+ so merging
+
+ {
+ 'key1': {
+ 'skey1': value1,
+ 'skey2': {'sskey1': value2},
+ },
+ 'key2': value3,
+ }
+
+ with
+
+ {
+ 'key1': {
+ 'skey1': value4,
+ 'skey2': {'sskey2': value5},
+ },
+ 'key3': value6,
+ }
+
+ will give:
+
+ {
+ 'key1': {
+ 'skey1': value4, # <-- note this
+ 'skey2': {
+ 'sskey1': value2,
+ 'sskey2': value5,
+ },
+ },
+ 'key2': value3,
+ 'key3': value6,
+ }
+
+ Note that no type checking is done for anything but dicts.
+ """
+ if not isinstance(base, dict) or not isinstance(other, dict):
+ raise TypeError(
+ 'Cannot merge a %s with a %s' % (type(base), type(other)))
+
+ output = {}
+ allkeys = set(chain(base.keys(), other.keys()))
+ for k in allkeys:
+ vb = base.get(k)
+ vo = other.get(k)
+
+ if isinstance(vo, dict):
+ output[k] = merge_configs(vb is not None and vb or {}, vo)
+ elif isinstance(vb, dict) and k in other and other[k] is not None:
+ output[k] = merge_configs(vb, vo is not None and vo or {})
+ elif k in other:
+ output[k] = deepcopy(vo)
+ else:
+ output[k] = deepcopy(vb)
+
+ return output
+
+
def swh_config_paths(base_filename):
"""Return the Software Heritage specific configuration paths for the given
filename."""
diff --git a/swh/core/tests/test_config.py b/swh/core/tests/test_config.py
index fd3b7005..8e5bbf80 100644
--- a/swh/core/tests/test_config.py
+++ b/swh/core/tests/test_config.py
@@ -223,3 +223,90 @@ def test_prepare_folder(tmp_path):
assert os.path.exists(conf['path1']), "path1 should still exist!"
assert os.path.exists(conf['path2']), "path2 should now exist."
+
+
+def test_merge_config():
+ cfg_a = {
+ 'a': 42,
+ 'b': [1, 2, 3],
+ 'c': None,
+ 'd': {'gheez': 27},
+ 'e': {
+ 'ea': 'Mr. Bungle',
+ 'eb': None,
+ 'ec': [11, 12, 13],
+ 'ed': {'eda': 'Secret Chief 3',
+ 'edb': 'Faith No More'},
+ 'ee': 451,
+ },
+ 'f': 'Janis',
+ }
+ cfg_b = {
+ 'a': 43,
+ 'b': [41, 42, 43],
+ 'c': 'Tom Waits',
+ 'd': None,
+ 'e': {
+ 'ea': 'Igorrr',
+ 'ec': [51, 52],
+ 'ed': {'edb': 'Sleepytime Gorilla Museum',
+ 'edc': 'Nils Peter Molvaer'},
+ },
+ 'g': 'Hüsker Dü',
+ }
+
+ # merge A, B
+ cfg_m = config.merge_configs(cfg_a, cfg_b)
+ assert cfg_m == {
+ 'a': 43, # b takes precedence
+ 'b': [41, 42, 43], # b takes precedence
+ 'c': 'Tom Waits', # b takes precedence
+ 'd': None, # b['d'] takes precedence (explicit None)
+ 'e': {
+ 'ea': 'Igorrr', # a takes precedence
+ 'eb': None, # only in a
+ 'ec': [51, 52], # b takes precedence
+ 'ed': {
+ 'eda': 'Secret Chief 3', # only in a
+ 'edb': 'Sleepytime Gorilla Museum', # b takes precedence
+ 'edc': 'Nils Peter Molvaer'}, # only defined in b
+ 'ee': 451,
+ },
+ 'f': 'Janis', # only defined in a
+ 'g': 'Hüsker Dü', # only defined in b
+ }
+
+ # merge B, A
+ cfg_m = config.merge_configs(cfg_b, cfg_a)
+ assert cfg_m == {
+ 'a': 42, # a takes precedence
+ 'b': [1, 2, 3], # a takes precedence
+ 'c': None, # a takes precedence
+ 'd': {'gheez': 27}, # a takes precedence
+ 'e': {
+ 'ea': 'Mr. Bungle', # a takes precedence
+ 'eb': None, # only defined in a
+ 'ec': [11, 12, 13], # a takes precedence
+ 'ed': {
+ 'eda': 'Secret Chief 3', # only in a
+ 'edb': 'Faith No More', # a takes precedence
+ 'edc': 'Nils Peter Molvaer'}, # only in b
+ 'ee': 451,
+ },
+ 'f': 'Janis', # only in a
+ 'g': 'Hüsker Dü', # only in b
+ }
+
+
+def test_merge_config_type_error():
+ for v in (1, 'str', None):
+ with pytest.raises(TypeError):
+ config.merge_configs(v, {})
+ with pytest.raises(TypeError):
+ config.merge_configs({}, v)
+
+ for v in (1, 'str'):
+ with pytest.raises(TypeError):
+ config.merge_configs({'a': v}, {'a': {}})
+ with pytest.raises(TypeError):
+ config.merge_configs({'a': {}}, {'a': v})
--
GitLab