swh/toolbox: Refactor to explicitely declare the configurations
It makes clear witch configuration will be expanded in the configmap. It also remove all the service name manipulations and bending to generate the correct configuration for the special cases like indexer-storage or all the different scrubber configurations
Related to (/ pre-requisite of) swh/infra/sysadm-environment#5145 (closed)
helm diff
[swh] Comparing changes between branches production and toolbox-config-refactor (per environment)...
Your branch is behind 'origin/production' by 3 commits, and can be fast-forwarded.
(use "git pull" to update your local branch)
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in toolbox-config-refactor branch for environment staging...
[swh] Generate config in toolbox-config-refactor branch for environment staging...
[swh] Generate config in toolbox-config-refactor branch for environment staging...
Your branch is behind 'origin/production' by 3 commits, and can be fast-forwarded.
(use "git pull" to update your local branch)
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in toolbox-config-refactor branch for environment production...
[swh] Generate config in toolbox-config-refactor branch for environment production...
[swh] Generate config in toolbox-config-refactor branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.ILQPNd84/staging-swh.before 2023-11-09 16:39:54.679753893 +0100
+++ /tmp/swh-chart.swh.ILQPNd84/staging-swh.after 2023-11-09 16:39:55.231755869 +0100
@@ -3968,49 +3968,46 @@
port: 6379
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-indexer-storage-template
namespace: swh
data:
config.yml.template: |
-
indexer_storage:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-indexer dbname=swh-indexer password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scheduler-template
namespace: swh
data:
config.yml.template: |
-
scheduler:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-scheduler dbname=swh-scheduler password=${POSTGRESQL_PASSWORD}
celery:
task_broker: amqp://swhproducer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scrubber-journal-template
namespace: swh
data:
config.yml.template: |
-
scrubber:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- journal1.internal.staging.swh.network:9094
- journal2.internal.staging.swh.network:9094
cls: kafka
group_id: swh-archive-stg-journalchecker
on_eof: restart
@@ -4021,34 +4018,32 @@
security.protocol: SASL_SSL
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-storage-template
namespace: swh
data:
config.yml.template: |
-
storage:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh dbname=swh password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-vault-template
namespace: swh
data:
config.yml.template: |
-
vault:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-vault dbname=swh-vault password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/script-utils-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-script-utils
namespace: swh
@@ -12908,21 +12903,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 613fdfd914645f2cb235466a4ce8d78239fd05f84bf4971f04a6653c9eab1e62
+ checksum/config: 5813a438f1dc196e606fd15e8c7827cb601b49623c6f76cb6a4b6b4f08504aa2
checksum/configScript: 2f52c9c95b13a0c755571c6a81681958552538cbf149b458241f7c36a8bbb01f
spec:
priorityClassName: swh-tools
initContainers:
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.ILQPNd84/production-swh.before 2023-11-09 16:39:55.715757602 +0100
+++ /tmp/swh-chart.swh.ILQPNd84/production-swh.after 2023-11-09 16:39:56.083758920 +0100
@@ -5181,49 +5181,46 @@
- 1.
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-indexer-storage-template
namespace: swh
data:
config.yml.template: |
-
indexer_storage:
cls: postgresql
db: host=db.internal.softwareheritage.org port=5432 user=swhstorage dbname=softwareheritage-indexer password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scheduler-template
namespace: swh
data:
config.yml.template: |
-
scheduler:
cls: postgresql
db: host=db.internal.softwareheritage.org port=5432 user=swhscheduler dbname=softwareheritage-scheduler password=${POSTGRESQL_PASSWORD}
celery:
task_broker: amqp://swhproducer:${AMQP_PASSWORD}@rabbitmq.internal.softwareheritage.org:5672/%2f
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scrubber-journal-template
namespace: swh
data:
config.yml.template: |
-
scrubber:
cls: postgresql
db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
journal:
brokers:
- kafka1.internal.softwareheritage.org:9094
- kafka2.internal.softwareheritage.org:9094
- kafka3.internal.softwareheritage.org:9094
- kafka4.internal.softwareheritage.org:9094
cls: kafka
@@ -5236,21 +5233,20 @@
security.protocol: SASL_SSL
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-storage-template
namespace: swh
data:
config.yml.template: |
-
storage:
cls: postgresql
db: host=db.internal.softwareheritage.org port=5432 user=swhstorage dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/script-utils-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-script-utils
namespace: swh
@@ -14814,21 +14810,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 9ca7316e5437c286d273205572f3f180c44cdf024927f55aeda023478dd59ed1
+ checksum/config: 0fda6c829387563faf59a9d1f01c3c511ed3cf5d06f74a7e1fef820fadf11ce5
checksum/configScript: efeecbb356cb04184a3c2ca2f60f2d08c634f34f1ce4bc6a6ebf937b34fadcbf
spec:
priorityClassName: swh-tools
initContainers:
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
------------- diff for environment production namespace swh-cassandra -------------
No differences