storage: Align configuration to manage journal writer & objstorage config
This aligns the storage configuration using configuration refs (to allow configuration reuse). This adapted the storage template to allow journal writer & objstorage configuration to be provided. It simplified the template which used to do some introspection to indent properly the configuration (depending on the actual storage implementation).
This also explicits the current objstorage configuration which was hidden in the secrets repository. This is a preparatory work which will help in migrating further the rpc services (storage then objstorage).
The diff is mostly about the objstorage configuration which got explicited (inlined in the diff). It should end up in functional noops (even though, pods will get restarted).
make swh-helm-diff
[swh] Comparing changes between branches production and migrate-storage-postgres (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-storage-postgres branch for environment staging...
[swh] Generate config in migrate-storage-postgres branch for environment staging...
[swh] Generate config in migrate-storage-postgres branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-storage-postgres branch for environment production...
[swh] Generate config in migrate-storage-postgres branch for environment production...
[swh] Generate config in migrate-storage-postgres branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.LKe6Kyd5/staging-swh-cassandra.before 2023-11-21 18:13:57.622246279 +0100
+++ /tmp/swh-chart.swh.LKe6Kyd5/staging-swh-cassandra.after 2023-11-21 18:13:58.266245395 +0100
@@ -4028,29 +4028,30 @@
keyspace: swh
consistency_level: LOCAL_QUORUM
auth_provider:
cls: cassandra.auth.PlainTextAuthProvider
password: ${CASSANDRA_PASSWORD}
username: swh-rw
directory_entries_insert_algo: batch
objstorage:
cls: remote
url: http://storage1.internal.staging.swh.network:5003/
+
journal_writer:
cls: kafka
brokers:
- - journal1.internal.staging.swh.network
- - journal2.internal.staging.swh.network
+ - journal1.internal.staging.swh.network
+ - journal2.internal.staging.swh.network
prefix: swh.journal.objects
client_id: swh.storage-cassandra.journal_writer.storage
anonymize: true
- producer_config:
+ producer_config:
message.max.bytes: 1000000000
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scrubber-storage-template
namespace: swh-cassandra
data:
config.yml.template: |
@@ -21826,21 +21827,21 @@
app: storage
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
- checksum/config: bce424b10db0b622f5b8050c99eac996142552bf9473bfba9d9e038cb6b61ece
+ checksum/config: 74608f339a88516dcf9b3d26b060ab6a34a88d5bdb261ac705b366ac7bf52f69
checksum/config-utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
@@ -21849,20 +21850,21 @@
initContainers:
- name: prepare-configuration
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
env:
+
- name: CASSANDRA_PASSWORD
valueFrom:
secretKeyRef:
name: common-secrets
key: cassandra-swh-rw-password
# 'name' secret must exist & include that ^ key
optional: false
volumeMounts:
- name: configuration
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.LKe6Kyd5/staging-swh-cassandra-next-version.before 2023-11-21 18:13:57.814246015 +0100
+++ /tmp/swh-chart.swh.LKe6Kyd5/staging-swh-cassandra-next-version.after 2023-11-21 18:13:58.482245098 +0100
@@ -3706,29 +3706,30 @@
keyspace: swh
consistency_level: LOCAL_QUORUM
auth_provider:
cls: cassandra.auth.PlainTextAuthProvider
password: ${CASSANDRA_PASSWORD}
username: swh-rw
directory_entries_insert_algo: batch
objstorage:
cls: remote
url: http://storage1.internal.staging.swh.network:5003/
+
journal_writer:
cls: kafka
brokers:
- - journal1.internal.staging.swh.network
- - journal2.internal.staging.swh.network
+ - journal1.internal.staging.swh.network
+ - journal2.internal.staging.swh.network
prefix: swh.journal.objects
client_id: swh.storage-cassandra.journal_writer.storage
anonymize: true
- producer_config:
+ producer_config:
message.max.bytes: 1000000000
---
# Source: swh/templates/utils/database-utils.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: database-utils
namespace: swh-cassandra-next-version
data:
init-keyspace.py: |
@@ -20370,21 +20371,21 @@
app: storage
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
- checksum/config: 54d5bdf8fcab3e26c7fa46f3b8562521315657c36c880d4ab8f2527152beb09d
+ checksum/config: eb77f5507bea433c87d5dd9120d3b3940ba1865f8ce868f955887c431c7ee068
checksum/config-utils: 908f9182febd57f799c59c25abdfbd7cfd832e1f1edc150c536d5f8087dd9486
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
@@ -20393,20 +20394,21 @@
initContainers:
- name: prepare-configuration
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
env:
+
- name: CASSANDRA_PASSWORD
valueFrom:
secretKeyRef:
name: common-secrets
key: cassandra-swh-rw-password
# 'name' secret must exist & include that ^ key
optional: false
volumeMounts:
- name: configuration
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.LKe6Kyd5/production-swh-cassandra.before 2023-11-21 18:13:58.870244565 +0100
+++ /tmp/swh-chart.swh.LKe6Kyd5/production-swh-cassandra.after 2023-11-21 18:13:59.270244016 +0100
@@ -1067,21 +1067,96 @@
keyspace: swh
consistency_level: LOCAL_QUORUM
auth_provider:
cls: cassandra.auth.PlainTextAuthProvider
password: ${CASSANDRA_PASSWORD}
username: swh-rw
directory_entries_insert_algo: batch
objstorage:
cls: multiplexer
objstorages:
- ${OBJSTORAGECONFIG}
+ - cls: filtered
+ filters_conf:
+ - type: readonly
+ storage_conf:
+ accounts:
+ "0":
+ account_name: ${0_ACCOUNT_NAME}
+ api_secret_key: ${0_API_SECRET_KEY}
+ container_name: contents
+ "1":
+ account_name: ${1_ACCOUNT_NAME}
+ api_secret_key: ${1_API_SECRET_KEY}
+ container_name: contents
+ "2":
+ account_name: ${2_ACCOUNT_NAME}
+ api_secret_key: ${2_API_SECRET_KEY}
+ container_name: contents
+ "3":
+ account_name: ${3_ACCOUNT_NAME}
+ api_secret_key: ${3_API_SECRET_KEY}
+ container_name: contents
+ "4":
+ account_name: ${4_ACCOUNT_NAME}
+ api_secret_key: ${4_API_SECRET_KEY}
+ container_name: contents
+ "5":
+ account_name: ${5_ACCOUNT_NAME}
+ api_secret_key: ${5_API_SECRET_KEY}
+ container_name: contents
+ "6":
+ account_name: ${6_ACCOUNT_NAME}
+ api_secret_key: ${6_API_SECRET_KEY}
+ container_name: contents
+ "7":
+ account_name: ${7_ACCOUNT_NAME}
+ api_secret_key: ${7_API_SECRET_KEY}
+ container_name: contents
+ "8":
+ account_name: ${8_ACCOUNT_NAME}
+ api_secret_key: ${8_API_SECRET_KEY}
+ container_name: contents
+ "9":
+ account_name: ${9_ACCOUNT_NAME}
+ api_secret_key: ${9_API_SECRET_KEY}
+ container_name: contents
+ a:
+ account_name: ${10_ACCOUNT_NAME}
+ api_secret_key: ${10_API_SECRET_KEY}
+ container_name: contents
+ b:
+ account_name: ${11_ACCOUNT_NAME}
+ api_secret_key: ${11_API_SECRET_KEY}
+ container_name: contents
+ c:
+ account_name: ${12_ACCOUNT_NAME}
+ api_secret_key: ${12_API_SECRET_KEY}
+ container_name: contents
+ d:
+ account_name: ${13_ACCOUNT_NAME}
+ api_secret_key: ${13_API_SECRET_KEY}
+ container_name: contents
+ e:
+ account_name: ${14_ACCOUNT_NAME}
+ api_secret_key: ${14_API_SECRET_KEY}
+ container_name: contents
+ f:
+ account_name: ${15_ACCOUNT_NAME}
+ api_secret_key: ${15_API_SECRET_KEY}
+ container_name: contents
+ cls: azure-prefixed
+ - cls: filtered
+ filters_conf:
+ - type: readonly
+ storage_conf:
+ cls: remote
+ url: http://objstorage.internal.softwareheritage.org:5003/
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scrubber-storage-template
namespace: swh-cassandra
data:
config.yml.template: |
storage:
@@ -13808,21 +13883,21 @@
app: storage
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
- checksum/config: 404b00dbad2af855b6bfca45fa8048c82a0a5658fc1a82d8a93944c58ff3cf22
+ checksum/config: 56b231d85cd40258490dc0017121e8a908422445ff94bde0d67182a7a39fcd62
checksum/config-utils: 885f4088d8181fabbd02e146f85462caced4878849cda6c1aea2f6b5ebc6e4e2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
@@ -13831,31 +13906,29 @@
initContainers:
- name: prepare-configuration
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
env:
+
- name: CASSANDRA_PASSWORD
valueFrom:
secretKeyRef:
name: common-secrets
key: cassandra-swh-rw-password
# 'name' secret must exist & include that ^ key
optional: false
- envFrom:
- - secretRef:
- name: swh-cassandra-objstorage-config
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
containers:
- name: storage
resources: