indexer-storage: move journal_writer to the proper indent level
The journal_writer config needs to be under the indexer_storage key, rather than at the toplevel.
Also give a per-pod clientId to all journal writers, instead of a hardcoded value for all pods.
Noticed while working on swh/infra/sysadm-environment#5200 (closed)
helm diff
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.fNFN4Av3/staging-swh.before 2024-01-11 19:50:49.718377187 +0100
+++ /tmp/swh-chart.swh.fNFN4Av3/staging-swh.after 2024-01-11 19:50:50.126382763 +0100
@@ -168,29 +168,28 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: indexer-storage-rpc-configuration-template
data:
config.yml.template: |
indexer_storage:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-indexer dbname=swh-indexer password=${POSTGRESQL_PASSWORD}
- journal_writer:
-
- client_id: swh.idx_storage.journal_writer.storage1
- cls: kafka
- prefix: swh.journal.indexed
- producer_config:
- message.max.bytes: 1000000000
- brokers:
- - journal2.internal.staging.swh.network
+ journal_writer:
+ client_id: swh.idx_storage.journal_writer.${HOSTNAME}
+ cls: kafka
+ prefix: swh.journal.indexed
+ producer_config:
+ message.max.bytes: 1000000000
+ brokers:
+ - journal2.internal.staging.swh.network
---
# Source: swh/templates/indexers/configmap-utils.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: indexer-utils
namespace: swh
data:
pre-stop-idempotent.sh: |
#!/bin/bash
@@ -4009,21 +4008,21 @@
db: host=db1.internal.staging.swh.network port=5432 user=swh dbname=swh password=${POSTGRESQL_PASSWORD}
objstorage:
cls: remote
url: http://objstorage-read-write-rpc-ingress
journal_writer:
cls: kafka
brokers:
- journal2.internal.staging.swh.network
prefix: swh.journal.objects
- client_id: swh.storage.journal_writer.storage1
+ client_id: swh.storage.journal_writer.${HOSTNAME}
anonymize: true
producer_config:
message.max.bytes: 1000000000
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-indexer-storage-template
namespace: swh
@@ -5062,21 +5061,21 @@
app: indexer-storage-rpc
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: indexer-storage-rpc
annotations:
- checksum/config: ac4c875d8a03fa39a48eed188b4049cbcabbd435f51707fd069335ffcad9ac39
+ checksum/config: 56ab19e613bf228521fd9f136904bfea0fbbd7edff68e66e66bc98b0ab1e1188
checksum/database-utils: 931b7f92b50eae0ecb6203c3e0e48f7c363a99ac0064451187dc12c6eaf0a793
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
@@ -13789,21 +13788,21 @@
app: storage-postgresql-read-write
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage-postgresql-read-write
annotations:
- checksum/config: f41bf53c1a3df2d84c89b62efa58ba4ca7d74c9d1eb5265c7ad056fd62de5e42
+ checksum/config: 2bc23e8868305ed4ab563a7338240a0d309419477d87bbcdc0738acb0a123712
checksum/database-utils: 931b7f92b50eae0ecb6203c3e0e48f7c363a99ac0064451187dc12c6eaf0a793
checksum/config-utils: 0cce256531366ac8ea2a0bde2a8f10937284a0f599f45fabe6babbbc096e179d
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.fNFN4Av3/staging-swh-cassandra.before 2024-01-11 19:50:49.854379046 +0100
+++ /tmp/swh-chart.swh.fNFN4Av3/staging-swh-cassandra.after 2024-01-11 19:50:50.262384621 +0100
@@ -4017,21 +4017,21 @@
directory_entries_insert_algo: batch
objstorage:
cls: remote
url: http://objstorage-read-write-rpc-ingress
journal_writer:
cls: kafka
brokers:
- journal2.internal.staging.swh.network
prefix: swh.journal.objects
- client_id: swh.storage-cassandra.journal_writer.storage
+ client_id: swh.storage-cassandra.journal_writer.${HOSTNAME}
anonymize: true
producer_config:
message.max.bytes: 1000000000
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scrubber-storage-template
namespace: swh-cassandra
@@ -12472,21 +12472,21 @@
app: storage-cassandra
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage-cassandra
annotations:
- checksum/config: b0f34bb659c89e7c81f94018039c5c6c358e56f57abcbb876f49c2c8f804b8be
+ checksum/config: 63337dd276105a1988c8e077eac3ea9fe3918d28fcb42cb720e0674a18d21838
checksum/database-utils: 367f28d8854b3b188d7a45015752b10be8e691f0c3b2a21db02501e655b932f0
checksum/config-utils: 6293d71ddd4a7f401d1105bc7f7590fcead5a3a0c152fe46d7d407839ea9a1d6
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.fNFN4Av3/staging-swh-cassandra-next-version.before 2024-01-11 19:50:49.974380685 +0100
+++ /tmp/swh-chart.swh.fNFN4Av3/staging-swh-cassandra-next-version.after 2024-01-11 19:50:50.386386315 +0100
@@ -3689,21 +3689,21 @@
directory_entries_insert_algo: batch
objstorage:
cls: remote
url: http://objstorage-read-write-rpc-ingress
journal_writer:
cls: kafka
brokers:
- journal2.internal.staging.swh.network
prefix: swh.journal.objects
- client_id: swh.storage-cassandra.journal_writer.storage
+ client_id: swh.storage-cassandra.journal_writer.${HOSTNAME}
anonymize: true
producer_config:
message.max.bytes: 1000000000
---
# Source: swh/templates/utils/config-utils.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: config-utils
namespace: swh-cassandra-next-version
@@ -10790,21 +10790,21 @@
app: storage-cassandra
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage-cassandra
annotations:
- checksum/config: 5ddeb73cd2cdd437e49835607a9a59ad5d7fc4b4feb26369c6711e407db5f965
+ checksum/config: 2cddfaac605a2f5e9a8689326082009456503eb4d7792e92fa26a9260659cb31
checksum/database-utils: dd7d83d547731ea7598fa298565f3b1fe41eb657ae93bab75e15b722dbcc71ae
checksum/config-utils: 00e1b490a5d09e6ab4c9b40bae85197b945f800ba39c757d5c4c37b1be3f0193
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differences