swh/scrubber: Add the support of journal scrubber
It's far from perfect right now as there is no automatic registration of the clients, but it possible to start them anyway if they are registered from the toolbox (with some manual edits)
Related to swh/infra/sysadm-environment#5108 (closed)
helm-diff
[swh] Comparing changes between branches production and scrubber (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in scrubber branch for environment staging...
[swh] Generate config in scrubber branch for environment staging...
[swh] Generate config in scrubber branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in scrubber branch for environment production...
[swh] Generate config in scrubber branch for environment production...
[swh] Generate config in scrubber branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.6tdzz7kF/staging-swh.before 2023-10-26 14:49:36.495426116 +0200
+++ /tmp/swh-chart.swh.6tdzz7kF/staging-swh.after 2023-10-26 14:49:37.787429511 +0200
@@ -2954,23 +2954,24 @@
name: scheduler-journal-client-configuration-template
namespace: swh
data:
config.yml.template: |
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
celery:
task_broker: amqp://swhproducer:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/%2f
journal:
+ cls: kafka
brokers:
- - journal1.internal.staging.swh.network
- - journal2.internal.staging.swh.network
+ - journal1.internal.staging.swh.network
+ - journal2.internal.staging.swh.network
group_id: swh.scheduler.journal_client
---
# Source: swh/templates/scheduler/recurrent-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: scheduler-schedule-recurrent-configuration-template
data:
config.yml.template: |
@@ -3758,21 +3759,30 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scrubber-template
namespace: swh
data:
config.yml.template: |
scrubber:
cls: postgresql
- db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${POSTGRESQL_PASSWORD}
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh dbname=swh password=${POSTGRESQL_PASSWORD}
+ journal:
+ cls: kafka
+ brokers:
+ - journal1.internal.staging.swh.network
+ - journal2.internal.staging.swh.network
+ group_id: swh.scheduler.journal_client
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-storage-template
namespace: swh
data:
config.yml.template: |
@@ -4053,20 +4063,48 @@
if [ -e "${DB_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
fi
if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
echo "code and DB versions are different. Blocking the deployment"
exit 1
fi
+
+ check-scrubber-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ # checking the database status
+ swh db --config-file=${SWH_CONFIG_FILENAME} version ${MODULE} | tee "${TEMP_FILE}"
+
+ CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
+ DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
+
+ if [ -e "${CODE_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ -e "${DB_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
+ echo "code and DB versions are different. Blocking the deployment"
+ exit 1
+ fi
---
# Source: swh/templates/web/configmap-pgservice.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: pgservice-configuration-template
data:
pg-service-conf: |
[syncmailmaps]
@@ -4584,21 +4622,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: indexer-storage-rpc
annotations:
checksum/config: b71cc0136e069c1a6ad2041e5e764a0823db7d2b29692156e3358374ca9ba604
- checksum/config-utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config-utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
- "true"
@@ -9718,21 +9756,21 @@
app: scheduler-journal-client
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scheduler-journal-client
annotations:
- checksum/config: 3cfa0fa3d0ab8652cbe984983074e1b94049d0f10e1faf08091bcc7c8b3dec89
+ checksum/config: cab65250c5d1b0a92c0b8380e1d8913e7e655b21a28a0718af3270f6136d5e72
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scheduler
operator: In
values:
- "true"
@@ -10220,21 +10258,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-content
template:
metadata:
labels:
app: storage-replayer-content
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10344,21 +10382,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-directory
template:
metadata:
labels:
app: storage-replayer-directory
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10468,21 +10506,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-extid
template:
metadata:
labels:
app: storage-replayer-extid
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10592,21 +10630,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-metadata
template:
metadata:
labels:
app: storage-replayer-metadata
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10716,21 +10754,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin
template:
metadata:
labels:
app: storage-replayer-origin
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10840,21 +10878,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit
template:
metadata:
labels:
app: storage-replayer-origin-visit
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10964,21 +11002,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit-status
template:
metadata:
labels:
app: storage-replayer-origin-visit-status
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -11088,21 +11126,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-raw-extrinsic-metadata
template:
metadata:
labels:
app: storage-replayer-raw-extrinsic-metadata
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -11212,21 +11250,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-release
template:
metadata:
labels:
app: storage-replayer-release
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -11336,21 +11374,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-revision
template:
metadata:
labels:
app: storage-replayer-revision
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -11460,21 +11498,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-skipped-content
template:
metadata:
labels:
app: storage-replayer-skipped-content
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -11584,21 +11622,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-snapshot
template:
metadata:
labels:
app: storage-replayer-snapshot
annotations:
checksum/config: 84f4df6898bc72445a40aad2201ff2f398c9903d6434f641bacfd7574ded9c98
- checksum/config_utils: 16bb50946dbf0c0e6d3d0030efac1fe6e3735faa0762d7e9e5897937cf9ef574
+ checksum/config_utils: 68e76c159405718cc9d9b8526f177b8bd04a6055bcba5863110b7d605b94d2fc
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -11712,21 +11750,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 40581def6ca8cb0affdf3abadbcd821cb4090de870c3c7cd0d7b3574eaee3d54
+ checksum/config: 7e75f30ebf38e5a2889256fd0cdc3f4a89e09dbbfab7853747f3fd154b099322
checksum/configScript: 6ea1b2f2870da707a7699ac13460639d19dda6c6c89cf114c75286dede149823
spec:
priorityClassName: swh-tools
initContainers:
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
@@ -11780,27 +11818,36 @@
- name: prepare-configuration-scrubber
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber.yml
env:
- - name: POSTGRESQL_PASSWORD
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-postgresql-common-secret
+ key: postgres-swh-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-scrubber-template
mountPath: /etc/swh/configuration-template
- name: prepare-configuration-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.6tdzz7kF/staging-swh-cassandra.before 2023-10-26 14:49:36.923427240 +0200
+++ /tmp/swh-chart.swh.6tdzz7kF/staging-swh-cassandra.after 2023-10-26 14:49:38.299430857 +0200
@@ -3817,20 +3817,48 @@
if [ -e "${DB_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
fi
if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
echo "code and DB versions are different. Blocking the deployment"
exit 1
fi
+
+ check-scrubber-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ # checking the database status
+ swh db --config-file=${SWH_CONFIG_FILENAME} version ${MODULE} | tee "${TEMP_FILE}"
+
+ CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
+ DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
+
+ if [ -e "${CODE_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ -e "${DB_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
+ echo "code and DB versions are different. Blocking the deployment"
+ exit 1
+ fi
---
# Source: swh/templates/vault/rpc-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: vault-rpc-configuration-template
data:
config.yml.template: |
vault:
@@ -8873,21 +8901,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-content
template:
metadata:
labels:
app: storage-replayer-content
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -8997,21 +9025,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-directory
template:
metadata:
labels:
app: storage-replayer-directory
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9121,21 +9149,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-extid
template:
metadata:
labels:
app: storage-replayer-extid
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9245,21 +9273,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-metadata
template:
metadata:
labels:
app: storage-replayer-metadata
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9369,21 +9397,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin
template:
metadata:
labels:
app: storage-replayer-origin
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9493,21 +9521,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit
template:
metadata:
labels:
app: storage-replayer-origin-visit
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9617,21 +9645,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit-status
template:
metadata:
labels:
app: storage-replayer-origin-visit-status
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9741,21 +9769,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-raw-extrinsic-metadata
template:
metadata:
labels:
app: storage-replayer-raw-extrinsic-metadata
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9865,21 +9893,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-release
template:
metadata:
labels:
app: storage-replayer-release
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9989,21 +10017,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-revision
template:
metadata:
labels:
app: storage-replayer-revision
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10113,21 +10141,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-skipped-content
template:
metadata:
labels:
app: storage-replayer-skipped-content
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10237,21 +10265,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-snapshot
template:
metadata:
labels:
app: storage-replayer-snapshot
annotations:
checksum/config: d2022c15b389d403300f06ff35e69770cf4264e3bfd4df32060d4d073542c5a7
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -10365,21 +10393,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
checksum/config: bce424b10db0b622f5b8050c99eac996142552bf9473bfba9d9e038cb6b61ece
- checksum/config-utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config-utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.6tdzz7kF/staging-swh-cassandra-next-version.before 2023-10-26 14:49:37.279428176 +0200
+++ /tmp/swh-chart.swh.6tdzz7kF/staging-swh-cassandra-next-version.after 2023-10-26 14:49:38.683431866 +0200
@@ -3611,20 +3611,48 @@
if [ -e "${DB_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
fi
if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
echo "code and DB versions are different. Blocking the deployment"
exit 1
fi
+
+ check-scrubber-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ # checking the database status
+ swh db --config-file=${SWH_CONFIG_FILENAME} version ${MODULE} | tee "${TEMP_FILE}"
+
+ CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
+ DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
+
+ if [ -e "${CODE_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ -e "${DB_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
+ echo "code and DB versions are different. Blocking the deployment"
+ exit 1
+ fi
---
# Source: swh/templates/vault/rpc-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra-next-version
name: vault-rpc-configuration-template
data:
config.yml.template: |
vault:
@@ -8335,21 +8363,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-content
template:
metadata:
labels:
app: storage-replayer-content
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -8459,21 +8487,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-directory
template:
metadata:
labels:
app: storage-replayer-directory
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -8583,21 +8611,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-extid
template:
metadata:
labels:
app: storage-replayer-extid
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -8707,21 +8735,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-metadata
template:
metadata:
labels:
app: storage-replayer-metadata
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -8831,21 +8859,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin
template:
metadata:
labels:
app: storage-replayer-origin
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -8955,21 +8983,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit
template:
metadata:
labels:
app: storage-replayer-origin-visit
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9079,21 +9107,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit-status
template:
metadata:
labels:
app: storage-replayer-origin-visit-status
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9203,21 +9231,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-raw-extrinsic-metadata
template:
metadata:
labels:
app: storage-replayer-raw-extrinsic-metadata
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9327,21 +9355,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-release
template:
metadata:
labels:
app: storage-replayer-release
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9451,21 +9479,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-revision
template:
metadata:
labels:
app: storage-replayer-revision
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9575,21 +9603,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-skipped-content
template:
metadata:
labels:
app: storage-replayer-skipped-content
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9699,21 +9727,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-snapshot
template:
metadata:
labels:
app: storage-replayer-snapshot
annotations:
checksum/config: 50af44f2fd32b0b910604600db76b306aff39961ca79fa71587bec194bbe9488
- checksum/config_utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config_utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -9827,21 +9855,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
checksum/config: 54d5bdf8fcab3e26c7fa46f3b8562521315657c36c880d4ab8f2527152beb09d
- checksum/config-utils: 1527a420c1aabad293bc608b9ffd1549c47e5e6a4f4c5fe65c5a2b47b690a59b
+ checksum/config-utils: 194ff45d5c41bebac87e9f9c137d0dee806959c9e45ca37ce2bac65065dc2266
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.6tdzz7kF/production-swh.before 2023-10-26 14:49:39.427433822 +0200
+++ /tmp/swh-chart.swh.6tdzz7kF/production-swh.after 2023-10-26 14:49:40.283436071 +0200
@@ -4971,25 +4971,26 @@
name: scheduler-journal-client-configuration-template
namespace: swh
data:
config.yml.template: |
scheduler:
cls: remote
url: http://scheduler.internal.softwareheritage.org
celery:
task_broker: amqp://swhproducer:${AMQP_PASSWORD}@rabbitmq.internal.softwareheritage.org:5672/%2f
journal:
+ cls: kafka
brokers:
- - kafka1.internal.softwareheritage.org
- - kafka2.internal.softwareheritage.org
- - kafka3.internal.softwareheritage.org
- - kafka4.internal.softwareheritage.org
+ - kafka1.internal.softwareheritage.org
+ - kafka2.internal.softwareheritage.org
+ - kafka3.internal.softwareheritage.org
+ - kafka4.internal.softwareheritage.org
group_id: swh.scheduler.journal_client
---
# Source: swh/templates/scheduler/recurrent-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: scheduler-schedule-recurrent-configuration-template
data:
config.yml.template: |
@@ -5131,21 +5132,32 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scrubber-template
namespace: swh
data:
config.yml.template: |
scrubber:
cls: postgresql
- db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${POSTGRESQL_PASSWORD}
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swhstorage dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+ journal:
+ cls: kafka
+ brokers:
+ - kafka1.internal.softwareheritage.org
+ - kafka2.internal.softwareheritage.org
+ - kafka3.internal.softwareheritage.org
+ - kafka4.internal.softwareheritage.org
+ group_id: swh.scheduler.journal_client
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-storage-template
namespace: swh
data:
config.yml.template: |
@@ -5398,20 +5410,48 @@
if [ -e "${DB_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
fi
if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
echo "code and DB versions are different. Blocking the deployment"
exit 1
fi
+
+ check-scrubber-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ # checking the database status
+ swh db --config-file=${SWH_CONFIG_FILENAME} version ${MODULE} | tee "${TEMP_FILE}"
+
+ CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
+ DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
+
+ if [ -e "${CODE_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ -e "${DB_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
+ echo "code and DB versions are different. Blocking the deployment"
+ exit 1
+ fi
---
# Source: swh/templates/graphql/service.yaml
apiVersion: v1
kind: Service
metadata:
name: graphql
namespace: swh
spec:
type: ClusterIP
selector:
@@ -13880,21 +13920,21 @@
app: scheduler-journal-client
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: scheduler-journal-client
annotations:
- checksum/config: 95e03f4ca28530a2536cbfcf61e873dcdd7ac58441be766c9a5d571104fa0581
+ checksum/config: dd654fd74f168987ce84a19328031dab0ba2266f958b269c8271d312771f3239
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/scheduler
operator: In
values:
- "true"
@@ -14265,21 +14305,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: fdc36d26f1420f24416d2f3c61a80c2a24e51c14cbf919fe9652bab58d33bb01
+ checksum/config: 1c03cc66af5f38f0f57323396602918d5a94531b8a3b51f08262aee2f2d2dead
checksum/configScript: 3303cfe671f82f98541336ae9e53a936f3dff2016116784815730206e6114797
spec:
priorityClassName: swh-tools
initContainers:
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
@@ -14333,27 +14373,36 @@
- name: prepare-configuration-scrubber
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber.yml
env:
- - name: POSTGRESQL_PASSWORD
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-scrubber-postgresql-common-secret
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-swh-storage-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-scrubber-template
mountPath: /etc/swh/configuration-template
- name: prepare-configuration-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
------------- diff for environment production namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.6tdzz7kF/production-swh-cassandra.before 2023-10-26 14:49:39.719434589 +0200
+++ /tmp/swh-chart.swh.6tdzz7kF/production-swh-cassandra.after 2023-10-26 14:49:40.615436944 +0200
@@ -1014,20 +1014,48 @@
if [ -e "${DB_VERSION}" ]; then
echo "Unable to find the code version"
exit 1
fi
if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
echo "code and DB versions are different. Blocking the deployment"
exit 1
fi
+
+ check-scrubber-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ # checking the database status
+ swh db --config-file=${SWH_CONFIG_FILENAME} version ${MODULE} | tee "${TEMP_FILE}"
+
+ CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
+ DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
+
+ if [ -e "${CODE_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ -e "${DB_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ "$DB_VERSION" -ne "$CODE_VERSION" ]; then
+ echo "code and DB versions are different. Blocking the deployment"
+ exit 1
+ fi
---
# Source: swh/templates/web/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: web-configuration-template
data:
config.yml.template: |
instance_name: webapp-cassandra.internal.softwareheritage.org
@@ -1436,21 +1464,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-content
template:
metadata:
labels:
app: storage-replayer-content
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1547,21 +1575,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-directory
template:
metadata:
labels:
app: storage-replayer-directory
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1658,21 +1686,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-extid
template:
metadata:
labels:
app: storage-replayer-extid
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1769,21 +1797,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-metadata
template:
metadata:
labels:
app: storage-replayer-metadata
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1880,21 +1908,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin
template:
metadata:
labels:
app: storage-replayer-origin
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -1991,21 +2019,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit
template:
metadata:
labels:
app: storage-replayer-origin-visit
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2102,21 +2130,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-origin-visit-status
template:
metadata:
labels:
app: storage-replayer-origin-visit-status
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2213,21 +2241,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-raw-extrinsic-metadata
template:
metadata:
labels:
app: storage-replayer-raw-extrinsic-metadata
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2324,21 +2352,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-release
template:
metadata:
labels:
app: storage-replayer-release
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2435,21 +2463,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-revision
template:
metadata:
labels:
app: storage-replayer-revision
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2546,21 +2574,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-skipped-content
template:
metadata:
labels:
app: storage-replayer-skipped-content
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2657,21 +2685,21 @@
revisionHistoryLimit: 2
selector:
matchLabels:
app: storage-replayer-snapshot
template:
metadata:
labels:
app: storage-replayer-snapshot
annotations:
checksum/config: ab1824d6c0a84c4fee1f4f45f656d7fdf809d225d646856ef9a7e2322749e1d5
- checksum/config_utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config_utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/replayer
operator: In
values:
- "true"
@@ -2773,21 +2801,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage
annotations:
checksum/config: 404b00dbad2af855b6bfca45fa8048c82a0a5658fc1a82d8a93944c58ff3cf22
- checksum/config-utils: ac0a3fd6aeb2364ec99194860a70c43bdef3599ef8e2f4c1c4a6dad606694d18
+ checksum/config-utils: 2248b6494f8eab3e89ce1027e3c850054402f78af726830d1818a519d3cc04db
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"