staging: Install blocking proxy for read-write storage
This also adds the toolbox configuration so we can migrate its model.
In another commit, this drops the journal writer configuration from the postgresql read-write storage. We moved from the postgresql backend as the main storage. It's now the cassandra storage which is the main storage. And it does already have that configuration.
helm diff
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.PkgRktA8/staging-swh.before 2024-05-23 14:37:59.131757662 +0200
+++ /tmp/swh-chart.swh.PkgRktA8/staging-swh.after 2024-05-23 14:37:59.723732529 +0200
@@ -1022,32 +1022,26 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: storage-postgresql-read-write-configuration-template
data:
config.yml.template: |
storage:
cls: pipeline
steps:
+ - blocking_db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh-blocking
+ password=${BLOCKING_GUEST_PGPASSWORD}
+ cls: blocking
- cls: record_references
- cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh dbname=swh password=${POSTGRESQL_PASSWORD}
- journal_writer:
- anonymize: true
- brokers:
- - journal2.internal.staging.swh.network
- client_id: swh.storage.journal_writer.${HOSTNAME}
- cls: kafka
- prefix: swh.journal.objects
- producer_config:
- message.max.bytes: 1000000000
objstorage:
cls: remote
url: http://objstorage-multiplexer-read-write-rpc-ingress
---
# Source: swh/templates/storage/cronjob-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: cronjob-create-object-reference-partitions-configuration-template
@@ -14462,41 +14456,47 @@
app: storage-postgresql-read-write
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage-postgresql-read-write
annotations:
- checksum/config: bc3eb4204c59dfc69f47580ef1c3afbd454ad1566c517187b87cdde5fd62b3f6
+ checksum/config: d2863939aa44dcbffca26e7e5a44d9de374f971676ef18aba35f7dcce2313f1b
checksum/database-utils: 931b7f92b50eae0ecb6203c3e0e48f7c363a99ac0064451187dc12c6eaf0a793
checksum/config-utils: d75ca13b805bce6a8ab59c8e24c938f2283108f6a79134f6e71db86308651dc6
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"
priorityClassName: swh-frontend-rpc
initContainers:
- name: prepare-configuration
image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
- /entrypoints/prepare-configuration.sh
env:
+ - name: BLOCKING_GUEST_PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-guest-password
+ name: swh-postgresql-common-secret
+ optional: false
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
key: postgres-swh-password
name: swh-postgresql-common-secret
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
@@ -15043,20 +15043,26 @@
spec:
template:
spec:
initContainers:
- name: prepare-configuration
image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
- /entrypoints/prepare-configuration.sh
env:
+ - name: BLOCKING_GUEST_PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-guest-password
+ name: swh-postgresql-common-secret
+ optional: false
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
key: postgres-swh-password
name: swh-postgresql-common-secret
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.PkgRktA8/staging-swh-cassandra.before 2024-05-23 14:37:59.427745095 +0200
+++ /tmp/swh-chart.swh.PkgRktA8/staging-swh-cassandra.after 2024-05-23 14:38:00.019719960 +0200
@@ -5784,44 +5784,49 @@
---
# Source: swh/templates/storage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: storage-cassandra-configuration-template
data:
config.yml.template: |
storage:
- auth_provider:
- cls: cassandra.auth.PlainTextAuthProvider
- password: ${CASSANDRA_PASSWORD}
- username: swh-rw
- cls: cassandra
- consistency_level: LOCAL_QUORUM
- directory_entries_insert_algo: batch
- hosts:
- - cassandra1.internal.staging.swh.network
- - cassandra2.internal.staging.swh.network
- - cassandra3.internal.staging.swh.network
- journal_writer:
- anonymize: true
- brokers:
- - journal2.internal.staging.swh.network
- client_id: swh.storage-cassandra.journal_writer.${HOSTNAME}
- cls: kafka
- prefix: swh.journal.objects
- producer_config:
- message.max.bytes: 1000000000
- keyspace: swh
- objstorage:
- cls: remote
- url: http://objstorage-multiplexer-read-write-rpc-ingress
+ cls: pipeline
+ steps:
+ - blocking_db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh-blocking
+ password=${BLOCKING_GUEST_PGPASSWORD}
+ cls: blocking
+ - auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-rw
+ cls: cassandra
+ consistency_level: LOCAL_QUORUM
+ directory_entries_insert_algo: batch
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ journal_writer:
+ anonymize: true
+ brokers:
+ - journal2.internal.staging.swh.network
+ client_id: swh.storage-cassandra.journal_writer.${HOSTNAME}
+ cls: kafka
+ prefix: swh.journal.objects
+ producer_config:
+ message.max.bytes: 1000000000
+ keyspace: swh
+ objstorage:
+ cls: remote
+ url: http://objstorage-multiplexer-read-write-rpc-ingress
---
# Source: swh/templates/storage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: storage-cassandra-read-only-configuration-template
data:
config.yml.template: |
storage:
@@ -5842,20 +5847,36 @@
- cassandra3.internal.staging.swh.network
keyspace: swh
objstorage:
cls: remote
url: http://objstorage-read-only-rpc-ingress
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: toolbox-blocking-template
+ namespace: swh-cassandra
+data:
+ config.yml.template: |
+ storage:
+ cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh-blocking dbname=swh-blocking
+ password=${BLOCKING_ADMIN_PGPASSWORD}
+ blocking_admin:
+ blocking_db: host=db1.internal.staging.swh.network port=5432 user=swh-blocking dbname=swh-blocking
+ password=${BLOCKING_ADMIN_PGPASSWORD}
+---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: toolbox-indexer-storage-template
namespace: swh-cassandra
data:
config.yml.template: |
indexer_storage:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-indexer dbname=swh-indexer
password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
@@ -6041,20 +6062,35 @@
MODULE_NAME="$1"
CONFIG_FILE="${2:-/etc/swh/config-${MODULE_NAME}.yml}"
CONFIG_KEY="${3:-$MODULE_NAME}"
TO_VERSION="${4:-}"
exec swh db --config-file="${CONFIG_FILE}" upgrade "${MODULE_NAME}" \
--module-config-key="${CONFIG_KEY}" \
${TO_VERSION:+"--to-version=${TO_VERSION}"}
+ check-blocking-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ exec /opt/swh/bin/check-db-version.sh storage /etc/swh/config-blocking.yml storage
+
+ migrate-blocking-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ exec /opt/swh/bin/migrate-db-version.sh storage /etc/swh/config-blocking.yml storage
+
+
check-indexer-storage-db-version.sh: |
#!/bin/bash
set -eu
exec /opt/swh/bin/check-db-version.sh indexer /etc/swh/config-indexer-storage.yml indexer_storage
migrate-indexer-storage-db-version.sh: |
#!/bin/bash
@@ -29054,41 +29090,47 @@
app: storage-cassandra
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage-cassandra
annotations:
- checksum/config: e5d118ea7ba9b1886804d3a2f429f62629792d7ec5fb53034e3847a18e094a3f
+ checksum/config: 1ca562ff0709dc9174c02430e7d7dc3b4aae3067b1ac5d41d16b073b396cbf31
checksum/database-utils: 367f28d8854b3b188d7a45015752b10be8e691f0c3b2a21db02501e655b932f0
checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"
priorityClassName: swh-cassandra-frontend-rpc
initContainers:
- name: prepare-configuration
image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
- /entrypoints/prepare-configuration.sh
env:
+ - name: BLOCKING_GUEST_PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-guest-password
+ name: swh-postgresql-common-secret
+ optional: false
- name: CASSANDRA_PASSWORD
valueFrom:
secretKeyRef:
key: cassandra-swh-rw-password
name: common-secrets
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
@@ -29348,35 +29390,55 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: add10c6433efe5f60151f5c91c28cc741ebbfdbf627f7fe210d1d9e44e6b1210
- checksum/configScript: f6e562f44e0193007fe9fc7965d8b2289b35bf970cbedebb90745d3d9ea244f3
+ checksum/config: fbfcdca316609856a4c76556f0b5714bc2a3311ae21569692545f62a7220c027
+ checksum/configScript: 1dc88196c7a30bb832d013b213eef90d7e98171db3d7b7be6d50778123c45b6f
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/toolbox
operator: In
values:
- "true"
priorityClassName: swh-cassandra-tools
initContainers:
+ - name: prepare-configuration-blocking
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-blocking.yml
+ env:
+ - name: BLOCKING_ADMIN_PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-swh-blocking-password
+ name: swh-postgresql-blocking-secret
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-blocking-template
+ mountPath: /etc/swh/configuration-template
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-indexer-storage.yml
env:
- name: POSTGRESQL_PASSWORD
@@ -29518,20 +29580,28 @@
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: toolbox-script-utils
mountPath: /opt/swh/bin
readOnly: true
volumes:
- name: configuration
emptyDir: {}
+ - name: configuration-blocking-template
+ configMap:
+ name: toolbox-blocking-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
- name: configuration-indexer-storage-template
configMap:
name: toolbox-indexer-storage-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: configuration-masking-template
configMap:
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.PkgRktA8/staging-swh-cassandra-next-version.before 2024-05-23 14:37:59.579738642 +0200
+++ /tmp/swh-chart.swh.PkgRktA8/staging-swh-cassandra-next-version.after 2024-05-23 14:38:00.175713339 +0200
@@ -1948,44 +1948,49 @@
---
# Source: swh/templates/storage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra-next-version
name: storage-cassandra-configuration-template
data:
config.yml.template: |
storage:
- auth_provider:
- cls: cassandra.auth.PlainTextAuthProvider
- password: ${CASSANDRA_PASSWORD}
- username: swh-rw
- cls: cassandra
- consistency_level: LOCAL_QUORUM
- directory_entries_insert_algo: batch
- hosts:
- - cassandra1.internal.staging.swh.network
- - cassandra2.internal.staging.swh.network
- - cassandra3.internal.staging.swh.network
- journal_writer:
- anonymize: true
- brokers:
- - journal2.internal.staging.swh.network
- client_id: swh.storage-cassandra.journal_writer.${HOSTNAME}
- cls: kafka
- prefix: swh.journal.objects
- producer_config:
- message.max.bytes: 1000000000
- keyspace: swh
- objstorage:
- cls: remote
- url: http://objstorage-multiplexer-read-write-rpc-ingress
+ cls: pipeline
+ steps:
+ - blocking_db: host=db1.internal.staging.swh.network port=5432 user=guest dbname=swh-blocking
+ password=${BLOCKING_GUEST_PGPASSWORD}
+ cls: blocking
+ - auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-rw
+ cls: cassandra
+ consistency_level: LOCAL_QUORUM
+ directory_entries_insert_algo: batch
+ hosts:
+ - cassandra1.internal.staging.swh.network
+ - cassandra2.internal.staging.swh.network
+ - cassandra3.internal.staging.swh.network
+ journal_writer:
+ anonymize: true
+ brokers:
+ - journal2.internal.staging.swh.network
+ client_id: swh.storage-cassandra.journal_writer.${HOSTNAME}
+ cls: kafka
+ prefix: swh.journal.objects
+ producer_config:
+ message.max.bytes: 1000000000
+ keyspace: swh
+ objstorage:
+ cls: remote
+ url: http://objstorage-multiplexer-read-write-rpc-ingress
---
# Source: swh/templates/utils/config-utils.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: config-utils
namespace: swh-cassandra-next-version
data:
prepare-configuration.sh: |
#!/bin/bash
@@ -16066,41 +16071,47 @@
app: storage-cassandra
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: storage-cassandra
annotations:
- checksum/config: 5b1cacb6575952b97d1c8c6172583ec0b070bceb947b4c165b83f1d9e28a2540
+ checksum/config: 2fd93127b8c83b41f9a8aaa1889aec4fc60a57c117ef769a9a24904d5e55043e
checksum/database-utils: dd7d83d547731ea7598fa298565f3b1fe41eb657ae93bab75e15b722dbcc71ae
checksum/config-utils: 94d255131467f84bef964a4c72b2b792c5ebaf711bb1c77829d7cd1007a8ac22
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/storage
operator: In
values:
- "true"
priorityClassName: swh-cassandra-next-version-frontend-rpc
initContainers:
- name: prepare-configuration
image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
- /entrypoints/prepare-configuration.sh
env:
+ - name: BLOCKING_GUEST_PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-guest-password
+ name: swh-postgresql-common-secret
+ optional: false
- name: CASSANDRA_PASSWORD
valueFrom:
secretKeyRef:
key: cassandra-swh-rw-password
name: common-secrets
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differencesRefs. swh/infra/sysadm-environment#5315 (closed)
Requires !391 (merged) to be merged first (done)
Edited by Antoine R. Dumont