toolbox: Add deposit configuration to access deposit db
Activate configuration for environments on all toolbox
We are currently missing that.
It's helpful to connect to it from the toolbox [2]
helm diff
[swh] Comparing changes between branches production and mr/add-deposit-config-to-toolbox (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in mr/add-deposit-config-to-toolbox branch for environment staging...
[swh] Generate config in mr/add-deposit-config-to-toolbox branch for environment staging...
[swh] Generate config in mr/add-deposit-config-to-toolbox branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in mr/add-deposit-config-to-toolbox branch for environment production...
[swh] Generate config in mr/add-deposit-config-to-toolbox branch for environment production...
[swh] Generate config in mr/add-deposit-config-to-toolbox branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.p2G6sccu/staging-swh-cassandra.before 2024-12-11 14:05:38.126337027 +0100
+++ /tmp/swh-chart.swh.p2G6sccu/staging-swh-cassandra.after 2024-12-11 14:05:39.466290726 +0100
@@ -7225,20 +7225,33 @@
cls: noop
blocking_admin:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-blocking dbname=swh-blocking
password=${BLOCKING_ADMIN_PGPASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: toolbox-deposit-template
+ namespace: swh-cassandra
+data:
+ config.yml.template: |
+ deposit:
+ cls: postgresql
+ db: host=swh-deposit-rw.swh-cassandra port=5432 user=${POSTGRESQL_USERNAME} dbname=swh-deposit
+ password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: toolbox-indexer-storage-template
namespace: swh-cassandra
data:
config.yml.template: |
indexer_storage:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-indexer dbname=swh-indexer
password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
@@ -7555,20 +7568,35 @@
exec /opt/swh/bin/check-db-version.sh storage.proxies.blocking /etc/swh/config-blocking.yml blocking_admin
migrate-blocking-db-version.sh: |
#!/bin/bash
set -eu
exec /opt/swh/bin/migrate-db-version.sh storage.proxies.blocking /etc/swh/config-blocking.yml blocking_admin
+ check-deposit-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ exec /opt/swh/bin/check-db-version.sh deposit /etc/swh/config-deposit.yml deposit
+
+ migrate-deposit-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ exec /opt/swh/bin/migrate-db-version.sh deposit /etc/swh/config-deposit.yml deposit
+
+
check-indexer-storage-db-version.sh: |
#!/bin/bash
set -eu
exec /opt/swh/bin/check-db-version.sh indexer /etc/swh/config-indexer-storage.yml indexer_storage
migrate-indexer-storage-db-version.sh: |
#!/bin/bash
@@ -22839,22 +22867,22 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: b55e5efdea135506c4871c0cb4930d15eb57dc483ee82c75eea5bd628ddc3d8b
- checksum/configScript: c23b77094af994ace25c94bfe58070847ad8d53a772e0c96676f4a2074057152
+ checksum/config: 4f03d771adf8d45a1ddfac95129ecc1702287b8ac8880f18e50b30e2c5e95ba6
+ checksum/configScript: 993602a4dba5c91f6197959f6f846c3f8d7261acbb1333ea890038686eba9f1b
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/toolbox
operator: In
values:
- "true"
@@ -22874,20 +22902,46 @@
valueFrom:
secretKeyRef:
key: postgres-swh-blocking-password
name: swh-postgresql-blocking-secret
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-blocking-template
mountPath: /etc/swh/configuration-template
+ - name: prepare-configuration-deposit
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-deposit.yml
+ env:
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: swh-postgresql-swh-deposit-secret
+ optional: false
+ - name: POSTGRESQL_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: swh-postgresql-swh-deposit-secret
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-deposit-template
+ mountPath: /etc/swh/configuration-template
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-indexer-storage.yml
env:
- name: POSTGRESQL_PASSWORD
@@ -23127,20 +23181,28 @@
emptyDir: {}
- name: configuration-blocking-template
configMap:
name: toolbox-blocking-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
+ - name: configuration-deposit-template
+ configMap:
+ name: toolbox-deposit-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
- name: configuration-indexer-storage-template
configMap:
name: toolbox-indexer-storage-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: configuration-masking-template
configMap:
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.p2G6sccu/staging-swh-cassandra-next-version.before 2024-12-11 14:05:38.610320304 +0100
+++ /tmp/swh-chart.swh.p2G6sccu/staging-swh-cassandra-next-version.after 2024-12-11 14:05:40.030271239 +0100
@@ -6261,20 +6261,33 @@
cls: noop
blocking_admin:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-blocking dbname=swh-blocking
password=${BLOCKING_ADMIN_PGPASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: toolbox-deposit-template
+ namespace: swh-cassandra-next-version
+data:
+ config.yml.template: |
+ deposit:
+ cls: postgresql
+ db: host=next-version-dbs-rw.swh-cassandra-next-version port=5432 user=${POSTGRESQL_USERNAME}
+ dbname=swh-deposit password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: toolbox-indexer-storage-template
namespace: swh-cassandra-next-version
data:
config.yml.template: |
indexer_storage:
cls: postgresql
db: host=next-version-swh-indexer-rw.swh-cassandra-next-version port=5432 user=${POSTGRESQL_IDX_USERNAME}
dbname=swh-indexer password=${POSTGRESQL_IDX_PASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
@@ -6598,20 +6611,35 @@
exec /opt/swh/bin/check-db-version.sh storage.proxies.blocking /etc/swh/config-blocking.yml blocking_admin
migrate-blocking-db-version.sh: |
#!/bin/bash
set -eu
exec /opt/swh/bin/migrate-db-version.sh storage.proxies.blocking /etc/swh/config-blocking.yml blocking_admin
+ check-deposit-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ exec /opt/swh/bin/check-db-version.sh deposit /etc/swh/config-deposit.yml deposit
+
+ migrate-deposit-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ exec /opt/swh/bin/migrate-db-version.sh deposit /etc/swh/config-deposit.yml deposit
+
+
check-indexer-storage-db-version.sh: |
#!/bin/bash
set -eu
exec /opt/swh/bin/check-db-version.sh indexer /etc/swh/config-indexer-storage.yml indexer_storage
migrate-indexer-storage-db-version.sh: |
#!/bin/bash
@@ -20273,22 +20301,22 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: e3c132d2c0ee6d34870c4e042c2357e7c15df400643bc33bd7bcbd1c2b8419dc
- checksum/configScript: 9fe2fc1c0533e8b849299c9a7e8ef7cbe6fc621786d1d29cef42d5f5edb65557
+ checksum/config: c323d14eb1e7945454c10209edd280a3a6a706170b89dc1aca716defd5ff4e24
+ checksum/configScript: 2dbd049506e794d641df93350a8fb7dd7ab17a8371aceea7b787d9a0cb879680
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/toolbox
operator: In
values:
- "true"
@@ -20308,20 +20336,46 @@
valueFrom:
secretKeyRef:
key: postgres-swh-blocking-password
name: swh-postgresql-blocking-secret
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-blocking-template
mountPath: /etc/swh/configuration-template
+ - name: prepare-configuration-deposit
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-deposit.yml
+ env:
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: password
+ name: swh-postgresql-swh-swh-secret
+ optional: false
+ - name: POSTGRESQL_USERNAME
+ valueFrom:
+ secretKeyRef:
+ key: username
+ name: swh-postgresql-swh-swh-secret
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-deposit-template
+ mountPath: /etc/swh/configuration-template
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-indexer-storage.yml
env:
- name: POSTGRESQL_IDX_PASSWORD
@@ -20599,20 +20653,28 @@
emptyDir: {}
- name: configuration-blocking-template
configMap:
name: toolbox-blocking-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
+ - name: configuration-deposit-template
+ configMap:
+ name: toolbox-deposit-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
- name: configuration-indexer-storage-template
configMap:
name: toolbox-indexer-storage-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: configuration-masking-template
configMap:
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.p2G6sccu/production-swh-cassandra.before 2024-12-11 14:05:40.866242351 +0100
+++ /tmp/swh-chart.swh.p2G6sccu/production-swh-cassandra.after 2024-12-11 14:05:41.694213742 +0100
@@ -9033,20 +9033,33 @@
- cassandra12.internal.softwareheritage.org
- cassandra13.internal.softwareheritage.org
keyspace: swh
objstorage:
cls: noop
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: toolbox-deposit-template
+ namespace: swh-cassandra
+data:
+ config.yml.template: |
+ deposit:
+ cls: postgresql
+ db: host=postgresql-deposit-rw.internal.softwareheritage.org port=5432 user=swhstorage
+ dbname=softwareheritage-deposit password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: toolbox-indexer-storage-template
namespace: swh-cassandra
data:
config.yml.template: |
indexer_storage:
cls: postgresql
db: host=postgresql-indexer-rw.internal.softwareheritage.org port=5432 user=swhstorage
dbname=softwareheritage-indexer password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/configmap.yaml
@@ -9271,20 +9284,35 @@
MODULE_NAME="$1"
CONFIG_FILE="${2:-/etc/swh/config-${MODULE_NAME}.yml}"
CONFIG_KEY="${3:-$MODULE_NAME}"
TO_VERSION="${4:-}"
exec swh db --config-file="${CONFIG_FILE}" upgrade "${MODULE_NAME}" \
--module-config-key="${CONFIG_KEY}" \
${TO_VERSION:+"--to-version=${TO_VERSION}"}
+ check-deposit-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ exec /opt/swh/bin/check-db-version.sh deposit /etc/swh/config-deposit.yml deposit
+
+ migrate-deposit-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ exec /opt/swh/bin/migrate-db-version.sh deposit /etc/swh/config-deposit.yml deposit
+
+
check-indexer-storage-db-version.sh: |
#!/bin/bash
set -eu
exec /opt/swh/bin/check-db-version.sh indexer /etc/swh/config-indexer-storage.yml indexer_storage
migrate-indexer-storage-db-version.sh: |
#!/bin/bash
@@ -27373,35 +27401,55 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 0e2dc0cbdb8f2c583d0f15f244e9998d1d65aab314ebf068af4a186791db7861
- checksum/configScript: 49e52cfc96c983af5c98e386c1b8abd596b83e90de199c6e1da17864932fa869
+ checksum/config: 39482e38f48944eb88c44ec8282731d891007dc2b7e642a3c42621588b8a77df
+ checksum/configScript: 06b664e1a92047fb20a80d90b5a23bb46345ae787a5dafa05b07b910961de3c9
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/toolbox
operator: In
values:
- "true"
priorityClassName: swh-cassandra-tools
initContainers:
+ - name: prepare-configuration-deposit
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-deposit.yml
+ env:
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-swh-deposit-password
+ name: swh-postgresql-deposit-secrets
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-deposit-template
+ mountPath: /etc/swh/configuration-template
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-indexer-storage.yml
env:
- name: POSTGRESQL_PASSWORD
@@ -27555,20 +27603,28 @@
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: toolbox-script-utils
mountPath: /opt/swh/bin
readOnly: true
volumes:
- name: configuration
emptyDir: {}
+ - name: configuration-deposit-template
+ configMap:
+ name: toolbox-deposit-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
- name: configuration-indexer-storage-template
configMap:
name: toolbox-indexer-storage-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: configuration-scheduler-template
configMap:
[2] toolbox check in local-cluster is fine
swh@swh-toolbox-55cbfb4c94-sljkq:~$ swh db shell deposit
INFO:swh.core.cli.db:Opening database shell for 'host=deposit-rw port=5432 user=swh-deposit dbname=swh-deposit'
psql (13.18 (Debian 13.18-0+deb11u1), server 17.0 (Debian 17.0-1.pgdg110+1))
WARNING: psql major version 13, server major version 17.
Some psql features might not work.
SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, bits: 256, compression: off)
Type "help" for help.
swh-deposit=> \d
List of relations
Schema | Name | Type | Owner
--------+-----------------------------------+----------+-------------
public | auth_group | table | swh-deposit
public | auth_group_id_seq | sequence | swh-deposit
public | auth_group_permissions | table | swh-deposit
public | auth_group_permissions_id_seq | sequence | swh-deposit
public | auth_permission | table | swh-deposit
public | auth_permission_id_seq | sequence | swh-deposit
public | auth_user | table | swh-deposit
public | auth_user_groups | table | swh-deposit
public | auth_user_groups_id_seq | sequence | swh-deposit
public | auth_user_id_seq | sequence | swh-deposit
public | auth_user_user_permissions | table | swh-deposit
public | auth_user_user_permissions_id_seq | sequence | swh-deposit
public | dbversion | table | swh-deposit
public | deposit | table | swh-deposit
public | deposit_client | table | swh-deposit
public | deposit_collection | table | swh-deposit
public | deposit_collection_id_seq | sequence | swh-deposit
public | deposit_id_seq | sequence | swh-deposit
public | deposit_request | table | swh-deposit
public | deposit_request_id_seq | sequence | swh-deposit
public | django_content_type | table | swh-deposit
public | django_content_type_id_seq | sequence | swh-deposit
public | django_migrations | table | swh-deposit
public | django_migrations_id_seq | sequence | swh-deposit
public | django_session | table | swh-deposit
(25 rows)
swh-deposit=> \conninfo
You are connected to database "swh-deposit" as user "swh-deposit" on host "deposit-rw" (address "10.96.112.202") at port "5432".
SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, bits: 256, compression: off)Edited by Antoine R. Dumont