From d516a3e0e33326e8a0baef65b0f9e9fe80402d6d Mon Sep 17 00:00:00 2001
From: Guillaume Samson <guillaume.samson@softwareheritage.org>
Date: Tue, 16 May 2023 13:44:48 +0200
Subject: [PATCH] swh/cassandra-replay: update configmap to support pg backend
and filter/retry configuration
Related to swh/infra/sysadm-environment#4879
---
swh/templates/storage-replayer/configmap.yaml | 41 ++++++++++++++-----
.../storage-replayer/deployment.yaml | 12 ++++++
swh/values.yaml | 17 +++++++-
swh/values/default.yaml | 1 -
swh/values/production-cassandra.yaml | 1 +
swh/values/staging-cassandra.yaml | 1 +
6 files changed, 59 insertions(+), 14 deletions(-)
diff --git a/swh/templates/storage-replayer/configmap.yaml b/swh/templates/storage-replayer/configmap.yaml
index 37ae527bf..e56672201 100644
--- a/swh/templates/storage-replayer/configmap.yaml
+++ b/swh/templates/storage-replayer/configmap.yaml
@@ -1,6 +1,7 @@
{{ if .Values.storage_replayer.enabled -}}
{{- $error_reporter := .Values.storage_replayer.error_reporter }}
{{- range $deployment, $deployment_config := .Values.storage_replayer.deployments -}}
+{{- $storageConfigIndent := ternary " " "" $.Values.storage_replayer.pipelineFilterRetry }}
---
apiVersion: v1
kind: ConfigMap
@@ -10,20 +11,38 @@ metadata:
data:
config.yml.template: |
storage:
- cls: {{ $.Values.storage_replayer.storageClass }}
- hosts:
- {{- range $seed := $.Values.storage_replayer.cassandra.seeds }}
- - {{ $seed }}
+ {{- with $.Values.storage_replayer }}
+ {{- if not .pipelineFilterRetry }}
+ cls: {{ .storageClass }}
+ {{- else if .pipelineFilterRetry }}
+ cls: pipeline
+ steps:
+ - cls: filter
+ - cls: retry
+ - cls: {{ .storageClass }}
{{- end }}
- keyspace: {{ $.Values.storage_replayer.cassandra.keySpace }}
- consistency_level: {{ $.Values.storage_replayer.cassandra.consistencyLevel }}
- {{- if $deployment_config.specific_options -}}
- {{- range $option, $value := $deployment_config.specific_options }}
- {{ $option }}: {{ $value }}
+ {{- end }}
+ {{- if eq $.Values.storage_replayer.storageClass "cassandra" }}
+ {{- with $.Values.storage_replayer.cassandra }}
+ {{ $storageConfigIndent }}hosts:
+ {{- range $seed := .seeds }}
+ {{ $storageConfigIndent }} - {{ $seed }}
{{- end }}
+ {{ $storageConfigIndent }}keyspace: {{ .keySpace }}
+ {{ $storageConfigIndent }}consistency_level: {{ .consistencyLevel }}
+ {{- end }}
+ {{- else if eq $.Values.storage_replayer.storageClass "postgresql" }}
+ {{- with $.Values.storage_replayer.postgresql }}
+ {{ $storageConfigIndent }}db: host={{ .host }} port={{ .port | default "5432" }} user={{ .user | default "guest" }} dbname={{ .db | default "swh" }} password=${POSTGRESQL_PASSWORD}
{{- end }}
- objstorage:
- cls: noop
+ {{- end }}
+ {{- if $deployment_config.specific_options -}}
+ {{- range $option, $value := $deployment_config.specific_options }}
+ {{ $storageConfigIndent }}{{ $option }}: {{ $value }}
+ {{- end }}
+ {{- end }}
+ {{ $storageConfigIndent }} objstorage:
+ {{ $storageConfigIndent }} cls: noop
journal_client:
cls: kafka
diff --git a/swh/templates/storage-replayer/deployment.yaml b/swh/templates/storage-replayer/deployment.yaml
index ffef9bea5..31e27d159 100644
--- a/swh/templates/storage-replayer/deployment.yaml
+++ b/swh/templates/storage-replayer/deployment.yaml
@@ -37,6 +37,18 @@ spec:
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ {{- if eq $.Values.storage_replayer.storageClass "postgresql" }}
+ {{ with .Values.storage_replayer.postgresql }}
+ env:
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ .secretKeyRef }}
+ key: {{ .secretKeyName }}
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ {{- end }}
+ {{- end }}
volumeMounts:
- name: configuration
mountPath: /etc/swh
diff --git a/swh/values.yaml b/swh/values.yaml
index aac6d1881..0d23b47ae 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -69,7 +69,6 @@ tmpEphemeralStorage:
storage_replayer:
enabled: false
- storageClass: cassandra # only cassandra is currently supported
maxMessagesBytes: "524288000"
journalBrokers:
# The name of the secret containing the BROKER_USER_PASSWORD value
@@ -78,6 +77,9 @@ storage_replayer:
- broker1
- broker2
user: myuser
+ ## storage backend must be cassandra or postgresql
+ storageClass: cassandra
+ ## if storageClass is cassandra
cassandra:
initKeyspace: false # useful to ensure the cassandra database is bootstrapped
seeds:
@@ -85,6 +87,17 @@ storage_replayer:
- seed2
keySpace: swh
consistencyLevel: LOCAL_QUORUM
+ ## if storageClass is postgresql
+ #postgresql:
+ # host: db1
+ # port: '5432'
+ # user: pg_user
+ # dbname: swh
+ ## name of the secret containing the $secretKeyName value
+ # secretKeyRef: swh-postgresql-common-secrets
+ # secretKeyName: storage-postgresql-password
+ ## if pipeline/filter/retry is needed in storage configuration
+ pipelineFilterRetry: false
# optional 'replayer' configuration may be specified with a 'error_reporter' config
# entry allowing to specify redis connection parameters. It will be used to report
# non-recoverable replaying issues
@@ -205,7 +218,7 @@ storage:
# db: softwareheritage
# user: guest
# # name of the secret containing the $secretKeyName value
- # secretKeyRef: common-secrets
+ # secretKeyRef: swh-postgresql-common-secrets
# secretKeyName: storage-postgresql-password
objstorage:
cls: noop
diff --git a/swh/values/default.yaml b/swh/values/default.yaml
index 4c272624b..a75cfb7c5 100644
--- a/swh/values/default.yaml
+++ b/swh/values/default.yaml
@@ -15,7 +15,6 @@ tmpEphemeralStorage:
storage: 100Gi # no effects
storage_replayer:
- storageClass: cassandra # only cassandra is currently supported
maxMessagesBytes: "524288000"
journalBrokers:
# The name of the secret containing the BROKER_USER_PASSWORD value
diff --git a/swh/values/production-cassandra.yaml b/swh/values/production-cassandra.yaml
index f333c7064..82fddef5f 100644
--- a/swh/values/production-cassandra.yaml
+++ b/swh/values/production-cassandra.yaml
@@ -21,6 +21,7 @@ storage_replayer:
- kafka3.internal.softwareheritage.org:9094
- kafka4.internal.softwareheritage.org:9094
user: swh-cassandra-prod
+ storageClass: cassandra
cassandra:
seeds:
- cassandra01.internal.softwareheritage.org
diff --git a/swh/values/staging-cassandra.yaml b/swh/values/staging-cassandra.yaml
index 0985d192c..1d6376c57 100644
--- a/swh/values/staging-cassandra.yaml
+++ b/swh/values/staging-cassandra.yaml
@@ -19,6 +19,7 @@ storage_replayer:
- journal1.internal.staging.swh.network:9094
user: swh-cassandra-stg
secretName: swh-cassandra-broker-secret
+ storageClass: cassandra
cassandra:
initKeyspace: true # only to bootstrap a new cassandra database
seeds:
--
GitLab