From 260c12419362049e3fbf88194cec888df9ed50ff Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Wed, 13 Dec 2023 13:50:50 +0100
Subject: [PATCH] indexer_storage: Make template multi-instance
Refs. swh/infra/sysadm-environment#5183
---
.../indexer-storage/_helper_configmap.yaml | 20 +++++
.../indexer-storage/autoscaling.yaml | 14 ++-
swh/templates/indexer-storage/configmap.yaml | 23 ++---
swh/templates/indexer-storage/deployment.yaml | 88 +++++++++++--------
swh/templates/indexer-storage/ingress.yaml | 15 ++--
swh/templates/indexer-storage/service.yaml | 15 +++-
swh/templates/storage/ingress.yaml | 2 +-
swh/values.yaml | 82 ++++++++---------
swh/values/minikube.yaml | 34 +++----
swh/values/production/swh.yaml | 34 +++----
swh/values/staging/swh.yaml | 58 ++++++------
11 files changed, 217 insertions(+), 168 deletions(-)
create mode 100644 swh/templates/indexer-storage/_helper_configmap.yaml
diff --git a/swh/templates/indexer-storage/_helper_configmap.yaml b/swh/templates/indexer-storage/_helper_configmap.yaml
new file mode 100644
index 000000000..8f6b68f72
--- /dev/null
+++ b/swh/templates/indexer-storage/_helper_configmap.yaml
@@ -0,0 +1,20 @@
+{{/*
+ * Create an storage configmap for service .serviceType
+ */}}
+{{ define "swh.indexer.storage.configmap" }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: {{ .Values.namespace }}
+ name: {{ .serviceType }}-configuration-template
+data:
+ config.yml.template: |
+ {{- include "swh.postgresql" (dict "serviceType" "indexer_storage"
+ "configurationRef" .configuration.indexerStorageConfigurationRef
+ "Values" .Values) | nindent 4 -}}
+
+ {{- include "swh.journal.configuration" (dict "serviceType" "journal_writer"
+ "configurationRef" .configuration.journalWriterConfigurationRef
+ "Values" .Values) | nindent 4 -}}
+{{- end -}}
diff --git a/swh/templates/indexer-storage/autoscaling.yaml b/swh/templates/indexer-storage/autoscaling.yaml
index bb177574e..4d7391eeb 100644
--- a/swh/templates/indexer-storage/autoscaling.yaml
+++ b/swh/templates/indexer-storage/autoscaling.yaml
@@ -1,5 +1,11 @@
-{{- if and .Values.indexerStorage.enabled .Values.indexerStorage.autoScaling -}}
-{{- include "swh.autoscale" (dict "Values" .Values
- "serviceType" "indexer-storage-rpc"
- "configuration" .Values.indexerStorage) -}}
+{{- if .Values.indexerStorage.enabled -}}
+{{ range $storage_type, $storage_config := .Values.indexerStorage.deployments }}
+{{- if and (or (not (hasKey $storage_config "enabled")) (get $storage_config "enabled"))
+ (hasKey $storage_config "autoScaling") -}}
+{{- include "swh.autoscale" (dict "Values" $.Values
+ "serviceType" ( print "indexer-storage-" $storage_type )
+ "configuration" $storage_config) -}}
{{- end -}}
+{{ end -}}
+{{- end -}}
+
diff --git a/swh/templates/indexer-storage/configmap.yaml b/swh/templates/indexer-storage/configmap.yaml
index 8218addd0..05ddd8e2c 100644
--- a/swh/templates/indexer-storage/configmap.yaml
+++ b/swh/templates/indexer-storage/configmap.yaml
@@ -1,17 +1,10 @@
{{ if .Values.indexerStorage.enabled -}}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: {{ .Values.namespace }}
- name: indexer-storage-configuration-template
-data:
- config.yml.template: |
- {{- include "swh.postgresql" (dict "serviceType" "indexer_storage"
- "configurationRef" .Values.indexerStorage.indexerStorageConfigurationRef
- "Values" .Values) | nindent 4 -}}
-
- {{- include "swh.journal.configuration" (dict "serviceType" "journal_writer"
- "configurationRef" .Values.indexerStorage.journalWriterConfigurationRef
- "Values" .Values) | nindent 4 -}}
+{{ range $storage_type, $storage_config := .Values.indexerStorage.deployments }}
+{{- if or (not (hasKey $storage_config "enabled")) (get $storage_config "enabled") -}}
+{{ include "swh.indexer.storage.configmap" (dict "serviceType" ( print "indexer-storage-" $storage_type )
+ "configuration" $storage_config
+ "Values" $.Values) }}
+{{- end -}}
+{{ end -}}
{{- end -}}
+
diff --git a/swh/templates/indexer-storage/deployment.yaml b/swh/templates/indexer-storage/deployment.yaml
index 70db1820a..fa6a24caf 100644
--- a/swh/templates/indexer-storage/deployment.yaml
+++ b/swh/templates/indexer-storage/deployment.yaml
@@ -1,20 +1,28 @@
{{ if .Values.indexerStorage.enabled -}}
+{{- $databaseUtilsPathChecksum := include (print $.Template.BasePath "/utils/database-utils.yaml") . | sha256sum -}}
+{{ range $storage_type, $storage_config := .Values.indexerStorage.deployments }}
+{{- if or (not (hasKey $storage_config "enabled")) (get $storage_config "enabled") -}}
+{{- $serviceType := ( print "indexer-storage-" $storage_type ) -}}
+{{- $configChecksum := include "swh.indexer.storage.configmap"
+ (dict "serviceType" $serviceType
+ "configuration" $storage_config
+ "Values" $.Values) | sha256sum -}}
---
apiVersion: apps/v1
kind: Deployment
metadata:
- namespace: {{ .Values.namespace }}
- name: indexer-storage-rpc
+ namespace: {{ $.Values.namespace }}
+ name: {{ $serviceType }}
labels:
- app: indexer-storage-rpc
+ app: {{ $serviceType }}
spec:
revisionHistoryLimit: 2
- {{ if .Values.indexerStorage.replicas -}}
- replicas: {{ .Values.indexerStorage.replicas }}
+ {{ if $storage_config.replicas -}}
+ replicas: {{ $storage_config.replicas }}
{{ end -}}
selector:
matchLabels:
- app: indexer-storage-rpc
+ app: {{ $serviceType }}
strategy:
type: RollingUpdate
rollingUpdate:
@@ -22,18 +30,18 @@ spec:
template:
metadata:
labels:
- app: indexer-storage-rpc
+ app: {{ $serviceType }}
annotations:
- checksum/config: {{ include (print $.Template.BasePath "/indexer-storage/configmap.yaml") . | sha256sum }}
- checksum/config-utils: {{ include (print $.Template.BasePath "/utils/database-utils.yaml") . | sha256sum }}
+ checksum/config: {{ $configChecksum }}
+ checksum/database-utils: {{ $databaseUtilsPathChecksum }}
spec:
- {{- if .Values.indexerStorage.affinity }}
+ {{- if $.Values.indexerStorage.affinity }}
affinity:
- {{- toYaml .Values.indexerStorage.affinity | nindent 8 }}
+ {{- toYaml $.Values.indexerStorage.affinity | nindent 8 }}
{{- end }}
- {{- if and .Values.podPriority.enabled .Values.indexerStorage.priorityClassName }}
- priorityClassName: {{ .Values.namespace }}-{{ .Values.indexerStorage.priorityClassName }}
+ {{- if and $.Values.podPriority.enabled $.Values.indexerStorage.priorityClassName }}
+ priorityClassName: {{ $.Values.namespace }}-{{ $.Values.indexerStorage.priorityClassName }}
{{- end }}
initContainers:
- name: prepare-configuration
@@ -45,37 +53,37 @@ spec:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
env:
- {{ include "swh.secrets.environment" (dict "configurationRef" .Values.indexerStorage.indexerStorageConfigurationRef
- "Values" .Values) | nindent 10 }}
+ {{ include "swh.secrets.environment" (dict "configurationRef" $storage_config.indexerStorageConfigurationRef
+ "Values" $.Values) | nindent 10 }}
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- {{- if .Values.indexerStorage.checkDbVersion }}
- {{ include "swh.checkDatabaseVersionContainer" (dict "Values" .Values
+ {{- if $storage_config.checkDbVersion }}
+ {{ include "swh.checkDatabaseVersionContainer" (dict "Values" $.Values
"imagePrefixName" "swh_indexer_storage_image"
"module" "indexer-storage") | nindent 8 }}
{{- end }}
containers:
- - name: indexer-storage-rpc
+ - name: {{ $serviceType }}
resources:
requests:
- memory: {{ .Values.indexerStorage.requestedMemory | default "512Mi" }}
- cpu: {{ .Values.indexerStorage.requestedCpu | default "500m" }}
- {{- if or .Values.indexerStorage.limitedMemory .Values.indexerStorage.limitedCpu }}
+ memory: {{ $storage_config.requestedMemory | default "512Mi" }}
+ cpu: {{ $storage_config.requestedCpu | default "500m" }}
+ {{- if or $storage_config.limitedMemory $storage_config.limitedCpu }}
limits:
- {{- if .Values.indexerStorage.limitedMemory }}
- memory: {{ .Values.indexerStorage.limitedMemory }}
+ {{- if $storage_config.limitedMemory }}
+ memory: {{ $storage_config.limitedMemory }}
{{- end }}
- {{- if .Values.indexerStorage.limitedCpu }}
- cpu: {{ .Values.indexerStorage.limitedCpu }}
+ {{- if $storage_config.limitedCpu }}
+ cpu: {{ $storage_config.limitedCpu }}
{{- end }}
{{- end }}
- image: {{ .Values.swh_indexer_storage_image }}:{{ .Values.swh_indexer_storage_image_version }}
+ image: {{ $.Values.swh_indexer_storage_image }}:{{ $.Values.swh_indexer_storage_image_version }}
imagePullPolicy: IfNotPresent
ports:
- - containerPort: {{ .Values.indexerStorage.port }}
+ - containerPort: {{ $storage_config.port | default $.Values.indexerStorage.port }}
name: rpc
readinessProbe:
httpGet:
@@ -96,30 +104,30 @@ spec:
- -c
- /opt/swh/entrypoint.sh
env:
- {{ if .Values.indexerStorage.gunicorn -}}
+ {{ if $storage_config.gunicorn -}}
- name: THREADS
- value: {{ .Values.indexerStorage.gunicorn.threads | default 5 | quote }}
+ value: {{ $storage_config.gunicorn.threads | default 5 | quote }}
- name: WORKERS
- value: {{ .Values.indexerStorage.gunicorn.workers | default 2 | quote }}
+ value: {{ $storage_config.gunicorn.workers | default 2 | quote }}
- name: TIMEOUT
- value: {{ .Values.indexerStorage.gunicorn.timeout | default 60 | quote }}
+ value: {{ $storage_config.gunicorn.timeout | default 60 | quote }}
{{ end -}}
- name: STATSD_HOST
- value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
- name: STATSD_PORT
- value: {{ .Values.statsdPort | default "9125" | quote }}
+ value: {{ $.Values.statsdPort | default "9125" | quote }}
- name: LOG_LEVEL
- value: {{ .Values.indexerStorage.logLevel | quote }}
- {{- if .Values.indexerStorage.sentry.enabled }}
+ value: {{ $storage_config.logLevel | default $.Values.indexerStorage.logLevel | quote }}
+ {{- if $.Values.indexerStorage.sentry.enabled }}
- name: SWH_SENTRY_ENVIRONMENT
- value: {{ .Values.sentry.environment }}
+ value: {{ $.Values.sentry.environment }}
- name: SWH_MAIN_PACKAGE
value: swh.indexer
- name: SWH_SENTRY_DSN
valueFrom:
secretKeyRef:
- name: {{ .Values.indexerStorage.sentry.secretKeyRef }}
- key: {{ .Values.indexerStorage.sentry.secretKeyName }}
+ name: {{ $.Values.indexerStorage.sentry.secretKeyRef }}
+ key: {{ $.Values.indexerStorage.sentry.secretKeyName }}
# 'name' secret should exist & include key
# if the setting doesn't exist, sentry pushes will be disabled
optional: true
@@ -134,7 +142,7 @@ spec:
emptyDir: {}
- name: configuration-template
configMap:
- name: indexer-storage-configuration-template
+ name: {{ $serviceType }}-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
@@ -142,4 +150,6 @@ spec:
configMap:
name: database-utils
defaultMode: 0555
+{{ end }}
+{{ end -}}
{{- end -}}
diff --git a/swh/templates/indexer-storage/ingress.yaml b/swh/templates/indexer-storage/ingress.yaml
index 4866ee68a..6f44eeb4b 100644
--- a/swh/templates/indexer-storage/ingress.yaml
+++ b/swh/templates/indexer-storage/ingress.yaml
@@ -1,5 +1,10 @@
-{{- if and .Values.indexerStorage.enabled .Values.indexerStorage.ingress.enabled }}
-{{- include "swh.ingress" (dict "Values" .Values
- "serviceType" "indexer-storage-rpc"
- "configuration" .Values.indexerStorage) -}}
-{{ end }}
+{{- if .Values.indexerStorage.enabled -}}
+{{ range $storage_type, $storage_config := .Values.indexerStorage.deployments }}
+{{- if and (or (not (hasKey $storage_config "enabled")) (get $storage_config "enabled"))
+ (and (hasKey $storage_config "ingress") $storage_config.ingress.enabled) -}}
+{{- include "swh.ingress" (dict "Values" $.Values
+ "serviceType" ( print "indexer-storage-" $storage_type )
+ "configuration" $storage_config) -}}
+{{- end -}}
+{{ end -}}
+{{- end -}}
diff --git a/swh/templates/indexer-storage/service.yaml b/swh/templates/indexer-storage/service.yaml
index e51023d79..e0d4b08d4 100644
--- a/swh/templates/indexer-storage/service.yaml
+++ b/swh/templates/indexer-storage/service.yaml
@@ -1,5 +1,12 @@
{{ if .Values.indexerStorage.enabled -}}
-{{- include "swh.service" (dict "Values" .Values
- "serviceType" "indexer-storage-rpc"
- "configuration" .Values.indexerStorage) -}}
-{{ end }}
+{{ range $storage_type, $storage_config := .Values.indexerStorage.deployments }}
+{{- if or (not (hasKey $storage_config "enabled")) (get $storage_config "enabled") -}}
+{{- if not (hasKey $storage_config "port") }}
+{{- $storage_config := set $storage_config "port" $.Values.indexerStorage.port -}}
+{{- end -}}
+{{- include "swh.service" (dict "serviceType" ( print "indexer-storage-" $storage_type )
+ "configuration" $storage_config
+ "Values" $.Values) -}}
+{{- end -}}
+{{ end -}}
+{{- end -}}
diff --git a/swh/templates/storage/ingress.yaml b/swh/templates/storage/ingress.yaml
index 9120b6b74..77fff8867 100644
--- a/swh/templates/storage/ingress.yaml
+++ b/swh/templates/storage/ingress.yaml
@@ -1,4 +1,4 @@
-{{ if .Values.storage.enabled -}}
+{{- if .Values.storage.enabled -}}
{{ range $storage_type, $deployment_config := .Values.storage.deployments }}
{{- if and (or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled"))
(and (hasKey $deployment_config "ingress") $deployment_config.ingress.enabled) -}}
diff --git a/swh/values.yaml b/swh/values.yaml
index 358e5e185..350639b53 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -554,28 +554,6 @@ indexerStorage:
port: 5007
priorityClassName: frontend-rpc
logLevel: INFO
-# Don't configure the replicas if autoscaling is configured
-# and vice-versa
-# replicas: 1
- requestedCpu: 50m
- requestedMemory: 100Mi
-# autoScaling:
-# minReplicaCount: 2
-# maxReplicaCount: 10
-# cpuPercentageUsage: 100
-# affinity:
-# nodeAffinity:
-# requiredDuringSchedulingIgnoredDuringExecution:
-# nodeSelectorTerms:
-# - matchExpressions:
-# - key: "swh/rpc"
-# operator: In
-# values:
-# - "true"
-# gunicorn:
-# threads: 5
-# workers: 2
-# timeout: 60
sentry:
enabled: false
# name of the secret containing the $secretKeyName value
@@ -583,24 +561,48 @@ indexerStorage:
# like https://token@sentry.host/id
secretKeyRef: common-secrets
secretKeyName: indexer-storage-sentry-dsn
- # mandatory
- # check defaultStorageConfiguration example to configure your storage
- indexerStorageConfigurationRef: defaultIndexerStorageConfiguration
- journalWriteConfigurationRef: defaultJournalWriterConfiguration
- # When true, check db & code versions and do not start if they diverge
- checkDbVersion: false
- # Deploy an ingress to access the storage
- ingress:
- enabled: false
- # Optional: the ingress classname to use
- # className: nginx
- # mandatory if ingress is enabled
- # the hostname on which the storage must be reachable
- # host: mystorage.localdomain
- endpoints:
- default:
- paths:
- - path: /
+ deployments:
+ # rpc:
+ # # Don't configure the replicas if autoscaling is configured
+ # # and vice-versa
+ # # replicas: 1
+ # requestedCpu: 50m
+ # requestedMemory: 100Mi
+ # # autoScaling:
+ # # minReplicaCount: 2
+ # # maxReplicaCount: 10
+ # # cpuPercentageUsage: 100
+ # # affinity:
+ # # nodeAffinity:
+ # # requiredDuringSchedulingIgnoredDuringExecution:
+ # # nodeSelectorTerms:
+ # # - matchExpressions:
+ # # - key: "swh/rpc"
+ # # operator: In
+ # # values:
+ # # - "true"
+ # # gunicorn:
+ # # threads: 5
+ # # workers: 2
+ # # timeout: 60
+ # # mandatory
+ # # check defaultStorageConfiguration example to configure your storage
+ # # indexerStorageConfigurationRef: defaultIndexerStorageConfiguration
+ # # journalWriteConfigurationRef: defaultJournalWriterConfiguration
+ # # When true, check db & code versions and do not start if they diverge
+ # checkDbVersion: false
+ # # Deploy an ingress to access the storage
+ # ingress:
+ # enabled: false
+ # # Optional: the ingress classname to use
+ # # className: nginx
+ # # mandatory if ingress is enabled
+ # # the hostname on which the storage must be reachable
+ # # host: mystorage.localdomain
+ # endpoints:
+ # default:
+ # paths:
+ # - path: /
storage:
enabled: false
diff --git a/swh/values/minikube.yaml b/swh/values/minikube.yaml
index ce9ccf247..b74eda04a 100644
--- a/swh/values/minikube.yaml
+++ b/swh/values/minikube.yaml
@@ -741,22 +741,24 @@ indexerJournalWriterConfiguration:
indexerStorage:
enabled: false
- checkDbVersion: false
- autoScaling:
- minReplicaCount: 2
- maxReplicaCount: 2
- cpuPercentageUsage: 90
- requestedCpu: 500m
- requestedMemory: 512Mi
- indexerStorageConfigurationRef: fakePostgresqlIndexerStorageConfiguration
- journalWriterConfigurationRef: indexerJournalWriterConfiguration
- hosts:
- - indexer-storage.i.s.s.n
- ingress:
- enabled: true
- gunicorn:
- threads: 2
- workers: 4
+ deployments:
+ read-write:
+ checkDbVersion: false
+ autoScaling:
+ minReplicaCount: 2
+ maxReplicaCount: 2
+ cpuPercentageUsage: 90
+ requestedCpu: 500m
+ requestedMemory: 512Mi
+ indexerStorageConfigurationRef: fakePostgresqlIndexerStorageConfiguration
+ journalWriterConfigurationRef: indexerJournalWriterConfiguration
+ hosts:
+ - indexer-storage.i.s.s.n
+ ingress:
+ enabled: true
+ gunicorn:
+ threads: 2
+ workers: 4
defaultElasticsearchHosts:
- host: search-esnode0.i.s.s.n
diff --git a/swh/values/production/swh.yaml b/swh/values/production/swh.yaml
index 9e63cf53f..09a05a03e 100644
--- a/swh/values/production/swh.yaml
+++ b/swh/values/production/swh.yaml
@@ -688,22 +688,24 @@ toolbox:
indexerStorage:
enabled: false
- checkDbVersion: true
- autoScaling:
- minReplicaCount: 2
- maxReplicaCount: 4
- cpuPercentageUsage: 90
- requestedCpu: 500m
- requestedMemory: 512Mi
- indexerStorageConfigurationRef: postgresqlIndexerStorageConfiguration
- journalWriterConfigurationRef: journalWriterConfiguration
- hosts:
- - saam.internal.softwareheritage.org
- ingress:
- enabled: true
- gunicorn:
- threads: 2
- workers: 4
+ deployments:
+ read-write:
+ checkDbVersion: true
+ autoScaling:
+ minReplicaCount: 2
+ maxReplicaCount: 4
+ cpuPercentageUsage: 90
+ requestedCpu: 500m
+ requestedMemory: 512Mi
+ indexerStorageConfigurationRef: postgresqlIndexerStorageConfiguration
+ journalWriterConfigurationRef: journalWriterConfiguration
+ hosts:
+ - saam.internal.softwareheritage.org
+ ingress:
+ enabled: true
+ gunicorn:
+ threads: 2
+ workers: 4
search:
enabled: true
diff --git a/swh/values/staging/swh.yaml b/swh/values/staging/swh.yaml
index 0b42c1d36..097cf9cba 100644
--- a/swh/values/staging/swh.yaml
+++ b/swh/values/staging/swh.yaml
@@ -654,34 +654,36 @@ scheduler:
indexerStorage:
enabled: true
- checkDbVersion: true
- autoScaling:
- minReplicaCount: 2
- maxReplicaCount: 4
- cpuPercentageUsage: 90
- requestedCpu: 10m
- requestedMemory: 200Mi
- indexerStorageConfigurationRef: postgresqlIndexerStorageConfiguration
- journalWriterConfigurationRef: indexerJournalWriterConfiguration
- hosts:
- - indexer-storage-rpc-ingress
- ingress:
- enabled: true
- extraAnnotations:
- nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
- nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
- nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
- nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
- nginx.ingress.kubernetes.io/proxy-body-size: "4G"
- # Default allowed ip ranges that can be extended per ingress definitions paths
- whitelistSourceRangeRef: clusterNetworkRanges
- endpoints:
- default:
- paths:
- - path: /
- gunicorn:
- threads: 2
- workers: 4
+ deployments:
+ rpc:
+ checkDbVersion: true
+ autoScaling:
+ minReplicaCount: 2
+ maxReplicaCount: 4
+ cpuPercentageUsage: 90
+ requestedCpu: 10m
+ requestedMemory: 200Mi
+ indexerStorageConfigurationRef: postgresqlIndexerStorageConfiguration
+ journalWriterConfigurationRef: indexerJournalWriterConfiguration
+ hosts:
+ - indexer-storage-rpc-ingress
+ ingress:
+ enabled: true
+ extraAnnotations:
+ nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
+ nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
+ nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+ nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
+ nginx.ingress.kubernetes.io/proxy-body-size: "4G"
+ # Default allowed ip ranges that can be extended per ingress definitions paths
+ whitelistSourceRangeRef: clusterNetworkRanges
+ endpoints:
+ default:
+ paths:
+ - path: /
+ gunicorn:
+ threads: 2
+ workers: 4
search:
enabled: true
--
GitLab