From f9cb5066eff6ca155d61070b0c5ca12a4b46b335 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Tue, 28 Nov 2023 15:27:01 +0100
Subject: [PATCH] Add objstorage template
Refs. swh/infra/sysadm-environment#5164
---
.../objstorage/_helper_configmap.yaml | 15 ++
.../objstorage/_helper_deployment.yaml | 140 ++++++++++++++++++
swh/templates/objstorage/autoscaling.yaml | 10 ++
swh/templates/objstorage/configmap.yaml | 9 ++
swh/templates/objstorage/deployment.yaml | 11 ++
swh/templates/objstorage/ingress.yaml | 11 ++
swh/templates/objstorage/service.yaml | 12 ++
swh/values.yaml | 82 ++++++++++
swh/values/default.yaml | 13 ++
9 files changed, 303 insertions(+)
create mode 100644 swh/templates/objstorage/_helper_configmap.yaml
create mode 100644 swh/templates/objstorage/_helper_deployment.yaml
create mode 100644 swh/templates/objstorage/autoscaling.yaml
create mode 100644 swh/templates/objstorage/configmap.yaml
create mode 100644 swh/templates/objstorage/deployment.yaml
create mode 100644 swh/templates/objstorage/ingress.yaml
create mode 100644 swh/templates/objstorage/service.yaml
diff --git a/swh/templates/objstorage/_helper_configmap.yaml b/swh/templates/objstorage/_helper_configmap.yaml
new file mode 100644
index 000000000..e04cdd2b7
--- /dev/null
+++ b/swh/templates/objstorage/_helper_configmap.yaml
@@ -0,0 +1,15 @@
+{{/*
+ * Create an objstorage configmap for service .serviceType
+ */}}
+{{ define "swh.objstorage.configmap" }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: {{ .Values.namespace }}
+ name: {{ .serviceType }}-configuration-template
+data:
+ config.yml.template: |
+ {{- include "swh.objstorageConfiguration" (dict "configurationRef" .configuration.objstorageConfigurationRef
+ "Values" .Values) | nindent 4 }}
+{{- end -}}
diff --git a/swh/templates/objstorage/_helper_deployment.yaml b/swh/templates/objstorage/_helper_deployment.yaml
new file mode 100644
index 000000000..c164b4974
--- /dev/null
+++ b/swh/templates/objstorage/_helper_deployment.yaml
@@ -0,0 +1,140 @@
+{{/*
+ * Create an objstorage deployment for service .serviceType
+ */}}
+{{- define "swh.objstorage.deployment" -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: {{ .Values.namespace }}
+ name: {{ .serviceType }}
+ labels:
+ app: {{ .serviceType }}
+spec:
+ revisionHistoryLimit: 2
+ {{ if .configuration.replicas -}}
+ replicas: {{ .configuration.replicas }}
+ {{ end -}}
+ selector:
+ matchLabels:
+ app: {{ .serviceType }}
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: {{ .serviceType }}
+ annotations:
+ checksum/config: {{ include "swh.objstorage.configmap" (dict "serviceType" .serviceType
+ "configuration" .configuration
+ "Values" .Values) | sha256sum }}
+ spec:
+ {{- if .Values.objstorage.affinity }}
+ affinity:
+ {{- toYaml .Values.objstorage.affinity | nindent 8 }}
+ {{- end }}
+
+ {{- if and .Values.podPriority.enabled .Values.objstorage.priorityClassName }}
+ priorityClassName: {{ .Values.namespace }}-{{ .Values.objstorage.priorityClassName }}
+ {{- end }}
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ env:
+ {{ include "swh.secrets.environment.inline" (dict "configurationRef" .configuration.objstorageConfigurationRef
+ "Values" .Values) | nindent 10 }}
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: {{ .serviceType }}
+ resources:
+ requests:
+ memory: {{ .configuration.requestedMemory | default "512Mi" }}
+ cpu: {{ .configuration.requestedCpu | default "500m" }}
+ {{- if or .configuration.limitedMemory .configuration.limitedCpu }}
+ limits:
+ {{- if .configuration.limitedMemory }}
+ memory: {{ .configuration.limitedMemory }}
+ {{- end }}
+ {{- if .configuration.limitedCpu }}
+ cpu: {{ .configuration.limitedCpu }}
+ {{- end }}
+ {{- end }}
+ image: {{ .Values.swh_objstorage_image }}:{{ .Values.swh_objstorage_image_version }}
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: {{ .Values.objstorage.port | default .configuration.port }}
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ {{ if .configuration.gunicorn -}}
+ - name: THREADS
+ value: {{ .configuration.gunicorn.threads | default 5 | quote }}
+ - name: WORKERS
+ value: {{ .configuration.gunicorn.workers | default 2 | quote }}
+ - name: TIMEOUT
+ value: {{ .configuration.gunicorn.timeout | default 60 | quote }}
+ {{ end -}}
+ - name: STATSD_HOST
+ value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ - name: STATSD_PORT
+ value: {{ .Values.statsdPort | default "9125" | quote }}
+ - name: LOG_LEVEL
+ value: {{ .configuration.logLevel | default "INFO" | quote }}
+ {{- if .Values.objstorage.sentry.enabled }}
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: {{ .Values.sentry.environment }}
+ - name: SWH_MAIN_PACKAGE
+ value: swh.objstorage
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.objstorage.sentry.secretKeyRef }}
+ key: {{ .Values.objstorage.sentry.secretKeyName }}
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ {{- end }}
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: {{ .serviceType }}-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+{{- end -}}
diff --git a/swh/templates/objstorage/autoscaling.yaml b/swh/templates/objstorage/autoscaling.yaml
new file mode 100644
index 000000000..f1f63085c
--- /dev/null
+++ b/swh/templates/objstorage/autoscaling.yaml
@@ -0,0 +1,10 @@
+{{ if .Values.objstorage.enabled -}}
+{{ range $objstorage_type, $deployment_config := .Values.objstorage.deployments }}
+{{- if and (or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled"))
+ (hasKey $deployment_config "autoScale") -}}
+{{- include "swh.autoscale" (dict "serviceType" ( print "objstorage-" $objstorage_type )
+ "configuration" $deployment_config
+ "Values" $.Values) -}}
+{{- end -}}
+{{ end -}}
+{{- end -}}
diff --git a/swh/templates/objstorage/configmap.yaml b/swh/templates/objstorage/configmap.yaml
new file mode 100644
index 000000000..b4e44a787
--- /dev/null
+++ b/swh/templates/objstorage/configmap.yaml
@@ -0,0 +1,9 @@
+{{ if .Values.objstorage.enabled -}}
+{{ range $objstorage_type, $deployment_config := .Values.objstorage.deployments }}
+{{- if or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled") -}}
+{{ include "swh.objstorage.configmap" (dict "serviceType" (print "objstorage-" $objstorage_type)
+ "configuration" $deployment_config
+ "Values" $.Values) }}
+{{- end -}}
+{{ end -}}
+{{- end -}}
diff --git a/swh/templates/objstorage/deployment.yaml b/swh/templates/objstorage/deployment.yaml
new file mode 100644
index 000000000..8c940a617
--- /dev/null
+++ b/swh/templates/objstorage/deployment.yaml
@@ -0,0 +1,11 @@
+{{ if .Values.objstorage.enabled -}}
+{{ range $objstorage_type, $deployment_config := .Values.objstorage.deployments }}
+{{- if or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled") -}}
+{{- include "swh.objstorage.deployment"
+ (dict "serviceType" ( print "objstorage-" $objstorage_type )
+ "configuration" $deployment_config
+ "Values" $.Values) -}}
+{{- end -}}
+{{ end -}}
+{{- end -}}
+
diff --git a/swh/templates/objstorage/ingress.yaml b/swh/templates/objstorage/ingress.yaml
new file mode 100644
index 000000000..1fc0a3e90
--- /dev/null
+++ b/swh/templates/objstorage/ingress.yaml
@@ -0,0 +1,11 @@
+{{ if .Values.objstorage.enabled -}}
+{{ range $objstorage_type, $deployment_config := .Values.objstorage.deployments }}
+{{- if and (or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled"))
+ (and (hasKey $deployment_config "ingress") $deployment_config.ingress.enabled) -}}
+{{- include "swh.ingress" (dict "serviceType" ( print "objstorage-" $objstorage_type )
+ "configuration" $deployment_config
+ "Values" $.Values) -}}
+{{- end -}}
+{{ end -}}
+{{- end -}}
+
diff --git a/swh/templates/objstorage/service.yaml b/swh/templates/objstorage/service.yaml
new file mode 100644
index 000000000..a1edeef70
--- /dev/null
+++ b/swh/templates/objstorage/service.yaml
@@ -0,0 +1,12 @@
+{{ if .Values.objstorage.enabled -}}
+{{ range $objstorage_type, $deployment_config := .Values.objstorage.deployments }}
+{{- if or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled") -}}
+{{- if not (hasKey $deployment_config "port") }}
+{{- $deployment_config := set $deployment_config "port" $.Values.objstorage.port -}}
+{{- end -}}
+{{- include "swh.service" (dict "serviceType" ( print "objstorage-" $objstorage_type )
+ "configuration" $deployment_config
+ "Values" $.Values) -}}
+{{- end -}}
+{{ end -}}
+{{- end -}}
diff --git a/swh/values.yaml b/swh/values.yaml
index 0592a4029..5cfe6470c 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -135,6 +135,27 @@ sentry:
# noopObjectStorage:
# cls: noop
+# readWriteObjstorageConfiguration:
+# cls: pathslicing
+# root: "/srv/softwareheritage/objects"
+# slicing: 0:1/1:5
+# client_max_size: 1073741824
+
+# readOnlyObjstorageConfiguration:
+# cls: multiplexer
+# objstorages:
+# storage_conf:
+# cls: remote
+# url: http://banco.internal.softwareheritage.org:5003/
+# filters_conf:
+# - type: readonly
+# - cls: filtered
+# storage_conf:
+# cls: remote
+# url: http://saam.internal.softwareheritage.org:5003/
+# filters_conf:
+# - type: readonly
+
# Typical storage configuration
# Use references to other entries to assemble
# a complete storage configuration
@@ -1206,6 +1227,67 @@ deposit:
# authentication_provider: keycloak
# cache_uri: 127.0.0.1:11211
+objstorage:
+ enabled: false
+ port: 5003
+ priorityClassName: frontend-rpc
+# affinity:
+# nodeAffinity:
+# requiredDuringSchedulingIgnoredDuringExecution:
+# nodeSelectorTerms:
+# - matchExpressions:
+# - key: "swh/objstorage"
+# operator: In
+# values:
+# - "true"
+ sentry:
+ enabled: false
+ # name of the secret containing the $secretKeyName value
+ # it defines the sentry token, host and projet to access
+ # like https://token@sentry.host/id
+ secretKeyRef: common-secrets
+ secretKeyName: objstorage-sentry-dsn
+
+ deployments:
+ # Each key will be an objstorage instance to be deployed
+ # The mandatory objstorageConfigurationRef key should target a dict
+ # with the specific objstorage configuration
+ read-write:
+ enabled: false
+ # # rpc port can be changed per deployment
+ # port: 5003
+ # logLevel: INFO
+ # requestedCpu: 50m
+ # requestedMemory: 100Mi
+ # gunicorn:
+ # threads: 5
+ # workers: 2
+ # timeout: 60
+ # autoScaling:
+ # minReplicaCount: 2
+ # maxReplicaCount: 10
+ # cpuPercentageUsage: 100
+ # # mandatory
+ # # check readWriteStorageConfiguration example to configure your storage
+ objstorageConfigurationRef: readWriteObjstorageConfiguration
+ # Deploy an ingress to access the objstorage
+ # hosts:
+ # - myobjstorage.local
+ ingress:
+ enabled: false
+ # mandatory if ingress is enabled
+ # the hostname on which the objstorage must be reachable
+ # Optional: the ingress classname to use
+ # className: nginx
+ endpoints:
+ default:
+ paths:
+ - path: /
+
+ read-only:
+ enabled: false
+ objstorageConfigurationRef: readOnlyObjstorageConfiguration
+
podPriority:
enabled: false
priorities:
diff --git a/swh/values/default.yaml b/swh/values/default.yaml
index d8fdfb670..10a746e2c 100644
--- a/swh/values/default.yaml
+++ b/swh/values/default.yaml
@@ -263,5 +263,18 @@ toolbox:
values:
- "true"
+objstorage:
+ sentry:
+ enabled: true
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/objstorage
+ operator: In
+ values:
+ - "true"
+
statsd_exporter:
enabled: true
--
GitLab