From e6e821c67cffd761e02dba7cf81ddaac0fdab355 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Wed, 4 Oct 2023 15:56:28 +0200
Subject: [PATCH] vault: Add template to expose a rpc service with or without
ingress
Refs. swh/infra/sysadm-environment#4780
---
.../swh-vault-postgresql-secret.yaml | 9 ++
swh/fake-secrets/vault-sentry-dsn-secret.yaml | 9 ++
swh/templates/vault/rpc-autoscale.yaml | 28 ++++
swh/templates/vault/rpc-configmap.yaml | 21 +++
swh/templates/vault/rpc-deployment.yaml | 133 ++++++++++++++++++
swh/templates/vault/rpc-ingress.yaml | 37 +++++
swh/templates/vault/rpc-service.yaml | 16 +++
swh/values.yaml | 89 +++++++++++-
swh/values/minikube.yaml | 87 ++++++++++++
values-swh-application-versions.yaml | 2 +
10 files changed, 430 insertions(+), 1 deletion(-)
create mode 100644 swh/fake-secrets/swh-vault-postgresql-secret.yaml
create mode 100644 swh/fake-secrets/vault-sentry-dsn-secret.yaml
create mode 100644 swh/templates/vault/rpc-autoscale.yaml
create mode 100644 swh/templates/vault/rpc-configmap.yaml
create mode 100644 swh/templates/vault/rpc-deployment.yaml
create mode 100644 swh/templates/vault/rpc-ingress.yaml
create mode 100644 swh/templates/vault/rpc-service.yaml
diff --git a/swh/fake-secrets/swh-vault-postgresql-secret.yaml b/swh/fake-secrets/swh-vault-postgresql-secret.yaml
new file mode 100644
index 000000000..07af0b1b0
--- /dev/null
+++ b/swh/fake-secrets/swh-vault-postgresql-secret.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: swh-postgresql-vault-secret
+type: Opaque
+stringData:
+ postgres-swh-vault-password: fake-swh-vault-password
+
diff --git a/swh/fake-secrets/vault-sentry-dsn-secret.yaml b/swh/fake-secrets/vault-sentry-dsn-secret.yaml
new file mode 100644
index 000000000..438a639f5
--- /dev/null
+++ b/swh/fake-secrets/vault-sentry-dsn-secret.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: vault-sentry-secrets
+type: Opaque
+stringData:
+ sentry-dsn: https://fake-id:fake-pass@sentry.s.o/12
+
diff --git a/swh/templates/vault/rpc-autoscale.yaml b/swh/templates/vault/rpc-autoscale.yaml
new file mode 100644
index 000000000..01e44afae
--- /dev/null
+++ b/swh/templates/vault/rpc-autoscale.yaml
@@ -0,0 +1,28 @@
+{{- if and .Values.vault.enabled .Values.vault.autoScaling -}}
+{{- with .Values.vault.autoScaling -}}
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ namespace: {{ $.Values.namespace }}
+ name: vault-rpc
+ labels:
+ app: vault-rpc
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: vault-rpc
+ minReplicas: {{ .minReplicaCount | default 2 }}
+ maxReplicas: {{ .maxReplicaCount | default 10 }}
+ metrics:
+ {{- if .cpuPercentageUsage }}
+ - type: Resource
+ resource:
+ name: cpu
+ target:
+ type: Utilization
+ averageUtilization: {{ .cpuPercentageUsage }}
+ {{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/swh/templates/vault/rpc-configmap.yaml b/swh/templates/vault/rpc-configmap.yaml
new file mode 100644
index 000000000..f54a09180
--- /dev/null
+++ b/swh/templates/vault/rpc-configmap.yaml
@@ -0,0 +1,21 @@
+{{ if .Values.vault.enabled -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: {{ .Values.namespace }}
+ name: vault-rpc-configuration-template
+data:
+ config.yml.template: |
+ {{- include "swh.postgresql" (dict "serviceType" "vault"
+ "configurationRef" .Values.vault.vaultConfigurationRef
+ "Values" .Values) | nindent 4 -}}
+ {{- include "swh.service.fromYaml" (dict "service" "scheduler"
+ "configurationRef" .Values.vault.schedulerConfigurationRef
+ "Values" .Values) | nindent 4 }}
+
+ {{- if .Values.vault.extraConfig -}}
+ {{ toYaml .Values.vault.extraConfig | nindent 4 }}
+ {{- end }}
+
+{{- end -}}
diff --git a/swh/templates/vault/rpc-deployment.yaml b/swh/templates/vault/rpc-deployment.yaml
new file mode 100644
index 000000000..8e3c56c57
--- /dev/null
+++ b/swh/templates/vault/rpc-deployment.yaml
@@ -0,0 +1,133 @@
+{{ if .Values.vault.enabled -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: {{ $.Values.namespace }}
+ name: vault-rpc
+ labels:
+ app: vault-rpc
+spec:
+ revisionHistoryLimit: 2
+ {{ if .Values.vault.replicas -}}
+ replicas: {{ .Values.vault.replicas }}
+ {{ end -}}
+ selector:
+ matchLabels:
+ app: vault-rpc
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: vault-rpc
+ annotations:
+ checksum/config: {{ include (print $.Template.BasePath "/vault/rpc-configmap.yaml") . | sha256sum }}
+ spec:
+ {{- if .Values.vault.affinity }}
+ affinity:
+ {{- toYaml .Values.vault.affinity | nindent 8 }}
+ {{- end }}
+ {{- if and .Values.podPriority.enabled .Values.vault.priorityClassName }}
+ priorityClassName: {{ .Values.namespace }}-{{ .Values.vault.priorityClassName }}
+ {{ end }}
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ env:
+ {{ include "swh.secrets.environment" (dict "configurationRef" .Values.vault.vaultConfigurationRef
+ "Values" .Values) | nindent 12 }}
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: vault-rpc
+ resources:
+ requests:
+ memory: {{ .Values.vault.requestedMemory | default "512Mi" }}
+ cpu: {{ .Values.vault.requestedCpu | default "500m" }}
+ {{- if or .Values.vault.limitedMemory .Values.vault.limitedCpu }}
+ limits:
+ {{- if .Values.vault.limitedMemory }}
+ memory: {{ .Values.vault.limitedMemory }}
+ {{- end }}
+ {{- if .Values.vault.limitedCpu }}
+ cpu: {{ .Values.vault.limitedCpu }}
+ {{- end }}
+ {{- end }}
+ image: {{ .Values.swh_vault_image }}:{{ .Values.swh_vault_image_version }}
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5005
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ {{ if .Values.vault.gunicorn -}}
+ - name: THREADS
+ value: {{ .Values.vault.gunicorn.threads | default 5 | quote }}
+ - name: WORKERS
+ value: {{ .Values.vault.gunicorn.workers | default 2 | quote }}
+ - name: TIMEOUT
+ value: {{ .Values.vault.gunicorn.timeout | default 60 | quote }}
+ {{ end -}}
+ - name: STATSD_HOST
+ value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ - name: STATSD_PORT
+ value: {{ .Values.statsdPort | default "9125" | quote }}
+ - name: LOG_LEVEL
+ value: {{ .Values.vault.logLevel | default "INFO" }}
+ {{- if .Values.vault.sentry.enabled }}
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: {{ .Values.sentry.environment }}
+ - name: SWH_MAIN_PACKAGE
+ value: swh.vault
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.vault.sentry.secretKeyRef }}
+ key: {{ .Values.vault.sentry.secretKeyName }}
+ # if the setting doesn't exist, sentry issue pushes will be disabled
+ optional: false
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ {{- end }}
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: vault-rpc-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+{{- end -}}
diff --git a/swh/templates/vault/rpc-ingress.yaml b/swh/templates/vault/rpc-ingress.yaml
new file mode 100644
index 000000000..0de2e9e1d
--- /dev/null
+++ b/swh/templates/vault/rpc-ingress.yaml
@@ -0,0 +1,37 @@
+{{- if and .Values.vault.enabled .Values.vault.ingress.enabled -}}
+{{- $defaultWhitelistSourceRange := .Values.vault.ingress.whitelistSourceRange | default list -}}
+{{- range $ingress_definition, $ingress_config := .Values.vault.ingress.ingressDefinitions -}}
+{{- $extraWhitelistSourceRange := get $ingress_config "extraWhitelistSourceRange" | default list -}}
+{{- $whitelistSourceRange := join "," (concat $defaultWhitelistSourceRange $extraWhitelistSourceRange) | default "" -}}
+{{- $paths := get $ingress_config "paths" -}}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: {{ $.Values.namespace }}
+ name: vault-rpc-ingress-{{ $ingress_definition }}
+ annotations:
+ {{- if $whitelistSourceRange }}
+ nginx.ingress.kubernetes.io/whitelist-source-range: {{ $whitelistSourceRange }}
+ {{- end }}
+{{ toYaml $.Values.vault.ingress.extraAnnotations | indent 4 }}
+
+spec:
+ {{- if $.Values.vault.ingress.className }}
+ ingressClassName: {{ $.Values.vault.ingress.className }}
+ {{- end }}
+ rules:
+ - host: {{ $.Values.vault.ingress.host }}
+ http:
+ paths:
+{{- range $path := $paths }}
+ - path: {{ $path }}
+ pathType: Prefix
+ backend:
+ service:
+ name: vault-rpc
+ port:
+ number: 5005
+{{ end }}
+{{ end }}
+{{ end }}
diff --git a/swh/templates/vault/rpc-service.yaml b/swh/templates/vault/rpc-service.yaml
new file mode 100644
index 000000000..f9b403969
--- /dev/null
+++ b/swh/templates/vault/rpc-service.yaml
@@ -0,0 +1,16 @@
+{{ if .Values.vault.enabled -}}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: vault-rpc
+ namespace: {{ .Values.namespace }}
+spec:
+ type: ClusterIP
+ selector:
+ app: vault-rpc
+ ports:
+ - port: 5005
+ targetPort: 5005
+ name: rpc
+{{ end }}
diff --git a/swh/values.yaml b/swh/values.yaml
index c142b3d85..4a6e6ad5d 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -143,6 +143,18 @@ sentry:
# secretKeyRef: swh-postgresql-common-secret
# secretKeyName: postgres-swh-password
+# vaultPostgresqlConfiguration:
+# cls: postgresql
+# host: mydatabasese.server
+# port: '5432'
+# db: swh-vault
+# user: swh-vault
+# password: ${POSTGRESQL_PASSWORD}
+# secrets:
+# POSTGRESQL_PASSWORD:
+# secretKeyRef: swh-postgresql-vault-secret
+# secretKeyName: postgres-swh-vault-password
+
# remoteSchedulerConfiguration:
# cls: remote
# url: http://scheduler.internal.staging.swh.network
@@ -642,6 +654,81 @@ toolbox:
# scrubber:
# scrubberConfigurationRef: postgresScrubberConfiguration
+vault:
+ enabled: false
+ priorityClassName: frontend-rpc
+ sentry:
+ enabled: false
+ secretKeyRef: scheduler-sentry-secrets
+ secretKeyName: sentry-dsn
+ # schedulerConfigurationRef: remoteSchedulerConfiguration
+ # vaultConfigurationRef: vaultPostgresqlConfiguration
+ # logLevel: INFO
+ # extraConfig:
+ # smtp:
+ # host: localhost
+ # port: 25
+ # cache:
+ # cls: pathslicing
+ # root: "/srv/softwareheritage/vault_cache"
+ # slicing: 0:1/1:5
+ # objstorage:
+ # cls: filtered
+ # storage_conf:
+ # cls: remote
+ # url: http://storage1.internal.staging.swh.network:5003/
+ # filters_conf:
+ # - type: readonly
+ # storage:
+ # cls: retry
+ # storage:
+ # cls: remote
+ # url: http://storage1.internal.staging.swh.network:5002/
+ # The scheduler instance to use for rpc must be a postgresql instance
+ # schedulerConfigurationRef: postgresqlSchedulerConfiguration
+ # replicas: 2
+ # gunicorn:
+ # threads: 5
+ # workers: 2
+ # timeout: 60
+ # RPC services may have different profiles than the rest so they need their specific
+ # setup
+ # requestedMemory: 512Mi
+ # requestedCpu: 500m
+ # limitedMemory: 512Mi
+ # limitedCpu: 500m
+ # autoScaling:
+ # minReplicaCount: 2
+ # maxReplicaCount: 10
+ # cpuPercentageUsage: 100
+ # ingress:
+ # enabled: false
+ # # Optional: the ingress classname to use
+ # # className: nginx
+ # # mandatory if ingress is enabled
+ # # the hostname on which the storage must be reachable
+ # # host: myscheduler.localdomain
+ # extraAnnotations:
+ # nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
+ # nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
+ # nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+ # nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
+ # nginx.ingress.kubernetes.io/proxy-body-size: "4G"
+ # # Default allowed ip ranges that can be extended per ingress definitions paths
+ # # Default allowed ip ranges that can be extended per ingress definitions paths
+ # # - xxx.xxx.xxx.xxx/24
+ # # ingressDefinitions:
+ # # default:
+ # # paths:
+ # # - /
+ # # read-only:
+ # # paths:
+ # # - /scheduler_metrics/get
+ # # - /visit_stats/get
+ # # # Extra allowed ip range for the paths above
+ # # extraWhitelistSourceRange:
+ # # - yyy.yyy.yyy.yyy/24
+
scheduler:
enabled: false
priorityClassName: frontend-rpc-workload
@@ -714,7 +801,7 @@ scheduler:
# ingressDefinitions:
# default:
# paths:
- # /
+ # - /
# read-only:
# paths:
# - /scheduler_metrics/get
diff --git a/swh/values/minikube.yaml b/swh/values/minikube.yaml
index a55544fdc..3a83fcd88 100644
--- a/swh/values/minikube.yaml
+++ b/swh/values/minikube.yaml
@@ -69,6 +69,18 @@ fakePostgresqlWebConfiguration:
secretKeyRef: swh-postgresql-web-secret
secretKeyName: postgres-swh-web-password
+fakePostgresqlVaultConfiguration:
+ cls: postgresql
+ host: db1.i.s.s.n
+ port: '5432'
+ db: swh-vault
+ user: swh-vault
+ pass: ${POSTGRESQL_PASSWORD}
+ secrets:
+ POSTGRESQL_PASSWORD:
+ secretKeyRef: swh-postgresql-vault-secret
+ secretKeyName: postgres-swh-vault-password
+
fakeRemoteStorageConfiguration:
cls: remote
host: http://fake-storage.i.s.s.n:8002
@@ -462,3 +474,78 @@ memcached:
podPriority:
enabled: true
+
+vault:
+ enabled: false
+ priorityClassName: frontend-rpc
+ sentry:
+ enabled: false
+ secretKeyRef: vault-sentry-secrets
+ secretKeyName: sentry-dsn
+ logLevel: INFO
+ schedulerConfigurationRef: fakeRemoteSchedulerConfiguration
+ vaultConfigurationRef: fakePostgresqlVaultConfiguration
+ extraConfig:
+ smtp:
+ host: localhost
+ port: 25
+ cache:
+ cls: pathslicing
+ root: "/srv/softwareheritage/vault_cache"
+ slicing: 0:1/1:5
+ objstorage:
+ cls: filtered
+ storage_conf:
+ cls: remote
+ url: http://storage1.i.s.s.n:5003/
+ filters_conf:
+ - type: readonly
+ storage:
+ cls: retry
+ storage:
+ cls: remote
+ url: http://storage1.i.s.s.n:5002/
+
+ # The scheduler instance to use for rpc must be a postgresql instance
+ # schedulerConfigurationRef: postgresqlSchedulerConfiguration
+ # replicas: 2
+ gunicorn:
+ threads: 5
+ workers: 2
+ timeout: 60
+ # RPC services may have different profiles than the rest so they need their specific
+ # setup
+ requestedMemory: 512Mi
+ requestedCpu: 500m
+ limitedMemory: 512Mi
+ limitedCpu: 500m
+ autoScaling:
+ minReplicaCount: 2
+ maxReplicaCount: 10
+ cpuPercentageUsage: 100
+ ingress:
+ enabled: true
+ # Optional: the ingress classname to use
+ # className: nginx
+ # mandatory if ingress is enabled
+ # the hostname on which the storage must be reachable
+ host: myscheduler.localdomain
+ extraAnnotations:
+ nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
+ nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
+ nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+ nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
+ nginx.ingress.kubernetes.io/proxy-body-size: "4G"
+ # Default allowed ip ranges that can be extended per ingress definitions paths
+ # Default allowed ip ranges that can be extended per ingress definitions paths
+ # - xxx.xxx.xxx.xxx/24
+ ingressDefinitions:
+ default:
+ paths:
+ - /
+ read-only:
+ paths:
+ - /vault_metrics/get
+ # Extra allowed ip range for the paths above
+ extraWhitelistSourceRange:
+ - 192.168.100.0/24
diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml
index 9d30a123f..edb785735 100644
--- a/values-swh-application-versions.yaml
+++ b/values-swh-application-versions.yaml
@@ -34,5 +34,7 @@ swh_toolbox_image: container-registry.softwareheritage.org/swh/infra/swh-apps/to
swh_toolbox_image_version: '20231006.2'
swh_vault_cookers_image: container-registry.softwareheritage.org/swh/infra/swh-apps/vault_cookers
swh_vault_cookers_image_version: '20231006.1'
+swh_vault_image: container-registry.softwareheritage.org/swh/infra/swh-apps/vault
+swh_vault_image_version: '20231004.2'
swh_web_image: container-registry.softwareheritage.org/swh/infra/swh-apps/web
swh_web_image_version: '20231006.1'
--
GitLab