vault: Add template to expose a rpc service with or without ingress

Refs. swh/infra/sysadm-environment#4780

swh/fake-secrets/swh-vault-postgresql-secret.yaml

+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: swh-postgresql-vault-secret
+type: Opaque
+stringData:
+  postgres-swh-vault-password: fake-swh-vault-password
+

swh/fake-secrets/vault-sentry-dsn-secret.yaml

+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vault-sentry-secrets
+type: Opaque
+stringData:
+  sentry-dsn: https://fake-id:fake-pass@sentry.s.o/12
+

swh/templates/vault/rpc-autoscale.yaml

+{{- if and .Values.vault.enabled .Values.vault.autoScaling -}}
+{{- with .Values.vault.autoScaling -}}
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  namespace: {{ $.Values.namespace }}
+  name: vault-rpc
+  labels:
+    app: vault-rpc
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: vault-rpc
+  minReplicas: {{ .minReplicaCount | default 2 }}
+  maxReplicas: {{ .maxReplicaCount | default 10 }}
+  metrics:
+  {{- if .cpuPercentageUsage }}
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: {{ .cpuPercentageUsage }}
+  {{- end -}}
+{{- end -}}
+{{- end -}}

swh/templates/vault/rpc-configmap.yaml

+{{ if .Values.vault.enabled -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: {{ .Values.namespace }}
+  name: vault-rpc-configuration-template
+data:
+  config.yml.template: |
+    {{- include "swh.postgresql" (dict "serviceType" "vault"
+                                      "configurationRef" .Values.vault.vaultConfigurationRef
+                                       "Values" .Values) | nindent 4 -}}
+    {{- include "swh.service.fromYaml" (dict "service" "scheduler"
+                                             "configurationRef" .Values.vault.schedulerConfigurationRef
+                                             "Values" .Values) | nindent 4 }}
+
+    {{- if .Values.vault.extraConfig -}}
+    {{ toYaml .Values.vault.extraConfig | nindent 4 }}
+    {{- end }}
+
+{{- end -}}

swh/templates/vault/rpc-deployment.yaml

+{{ if .Values.vault.enabled -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: {{ $.Values.namespace }}
+  name: vault-rpc
+  labels:
+    app: vault-rpc
+spec:
+  revisionHistoryLimit: 2
+  {{ if .Values.vault.replicas -}}
+  replicas: {{ .Values.vault.replicas }}
+  {{ end -}}
+  selector:
+    matchLabels:
+      app: vault-rpc
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: vault-rpc
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/vault/rpc-configmap.yaml") . | sha256sum }}
+    spec:
+      {{- if .Values.vault.affinity }}
+      affinity:
+        {{- toYaml .Values.vault.affinity | nindent 8 }}
+      {{- end }}
+      {{- if and .Values.podPriority.enabled .Values.vault.priorityClassName }}
+      priorityClassName: {{ .Values.namespace }}-{{ .Values.vault.priorityClassName }}
+      {{ end }}
+      initContainers:
+        - name: prepare-configuration
+          image: debian:bullseye
+          imagePullPolicy: IfNotPresent
+          command:
+          - /bin/bash
+          args:
+          - -c
+          - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+          env:
+            {{ include "swh.secrets.environment" (dict "configurationRef" .Values.vault.vaultConfigurationRef
+                                                       "Values" .Values) | nindent 12 }}
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+      containers:
+        - name: vault-rpc
+          resources:
+            requests:
+              memory: {{ .Values.vault.requestedMemory | default "512Mi" }}
+              cpu: {{ .Values.vault.requestedCpu | default "500m" }}
+          {{- if or .Values.vault.limitedMemory .Values.vault.limitedCpu }}
+            limits:
+            {{- if .Values.vault.limitedMemory }}
+              memory: {{ .Values.vault.limitedMemory }}
+            {{- end }}
+            {{- if .Values.vault.limitedCpu }}
+              cpu: {{ .Values.vault.limitedCpu }}
+            {{- end }}
+        {{- end }}
+          image: {{ .Values.swh_vault_image }}:{{ .Values.swh_vault_image_version }}
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 5005
+              name: rpc
+          readinessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 15
+            failureThreshold: 30
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 10
+            periodSeconds: 5
+          command:
+            - /bin/bash
+          args:
+            - -c
+            - /opt/swh/entrypoint.sh
+          env:
+            {{ if .Values.vault.gunicorn -}}
+            - name: THREADS
+              value: {{ .Values.vault.gunicorn.threads | default 5 | quote }}
+            - name: WORKERS
+              value: {{ .Values.vault.gunicorn.workers | default 2 | quote }}
+            - name: TIMEOUT
+              value: {{ .Values.vault.gunicorn.timeout | default 60 | quote }}
+            {{ end -}}
+            - name: STATSD_HOST
+              value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+            - name: STATSD_PORT
+              value: {{ .Values.statsdPort | default "9125" | quote }}
+            - name: LOG_LEVEL
+              value: {{ .Values.vault.logLevel | default "INFO" }}
+          {{- if .Values.vault.sentry.enabled }}
+            - name: SWH_SENTRY_ENVIRONMENT
+              value: {{ .Values.sentry.environment }}
+            - name: SWH_MAIN_PACKAGE
+              value: swh.vault
+            - name: SWH_SENTRY_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.vault.sentry.secretKeyRef }}
+                  key: {{ .Values.vault.sentry.secretKeyName }}
+                  # if the setting doesn't exist, sentry issue pushes will be disabled
+                  optional: false
+            - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+              value: "true"
+          {{- end }}
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+      volumes:
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: vault-rpc-configuration-template
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+{{- end -}}

swh/templates/vault/rpc-ingress.yaml

+{{- if and .Values.vault.enabled .Values.vault.ingress.enabled -}}
+{{- $defaultWhitelistSourceRange := .Values.vault.ingress.whitelistSourceRange | default list -}}
+{{- range $ingress_definition, $ingress_config := .Values.vault.ingress.ingressDefinitions -}}
+{{- $extraWhitelistSourceRange := get $ingress_config "extraWhitelistSourceRange" | default list -}}
+{{- $whitelistSourceRange := join "," (concat $defaultWhitelistSourceRange $extraWhitelistSourceRange) | default "" -}}
+{{- $paths := get $ingress_config "paths" -}}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: {{ $.Values.namespace }}
+  name: vault-rpc-ingress-{{ $ingress_definition }}
+  annotations:
+  {{- if $whitelistSourceRange }}
+    nginx.ingress.kubernetes.io/whitelist-source-range: {{ $whitelistSourceRange }}
+  {{- end }}
+{{ toYaml $.Values.vault.ingress.extraAnnotations | indent 4 }}
+
+spec:
+  {{- if $.Values.vault.ingress.className }}
+  ingressClassName: {{ $.Values.vault.ingress.className }}
+  {{- end }}
+  rules:
+  - host: {{ $.Values.vault.ingress.host }}
+    http:
+      paths:
+{{- range $path := $paths }}
+      - path: {{ $path }}
+        pathType: Prefix
+        backend:
+          service:
+            name: vault-rpc
+            port:
+              number: 5005
+{{ end }}
+{{ end }}
+{{ end }}

swh/templates/vault/rpc-service.yaml

+{{ if .Values.vault.enabled -}}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: vault-rpc
+  namespace: {{ .Values.namespace }}
+spec:
+  type: ClusterIP
+  selector:
+    app: vault-rpc
+  ports:
+    - port: 5005
+      targetPort: 5005
+      name: rpc
+{{ end }}

swh/values.yaml

@@ -143,6 +143,18 @@ sentry:
 #       secretKeyRef: swh-postgresql-common-secret
 #       secretKeyName: postgres-swh-password
 
+# vaultPostgresqlConfiguration:
+#   cls: postgresql
+#   host: mydatabasese.server
+#   port: '5432'
+#   db: swh-vault
+#   user: swh-vault
+#   password: ${POSTGRESQL_PASSWORD}
+#   secrets:
+#     POSTGRESQL_PASSWORD:
+#       secretKeyRef: swh-postgresql-vault-secret
+#       secretKeyName: postgres-swh-vault-password
+
 # remoteSchedulerConfiguration:
 #   cls: remote
 #   url: http://scheduler.internal.staging.swh.network
@@ -642,6 +654,81 @@ toolbox:
   #   scrubber:
   #     scrubberConfigurationRef: postgresScrubberConfiguration
 
+vault:
+  enabled: false
+  priorityClassName: frontend-rpc
+  sentry:
+    enabled: false
+    secretKeyRef: scheduler-sentry-secrets
+    secretKeyName: sentry-dsn
+  # schedulerConfigurationRef: remoteSchedulerConfiguration
+  # vaultConfigurationRef: vaultPostgresqlConfiguration
+  # logLevel: INFO
+  # extraConfig:
+  #   smtp:
+  #     host: localhost
+  #     port: 25
+  #   cache:
+  #     cls: pathslicing
+  #     root: "/srv/softwareheritage/vault_cache"
+  #     slicing: 0:1/1:5
+  #   objstorage:
+  #     cls: filtered
+  #     storage_conf:
+  #       cls: remote
+  #       url: http://storage1.internal.staging.swh.network:5003/
+  #     filters_conf:
+  #     - type: readonly
+  #   storage:
+  #     cls: retry
+  #     storage:
+  #       cls: remote
+  #       url: http://storage1.internal.staging.swh.network:5002/
+  # The scheduler instance to use for rpc must be a postgresql instance
+  # schedulerConfigurationRef: postgresqlSchedulerConfiguration
+  # replicas: 2
+  # gunicorn:
+  #   threads: 5
+  #   workers: 2
+  #   timeout: 60
+  # RPC services may have different profiles than the rest so they need their specific
+  # setup
+  # requestedMemory: 512Mi
+  # requestedCpu: 500m
+  # limitedMemory: 512Mi
+  # limitedCpu: 500m
+  # autoScaling:
+  #   minReplicaCount: 2
+  #   maxReplicaCount: 10
+  #   cpuPercentageUsage: 100
+  # ingress:
+  #   enabled: false
+  #   # Optional: the ingress classname to use
+  #   # className: nginx
+  #   # mandatory if ingress is enabled
+  #   # the hostname on which the storage must be reachable
+  #   # host: myscheduler.localdomain
+  #   extraAnnotations:
+  #     nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
+  #     nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
+  #     nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+  #     nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
+  #     nginx.ingress.kubernetes.io/proxy-body-size: "4G"
+  #   # Default allowed ip ranges that can be extended per ingress definitions paths
+  #   # Default allowed ip ranges that can be extended per ingress definitions paths
+  #   #   - xxx.xxx.xxx.xxx/24
+  #   # ingressDefinitions:
+  #   #   default:
+  #   #     paths:
+  #   #       - /
+  #   #   read-only:
+  #   #     paths:
+  #   #       - /scheduler_metrics/get
+  #   #       - /visit_stats/get
+  #   #     # Extra allowed ip range for the paths above
+  #   #     extraWhitelistSourceRange:
+  #   #       - yyy.yyy.yyy.yyy/24
+
 scheduler:
   enabled: false
   priorityClassName: frontend-rpc-workload
@@ -714,7 +801,7 @@ scheduler:
       # ingressDefinitions:
       #   default:
       #     paths:
-      #       /
+      #       - /
       #   read-only:
       #     paths:
       #       - /scheduler_metrics/get

swh/values/minikube.yaml

@@ -69,6 +69,18 @@ fakePostgresqlWebConfiguration:
       secretKeyRef: swh-postgresql-web-secret
       secretKeyName: postgres-swh-web-password
 
+fakePostgresqlVaultConfiguration:
+  cls: postgresql
+  host: db1.i.s.s.n
+  port: '5432'
+  db: swh-vault
+  user: swh-vault
+  pass: ${POSTGRESQL_PASSWORD}
+  secrets:
+    POSTGRESQL_PASSWORD:
+      secretKeyRef: swh-postgresql-vault-secret
+      secretKeyName: postgres-swh-vault-password
+
 fakeRemoteStorageConfiguration:
   cls: remote
   host: http://fake-storage.i.s.s.n:8002
@@ -462,3 +474,78 @@ memcached:
 
 podPriority:
   enabled: true
+
+vault:
+  enabled: false
+  priorityClassName: frontend-rpc
+  sentry:
+    enabled: false
+    secretKeyRef: vault-sentry-secrets
+    secretKeyName: sentry-dsn
+  logLevel: INFO
+  schedulerConfigurationRef: fakeRemoteSchedulerConfiguration
+  vaultConfigurationRef: fakePostgresqlVaultConfiguration
+  extraConfig:
+    smtp:
+      host: localhost
+      port: 25
+    cache:
+      cls: pathslicing
+      root: "/srv/softwareheritage/vault_cache"
+      slicing: 0:1/1:5
+    objstorage:
+      cls: filtered
+      storage_conf:
+        cls: remote
+        url: http://storage1.i.s.s.n:5003/
+      filters_conf:
+      - type: readonly
+    storage:
+      cls: retry
+      storage:
+        cls: remote
+        url: http://storage1.i.s.s.n:5002/
+
+  # The scheduler instance to use for rpc must be a postgresql instance
+  # schedulerConfigurationRef: postgresqlSchedulerConfiguration
+  # replicas: 2
+  gunicorn:
+    threads: 5
+    workers: 2
+    timeout: 60
+  # RPC services may have different profiles than the rest so they need their specific
+  # setup
+  requestedMemory: 512Mi
+  requestedCpu: 500m
+  limitedMemory: 512Mi
+  limitedCpu: 500m
+  autoScaling:
+    minReplicaCount: 2
+    maxReplicaCount: 10
+    cpuPercentageUsage: 100
+  ingress:
+    enabled: true
+    # Optional: the ingress classname to use
+    # className: nginx
+    # mandatory if ingress is enabled
+    # the hostname on which the storage must be reachable
+    host: myscheduler.localdomain
+    extraAnnotations:
+      nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
+      nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
+      nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+      nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
+      nginx.ingress.kubernetes.io/proxy-body-size: "4G"
+    # Default allowed ip ranges that can be extended per ingress definitions paths
+    # Default allowed ip ranges that can be extended per ingress definitions paths
+    #   - xxx.xxx.xxx.xxx/24
+    ingressDefinitions:
+      default:
+        paths:
+          - /
+      read-only:
+        paths:
+          - /vault_metrics/get
+        # Extra allowed ip range for the paths above
+        extraWhitelistSourceRange:
+          - 192.168.100.0/24

values-swh-application-versions.yaml

@@ -34,5 +34,7 @@ swh_toolbox_image: container-registry.softwareheritage.org/swh/infra/swh-apps/to
 swh_toolbox_image_version: '20231006.2'
 swh_vault_cookers_image: container-registry.softwareheritage.org/swh/infra/swh-apps/vault_cookers
 swh_vault_cookers_image_version: '20231006.1'
+swh_vault_image: container-registry.softwareheritage.org/swh/infra/swh-apps/vault
+swh_vault_image_version: '20231004.2'
 swh_web_image: container-registry.softwareheritage.org/swh/infra/swh-apps/web
 swh_web_image_version: '20231006.1'