Add new chart for swh-loader-metadata

Refs. swh/infra/sysadm-environment#4794

swh/templates/loader-metadata/configmap-utils.yaml

+{{ if .Values.loader_metadata.enabled -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loader-metadata-utils
+  namespace: {{ .Values.namespace }}
+data:
+  pre-stop-idempotent.sh: |
+    #!/bin/bash
+
+    # pre-stop hook can be triggered multiple times but we want it to be applied only
+    # once so container can warm-shutdown properly.
+
+    # When celery receives multiple times the sigterm signal, this ends up doing an
+    # immediate shutdown which prevents long-standing tasks to finish properly.
+
+    set -ex
+
+    WITNESS_FILE=/tmp/already-stopped
+
+    # Seed awk with the number of nanoseconds since epoch
+    # and have it generate a number between 0 and 1
+    sleep $(date +%s%N | awk '{srand($1); print rand()}')
+
+    if [ ! -e $WITNESS_FILE ]; then
+      touch $WITNESS_FILE
+      # journal clients expect a SIGINT, not a SIGTERM
+      kill -INT 1
+    fi
+{{ end }}

swh/templates/loader-metadata/configmap.yaml

+{{ if .Values.loader_metadata.enabled -}}
+{{- $journalUser := .Values.loader_metadata.journalBrokers.user -}}
+{{- $consumerGroup := .Values.loader_metadata.consumerGroup -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loader-metadata-template
+  namespace: {{ .Values.namespace }}
+data:
+  config.yml.template: |
+    storage:
+      cls: pipeline
+      steps:
+      - cls: retry
+      - cls: remote
+        url: http://{{ .Values.loader_metadata.storage.host }}:{{ .Values.loader_metadata.storage.port }}/
+    scheduler:
+      cls: remote
+      url: http://{{ .Values.loader_metadata.scheduler.host }}:{{ .Values.loader_metadata.scheduler.port }}/
+    journal:
+      brokers: {{ toYaml .Values.loader_metadata.journalBrokers.hosts | nindent 8 }}
+      group_id: {{ $journalUser }}-{{ $consumerGroup }}
+      prefix: {{ .Values.loader_metadata.prefix }}
+      sasl.mechanism: SCRAM-SHA-512
+      security.protocol: SASL_SSL
+      sasl.username: {{ $journalUser }}
+      sasl.password: ${JOURNAL_PASSWORD}
+    metadata_fetcher_credentials:
+
+  init-container-entrypoint.sh: |
+    #!/bin/bash
+
+    set -e
+
+    CONFIG_FILE=/etc/swh/config.yml
+
+    # substitute environment variables when creating the default config.yml
+    eval echo \""$(</etc/swh/configuration-template/config.yml.template)"\" \
+      > $CONFIG_FILE
+
+    CREDS_PATH=/etc/credentials/metadata-fetcher/credentials
+    [ -f $CREDS_PATH ] && \
+      sed 's/^/  /g' $CREDS_PATH >> $CONFIG_FILE
+
+    exit 0
+{{ end }}

swh/templates/loader-metadata/deployment.yaml

+{{ if .Values.loader_metadata.enabled -}}
+{{- $configurationChecksum := include (print .Template.BasePath "/loader-metadata/configmap.yaml") . -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loader-metadata
+  namespace: {{ .Values.namespace }}
+  labels:
+    app: loader-metadata
+spec:
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: loader-metadata
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: loader-metadata
+      annotations:
+        # Force a rollout upgrade if the configuration changes
+        checksum/config: {{ $configurationChecksum | sha256sum }}
+    spec:
+      {{- if .Values.loader_metadata.affinity }}
+      affinity:
+        {{ toYaml .Values.loader_metadata.affinity | nindent 8 }}
+      {{- end }}
+      terminationGracePeriodSeconds: 3600
+      initContainers:
+        - name: prepare-configuration
+          image: debian:bullseye
+          imagePullPolicy: Always
+          env:
+          - name: JOURNAL_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: common-secrets
+                key: journal-password
+                optional: true
+          command:
+            - /entrypoint.sh
+          volumeMounts:
+          - name: configuration-template
+            mountPath: /entrypoint.sh
+            subPath: "init-container-entrypoint.sh"
+            readOnly: true
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+      containers:
+      - name: loader_metadata
+        image: {{ .Values.swh_loader_metadata_image }}:{{ .Values.swh_loader_metadata_image_version }}
+        imagePullPolicy: Always
+        command:
+          - /opt/swh/entrypoint.sh
+        resources:
+          requests:
+            memory: {{ .Values.loader_metadata.requestedMemory | default "512Mi" }}
+            cpu: {{ .Values.loader_metadata.requestedCpu | default "500m" }}
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/pre-stop.sh"]
+        env:
+        - name: STATSD_HOST
+          value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+        - name: STATSD_PORT
+          value: {{ .Values.statsdPort | default "9125" | quote }}
+        - name: LOGLEVEL
+          value: {{ .Values.loader_metadata.logLevel | default "INFO" | quote }}
+        - name: SWH_CONFIG_FILENAME
+          value: /etc/swh/config.yml
+        - name: SWH_SENTRY_ENVIRONMENT
+          value: {{ .Values.sentry.environment }}
+        - name: SWH_MAIN_PACKAGE
+          value: {{ .Values.loader_metadata.sentrySwhPackage }}
+        - name: SWH_SENTRY_DSN
+          valueFrom:
+            secretKeyRef:
+              name: common-secrets
+              key: loader-metadata-sentry-dsn
+              # 'name' secret must exist & include key "host"
+              optional: true
+        volumeMounts:
+          - name: loader-metadata-utils
+            mountPath: /pre-stop.sh
+            subPath: "pre-stop.sh"
+          - name: configuration
+            mountPath: /etc/swh
+          - name: localstorage
+            mountPath: /tmp
+      volumes:
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: loader-metadata-template
+          defaultMode: 0777
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+          - key: "init-container-entrypoint.sh"
+            path: "init-container-entrypoint.sh"
+      - name: loader-metadata-utils
+        configMap:
+          name: loader-metadata-utils
+          defaultMode: 0777
+          items:
+          - key: "pre-stop-idempotent.sh"
+            path: "pre-stop.sh"
+{{ end }}

swh/templates/loader-metadata/keda-autoscaling.yaml

+{{ if .Values.loader_metadata.enabled -}}
+{{- $autoscalingConfig := $.Values.loader_metadata.autoScaling -}}
+{{ if $autoscalingConfig }}
+{{- $journalUser := .Values.loader_metadata.journalBrokers.user -}}
+{{- $consumerGroup := .Values.loader_metadata.consumerGroup -}}
+---
+# FIXME: Look into autoscaling from prometheus depending on api authentication
+# token use metrics. See: https://keda.sh/docs/2.9/scalers/prometheus/
+# https://docs.softwareheritage.org/devel/statsd.html#outgoing-requests
+# https://grafana.softwareheritage.org/d/FR9JAYhVk/outgoing-api-requests?orgId=1
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: loader-metadata-scaledobject
+  namespace: {{ .Values.namespace }}
+spec:
+  scaleTargetRef:
+    name: loader-metadata
+  pollingInterval: {{ get $autoscalingConfig "poolInterval" | default 120 }}
+  minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 1 }}
+  maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }}
+  triggers:
+  - type: kafka
+    metadata:
+      bootstrapServers: {{ first .Values.loader_metadata.journalBrokers.hosts }}
+      {{ if $journalUser }}
+      consumerGroup: {{ $journalUser }}-{{ $consumerGroup }}
+      {{ else }}
+      consumerGroup: {{ $consumerGroup }}
+      {{ end }}
+      lagThreshold: {{ get $autoscalingConfig "lagThreshold" | default 1000 | quote }}
+      offsetResetPolicy: earliest
+    authenticationRef:
+      name: {{ .Values.loader_metadata.authenticationRef }}
+{{ end }}
+{{ end }}

swh/values.yaml

@@ -102,6 +102,28 @@ storage_replayer:
   #       lagThreashold: 1000
   #       minReplicaCount: 1
   #       maxReplicaCount: 10
+
+loader_metadata:
+  enabled: false
+  authenticationRef: keda-storage-replayer-trigger-authentication
+  # storage:
+  #   host: ...
+  #   port: 5002
+  # scheduler:
+  #   host: ...
+  #   port: 5008
+  # consumerGroup: ...
+  # prefix: swh.journal.objects
+  # journalBrokers:
+  #   hosts:
+  #     - ...
+  #   user: ...
+  # autoScaling:
+  #   poolInterval: 120
+  #   lagThreashold: 1000
+  #   minReplicaCount: 1
+  #   maxReplicaCount: 10
+
 loaders:
   enabled: false
   deployments:

swh/values/default.yaml

@@ -44,6 +44,18 @@ cookers:
             values:
             - "true"
 
+loader_metadata:
+  sentrySwhPackage: swh.loader.metadata
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "swh/loader_metadata"
+            operator: In
+            values:
+            - "true"
+
 indexers:
   sentrySwhPackage: swh.indexer
   affinity:

swh/values/staging.yaml

@@ -339,6 +339,22 @@ checker_deposit:
   autoScaling:
     maxReplicaCount: 2
 
+loader_metadata:
+  enabled: true
+  storage:
+    host: storage1.internal.staging.swh.network
+    port: 5002
+  scheduler:
+    host: scheduler0.internal.staging.swh.network
+    port: 5008
+  consumerGroup: swh.loader_metadata.journal_client
+  prefix: swh.journal.objects
+  journalBrokers:
+    hosts:
+      - journal1.internal.staging.swh.network:9092
+  autoScaling:
+    maxReplicaCount: 2
+
 indexers:
   enabled: true
   storage:

values-swh-application-versions.yaml

@@ -16,6 +16,8 @@ swh_loader_highpriority_image: container-registry.softwareheritage.org/swh/infra
 swh_loader_highpriority_image_version: '20230313.1'
 swh_loader_mercurial_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_mercurial
 swh_loader_mercurial_image_version: '20230203.1'
+swh_loader_metadata_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata
+swh_loader_metadata_image_version: '20230309.1'
 swh_loader_package_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_package
 swh_loader_package_image_version: '20230220.1'
 swh_loader_svn_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_svn