diff --git a/swh/templates/loader-metadata/configmap-utils.yaml b/swh/templates/loader-metadata/configmap-utils.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b9e4933f07624dc0c7ecdbb994849800a3060a77
--- /dev/null
+++ b/swh/templates/loader-metadata/configmap-utils.yaml
@@ -0,0 +1,31 @@
+{{ if .Values.loader_metadata.enabled -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: loader-metadata-utils
+ namespace: {{ .Values.namespace }}
+data:
+ pre-stop-idempotent.sh: |
+ #!/bin/bash
+
+ # pre-stop hook can be triggered multiple times but we want it to be applied only
+ # once so container can warm-shutdown properly.
+
+ # When celery receives multiple times the sigterm signal, this ends up doing an
+ # immediate shutdown which prevents long-standing tasks to finish properly.
+
+ set -ex
+
+ WITNESS_FILE=/tmp/already-stopped
+
+ # Seed awk with the number of nanoseconds since epoch
+ # and have it generate a number between 0 and 1
+ sleep $(date +%s%N | awk '{srand($1); print rand()}')
+
+ if [ ! -e $WITNESS_FILE ]; then
+ touch $WITNESS_FILE
+ # journal clients expect a SIGINT, not a SIGTERM
+ kill -INT 1
+ fi
+{{ end }}
diff --git a/swh/templates/loader-metadata/configmap.yaml b/swh/templates/loader-metadata/configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f036770d7eddef63b248089021e53ff614e80ad3
--- /dev/null
+++ b/swh/templates/loader-metadata/configmap.yaml
@@ -0,0 +1,54 @@
+{{ if .Values.loader_metadata.enabled -}}
+{{- $journalUser := .Values.loader_metadata.journalBrokers.user -}}
+{{- $consumerGroup := .Values.loader_metadata.consumerGroup -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: loader-metadata-template
+ namespace: {{ .Values.namespace }}
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: retry
+ - cls: remote
+ url: http://{{ .Values.loader_metadata.storage.host }}:{{ .Values.loader_metadata.storage.port }}/
+ scheduler:
+ cls: remote
+ url: http://{{ .Values.loader_metadata.scheduler.host }}:{{ .Values.loader_metadata.scheduler.port }}/
+ journal:
+ brokers: {{ toYaml .Values.loader_metadata.journalBrokers.hosts | nindent 8 }}
+ {{ if $journalUser }}
+ group_id: {{ $journalUser }}-{{ $consumerGroup }}
+ {{ else }}
+ group_id: {{ $consumerGroup }}
+ {{ end -}}
+ prefix: {{ .Values.loader_metadata.prefix }}
+
+ {{ if $journalUser }}
+ sasl.mechanism: SCRAM-SHA-512
+ security.protocol: SASL_SSL
+ sasl.username: {{ $journalUser }}
+ sasl.password: ${JOURNAL_PASSWORD}
+ {{ end -}}
+ metadata_fetcher_credentials:
+
+ init-container-entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ CONFIG_FILE=/etc/swh/config.yml
+
+ # substitute environment variables when creating the default config.yml
+ eval echo \""$(</etc/swh/configuration-template/config.yml.template)"\" \
+ > $CONFIG_FILE
+
+ CREDS_PATH=/etc/credentials/metadata-fetcher/credentials
+ [ -f $CREDS_PATH ] && \
+ sed 's/^/ /g' $CREDS_PATH >> $CONFIG_FILE
+
+ exit 0
+{{ end }}
diff --git a/swh/templates/loader-metadata/deployment.yaml b/swh/templates/loader-metadata/deployment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fcc5156615b49f316683f02597e21c3dfbee9a21
--- /dev/null
+++ b/swh/templates/loader-metadata/deployment.yaml
@@ -0,0 +1,116 @@
+{{ if .Values.loader_metadata.enabled -}}
+{{- $configurationChecksum := include (print .Template.BasePath "/loader-metadata/configmap.yaml") . -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: loader-metadata
+ namespace: {{ .Values.namespace }}
+ labels:
+ app: loader-metadata
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: loader-metadata
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: loader-metadata
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: {{ $configurationChecksum | sha256sum }}
+ spec:
+ {{- if .Values.loader_metadata.affinity }}
+ affinity:
+ {{ toYaml .Values.loader_metadata.affinity | nindent 8 }}
+ {{- end }}
+ terminationGracePeriodSeconds: 3600
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: Always
+ env:
+ - name: JOURNAL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: journal-password
+ optional: true
+ command:
+ - /entrypoint.sh
+ volumeMounts:
+ - name: configuration-template
+ mountPath: /entrypoint.sh
+ subPath: "init-container-entrypoint.sh"
+ readOnly: true
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: loader_metadata
+ image: {{ .Values.swh_loader_metadata_image }}:{{ .Values.swh_loader_metadata_image_version }}
+ imagePullPolicy: Always
+ command:
+ - /opt/swh/entrypoint.sh
+ resources:
+ requests:
+ memory: {{ .Values.loader_metadata.requestedMemory | default "512Mi" }}
+ cpu: {{ .Values.loader_metadata.requestedCpu | default "500m" }}
+ lifecycle:
+ preStop:
+ exec:
+ command: ["/pre-stop.sh"]
+ env:
+ - name: STATSD_HOST
+ value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ - name: STATSD_PORT
+ value: {{ .Values.statsdPort | default "9125" | quote }}
+ - name: LOGLEVEL
+ value: {{ .Values.loader_metadata.logLevel | default "INFO" | quote }}
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: {{ .Values.sentry.environment }}
+ - name: SWH_MAIN_PACKAGE
+ value: {{ .Values.loader_metadata.sentrySwhPackage }}
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: loader-metadata-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: true
+ volumeMounts:
+ - name: loader-metadata-utils
+ mountPath: /pre-stop.sh
+ subPath: "pre-stop.sh"
+ - name: configuration
+ mountPath: /etc/swh
+ - name: localstorage
+ mountPath: /tmp
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: loader-metadata-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - key: "init-container-entrypoint.sh"
+ path: "init-container-entrypoint.sh"
+ - name: loader-metadata-utils
+ configMap:
+ name: loader-metadata-utils
+ defaultMode: 0777
+ items:
+ - key: "pre-stop-idempotent.sh"
+ path: "pre-stop.sh"
+{{ end }}
diff --git a/swh/templates/loader-metadata/keda-autoscaling.yaml b/swh/templates/loader-metadata/keda-autoscaling.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e3f79886f118cc1b7a52b94e01289116c2df7e01
--- /dev/null
+++ b/swh/templates/loader-metadata/keda-autoscaling.yaml
@@ -0,0 +1,36 @@
+{{ if .Values.loader_metadata.enabled -}}
+{{- $autoscalingConfig := $.Values.loader_metadata.autoScaling -}}
+{{ if $autoscalingConfig }}
+{{- $journalUser := .Values.loader_metadata.journalBrokers.user -}}
+{{- $consumerGroup := .Values.loader_metadata.consumerGroup -}}
+---
+# FIXME: Look into autoscaling from prometheus depending on api authentication
+# token use metrics. See: https://keda.sh/docs/2.9/scalers/prometheus/
+# https://docs.softwareheritage.org/devel/statsd.html#outgoing-requests
+# https://grafana.softwareheritage.org/d/FR9JAYhVk/outgoing-api-requests?orgId=1
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: loader-metadata-scaledobject
+ namespace: {{ .Values.namespace }}
+spec:
+ scaleTargetRef:
+ name: loader-metadata
+ pollingInterval: {{ get $autoscalingConfig "poolInterval" | default 120 }}
+ minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 1 }}
+ maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }}
+ triggers:
+ - type: kafka
+ metadata:
+ bootstrapServers: {{ first .Values.loader_metadata.journalBrokers.hosts }}
+ {{ if $journalUser }}
+ consumerGroup: {{ $journalUser }}-{{ $consumerGroup }}
+ {{ else }}
+ consumerGroup: {{ $consumerGroup }}
+ {{ end }}
+ lagThreshold: {{ get $autoscalingConfig "lagThreshold" | default 1000 | quote }}
+ offsetResetPolicy: earliest
+ authenticationRef:
+ name: {{ .Values.loader_metadata.authenticationRef }}
+{{ end }}
+{{ end }}
diff --git a/swh/values.yaml b/swh/values.yaml
index 3c11b620f2f818e1c2e0c36d55785bc00b9dd79c..00cb78b5c41624265799a4f17d132ed1cde7d97b 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -102,6 +102,28 @@ storage_replayer:
# lagThreashold: 1000
# minReplicaCount: 1
# maxReplicaCount: 10
+
+loader_metadata:
+ enabled: false
+ authenticationRef: keda-storage-replayer-trigger-authentication
+ # storage:
+ # host: ...
+ # port: 5002
+ # scheduler:
+ # host: ...
+ # port: 5008
+ # consumerGroup: ...
+ # prefix: swh.journal.objects
+ # journalBrokers:
+ # hosts:
+ # - ...
+ # user: ...
+ # autoScaling:
+ # poolInterval: 120
+ # lagThreashold: 1000
+ # minReplicaCount: 1
+ # maxReplicaCount: 10
+
loaders:
enabled: false
deployments:
diff --git a/swh/values/default.yaml b/swh/values/default.yaml
index da5afba88661cb06e90bcb562e4b48a9c89f9e58..3e758c8e05a77064ca65cbe41bb602ae0de85c87 100644
--- a/swh/values/default.yaml
+++ b/swh/values/default.yaml
@@ -44,6 +44,18 @@ cookers:
values:
- "true"
+loader_metadata:
+ sentrySwhPackage: swh.loader.metadata
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: "swh/loader_metadata"
+ operator: In
+ values:
+ - "true"
+
indexers:
sentrySwhPackage: swh.indexer
affinity:
diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml
index 4f83aa649fd4cf950748ded7a600e06af9861f48..720c81950f3f771ea001559d78bfbf7818ff723c 100644
--- a/swh/values/staging.yaml
+++ b/swh/values/staging.yaml
@@ -339,6 +339,22 @@ checker_deposit:
autoScaling:
maxReplicaCount: 2
+loader_metadata:
+ enabled: true
+ storage:
+ host: storage1.internal.staging.swh.network
+ port: 5002
+ scheduler:
+ host: scheduler0.internal.staging.swh.network
+ port: 5008
+ consumerGroup: swh.loader_metadata.journal_client
+ prefix: swh.journal.objects
+ journalBrokers:
+ hosts:
+ - journal1.internal.staging.swh.network:9092
+ autoScaling:
+ maxReplicaCount: 2
+
indexers:
enabled: true
storage:
diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml
index 7a91353ee9e4fc26bb505f71f3e7d3d0cfb5ede7..ec465c500c5ffec2b10b8680c6cf25284364bbdc 100644
--- a/values-swh-application-versions.yaml
+++ b/values-swh-application-versions.yaml
@@ -16,6 +16,8 @@ swh_loader_highpriority_image: container-registry.softwareheritage.org/swh/infra
swh_loader_highpriority_image_version: '20230313.1'
swh_loader_mercurial_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_mercurial
swh_loader_mercurial_image_version: '20230203.1'
+swh_loader_metadata_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata
+swh_loader_metadata_image_version: '20230309.1'
swh_loader_package_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_package
swh_loader_package_image_version: '20230220.1'
swh_loader_svn_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_svn