diff --git a/swh/templates/loader-metadata/keda-autoscaling.yaml b/swh/templates/loader-metadata/keda-autoscaling.yaml
index e3f79886f118cc1b7a52b94e01289116c2df7e01..5fb49595312c7be6b79162f37700aee104b28e7c 100644
--- a/swh/templates/loader-metadata/keda-autoscaling.yaml
+++ b/swh/templates/loader-metadata/keda-autoscaling.yaml
@@ -1,6 +1,5 @@
{{ if .Values.loader_metadata.enabled -}}
{{- $autoscalingConfig := $.Values.loader_metadata.autoScaling -}}
-{{ if $autoscalingConfig }}
{{- $journalUser := .Values.loader_metadata.journalBrokers.user -}}
{{- $consumerGroup := .Values.loader_metadata.consumerGroup -}}
---
@@ -23,14 +22,9 @@ spec:
- type: kafka
metadata:
bootstrapServers: {{ first .Values.loader_metadata.journalBrokers.hosts }}
- {{ if $journalUser }}
consumerGroup: {{ $journalUser }}-{{ $consumerGroup }}
- {{ else }}
- consumerGroup: {{ $consumerGroup }}
- {{ end }}
lagThreshold: {{ get $autoscalingConfig "lagThreshold" | default 1000 | quote }}
offsetResetPolicy: earliest
authenticationRef:
- name: {{ .Values.loader_metadata.authenticationRef }}
-{{ end }}
-{{ end }}
+ name: keda-loader-metadata-trigger-authentication
+{{- end -}}
diff --git a/swh/templates/loader-metadata/keda-secrets.yaml b/swh/templates/loader-metadata/keda-secrets.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..50811d07696f0f49120bd1a60783cce0871b6642
--- /dev/null
+++ b/swh/templates/loader-metadata/keda-secrets.yaml
@@ -0,0 +1,33 @@
+{{ if .Values.loader_metadata.enabled -}}
+{{- $journalUser := .Values.loader_metadata.journalBrokers.user -}}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: keda-loader-metadata-kafka-secrets
+ namespace: {{ .Values.namespace }}
+type: Opaque
+stringData:
+ sasl: "scram_sha512"
+ username: {{ $journalUser }}
+ tls: "enable"
+---
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+ name: keda-loader-metadata-trigger-authentication
+ namespace: {{ .Values.namespace }}
+spec:
+ secretTargetRef:
+ - parameter: sasl
+ name: keda-loader-metadata-kafka-secrets
+ key: sasl
+ - parameter: username
+ name: keda-loader-metadata-kafka-secrets
+ key: username
+ - parameter: tls
+ name: keda-loader-metadata-kafka-secrets
+ key: tls
+ - parameter: password
+ name: {{ .Values.loader_metadata.journalBrokers.secretName }}
+ key: BROKER_USER_PASSWORD
+{{- end -}}
diff --git a/swh/tests/loader_metadata_configmap_test.yaml b/swh/tests/loader_metadata_configmap_test.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cf3c985ccb95ee3a69073e9e89bdeda0845aa5a5
--- /dev/null
+++ b/swh/tests/loader_metadata_configmap_test.yaml
@@ -0,0 +1,37 @@
+suite: test loader_metadata configmap deployment
+templates:
+ - loader-metadata/configmap.yaml
+tests:
+ - it: loader-metadata configmap should be deployed when activated
+ # set:
+ # loader_metadata.enabled: true
+ values:
+ - values/loader-metadata.yaml
+ asserts:
+ # - containsDocument:
+ # kind: ConfigMap
+ # apiVersion: v1
+ # name: loader-metadata-utils
+ # namespace: swh
+ - containsDocument:
+ kind: ConfigMap
+ apiVersion: v1
+ name: loader-metadata-template
+ namespace: swh
+ # not testable as there is a dot in the entry name
+ # - contains:
+ # path: data.config.yml
+ # content:
+ # debug: false
+ # count: 1
+ - it: loader metadata configmap namespace should be overridable
+ values:
+ - values/loader-metadata.yaml
+ set:
+ namespace: mynamespace
+ asserts:
+ - equal:
+ path: metadata.namespace
+ value: mynamespace
+ - hasDocuments:
+ count: 1
diff --git a/swh/tests/loader_metadata_configmap_utils_test.yaml b/swh/tests/loader_metadata_configmap_utils_test.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..48c1975501709480abd5564971d81c2ed7577bd1
--- /dev/null
+++ b/swh/tests/loader_metadata_configmap_utils_test.yaml
@@ -0,0 +1,24 @@
+suite: test loader_metadata configmap utils deployment
+templates:
+ - loader-metadata/configmap-utils.yaml
+tests:
+ - it: loader-metadata configmap utils should be deployed when activated
+ values:
+ - values/loader-metadata.yaml
+ asserts:
+ - containsDocument:
+ kind: ConfigMap
+ apiVersion: v1
+ name: loader-metadata-utils
+ namespace: swh
+ - it: loader metadata configmap utils namespace should be overridable
+ values:
+ - values/loader-metadata.yaml
+ set:
+ namespace: mynamespace
+ asserts:
+ - equal:
+ path: metadata.namespace
+ value: mynamespace
+ - hasDocuments:
+ count: 1
diff --git a/swh/tests/loader_metadata_deployment_test.yaml b/swh/tests/loader_metadata_deployment_test.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f362f359a9ba01466070e96e1cc7432d094168ed
--- /dev/null
+++ b/swh/tests/loader_metadata_deployment_test.yaml
@@ -0,0 +1,124 @@
+suite: test loader_metadata deployment
+templates:
+ - loader-metadata/configmap.yaml
+ - loader-metadata/configmap-utils.yaml
+ - loader-metadata/deployment.yaml
+tests:
+ - it: Loader metadata deployment should be deployed with default values
+ templates:
+ - loader-metadata/deployment.yaml
+ values:
+ - ./values/loader-metadata.yaml
+ asserts:
+ - containsDocument:
+ kind: Deployment
+ apiVersion: apps/v1
+ namespace: swh
+ name: loader-metadata
+ - lengthEqual:
+ path: spec.template.spec.containers
+ count: 1
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].image
+ value: image:version
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].resources.requests.memory
+ value: "512Mi"
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].resources.requests.cpu
+ value: "500m"
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "STATSD_HOST")].value
+ value: "prometheus-statsd-exporter"
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "STATSD_PORT")].value
+ value: "9125"
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "LOGLEVEL")].value
+ value: INFO
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "SWH_SENTRY_ENVIRONMENT")].value
+ value: production
+ - isNull:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "SWH_MAIN_PACKAGE")].value
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "SWH_SENTRY_DSN")].valueFrom.secretKeyRef.name
+ value: common-secrets
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "SWH_SENTRY_DSN")].valueFrom.secretKeyRef.key
+ value: loader-metadata-sentry-dsn
+
+ - it: Loader metadata deployment should be deployed with overriden values
+ templates:
+ - loader-metadata/deployment.yaml
+ set:
+ # global values
+ namespace: test
+ statsdExternalHost: testing-statsd-exporter
+ statsdPort: 5219
+ sentry:
+ environment: testing
+ swh_loader_metadata_image: test.image
+ swh_loader_metadata_image_version: test.version
+
+ # specific values
+ loader_metadata:
+ enabled: true
+ logLevel: DEBUG
+ storage:
+ host: test.storage.org
+ port: 2005
+ scheduler:
+ host: test.scheduler.org
+ port: 8005
+ consumerGroup: test.loader_metadata.journal_client
+ prefix: test.journal.objects
+ journalBrokers:
+ hosts:
+ - journal.test.org
+ user: ""
+ autoScaling:
+ maxReplicaCount: 2
+ requestedMemory: "256Mi"
+ requestedCpu: "250m"
+ sentrySwhPackage: test.swh.loader.metadata
+
+ asserts:
+ - containsDocument:
+ kind: Deployment
+ apiVersion: apps/v1
+ namespace: test
+ name: loader-metadata
+ - lengthEqual:
+ path: spec.template.spec.containers
+ count: 1
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].image
+ value: test.image:test.version
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].resources.requests.memory
+ value: "256Mi"
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].resources.requests.cpu
+ value: "250m"
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "STATSD_HOST")].value
+ value: "testing-statsd-exporter"
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "STATSD_PORT")].value
+ value: "5219"
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "LOGLEVEL")].value
+ value: DEBUG
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "SWH_SENTRY_ENVIRONMENT")].value
+ value: testing
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "SWH_MAIN_PACKAGE")].value
+ value: test.swh.loader.metadata
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "SWH_SENTRY_DSN")].valueFrom.secretKeyRef.name
+ value: common-secrets
+ - equal:
+ path: spec.template.spec.containers[?(@.name == "loader_metadata")].env[?(@.name == "SWH_SENTRY_DSN")].valueFrom.secretKeyRef.key
+ value: loader-metadata-sentry-dsn
diff --git a/swh/tests/loader_metadata_global_test.yaml b/swh/tests/loader_metadata_global_test.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ce58ee359692d365984700dfedab2692b3e780a8
--- /dev/null
+++ b/swh/tests/loader_metadata_global_test.yaml
@@ -0,0 +1,14 @@
+suite: test global loader metadata deployment
+templates:
+ - loader-metadata/deployment.yaml
+ - loader-metadata/configmap.yaml
+ - loader-metadata/configmap-utils.yaml
+ - loader-metadata/keda-autoscaling.yaml
+ - loader-metadata/keda-secrets.yaml
+tests:
+ - it: Metadata Loader is not deployed by default
+ values:
+ - ../values.yaml
+ asserts:
+ - hasDocuments:
+ count: 0
diff --git a/swh/tests/loader_metadata_keda_autoscaling_test.yaml b/swh/tests/loader_metadata_keda_autoscaling_test.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4fbef3abf91ad70e3bc3f9037ccdf5d95a5e3943
--- /dev/null
+++ b/swh/tests/loader_metadata_keda_autoscaling_test.yaml
@@ -0,0 +1,60 @@
+suite: test loader_metadata keda-scaling
+tests:
+ - it: Loader metadata keda scaling deployment should be deployed
+ templates:
+ - loader-metadata/keda-autoscaling.yaml
+ values:
+ - ./values/loader-metadata.yaml
+ set:
+ namespace: test
+ loader_metadata:
+ enabled: true
+ logLevel: DEBUG
+ requestedMemory: "256Mi"
+ requestedCpu: "250m"
+ sentrySwhPackage: test.swh.loader.metadata
+ storage:
+ host: test.storage.org
+ port: 2005
+ scheduler:
+ host: test.scheduler.org
+ port: 8005
+ prefix: test.journal.objects
+ consumerGroup: test.consumerGroup
+ journalBrokers:
+ hosts:
+ - journal.test.org
+ user: testuser
+ secretName: test-secret-key
+
+ autoScaling:
+ poolInterval: 10
+ minReplicaCount: 10
+ maxReplicaCount: 50
+ lagThreshold: 200
+
+ asserts:
+ - containsDocument:
+ kind: ScaledObject
+ apiVersion: keda.sh/v1alpha1
+ namespace: test
+ name: loader-metadata-scaledobject
+
+ - equal:
+ path: spec.pollingInterval
+ value: 10
+ - equal:
+ path: spec.minReplicaCount
+ value: 10
+ - equal:
+ path: spec.maxReplicaCount
+ value: 50
+ - equal:
+ path: spec.triggers[?(@.type == "kafka")].metadata.bootstrapServers
+ value: journal.test.org
+ - equal:
+ path: spec.triggers[?(@.type == "kafka")].metadata.consumerGroup
+ value: testuser-test.consumerGroup
+ - equal:
+ path: spec.triggers[?(@.type == "kafka")].metadata.lagThreshold
+ value: "200"
diff --git a/swh/tests/loader_metadata_keda_secrets_test.yaml b/swh/tests/loader_metadata_keda_secrets_test.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6500d57ea6ec6757a24803b6dd560e386b522e4a
--- /dev/null
+++ b/swh/tests/loader_metadata_keda_secrets_test.yaml
@@ -0,0 +1,20 @@
+suite: test loader_metadata keda-secrets
+tests:
+ - it: Loader metadata keda secrets deployment should be deployed
+ templates:
+ - loader-metadata/keda-secrets.yaml
+ set:
+ namespace: test
+ values:
+ - ./values/loader-metadata.yaml
+ asserts:
+ - containsDocument:
+ kind: Secret
+ apiVersion: v1
+ namespace: test
+ name: keda-loader-metadata-kafka-secrets
+ - containsDocument:
+ kind: TriggerAuthentication
+ apiVersion: keda.sh/v1alpha1
+ namespace: test
+ name: keda-loader-metadata-trigger-authentication
diff --git a/swh/tests/values/loader-metadata.yaml b/swh/tests/values/loader-metadata.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a26b56c54114f3e0f671937da403346133e8d62a
--- /dev/null
+++ b/swh/tests/values/loader-metadata.yaml
@@ -0,0 +1,20 @@
+loader_metadata:
+ enabled: true
+ storage:
+ host: storage.test.org
+ port: 2005
+ scheduler:
+ host: scheduler.test.org
+ port: 8005
+ consumerGroup: test.loader_metadata.journal_client
+ prefix: test.journal.objects
+ journalBrokers:
+ secretName: test-loader-metadata-secret
+ hosts:
+ - journal.test.org
+ user: test
+ autoScaling:
+ maxReplicaCount: 2
+
+swh_loader_metadata_image: image
+swh_loader_metadata_image_version: version
diff --git a/swh/values.yaml b/swh/values.yaml
index 00cb78b5c41624265799a4f17d132ed1cde7d97b..2af794c46e6a0c3f3e46304399508145f54c2178 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -105,7 +105,6 @@ storage_replayer:
loader_metadata:
enabled: false
- authenticationRef: keda-storage-replayer-trigger-authentication
# storage:
# host: ...
# port: 5002
diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml
index 720c81950f3f771ea001559d78bfbf7818ff723c..14719ddf298f2e3a798f47a9c68546798144b4b6 100644
--- a/swh/values/staging.yaml
+++ b/swh/values/staging.yaml
@@ -350,10 +350,13 @@ loader_metadata:
consumerGroup: swh.loader_metadata.journal_client
prefix: swh.journal.objects
journalBrokers:
+ secretName: swh-archive-broker-secret
hosts:
- journal1.internal.staging.swh.network:9092
+ user: swh-archive-stg
autoScaling:
maxReplicaCount: 2
+
indexers:
enabled: true