diff --git a/swh/templates/indexers/keda-autoscaling.yaml b/swh/templates/indexers/keda-autoscaling.yaml index da0b702e8664eb896d91bf25c889f4199d5e0fef..919f60fbe3d164e355c5dc1bd7e443f41ba032ad 100644 --- a/swh/templates/indexers/keda-autoscaling.yaml +++ b/swh/templates/indexers/keda-autoscaling.yaml @@ -20,15 +20,11 @@ spec: - type: kafka metadata: bootstrapServers: {{ first $.Values.indexers.journalBrokers.hosts }} - {{ if $journalUser }} consumerGroup: {{ $journalUser }}-{{ $consumerGroup }} - {{ else }} - consumerGroup: {{ $consumerGroup }} - {{ end }} lagThreshold: {{ get $autoscalingConfig "lagThreshold" | default 1000 | quote }} offsetResetPolicy: earliest authenticationRef: - name: keda-storage-replayer-trigger-authentication + name: keda-indexers-kafka-secrets {{ end }} {{ end }} {{- end -}} diff --git a/swh/templates/indexers/keda-secrets.yaml b/swh/templates/indexers/keda-secrets.yaml new file mode 100644 index 0000000000000000000000000000000000000000..511c95b6bb288bbac827f748e430be92e7bc6e82 --- /dev/null +++ b/swh/templates/indexers/keda-secrets.yaml @@ -0,0 +1,33 @@ +{{ if .Values.indexers.enabled -}} +{{- $journalUser := .Values.indexers.journalBrokers.user -}} +apiVersion: v1 +kind: Secret +metadata: + name: keda-indexers-kafka-secrets + namespace: {{ .Values.namespace }} +type: Opaque +stringData: + sasl: "scram_sha512" + username: {{ $journalUser }} + tls: "enable" +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-indexers-trigger-authentication + namespace: {{ .Values.namespace }} +spec: + secretTargetRef: + - parameter: sasl + name: keda-indexers-kafka-secrets + key: sasl + - parameter: username + name: keda-indexers-kafka-secrets + key: username + - parameter: tls + name: keda-indexers-kafka-secrets + key: tls + - parameter: password + name: {{ .Values.indexers.journalBrokers.secretName }} + key: BROKER_USER_PASSWORD +{{- end -}} diff --git a/swh/tests/indexers_keda_autoscaling.yaml b/swh/tests/indexers_keda_autoscaling.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b09e8d3ba5761371b3a78eded8841c6776bc643 --- /dev/null +++ b/swh/tests/indexers_keda_autoscaling.yaml @@ -0,0 +1,16 @@ +suite: test indexer keda-scaling +tests: + - it: Indexer keda scaling deployment should be deployed + templates: + - indexers/keda-autoscaling.yaml + asserts: + - containsDocument: + kind: ScaledObject + apiVersion: keda.sh/v1alpha1 + namespace: test + name: indexer-idx_test0-scaledobject + - containsDocument: + kind: ScaledObject + apiVersion: keda.sh/v1alpha1 + namespace: test + name: indexer-idx_test1-scaledobject diff --git a/swh/tests/indexers_keda_secrets_test.yaml b/swh/tests/indexers_keda_secrets_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1f0b6643a0de560f21ea673592e390a349119a1 --- /dev/null +++ b/swh/tests/indexers_keda_secrets_test.yaml @@ -0,0 +1,20 @@ +suite: test indexers keda-secrets +tests: + - it: Indexer keda secrets deployment should be deployed + templates: + - indexers/keda-secrets.yaml + set: + namespace: test + values: + - ./values/indexers.yaml + asserts: + - containsDocument: + kind: Secret + apiVersion: v1 + namespace: test + name: keda-indexers-kafka-secrets + - containsDocument: + kind: TriggerAuthentication + apiVersion: keda.sh/v1alpha1 + namespace: test + name: keda-indexers-trigger-authentication diff --git a/swh/tests/values/indexers.yaml b/swh/tests/values/indexers.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e2f433c6289d8f7c42c0912a4c558327edf5e21 --- /dev/null +++ b/swh/tests/values/indexers.yaml @@ -0,0 +1,56 @@ +swh_indexer_image: image +swh_indexer_image_version: version + +indexers: + enabled: true + storage: + host: storage.test.org + port: 2005 + scheduler: + host: scheduler.test.org + port: 8005 + indexer_storage: + host: indexer.test.org + port: 7005 + objstorage: + host: objstorage.test.org + port: 3005 + journalBrokers: + secretName: swh-archive-broker-secret + hosts: + - test.journal:2909 + user: test-swh + deployments: + indexer0: + indexer_type: idx_test0 + consumerGroup: test.swh.idx.content_mimetype + prefix: swh.journal.objects + autoScaling: + poolInterval: 10 + minReplicaCount: 10 + maxReplicaCount: 50 + lagThreshold: 200 + extraConfig: + tools: + name: file + # FIXME: Push this version to be autodetected by indexer + version: 2:0.4.15-2 + configuration: + type: library + debian-package: python3-magic + write_batch_size: 1000 + indexer1: + indexer_type: idx_test1 + consumerGroup: test.swh.idx.content_fossology + prefix: swh.journal.objects + autoScaling: + poolInterval: 20 + minReplicaCount: 20 + maxReplicaCount: 100 + lagThreshold: 250 + extraConfig: + tools: + name: swh-metadata-detector + version: 0.0.2 + configuration: {} + write_batch_size: 1000 diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml index 14719ddf298f2e3a798f47a9c68546798144b4b6..d0734068b9b21c17522f16d6dd7f1d77f3191733 100644 --- a/swh/values/staging.yaml +++ b/swh/values/staging.yaml @@ -356,7 +356,6 @@ loader_metadata: user: swh-archive-stg autoScaling: maxReplicaCount: 2 - indexers: enabled: true @@ -373,8 +372,10 @@ indexers: host: storage1.internal.staging.swh.network port: 5003 journalBrokers: + secretName: swh-archive-broker-secret hosts: - journal1.internal.staging.swh.network:9092 + user: swh-archive-stg deployments: # content-mimetype: # indexer_type: content_mimetype