production: Deploy one instance of origin intrinsic metadata indexer

This deploys one instance of indexer origin intrinsic metadata.

helm diff

[swh] Comparing changes between branches production and deploy-origin-intrinsic-metadata (per environment)...
Your branch is ahead of 'origin/production' by 2 commits.
  (use "git push" to publish your local commits)
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment staging...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment staging...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment staging...
Your branch is ahead of 'origin/production' by 2 commits.
  (use "git push" to publish your local commits)
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment production...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment production...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment production...


------------- diff for environment staging namespace swh -------------

No differences


------------- diff for environment staging namespace swh-cassandra -------------

No differences


------------- diff for environment staging namespace swh-cassandra-next-version -------------

No differences


------------- diff for environment production namespace swh -------------

--- /tmp/swh-chart.swh.dMNQLih4/production-swh.before   2024-02-09 14:57:41.259760120 +0100
+++ /tmp/swh-chart.swh.dMNQLih4/production-swh.after    2024-02-09 14:57:41.763759426 +0100
@@ -139,20 +139,31 @@
 apiVersion: v1
 kind: Secret
 metadata:
   name: keda-indexer-extrinsic-secrets
   namespace: swh
 type: Opaque
 stringData:
   sasl: "scram_sha512"
   tls: "enable"
 ---
+# Source: swh/templates/indexers/keda-autoscaling.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: keda-indexer-origin-intrinsic-secrets
+  namespace: swh
+type: Opaque
+stringData:
+  sasl: "scram_sha512"
+  tls: "enable"
+---
 # Source: swh/templates/loader-metadata/keda-secrets.yaml
 apiVersion: v1
 kind: Secret
 metadata:
   name: keda-loader-metadata-kafka-secrets
   namespace: swh
 type: Opaque
 stringData:
   sasl: "scram_sha512"
   username: swh-archive-prod
@@ -1172,20 +1183,146 @@
       prefix: swh.journal.objects
       sasl.mechanism: SCRAM-SHA-512
       sasl.password: ${BROKER_USER_PASSWORD}
       sasl.username: ${BROKER_USER}
       security.protocol: SASL_SSL
     tools:
       configuration: {}
       name: swh-metadata-detector
       version: 0.0.2
 ---
+# Source: swh/templates/indexers/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: indexer-origin-intrinsic-configuration-template
+  namespace: swh
+data:
+  config.yml.template: |
+    storage:
+      cls: pipeline
+      steps:
+      - cls: retry
+      - cls: remote
+        url: http://storage-azure-read-only-rpc-ingress
+    scheduler:
+      cls: remote
+      url: http://scheduler.internal.softwareheritage.org
+    indexer_storage:
+      cls: remote
+      url: http://indexer-storage-read-write-rpc-ingress
+    objstorage:
+      cls: multiplexer
+      objstorages:
+      - cls: filtered
+        filters_conf:
+        - type: readonly
+        storage_conf:
+          cls: http
+          compression: gzip
+          timeout: 120
+          url: https://softwareheritage.s3.amazonaws.com/content/
+      - cls: filtered
+        filters_conf:
+        - type: readonly
+        storage_conf:
+          accounts:
+            "0":
+              account_name: ${ACCOUNT_NAME_0}
+              api_secret_key: ${API_SECRET_KEY_0}
+              container_name: contents
+            "1":
+              account_name: ${ACCOUNT_NAME_1}
+              api_secret_key: ${API_SECRET_KEY_1}
+              container_name: contents
+            "2":
+              account_name: ${ACCOUNT_NAME_2}
+              api_secret_key: ${API_SECRET_KEY_2}
+              container_name: contents
+            "3":
+              account_name: ${ACCOUNT_NAME_3}
+              api_secret_key: ${API_SECRET_KEY_3}
+              container_name: contents
+            "4":
+              account_name: ${ACCOUNT_NAME_4}
+              api_secret_key: ${API_SECRET_KEY_4}
+              container_name: contents
+            "5":
+              account_name: ${ACCOUNT_NAME_5}
+              api_secret_key: ${API_SECRET_KEY_5}
+              container_name: contents
+            "6":
+              account_name: ${ACCOUNT_NAME_6}
+              api_secret_key: ${API_SECRET_KEY_6}
+              container_name: contents
+            "7":
+              account_name: ${ACCOUNT_NAME_7}
+              api_secret_key: ${API_SECRET_KEY_7}
+              container_name: contents
+            "8":
+              account_name: ${ACCOUNT_NAME_8}
+              api_secret_key: ${API_SECRET_KEY_8}
+              container_name: contents
+            "9":
+              account_name: ${ACCOUNT_NAME_9}
+              api_secret_key: ${API_SECRET_KEY_9}
+              container_name: contents
+            a:
+              account_name: ${ACCOUNT_NAME_10}
+              api_secret_key: ${API_SECRET_KEY_10}
+              container_name: contents
+            b:
+              account_name: ${ACCOUNT_NAME_11}
+              api_secret_key: ${API_SECRET_KEY_11}
+              container_name: contents
+            c:
+              account_name: ${ACCOUNT_NAME_12}
+              api_secret_key: ${API_SECRET_KEY_12}
+              container_name: contents
+            d:
+              account_name: ${ACCOUNT_NAME_13}
+              api_secret_key: ${API_SECRET_KEY_13}
+              container_name: contents
+            e:
+              account_name: ${ACCOUNT_NAME_14}
+              api_secret_key: ${API_SECRET_KEY_14}
+              container_name: contents
+            f:
+              account_name: ${ACCOUNT_NAME_15}
+              api_secret_key: ${API_SECRET_KEY_15}
+              container_name: contents
+          cls: azure-prefixed
+      - cls: filtered
+        filters_conf:
+        - type: readonly
+        storage_conf:
+          cls: remote
+          url: http://objstorage-read-only-rpc-ingress
+    journal_client:
+      brokers:
+        - kafka1.internal.softwareheritage.org:9094
+        - kafka2.internal.softwareheritage.org:9094
+        - kafka3.internal.softwareheritage.org:9094
+        - kafka4.internal.softwareheritage.org:9094
+      batch_size: 200
+      cls: kafka
+      group_id: swh-indexer-prod-01-swh.indexer.journal_client.origin_intrinsic_metadata
+      prefix: swh.journal.objects
+      sasl.mechanism: SCRAM-SHA-512
+      sasl.password: ${BROKER_USER_PASSWORD}
+      sasl.username: ${BROKER_USER}
+      security.protocol: SASL_SSL
+    tools:
+      configuration: {}
+      name: swh-metadata-detector
+      version: 0.0.2
+---
 # Source: swh/templates/listers/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: lister-utils
   namespace: swh
 data:
   pre-stop-idempotent.sh: |
     #!/bin/bash

@@ -21004,20 +21141,396 @@
           name: config-utils
           defaultMode: 0555
       - name: indexer-utils
         configMap:
           name: indexer-utils
           defaultMode: 0777
           items:
           - key: "pre-stop-idempotent.sh"
             path: "pre-stop.sh"
 ---
+# Source: swh/templates/indexers/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: indexer-origin-intrinsic
+  namespace: swh
+  labels:
+    app: indexer-origin-intrinsic
+spec:
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: indexer-origin-intrinsic
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: indexer-origin-intrinsic
+      annotations:
+        # Force a rollout upgrade if the configuration changes
+        checksum/config: 8952686da7734512383cadbd120899b7cf772bc40fad1597d7fe06659e7b8561
+        checksum/config-utils: d75ca13b805bce6a8ab59c8e24c938f2283108f6a79134f6e71db86308651dc6
+    spec:
+      affinity:
+
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: swh/indexer
+                operator: In
+                values:
+                - "true"
+      priorityClassName: swh-low-workload
+
+      terminationGracePeriodSeconds: 3600
+      initContainers:
+        - name: prepare-configuration
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+          imagePullPolicy: IfNotPresent
+          command:
+          - /entrypoints/prepare-configuration.sh
+          env:
+
+
+          - name: BROKER_USER
+            valueFrom:
+              secretKeyRef:
+                name: swh-indexer-prod-01-broker-secret
+                key: BROKER_USER
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: BROKER_USER_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: swh-indexer-prod-01-broker-secret
+                key: BROKER_USER_PASSWORD
+                # 'name' secret must exist & include that ^ key
+                optional: false
+
+
+          - name: ACCOUNT_NAME_0
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 0_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_1
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 1_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_10
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 10_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_11
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 11_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_12
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 12_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_13
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 13_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_14
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 14_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_15
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 15_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_2
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 2_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_3
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 3_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_4
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 4_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_5
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 5_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_6
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 6_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_7
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 7_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_8
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 8_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: ACCOUNT_NAME_9
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 9_account_name
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_0
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 0_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_1
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 1_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_10
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 10_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_11
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 11_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_12
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 12_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_13
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 13_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_14
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 14_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_15
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 15_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_2
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 2_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_3
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 3_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_4
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 4_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_5
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 5_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_6
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 6_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_7
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 7_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_8
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 8_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: API_SECRET_KEY_9
+            valueFrom:
+              secretKeyRef:
+                name: swh-objstorage-config
+                key: 9_api_secret_key
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+          - name: config-utils
+            mountPath: /entrypoints
+      containers:
+      - name: indexers
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer:20240202.1
+        imagePullPolicy: IfNotPresent
+        command:
+          - /opt/swh/entrypoint.sh
+        resources:
+          requests:
+            memory: 512Mi
+            cpu: 500m
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/pre-stop.sh"]
+        env:
+        - name: STATSD_HOST
+          value: prometheus-statsd-exporter
+        - name: STATSD_PORT
+          value: "9125"
+        - name: MAX_TASKS_PER_CHILD
+          value: "10"
+        - name: LOGLEVEL
+          value: "INFO"
+        - name: SWH_CONFIG_FILENAME
+          value: /etc/swh/config.yml
+        - name: SWH_INDEXER_TYPE
+          value: origin_intrinsic_metadata
+        - name: SWH_SENTRY_ENVIRONMENT
+          value: production
+        - name: SWH_MAIN_PACKAGE
+          value: swh.indexer
+        - name: SWH_SENTRY_DSN
+          valueFrom:
+            secretKeyRef:
+              name: common-secrets
+              key: indexer-sentry-dsn
+              # 'name' secret must exist & include key "host"
+              optional: false
+        volumeMounts:
+          - name: indexer-utils
+            mountPath: /pre-stop.sh
+            subPath: "pre-stop.sh"
+          - name: configuration
+            mountPath: /etc/swh
+          - name: localstorage
+            mountPath: /tmp
+      volumes:
+      - name: localstorage
+
+        ephemeral:
+          volumeClaimTemplate:
+            metadata:
+              labels:
+                type: ephemeral-volume
+            spec:
+              accessModes:
+              - ReadWriteOnce
+              resources:
+                requests:
+                  storage: 100Gi
+              storageClassName: local-path
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: indexer-origin-intrinsic-configuration-template
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+      - name: config-utils
+        configMap:
+          name: config-utils
+          defaultMode: 0555
+      - name: indexer-utils
+        configMap:
+          name: indexer-utils
+          defaultMode: 0777
+          items:
+          - key: "pre-stop-idempotent.sh"
+            path: "pre-stop.sh"
+---
 # Source: swh/templates/listers/deployment.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: lister-bitbucket
   namespace: swh
   labels:
     app: lister-bitbucket
 spec:
   revisionHistoryLimit: 2
@@ -37149,20 +37662,43 @@
   triggers:
   - type: kafka
     metadata:
       bootstrapServers: kafka1.internal.softwareheritage.org:9094,kafka2.internal.softwareheritage.org:9094,kafka3.internal.softwareheritage.org:9094,kafka4.internal.softwareheritage.org:9094
       consumerGroup: swh-archive-prod-swh.indexer.journal_client.extrinsic_metadata
       lagThreshold: "1000"
       offsetResetPolicy: earliest
     authenticationRef:
       name: keda-indexer-extrinsic-authentication
 ---
+# Source: swh/templates/indexers/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: indexer-origin-intrinsic-scaledobject
+  namespace: swh
+spec:
+  scaleTargetRef:
+    name: indexer-origin-intrinsic
+  pollingInterval: 120
+  minReplicaCount: 1
+  maxReplicaCount: 1
+  idleReplicaCount: 0
+  triggers:
+  - type: kafka
+    metadata:
+      bootstrapServers: kafka1.internal.softwareheritage.org:9094,kafka2.internal.softwareheritage.org:9094,kafka3.internal.softwareheritage.org:9094,kafka4.internal.softwareheritage.org:9094
+      consumerGroup: swh-indexer-prod-01-swh.indexer.journal_client.origin_intrinsic_metadata
+      lagThreshold: "1000"
+      offsetResetPolicy: earliest
+    authenticationRef:
+      name: keda-indexer-origin-intrinsic-authentication
+---
 # Source: swh/templates/listers/keda-autoscaling.yaml
 apiVersion: keda.sh/v1alpha1
 kind: ScaledObject
 metadata:
   name: lister-bitbucket-operators
   namespace: swh
 spec:
   scaleTargetRef:
     apiVersion:    apps/v1     # Optional. Default: apps/v1
     kind:          Deployment  # Optional. Default: Deployment
@@ -39976,20 +40512,41 @@
   - parameter: password
     name: swh-archive-broker-secret
     key: BROKER_USER_PASSWORD
   - parameter: sasl
     name: keda-indexer-extrinsic-secrets
     key: sasl
   - parameter: tls
     name: keda-indexer-extrinsic-secrets
     key: tls
 ---
+# Source: swh/templates/indexers/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: keda-indexer-origin-intrinsic-authentication
+  namespace: swh
+spec:
+  secretTargetRef:
+  - parameter: username
+    name: swh-indexer-prod-01-broker-secret
+    key: BROKER_USER
+  - parameter: password
+    name: swh-indexer-prod-01-broker-secret
+    key: BROKER_USER_PASSWORD
+  - parameter: sasl
+    name: keda-indexer-origin-intrinsic-secrets
+    key: sasl
+  - parameter: tls
+    name: keda-indexer-origin-intrinsic-secrets
+    key: tls
+---
 # Source: swh/templates/listers/keda-autoscaling.yaml
 apiVersion: keda.sh/v1alpha1
 kind: TriggerAuthentication
 metadata:
   name: amqp-authentication-lister-bitbucket
   namespace: swh
 spec:
   secretTargetRef:
   - parameter: host            # "host" is required by the scalerObject trigger metadata
     name: common-secrets


------------- diff for environment production namespace swh-cassandra -------------

No differences

Refs. swh/infra/sysadm-environment#5238 (closed)