indexer_storage: Make template multi-instance & deploy read-only instance to production

This makes the indexer storage template multi-instance. This also adapts slightly the template to use the prepare configuration utils image. This simplifies and unifies the init-container with the recent change on this.

Expectedly, the diff [1] only shows:

• the renaming of the rpc configmap (due to the multi-instance change).
• The image adaptation in the init-container

The last commit actually deploys a new indexer-storage-read-only instance in production environment. [2] This instance will be used by the soon to be released webapp on the dynamic infra.

[1]

make swh-helm-diff

[swh] Comparing changes between branches production and migrate-indexers-to-prod (per environment)...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
Switched to branch 'migrate-indexers-to-prod'
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
Switched to branch 'migrate-indexers-to-prod'
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...


------------- diff for environment staging namespace swh -------------

--- /tmp/swh-chart.swh.eUVAEZAd/staging-swh.before      2023-12-13 14:33:41.102880007 +0100
+++ /tmp/swh-chart.swh.eUVAEZAd/staging-swh.after       2023-12-13 14:33:41.838879725 +0100
@@ -287,21 +287,21 @@
       realm: SoftwareHeritageStaging
       client: swh-web
       cache:
         url: memcached://memcached:11211
 ---
 # Source: swh/templates/indexer-storage/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   namespace: swh
-  name: indexer-storage-configuration-template
+  name: indexer-storage-rpc-configuration-template
 data:
   config.yml.template: |
     indexer_storage:
       cls: postgresql
       db: host=db1.internal.staging.swh.network port=5432 user=swh-indexer dbname=swh-indexer password=${POSTGRESQL_PASSWORD}
     journal_writer:

       client_id: swh.idx_storage.journal_writer.storage1
       cls: kafka
       prefix: swh.journal.indexed
@@ -15096,57 +15096,57 @@
       app: indexer-storage-rpc
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: indexer-storage-rpc
       annotations:
-        checksum/config: b71cc0136e069c1a6ad2041e5e764a0823db7d2b29692156e3358374ca9ba604
-        checksum/config-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
+        checksum/config: e688ca7853c5421a857f84a4da8a5267fee3117c32587fd49510eaee2b23fb62
+        checksum/database-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/rpc
                 operator: In
                 values:
                 - "true"
       priorityClassName: swh-frontend-rpc
       initContainers:
         - name: prepare-configuration
-          image: debian:bullseye
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
           imagePullPolicy: IfNotPresent
           command:
-          - /bin/bash
-          args:
-          - -c
-          - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+          - /entrypoints/prepare-configuration.sh
           env:


           - name: POSTGRESQL_PASSWORD
             valueFrom:
               secretKeyRef:
                 name: swh-indexer-storage-postgresql-secret
                 key: postgres-swh-indexer-password
                 # 'name' secret must exist & include that ^ key
                 optional: false
           volumeMounts:
           - name: configuration
             mountPath: /etc/swh
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
+          - name: config-utils
+            mountPath: /entrypoints
+            readOnly: true

         - name: check-migration
           image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage:20231205.1
           command:
           - /entrypoints/check-indexer-storage-db-version.sh
           env:
           - name: MODULE
             value: indexer-storage
           volumeMounts:
           - name: configuration
@@ -15210,28 +15210,32 @@
             - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
               value: "true"
           volumeMounts:
           - name: configuration
             mountPath: /etc/swh
       volumes:
       - name: configuration
         emptyDir: {}
       - name: configuration-template
         configMap:
-          name: indexer-storage-configuration-template
+          name: indexer-storage-rpc-configuration-template
           items:
           - key: "config.yml.template"
             path: "config.yml.template"
       - name: database-utils
         configMap:
           name: database-utils
           defaultMode: 0555
+      - name: config-utils
+        configMap:
+          name: config-utils
+          defaultMode: 0555
 ---
 # Source: swh/templates/indexers/deployment.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: indexer-origin-intrinsic
   namespace: swh
   labels:
     app: indexer-origin-intrinsic
 spec:


------------- diff for environment staging namespace swh-cassandra -------------

No differences


------------- diff for environment staging namespace swh-cassandra-next-version -------------

No differences


------------- diff for environment production namespace swh -------------

No differences


------------- diff for environment production namespace swh-cassandra -------------

No differences

[2] (diff on top of the previous 2 commits to only concentrate on the deployment)

make swh-helm-diff

[swh] Comparing changes between branches production and migrate-indexers-to-prod (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...


------------- diff for environment staging namespace swh -------------

No differences


------------- diff for environment staging namespace swh-cassandra -------------

No differences


------------- diff for environment staging namespace swh-cassandra-next-version -------------

No differences


------------- diff for environment production namespace swh -------------

--- /tmp/swh-chart.swh.ugkTtDiH/production-swh.before   2023-12-13 16:05:23.529260972 +0100
+++ /tmp/swh-chart.swh.ugkTtDiH/production-swh.after    2023-12-13 16:05:23.985260878 +0100
@@ -270,20 +270,32 @@
       anonymous: 50
       user: 500

     auth:
       server: https://auth.softwareheritage.org/auth/
       realm: SoftwareHeritage
       client: swh-web
       cache:
         url: memcached://memcached:11211
 ---
+# Source: swh/templates/indexer-storage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: swh
+  name: indexer-storage-read-only-configuration-template
+data:
+  config.yml.template: |
+    indexer_storage:
+      cls: postgresql
+      db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage-indexer password=${POSTGRESQL_PASSWORD}
+---
 # Source: swh/templates/listers/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: lister-utils
   namespace: swh
 data:
   pre-stop-idempotent.sh: |
     #!/bin/bash

@@ -15575,20 +15587,30 @@
     port: 443
     protocol: TCP
     targetPort: 9443
   selector:
     app: keda-admission-webhooks
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  name: indexer-storage-read-only-rpc-ingress
+  namespace: swh
+spec:
+  type: ExternalName
+  externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
   name: search-rpc-ingress
   namespace: swh
 spec:
   type: ExternalName
   externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
@@ -15606,20 +15628,35 @@
   namespace: swh
 spec:
   type: ClusterIP
   selector:
     app: graphql
   ports:
     - port: 5013
       targetPort: 5013
       name: rpc
 ---
+# Source: swh/templates/indexer-storage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: indexer-storage-read-only
+  namespace: swh
+spec:
+  type: ClusterIP
+  selector:
+    app: indexer-storage-read-only
+  ports:
+    - port: 5007
+      targetPort: 5007
+      name: rpc
+---
 # Source: swh/templates/memcached/service.yaml
 apiVersion: v1
 kind: Service
 metadata:
   name: memcached
   namespace: swh
   labels:
     app: memcached
     chart: "swh-0.1.0"
 spec:
@@ -16287,20 +16324,169 @@
             - name: config
               mountPath: /etc/swh/config.yml
               subPath: config.yml
               readOnly: true
       volumes:
         - name: config
           configMap:
             name: graphql
             defaultMode: 0444
 ---
+# Source: swh/templates/indexer-storage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: swh
+  name: indexer-storage-read-only
+  labels:
+    app: indexer-storage-read-only
+spec:
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: indexer-storage-read-only
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: indexer-storage-read-only
+      annotations:
+        checksum/config: 0413c049af8223046084d7c4f880f076d11d09b13523036d5cdeb6a2d4f537bd
+        checksum/database-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: swh/rpc
+                operator: In
+                values:
+                - "true"
+      priorityClassName: swh-frontend-rpc
+      initContainers:
+        - name: prepare-configuration
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+          imagePullPolicy: IfNotPresent
+          command:
+          - /entrypoints/prepare-configuration.sh
+          env:
+
+
+          - name: POSTGRESQL_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: swh-indexer-storage-postgresql-secret
+                key: postgres-guest-password
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+          - name: config-utils
+            mountPath: /entrypoints
+            readOnly: true
+
+        - name: check-migration
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage:20231205.1
+          command:
+          - /entrypoints/check-indexer-storage-db-version.sh
+          env:
+          - name: MODULE
+            value: indexer-storage
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: database-utils
+            mountPath: /entrypoints
+      containers:
+        - name: indexer-storage-read-only
+          resources:
+            requests:
+              memory: 512Mi
+              cpu: 500m
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage:20231205.1
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 5007
+              name: rpc
+          readinessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 15
+            failureThreshold: 30
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 10
+            periodSeconds: 5
+          command:
+          - /bin/bash
+          args:
+          - -c
+          - /opt/swh/entrypoint.sh
+          env:
+            - name: THREADS
+              value: "2"
+            - name: WORKERS
+              value: "4"
+            - name: TIMEOUT
+              value: "60"
+            - name: STATSD_HOST
+              value: prometheus-statsd-exporter
+            - name: STATSD_PORT
+              value: "9125"
+            - name: LOG_LEVEL
+              value: "INFO"
+            - name: SWH_SENTRY_ENVIRONMENT
+              value: production
+            - name: SWH_MAIN_PACKAGE
+              value: swh.indexer
+            - name: SWH_SENTRY_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: common-secrets
+                  key: indexer-storage-sentry-dsn
+                  # 'name' secret should exist & include key
+                  # if the setting doesn't exist, sentry pushes will be disabled
+                  optional: true
+            - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+              value: "true"
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+      volumes:
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: indexer-storage-read-only-configuration-template
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+      - name: database-utils
+        configMap:
+          name: database-utils
+          defaultMode: 0555
+      - name: config-utils
+        configMap:
+          name: config-utils
+          defaultMode: 0555
+---
 # Source: swh/templates/listers/deployment.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: lister-bitbucket
   namespace: swh
   labels:
     app: lister-bitbucket
 spec:
   revisionHistoryLimit: 2
@@ -28105,20 +28291,43 @@
           defaultMode: 0777
           items:
           - key: "config.yml.template"
             path: "config.yml.template"

       - name: toolbox-script-utils
         configMap:
           name: toolbox-script-utils
           defaultMode: 0555
 ---
+# Source: swh/templates/indexer-storage/autoscaling.yaml
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  namespace: swh
+  name: indexer-storage-read-only
+  labels:
+    app: indexer-storage-read-only
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: indexer-storage-read-only
+  minReplicas: 2
+  maxReplicas: 4
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 90
+---
 # Source: swh/templates/search/rpc-autoscale.yaml
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
   namespace: swh
   name: search-rpc
   labels:
     app: search-rpc
 spec:
   scaleTargetRef:
@@ -28279,20 +28488,47 @@
     http:
       paths:
       - path: /
         pathType: Prefix
         backend:
           service:
             name: graphql
             port:
               number: 5013
 ---
+# Source: swh/templates/indexer-storage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: swh
+  name: indexer-storage-read-only-ingress-default
+  annotations:
+    nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16
+    nginx.ingress.kubernetes.io/proxy-body-size: 4G
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
+
+spec:
+  rules:
+  - host: indexer-storage-read-only-rpc-ingress
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: indexer-storage-read-only
+            port:
+              number: 5007
+---
 # Source: swh/templates/scheduler/rpc-ingress.yaml
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   namespace: swh
   name: scheduler-rpc-ingress-default
   annotations:
     nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,127.0.0.0/8,192.168.100.0/24,192.168.130.221,192.168.200.0/22
     nginx.ingress.kubernetes.io/proxy-body-size: 4G
     nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
@@ -28340,20 +28576,21 @@
             port:
               number: 5008
 ---
 # Source: swh/templates/search/rpc-ingress.yaml
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   namespace: swh
   name: search-rpc-ingress-default
   annotations:
+    nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16
     nginx.ingress.kubernetes.io/proxy-body-size: 4G
     nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
     nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
     nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "90"

 spec:
   rules:
   - host: search-rpc-ingress
     http:


------------- diff for environment production namespace swh-cassandra -------------

--- /tmp/swh-chart.swh.ugkTtDiH/production-swh-cassandra.before 2023-12-13 16:05:23.701260937 +0100
+++ /tmp/swh-chart.swh.ugkTtDiH/production-swh-cassandra.after  2023-12-13 16:05:24.157260843 +0100
@@ -11095,20 +11095,30 @@
     port: 443
     protocol: TCP
     targetPort: 9443
   selector:
     app: keda-admission-webhooks
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  name: indexer-storage-read-only-rpc-ingress
+  namespace: swh-cassandra
+spec:
+  type: ExternalName
+  externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
   name: search-rpc-ingress
   namespace: swh-cassandra
 spec:
   type: ExternalName
   externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:

Refs. swh/infra/sysadm-environment#5183 (closed)