staging: Activate indexer storage in elastic infra

make swh-helm-diff

[swh] Comparing changes between branches production and migrate-indexer-storage (per environment)...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
Switched to branch 'migrate-indexer-storage'
[swh] Generate config in migrate-indexer-storage branch for environment staging...
[swh] Generate config in migrate-indexer-storage branch for environment staging...
[swh] Generate config in migrate-indexer-storage branch for environment staging...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
Switched to branch 'migrate-indexer-storage'
[swh] Generate config in migrate-indexer-storage branch for environment production...
[swh] Generate config in migrate-indexer-storage branch for environment production...
[swh] Generate config in migrate-indexer-storage branch for environment production...


------------- diff for environment staging namespace swh -------------

--- /tmp/swh-chart.swh.JuhqJYOt/staging-swh.before      2023-10-25 17:11:44.983164235 +0200
+++ /tmp/swh-chart.swh.JuhqJYOt/staging-swh.after       2023-10-25 17:11:45.675163745 +0200
@@ -282,20 +282,42 @@
       anonymous: 50
       user: 500

     auth:
       server: https://auth.softwareheritage.org/auth/
       realm: SoftwareHeritageStaging
       client: swh-web
       cache:
         url: memcached://memcached:11211
 ---
+# Source: swh/templates/indexer-storage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: swh
+  name: indexer-storage-configuration-template
+data:
+  config.yml.template: |
+    indexer_storage:
+      cls: postgresql
+      db: host=db1.internal.staging.swh.network port=5432 user=swh-indexer dbname=swh-indexer password=${POSTGRESQL_PASSWORD}
+    journal_writer:
+
+      client_id: swh.idx_storage.journal_writer.storage1
+      cls: kafka
+      prefix: swh.journal.indexed
+      producer_config:
+        message.max.bytes: 1000000000
+      brokers:
+      - journal1.internal.staging.swh.network
+      - journal2.internal.staging.swh.network
+---
 # Source: swh/templates/indexers/configmap-utils.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: indexer-utils
   namespace: swh
 data:
   pre-stop-idempotent.sh: |
     #!/bin/bash

@@ -331,21 +353,21 @@
       cls: pipeline
       steps:
       - cls: retry
       - cls: remote
         url: http://storage1.internal.staging.swh.network:5002
     scheduler:
       cls: remote
       url: http://scheduler.internal.staging.swh.network
     indexer_storage:
       cls: remote
-      url: http://storage1.internal.staging.swh.network:5007/
+      url: http://indexer-storage-rpc-ingress
     objstorage:
       cls: remote
       url: http://storage1.internal.staging.swh.network:5003/
     journal:
       brokers:
         - journal1.internal.staging.swh.network:9094

       group_id: swh-archive-stg-swh.indexer.journal_client.origin_intrinsic_metadata
       prefix: swh.journal.objects

@@ -4066,21 +4088,21 @@
       cls: remote
       url: http://search0.internal.staging.swh.network:5010
     scheduler:
       cls: remote
       url: http://scheduler.internal.staging.swh.network
     vault:
       cls: remote
       url: http://vault-rpc-ingress
     indexer_storage:
       cls: remote
-      url: http://storage1.internal.staging.swh.network:5007/
+      url: http://indexer-storage-rpc-ingress
     counters_backend: swh-counters
     counters:
       cls: remote
       url: http://counters0.internal.staging.swh.network:5011/
     deposit:
       private_api_url: https://deposit-rp.internal.staging.swh.network/1/private/
       private_api_user: ${DEPOSIT_USERNAME}
       private_api_password: ${DEPOSIT_PASSWORD}

     secret_key: ${DJANGO_SECRET_KEY}
@@ -13710,20 +13732,30 @@
     port: 443
     protocol: TCP
     targetPort: 9443
   selector:
     app: keda-admission-webhooks
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  name: indexer-storage-rpc-ingress
+  namespace: swh
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
   name: vault-rpc-ingress
   namespace: swh
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/graphql/service.yaml
 apiVersion: v1
 kind: Service
 metadata:
@@ -13731,20 +13763,35 @@
   namespace: swh
 spec:
   type: ClusterIP
   selector:
     app: graphql
   ports:
     - port: 5013
       targetPort: 5013
       name: rpc
 ---
+# Source: swh/templates/indexer-storage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: indexer-storage-rpc
+  namespace: swh
+spec:
+  type: ClusterIP
+  selector:
+    app: indexer-storage-rpc
+  ports:
+    - port: 5007
+      targetPort: 5007
+      name: rpc
+---
 # Source: swh/templates/memcached/service.yaml
 apiVersion: v1
 kind: Service
 metadata:
   name: memcached
   namespace: swh
   labels:
     app: memcached
     chart: "swh-0.1.0"
 spec:
@@ -14382,20 +14429,165 @@
             - name: config
               mountPath: /etc/swh/config.yml
               subPath: config.yml
               readOnly: true
       volumes:
         - name: config
           configMap:
             name: graphql
             defaultMode: 0444
 ---
+# Source: swh/templates/indexer-storage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: swh
+  name: indexer-storage
+  labels:
+    app: indexer-storage
+spec:
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: indexer-storage
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: indexer-storage
+      annotations:
+        checksum/config: b71cc0136e069c1a6ad2041e5e764a0823db7d2b29692156e3358374ca9ba604
+        checksum/config-utils: 8d954649e8974c3ddeb97d0e37c29427f865a3c99a8f94b7860b26a15672c172
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: swh/rpc
+                operator: In
+                values:
+                - "true"
+      priorityClassName: swh-frontend-rpc
+      initContainers:
+        - name: prepare-configuration
+          image: debian:bullseye
+          imagePullPolicy: IfNotPresent
+          command:
+          - /bin/bash
+          args:
+          - -c
+          - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+          env:
+
+
+          - name: POSTGRESQL_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: swh-indexer-storage-postgresql-secret
+                key: postgres-swh-indexer-password
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+
+        - name: check-migration
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage:20231013.1
+          command:
+          - /entrypoints/check-indexer-storage-db-version.sh
+          env:
+          - name: MODULE
+            value: indexer-storage
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: database-utils
+            mountPath: /entrypoints
+      containers:
+        - name: indexer-storage
+          resources:
+            requests:
+              memory: 512Mi
+              cpu: 500m
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage:20231013.1
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 5007
+              name: rpc
+          readinessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 15
+            failureThreshold: 30
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 10
+            periodSeconds: 5
+          command:
+          - /bin/bash
+          args:
+          - -c
+          - /opt/swh/entrypoint.sh
+          env:
+            - name: THREADS
+              value: "2"
+            - name: WORKERS
+              value: "4"
+            - name: TIMEOUT
+              value: "60"
+            - name: STATSD_HOST
+              value: prometheus-statsd-exporter
+            - name: STATSD_PORT
+              value: "9125"
+            - name: LOG_LEVEL
+              value: "INFO"
+            - name: SWH_SENTRY_ENVIRONMENT
+              value: staging
+            - name: SWH_MAIN_PACKAGE
+              value: swh.indexer
+            - name: SWH_SENTRY_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: common-secrets
+                  key: indexer-storage-sentry-dsn
+                  # 'name' secret should exist & include key
+                  # if the setting doesn't exist, sentry pushes will be disabled
+                  optional: true
+            - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+              value: "true"
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+      volumes:
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: indexer-storage-configuration-template
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+      - name: database-utils
+        configMap:
+          name: database-utils
+          defaultMode: 0555
+---
 # Source: swh/templates/indexers/deployment.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: indexer-origin-intrinsic
   namespace: swh
   labels:
     app: indexer-origin-intrinsic
 spec:
   revisionHistoryLimit: 2
@@ -14405,21 +14597,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: indexer-origin-intrinsic
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 0b6c8039f89e2bc18a3f4b3bbf222e2c12e3a7dfd075d0dd254c9b3edba0fae5
+        checksum/config: a2ceebb08d978c9bb866a9adaf63e680e73877df2f045217e184b07d0d514292
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/indexer
                 operator: In
                 values:
@@ -21485,21 +21677,21 @@
       app: web
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: web
       annotations:
-        checksum/config: eff72d25b88f78efc4d7fd2bebc98cffceff5af247dd83ab2ea9dd151d563900
+        checksum/config: 7884686e868f39eef94bc70510995469c49cdf1ed96c9b0250e4c548f803eb51
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/web
                 operator: In
                 values:
                 - "true"
@@ -21708,20 +21900,43 @@
         emptyDir: {}
       - name: configuration-template
         configMap:
          name: web-configuration-template
          items:
          - key: "config.yml.template"
            path: "config.yml.template"
       - name: static
         emptyDir: {}
 ---
+# Source: swh/templates/indexer-storage/autoscaling.yaml
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  namespace: swh
+  name: indexer-storage-rpc
+  labels:
+    app: indexer-storage-rpc
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: indexer-storage-rpc
+  minReplicas: 2
+  maxReplicas: 4
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 90
+---
 # Source: swh/templates/web/autoscaling.yaml
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
   namespace: swh
   name: web
   labels:
     app: web
 spec:
   scaleTargetRef:
@@ -22165,20 +22380,41 @@
     http:
       paths:
       - path: /graphql/
         pathType: Prefix
         backend:
           service:
             name: graphql
             port:
               number: 5013
 ---
+# Source: swh/templates/indexer-storage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: swh
+  name: indexer-storage-rpc-ingress-default
+  annotations:
+
+spec:
+  rules:
+  - host: indexer-storage-rpc-ingress
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: indexer-storage-rpc
+            port:
+              number: 5007
+---
 # Source: swh/templates/scheduler/rpc-ingress.yaml
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   namespace: swh
   name: scheduler-rpc-ingress-default
   annotations:
     nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.130.0/24,192.168.50.0/24

     nginx.ingress.kubernetes.io/proxy-body-size: 4G


------------- diff for environment staging namespace swh-cassandra -------------

--- /tmp/swh-chart.swh.JuhqJYOt/staging-swh-cassandra.before    2023-10-25 17:11:45.223164065 +0200
+++ /tmp/swh-chart.swh.JuhqJYOt/staging-swh-cassandra.after     2023-10-25 17:11:45.899163586 +0200
@@ -3894,21 +3894,21 @@
       cls: remote
       url: http://search0.internal.staging.swh.network:5010
     scheduler:
       cls: remote
       url: http://scheduler.internal.staging.swh.network
     vault:
       cls: remote
       url: http://vault-rpc-ingress
     indexer_storage:
       cls: remote
-      url: http://storage1.internal.staging.swh.network:5007/
+      url: http://indexer-storage-rpc-ingress
     counters_backend: swh-counters
     counters:
       cls: remote
       url: http://counters0.internal.staging.swh.network:5011/

     secret_key: ${DJANGO_SECRET_KEY}
     production_db:

       host: db1.internal.staging.swh.network
       port: 5432
@@ -13534,20 +13534,30 @@
     port: 443
     protocol: TCP
     targetPort: 9443
   selector:
     app: keda-admission-webhooks
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  name: indexer-storage-rpc-ingress
+  namespace: swh-cassandra
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
   name: vault-rpc-ingress
   namespace: swh-cassandra
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/graphql/service.yaml
 apiVersion: v1
 kind: Service
 metadata:
@@ -20540,21 +20550,21 @@
       app: web
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: web
       annotations:
-        checksum/config: 09a3336045d642f75d9eb60ab89121c0ab2e8ca3fbe8e85c832b805fb789da09
+        checksum/config: 2eb28c73fcf0bef340dc3fdcded027efed17efc0b4039592a1bd853002fcc8fd
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/web
                 operator: In
                 values:
                 - "true"


------------- diff for environment staging namespace swh-cassandra-next-version -------------

--- /tmp/swh-chart.swh.JuhqJYOt/staging-swh-cassandra-next-version.before       2023-10-25 17:11:45.443163909 +0200
+++ /tmp/swh-chart.swh.JuhqJYOt/staging-swh-cassandra-next-version.after        2023-10-25 17:11:46.111163435 +0200
@@ -3688,21 +3688,21 @@
       cls: remote
       url: http://search0.internal.staging.swh.network:5010
     scheduler:
       cls: remote
       url: http://scheduler.internal.staging.swh.network
     vault:
       cls: remote
       url: http://vault-rpc-ingress-next-version
     indexer_storage:
       cls: remote
-      url: http://storage1.internal.staging.swh.network:5007/
+      url: http://indexer-storage-rpc-ingress
     counters_backend: swh-counters
     counters:
       cls: remote
       url: http://counters0.internal.staging.swh.network:5011/

     secret_key: ${DJANGO_SECRET_KEY}
     production_db:

       host: db1.internal.staging.swh.network
       port: 5432
@@ -13328,20 +13328,30 @@
     port: 443
     protocol: TCP
     targetPort: 9443
   selector:
     app: keda-admission-webhooks
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  name: indexer-storage-rpc-ingress
+  namespace: swh-cassandra-next-version
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
   name: vault-rpc-ingress
   namespace: swh-cassandra-next-version
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/graphql/service.yaml
 apiVersion: v1
 kind: Service
 metadata:
@@ -20004,21 +20014,21 @@
       app: web
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: web
       annotations:
-        checksum/config: f63342dcc16ae9ec1d6c764d085678aac2fa8ba6597ae180517edaf75993353f
+        checksum/config: ab9fb370e20d7f3e390d0bcb30dc4ffa3febdd79a5ba2739f278d2165df2ea07
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/web
                 operator: In
                 values:
                 - "true"


------------- diff for environment production namespace swh -------------

No differences


------------- diff for environment production namespace swh-cassandra -------------

No differences

Refs. swh/infra/sysadm-environment#4780 (closed)

swh/templates/indexer-storage/ingress.yaml

 {{- if and .Values.indexerStorage.enabled .Values.indexerStorage.ingress.enabled }}
 {{- include "swh.ingress" (dict "Values"        .Values
                                 "serviceType"   "indexer-storage-rpc"
-                                "configuration" .Values.search) -}}
+                                "configuration" .Values.indexerStorage) -}}
 {{ end }}