Add objstorage template & deploy objstorage read-write/read-only to staging (on storage1.staging)

In multiple commits, this adds:

• the multi-deployment objstorage template (up to the volume configuration)
• staging declaration to deploy 2 objstorage instances (read-write, read-only)

The objstorage instances are to be deployed in the storage1.internal.staging.swh.network node (which is a rancher agent now). So the access to /srv/softwareheritage/objects is possible. The docker image used by the objstorage deployed uses the 'swh' user (as usual) but specifically with the uid/gid pair 1005 (which corresponds to the actual swhstorage user in the storage1.staging node). This is to avoid having to do any kind of migration on the current path.

Note that this disables the configuration for the swh-next-version branch.

Tested through the usual diff tool [1] and minikube [2].

[2] !246 (comment 157550)

[1]

make swh-helm-diff

[swh] Comparing changes between branches production and add-objstorage-template (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in add-objstorage-template branch for environment staging...
[swh] Generate config in add-objstorage-template branch for environment staging...
[swh] Generate config in add-objstorage-template branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in add-objstorage-template branch for environment production...
[swh] Generate config in add-objstorage-template branch for environment production...
[swh] Generate config in add-objstorage-template branch for environment production...


------------- diff for environment staging namespace swh -------------

--- /tmp/swh-chart.swh.34m0XmrN/staging-swh.before      2023-11-30 15:02:53.083746877 +0100
+++ /tmp/swh-chart.swh.34m0XmrN/staging-swh.after       2023-11-30 15:02:53.771746455 +0100
@@ -14248,20 +14248,40 @@
   name: indexer-storage-rpc-ingress
   namespace: swh
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  name: objstorage-read-only-rpc-ingress
+  namespace: swh
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: objstorage-read-write-rpc-ingress
+  namespace: swh
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
   name: search-rpc-ingress
   namespace: swh
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:


------------- diff for environment staging namespace swh-cassandra -------------

--- /tmp/swh-chart.swh.34m0XmrN/staging-swh-cassandra.before    2023-11-30 15:02:53.331746725 +0100
+++ /tmp/swh-chart.swh.34m0XmrN/staging-swh-cassandra.after     2023-11-30 15:02:53.995746318 +0100
@@ -3329,20 +3329,48 @@
       swh:
         level: "INFO"
       celery.task:
         level: "INFO"

     root:
       level: "INFO"
       handlers:
       - console
 ---
+# Source: swh/templates/objstorage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: swh-cassandra
+  name: objstorage-read-only-configuration-template
+data:
+  config.yml.template: |
+    objstorage:
+      client_max_size: 1073741824
+      cls: pathslicing
+      root: /srv/softwareheritage/objects
+      slicing: 0:1/1:5
+---
+# Source: swh/templates/objstorage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: swh-cassandra
+  name: objstorage-read-write-configuration-template
+data:
+  config.yml.template: |
+    objstorage:
+      client_max_size: 1073741824
+      cls: pathslicing
+      root: /srv/softwareheritage/objects
+      slicing: 0:1/1:5
+---
 # Source: swh/templates/scrubber/storage-checker-configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
   namespace: swh-cassandra
   name: scrubber-storagechecker-directory-hashes-template
 data:
   config.yml.template: |
     scrubber:
       cls: postgresql
@@ -14066,20 +14094,40 @@
   name: indexer-storage-rpc-ingress
   namespace: swh-cassandra
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  name: objstorage-read-only-rpc-ingress
+  namespace: swh-cassandra
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: objstorage-read-write-rpc-ingress
+  namespace: swh-cassandra
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
   name: search-rpc-ingress
   namespace: swh-cassandra
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
@@ -14128,20 +14176,50 @@
   selector:
     app: memcached
   ports:
     - name: memcached
       port: 11211
       targetPort: 11211
     - name: metrics
       port: 9150
       targetPort: 9150
 ---
+# Source: swh/templates/objstorage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: objstorage-read-only
+  namespace: swh-cassandra
+spec:
+  type: ClusterIP
+  selector:
+    app: objstorage-read-only
+  ports:
+    - port: 5003
+      targetPort: 5003
+      name: rpc
+---
+# Source: swh/templates/objstorage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: objstorage-read-write
+  namespace: swh-cassandra
+spec:
+  type: ClusterIP
+  selector:
+    app: objstorage-read-write
+  ports:
+    - port: 5003
+      targetPort: 5003
+      name: rpc
+---
 # Source: swh/templates/statsd-exporter/service.yaml
 apiVersion: v1
 kind: Service
 metadata:
   name: prometheus-statsd-exporter
   namespace: swh-cassandra
   labels:
     app: prometheus-statsd-exporter
 spec:
   type: ClusterIP
@@ -19802,20 +19880,284 @@
                 port: 9150
             initialDelaySeconds: 5
             periodSeconds: 10
         livenessProbe:
             httpGet:
                 path: /metrics
                 port: 9150
             initialDelaySeconds: 5
             periodSeconds: 10
 ---
+# Source: swh/templates/objstorage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: swh-cassandra
+  name: objstorage-read-only
+  labels:
+    app: objstorage-read-only
+spec:
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: objstorage-read-only
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: objstorage-read-only
+      annotations:
+        checksum/config: aab957cf1745dce7137db591c863042fd32296081767ad821315da5d58b5a6f2
+        checksum/config-utils: 5368e835c6e31ce12e88013400913c86b168a594087648d03cd2ecead02114ab
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: swh/objstorage
+                operator: In
+                values:
+                - "true"
+              - key: kubernetes.io/hostname
+                operator: In
+                values:
+                - storage1
+      priorityClassName: swh-cassandra-frontend-rpc
+      initContainers:
+        - name: prepare-configuration
+          image: debian:bullseye
+          imagePullPolicy: IfNotPresent
+          command:
+          - /entrypoints/prepare-configuration.sh
+          env:
+
+
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+      containers:
+        - name: objstorage-read-only
+          resources:
+            requests:
+              memory: 512Mi
+              cpu: 500m
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage:20231123.1
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 5003
+              name: rpc
+          readinessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 15
+            failureThreshold: 30
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 10
+            periodSeconds: 5
+          command:
+          - /bin/bash
+          args:
+          - -c
+          - /opt/swh/entrypoint.sh
+          env:
+            - name: THREADS
+              value: "5"
+            - name: WORKERS
+              value: "2"
+            - name: TIMEOUT
+              value: "60"
+            - name: STATSD_HOST
+              value: prometheus-statsd-exporter
+            - name: STATSD_PORT
+              value: "9125"
+            - name: LOG_LEVEL
+              value: "INFO"
+            - name: SWH_SENTRY_ENVIRONMENT
+              value: staging
+            - name: SWH_MAIN_PACKAGE
+              value: swh.objstorage
+            - name: SWH_SENTRY_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: common-secrets
+                  key: objstorage-sentry-dsn
+                  # 'name' secret should exist & include key
+                  # if the setting doesn't exist, sentry pushes will be disabled
+                  optional: true
+            - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+              value: "true"
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: staging-pathslicing-ro
+            mountPath: /srv/softwareheritage/objects
+            readOnly: true
+
+      volumes:
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: objstorage-read-only-configuration-template
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+
+      - name: staging-pathslicing-ro
+
+        hostPath:
+          path: /srv/softwareheritage/objects
+          type: Directory
+---
+# Source: swh/templates/objstorage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: swh-cassandra
+  name: objstorage-read-write
+  labels:
+    app: objstorage-read-write
+spec:
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: objstorage-read-write
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: objstorage-read-write
+      annotations:
+        checksum/config: 9e35f258338db7f8ee53e44ec17a264d4171cb5d1a5ce29aeee782160b470b27
+        checksum/config-utils: 5368e835c6e31ce12e88013400913c86b168a594087648d03cd2ecead02114ab
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: swh/objstorage
+                operator: In
+                values:
+                - "true"
+              - key: kubernetes.io/hostname
+                operator: In
+                values:
+                - storage1
+      priorityClassName: swh-cassandra-frontend-rpc
+      initContainers:
+        - name: prepare-configuration
+          image: debian:bullseye
+          imagePullPolicy: IfNotPresent
+          command:
+          - /entrypoints/prepare-configuration.sh
+          env:
+
+
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+      containers:
+        - name: objstorage-read-write
+          resources:
+            requests:
+              memory: 512Mi
+              cpu: 500m
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage:20231123.1
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 5003
+              name: rpc
+          readinessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 15
+            failureThreshold: 30
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /
+              port: rpc
+            initialDelaySeconds: 10
+            periodSeconds: 5
+          command:
+          - /bin/bash
+          args:
+          - -c
+          - /opt/swh/entrypoint.sh
+          env:
+            - name: THREADS
+              value: "5"
+            - name: WORKERS
+              value: "2"
+            - name: TIMEOUT
+              value: "60"
+            - name: STATSD_HOST
+              value: prometheus-statsd-exporter
+            - name: STATSD_PORT
+              value: "9125"
+            - name: LOG_LEVEL
+              value: "INFO"
+            - name: SWH_SENTRY_ENVIRONMENT
+              value: staging
+            - name: SWH_MAIN_PACKAGE
+              value: swh.objstorage
+            - name: SWH_SENTRY_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: common-secrets
+                  key: objstorage-sentry-dsn
+                  # 'name' secret should exist & include key
+                  # if the setting doesn't exist, sentry pushes will be disabled
+                  optional: true
+            - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+              value: "true"
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: staging-pathslicing-rw
+            mountPath: /srv/softwareheritage/objects
+            readOnly: false
+
+      volumes:
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: objstorage-read-write-configuration-template
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+
+      - name: staging-pathslicing-rw
+
+        hostPath:
+          path: /srv/softwareheritage/objects
+          type: Directory
+---
 # Source: swh/templates/scrubber/storage-checker-deployment.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: scrubber-storagechecker-directory-hashes
   namespace: swh-cassandra
   labels:
     app: scrubber-storagechecker-directory-hashes
 spec:
   revisionHistoryLimit: 2
@@ -22910,20 +23252,70 @@
     http:
       paths:
       - path: /graphql/
         pathType: Prefix
         backend:
           service:
             name: graphql
             port:
               number: 5013
 ---
+# Source: swh/templates/objstorage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: swh-cassandra
+  name: objstorage-read-only-ingress-default
+  annotations:
+    nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
+    nginx.ingress.kubernetes.io/proxy-body-size: 4G
+    nginx.ingress.kubernetes.io/proxy-buffering: "on"
+
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: objstorage-read-only-rpc-ingress
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: objstorage-read-only
+            port:
+              number: 5003
+---
+# Source: swh/templates/objstorage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: swh-cassandra
+  name: objstorage-read-write-ingress-default
+  annotations:
+    nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.130.0/24,192.168.50.0/24
+    nginx.ingress.kubernetes.io/proxy-body-size: 4G
+    nginx.ingress.kubernetes.io/proxy-buffering: "on"
+
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: objstorage-read-write-rpc-ingress
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: objstorage-read-write
+            port:
+              number: 5003
+---
 # Source: swh/templates/storage/ingress.yaml
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   namespace: swh-cassandra
   name: storage-cassandra-ingress-default
   annotations:
     nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
     nginx.ingress.kubernetes.io/proxy-body-size: 4G
     nginx.ingress.kubernetes.io/proxy-buffering: "on"


------------- diff for environment staging namespace swh-cassandra-next-version -------------

--- /tmp/swh-chart.swh.34m0XmrN/staging-swh-cassandra-next-version.before       2023-11-30 15:02:53.531746602 +0100
+++ /tmp/swh-chart.swh.34m0XmrN/staging-swh-cassandra-next-version.after        2023-11-30 15:02:54.203746191 +0100
@@ -13616,20 +13616,40 @@
   name: indexer-storage-rpc-ingress
   namespace: swh-cassandra-next-version
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  name: objstorage-read-only-rpc-ingress
+  namespace: swh-cassandra-next-version
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: objstorage-read-write-rpc-ingress
+  namespace: swh-cassandra-next-version
+spec:
+  type: ExternalName
+  externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
   name: search-rpc-ingress
   namespace: swh-cassandra-next-version
 spec:
   type: ExternalName
   externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 ---
 # Source: swh/templates/external-services/cname.yaml
 apiVersion: v1
 kind: Service
 metadata:


------------- diff for environment production namespace swh -------------

No differences


------------- diff for environment production namespace swh-cassandra -------------

No differences

Refs. swh/infra/sysadm-environment#5164 (closed)

swh/values/staging/default.yaml

@@ -23,6 +23,12 @@ stagingNetworkRanges:
 noopObjectStorage:
   cls: noop
 
+readWriteObjstorageConfiguration:
+  cls: pathslicing
+  root: "/srv/softwareheritage/objects"
+  slicing: 0:1/1:5
+  client_max_size: 1073741824
+
 storagePipelineSteps:
   - &bufferStorage
     cls: buffer
@@ -294,6 +300,9 @@ externalServices:
     storage-postgresql:
       internalName: storage-postgresql-rpc-ingress
       target: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    # objstorage-read-write:
+    #   internalName: objstorage-read-write-rpc-ingress
+    #   target: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
 
 vault:
   hosts: