Add objstorage template & deploy objstorage read-write/read-only to staging (on storage1.staging)
In multiple commits, this adds:
- the multi-deployment objstorage template (up to the volume configuration)
- staging declaration to deploy 2 objstorage instances (read-write, read-only)
The objstorage instances are to be deployed in the storage1.internal.staging.swh.network node (which is a rancher agent now). So the access to /srv/softwareheritage/objects is possible. The docker image used by the objstorage deployed uses the 'swh' user (as usual) but specifically with the uid/gid pair 1005 (which corresponds to the actual swhstorage user in the storage1.staging node). This is to avoid having to do any kind of migration on the current path.
Note that this disables the configuration for the swh-next-version branch.
Tested through the usual diff tool [1] and minikube [2].
[1]
make swh-helm-diff
[swh] Comparing changes between branches production and add-objstorage-template (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in add-objstorage-template branch for environment staging...
[swh] Generate config in add-objstorage-template branch for environment staging...
[swh] Generate config in add-objstorage-template branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in add-objstorage-template branch for environment production...
[swh] Generate config in add-objstorage-template branch for environment production...
[swh] Generate config in add-objstorage-template branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.34m0XmrN/staging-swh.before 2023-11-30 15:02:53.083746877 +0100
+++ /tmp/swh-chart.swh.34m0XmrN/staging-swh.after 2023-11-30 15:02:53.771746455 +0100
@@ -14248,20 +14248,40 @@
name: indexer-storage-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: objstorage-read-only-rpc-ingress
+ namespace: swh
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: objstorage-read-write-rpc-ingress
+ namespace: swh
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: search-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.34m0XmrN/staging-swh-cassandra.before 2023-11-30 15:02:53.331746725 +0100
+++ /tmp/swh-chart.swh.34m0XmrN/staging-swh-cassandra.after 2023-11-30 15:02:53.995746318 +0100
@@ -3329,20 +3329,48 @@
swh:
level: "INFO"
celery.task:
level: "INFO"
root:
level: "INFO"
handlers:
- console
---
+# Source: swh/templates/objstorage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: objstorage-read-only-configuration-template
+data:
+ config.yml.template: |
+ objstorage:
+ client_max_size: 1073741824
+ cls: pathslicing
+ root: /srv/softwareheritage/objects
+ slicing: 0:1/1:5
+---
+# Source: swh/templates/objstorage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: objstorage-read-write-configuration-template
+data:
+ config.yml.template: |
+ objstorage:
+ client_max_size: 1073741824
+ cls: pathslicing
+ root: /srv/softwareheritage/objects
+ slicing: 0:1/1:5
+---
# Source: swh/templates/scrubber/storage-checker-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: scrubber-storagechecker-directory-hashes-template
data:
config.yml.template: |
scrubber:
cls: postgresql
@@ -14066,20 +14094,40 @@
name: indexer-storage-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: objstorage-read-only-rpc-ingress
+ namespace: swh-cassandra
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: objstorage-read-write-rpc-ingress
+ namespace: swh-cassandra
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: search-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -14128,20 +14176,50 @@
selector:
app: memcached
ports:
- name: memcached
port: 11211
targetPort: 11211
- name: metrics
port: 9150
targetPort: 9150
---
+# Source: swh/templates/objstorage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: objstorage-read-only
+ namespace: swh-cassandra
+spec:
+ type: ClusterIP
+ selector:
+ app: objstorage-read-only
+ ports:
+ - port: 5003
+ targetPort: 5003
+ name: rpc
+---
+# Source: swh/templates/objstorage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: objstorage-read-write
+ namespace: swh-cassandra
+spec:
+ type: ClusterIP
+ selector:
+ app: objstorage-read-write
+ ports:
+ - port: 5003
+ targetPort: 5003
+ name: rpc
+---
# Source: swh/templates/statsd-exporter/service.yaml
apiVersion: v1
kind: Service
metadata:
name: prometheus-statsd-exporter
namespace: swh-cassandra
labels:
app: prometheus-statsd-exporter
spec:
type: ClusterIP
@@ -19802,20 +19880,284 @@
port: 9150
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /metrics
port: 9150
initialDelaySeconds: 5
periodSeconds: 10
---
+# Source: swh/templates/objstorage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra
+ name: objstorage-read-only
+ labels:
+ app: objstorage-read-only
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: objstorage-read-only
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: objstorage-read-only
+ annotations:
+ checksum/config: aab957cf1745dce7137db591c863042fd32296081767ad821315da5d58b5a6f2
+ checksum/config-utils: 5368e835c6e31ce12e88013400913c86b168a594087648d03cd2ecead02114ab
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/objstorage
+ operator: In
+ values:
+ - "true"
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - storage1
+ priorityClassName: swh-cassandra-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: objstorage-read-only
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage:20231123.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5003
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: THREADS
+ value: "5"
+ - name: WORKERS
+ value: "2"
+ - name: TIMEOUT
+ value: "60"
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: LOG_LEVEL
+ value: "INFO"
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.objstorage
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: objstorage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: staging-pathslicing-ro
+ mountPath: /srv/softwareheritage/objects
+ readOnly: true
+
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: objstorage-read-only-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
+ - name: staging-pathslicing-ro
+
+ hostPath:
+ path: /srv/softwareheritage/objects
+ type: Directory
+---
+# Source: swh/templates/objstorage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra
+ name: objstorage-read-write
+ labels:
+ app: objstorage-read-write
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: objstorage-read-write
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: objstorage-read-write
+ annotations:
+ checksum/config: 9e35f258338db7f8ee53e44ec17a264d4171cb5d1a5ce29aeee782160b470b27
+ checksum/config-utils: 5368e835c6e31ce12e88013400913c86b168a594087648d03cd2ecead02114ab
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/objstorage
+ operator: In
+ values:
+ - "true"
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - storage1
+ priorityClassName: swh-cassandra-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: objstorage-read-write
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage:20231123.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5003
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: THREADS
+ value: "5"
+ - name: WORKERS
+ value: "2"
+ - name: TIMEOUT
+ value: "60"
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: LOG_LEVEL
+ value: "INFO"
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.objstorage
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: objstorage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: staging-pathslicing-rw
+ mountPath: /srv/softwareheritage/objects
+ readOnly: false
+
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: objstorage-read-write-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
+ - name: staging-pathslicing-rw
+
+ hostPath:
+ path: /srv/softwareheritage/objects
+ type: Directory
+---
# Source: swh/templates/scrubber/storage-checker-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: scrubber-storagechecker-directory-hashes
namespace: swh-cassandra
labels:
app: scrubber-storagechecker-directory-hashes
spec:
revisionHistoryLimit: 2
@@ -22910,20 +23252,70 @@
http:
paths:
- path: /graphql/
pathType: Prefix
backend:
service:
name: graphql
port:
number: 5013
---
+# Source: swh/templates/objstorage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh-cassandra
+ name: objstorage-read-only-ingress-default
+ annotations:
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-buffering: "on"
+
+spec:
+ ingressClassName: nginx
+ rules:
+ - host: objstorage-read-only-rpc-ingress
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: objstorage-read-only
+ port:
+ number: 5003
+---
+# Source: swh/templates/objstorage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh-cassandra
+ name: objstorage-read-write-ingress-default
+ annotations:
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.130.0/24,192.168.50.0/24
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-buffering: "on"
+
+spec:
+ ingressClassName: nginx
+ rules:
+ - host: objstorage-read-write-rpc-ingress
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: objstorage-read-write
+ port:
+ number: 5003
+---
# Source: swh/templates/storage/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh-cassandra
name: storage-cassandra-ingress-default
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
nginx.ingress.kubernetes.io/proxy-body-size: 4G
nginx.ingress.kubernetes.io/proxy-buffering: "on"
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.34m0XmrN/staging-swh-cassandra-next-version.before 2023-11-30 15:02:53.531746602 +0100
+++ /tmp/swh-chart.swh.34m0XmrN/staging-swh-cassandra-next-version.after 2023-11-30 15:02:54.203746191 +0100
@@ -13616,20 +13616,40 @@
name: indexer-storage-rpc-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: objstorage-read-only-rpc-ingress
+ namespace: swh-cassandra-next-version
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: objstorage-read-write-rpc-ingress
+ namespace: swh-cassandra-next-version
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: search-rpc-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differences