From 8f2e84cc8efe4d5d6964715504683e7fd2203a66 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Tue, 28 Nov 2023 16:40:30 +0100
Subject: [PATCH] staging: Add extra volume configuration for pathslicing
objstorage
Refs. swh/infra/sysadm-environment#5164
---
Makefile | 2 +-
.../objstorage/_helper_deployment.yaml | 11 ++-
swh/values/minikube.yaml | 82 +++++++++++++++++++
swh/values/staging/default.yaml | 24 +++++-
swh/values/staging/swh-cassandra.yaml | 49 ++++++++++-
5 files changed, 162 insertions(+), 6 deletions(-)
diff --git a/Makefile b/Makefile
index 76d82cd89..1d8041156 100644
--- a/Makefile
+++ b/Makefile
@@ -42,7 +42,7 @@ helm-diff: swh-helm-diff ccf-helm-diff cc-helm-diff ss-helm-diff
swh-minikube:
kubectl --context minikube create namespace swh ; \
- kubectl --context minikube --namespace swh apply -f '$(SWH_CHART)/fake-secrets/*.yaml' ; \
+ kubectl --context minikube --namespace swh apply -f '$(SWH_CHART)/fake-secrets/*.yaml'; \
helm --kube-context minikube upgrade --install $(SWH_CHART) $(SWH_CHART)/ --values values-swh-application-versions.yaml \
--values $(SWH_CHART)/values.yaml \
--values $(SWH_CHART)/values/minikube.yaml \
diff --git a/swh/templates/objstorage/_helper_deployment.yaml b/swh/templates/objstorage/_helper_deployment.yaml
index c164b4974..301caf406 100644
--- a/swh/templates/objstorage/_helper_deployment.yaml
+++ b/swh/templates/objstorage/_helper_deployment.yaml
@@ -128,6 +128,11 @@ spec:
volumeMounts:
- name: configuration
mountPath: /etc/swh
+ {{- range $volume_name, $volume_config := .configuration.volumes }}
+ - name: {{ $volume_name }}
+ mountPath: {{ $volume_config.mountPath }}
+ readOnly: {{ $volume_config.readOnly | default "false" }}
+ {{ end }}
volumes:
- name: configuration
emptyDir: {}
@@ -137,4 +142,8 @@ spec:
items:
- key: "config.yml.template"
path: "config.yml.template"
-{{- end -}}
+ {{ range $volume_name, $volume_config := .configuration.volumes }}
+ - name: {{ $volume_name }}
+ {{ toYaml $volume_config.volumeDefinition | nindent 8 }}
+ {{ end }}
+{{ end }}
diff --git a/swh/values/minikube.yaml b/swh/values/minikube.yaml
index 7aef2cd98..ee23bd50b 100644
--- a/swh/values/minikube.yaml
+++ b/swh/values/minikube.yaml
@@ -839,3 +839,85 @@ scrubber:
journalClientOverrides:
batch_size: 200
object: release
+
+fakeObjstorageConfiguration:
+ cls: pathslicing
+ root: "/srv/swh/objects"
+ slicing: 0:1/1:5
+ client_max_size: 1073741824
+
+objstorage:
+ enabled: false
+ port: 5003
+ priorityClassName: frontend-rpc
+ deployments:
+ # Each key will be an objstorage instance to be deployed
+ # The mandatory objstorageConfigurationRef key should target a dict
+ # with the specific objstorage configuration
+ read-write:
+ enabled: true
+ logLevel: INFO
+ requestedCpu: 50m
+ requestedMemory: 100Mi
+ gunicorn:
+ threads: 5
+ workers: 2
+ timeout: 60
+ replicas: 1
+ volumes:
+ pathslicing-rw:
+ mountPath: /srv/swh/objects
+ volumeDefinition:
+ hostPath:
+ path: /srv/swh/objects
+ type: DirectoryOrCreate # Use Directory for production/staging
+
+ # mandatory
+ # check readWriteStorageConfiguration example to configure your storage
+ objstorageConfigurationRef: fakeObjstorageConfiguration
+ # Deploy an ingress to access the objstorage
+ hosts:
+ - objstorage-rw.s.s.n
+ ingress:
+ enabled: false
+ # mandatory if ingress is enabled
+ # the hostname on which the objstorage must be reachable
+ # Optional: the ingress classname to use
+ className: nginx
+ endpoints:
+ default:
+ paths:
+ - path: /
+ read-only:
+ enabled: true
+ logLevel: INFO
+ requestedCpu: 50m
+ requestedMemory: 100Mi
+ gunicorn:
+ threads: 5
+ workers: 2
+ timeout: 60
+ replicas: 1
+ volumes:
+ pathslicing-ro:
+ mountPath: /srv/swh/objects
+ readOnly: true
+ volumeDefinition:
+ hostPath:
+ path: /srv/swh/objects
+ type: DirectoryOrCreate
+
+ objstorageConfigurationRef: fakeObjstorageConfiguration
+ # Deploy an ingress to access the objstorage
+ hosts:
+ - objstorage-ro.s.s.n
+ ingress:
+ enabled: false
+ # mandatory if ingress is enabled
+ # the hostname on which the objstorage must be reachable
+ # Optional: the ingress classname to use
+ className: nginx
+ endpoints:
+ default:
+ paths:
+ - path: /
diff --git a/swh/values/staging/default.yaml b/swh/values/staging/default.yaml
index 8b9f607a9..1d6264e45 100644
--- a/swh/values/staging/default.yaml
+++ b/swh/values/staging/default.yaml
@@ -300,9 +300,12 @@ externalServices:
storage-postgresql:
internalName: storage-postgresql-rpc-ingress
target: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
- # objstorage-read-write:
- # internalName: objstorage-read-write-rpc-ingress
- # target: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+ objstorage-read-write:
+ internalName: objstorage-read-write-rpc-ingress
+ target: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+ objstorage-read-only:
+ internalName: objstorage-read-only-rpc-ingress
+ target: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
vault:
hosts:
@@ -329,3 +332,18 @@ addForgeNowConfiguration:
GITLAB_AFN_TOKEN:
secretKeyRef: common-secrets
secretKeyName: gitlab_afn_token
+
+objstorage:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/objstorage
+ operator: In
+ values:
+ - "true"
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - "storage1"
diff --git a/swh/values/staging/swh-cassandra.yaml b/swh/values/staging/swh-cassandra.yaml
index 65a9d7c47..4fce80433 100644
--- a/swh/values/staging/swh-cassandra.yaml
+++ b/swh/values/staging/swh-cassandra.yaml
@@ -851,12 +851,59 @@ objstorage:
minReplicaCount: 2
maxReplicaCount: 10
cpuPercentageUsage: 100
+ volumes:
+ staging-pathslicing-rw:
+ mountPath: /srv/softwareheritage/objects
+ volumeDefinition:
+ hostPath:
+ path: /srv/softwareheritage/objects
+ type: Directory
+
+ # mandatory
+ # check readWriteStorageConfiguration example to configure your storage
+ objstorageConfigurationRef: readWriteObjstorageConfiguration
+ # Deploy an ingress to access the objstorage
+ hosts:
+ - objstorage-read-write-rpc-ingress
+ ingress:
+ enabled: true
+ # mandatory if ingress is enabled
+ # the hostname on which the objstorage must be reachable
+ # Optional: the ingress classname to use
+ className: nginx
+ whitelistSourceRangeRef: stagingNetworkRanges
+ endpoints:
+ default:
+ paths:
+ - path: /
+ read-only:
+ enabled: true
+ logLevel: INFO
+ requestedCpu: 50m
+ requestedMemory: 100Mi
+ gunicorn:
+ threads: 5
+ workers: 2
+ timeout: 60
+ autoScaling:
+ minReplicaCount: 2
+ maxReplicaCount: 10
+ cpuPercentageUsage: 100
+ volumes:
+ staging-pathslicing-ro:
+ mountPath: /srv/softwareheritage/objects
+ readOnly: true
+ volumeDefinition:
+ hostPath:
+ path: /srv/softwareheritage/objects
+ type: Directory
+
# mandatory
# check readWriteStorageConfiguration example to configure your storage
objstorageConfigurationRef: readWriteObjstorageConfiguration
# Deploy an ingress to access the objstorage
hosts:
- - objstorage.staging.swh.network
+ - objstorage-read-only-rpc-ingress
ingress:
enabled: true
# mandatory if ingress is enabled
--
GitLab