diff --git a/cluster-components/templates/volumes/persistent-volume-claims.yaml b/cluster-components/templates/volumes/persistent-volume-claims.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26716fb4996c383ff2d76d2b6329e7fac3975f51
--- /dev/null
+++ b/cluster-components/templates/volumes/persistent-volume-claims.yaml
@@ -0,0 +1,18 @@
+{{- range $pvcName, $pvcConfig := .Values.volumes.persistentVolumeClaims }}
+{{- if or (not (hasKey $pvcConfig "enabled")) (get $pvcConfig "enabled") -}}
+{{- $namespace := $pvcConfig.namespace | default $.Values.namespace -}}
+{{- $appName := $pvcConfig.appName -}}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ $pvcName }}
+ namespace: {{ $namespace }}
+ {{- if $appName }}
+ labels:
+ app: {{ $appName }}
+ {{- end }}
+spec:
+ {{- toYaml $pvcConfig.spec | nindent 2 }}
+{{ end }}
+{{ end }}
diff --git a/cluster-components/templates/volumes/persistent-volume.yaml b/cluster-components/templates/volumes/persistent-volume.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1d0f8db9027abe069e7962c29f3d04558a65a48f
--- /dev/null
+++ b/cluster-components/templates/volumes/persistent-volume.yaml
@@ -0,0 +1,16 @@
+{{- range $pvName, $pvConfig := .Values.volumes.persistentVolumes }}
+{{- if or (not (hasKey $pvConfig "enabled")) (get $pvConfig "enabled") -}}
+{{- $appName := $pvConfig.appName -}}
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: {{ $pvName }}
+ {{- if $appName }}
+ labels:
+ app: {{ $appName }}
+ {{- end }}
+spec:
+ {{- toYaml $pvConfig.spec | nindent 2 }}
+{{ end }}
+{{ end }}
diff --git a/cluster-components/values.yaml b/cluster-components/values.yaml
index 650a5e35c3d9d16b7f244e6714b42fb601b067a9..2d2f90eb393b5e94af03397e1bc79a0d26f2c197 100644
--- a/cluster-components/values.yaml
+++ b/cluster-components/values.yaml
@@ -533,3 +533,57 @@ redis:
# # [optional] Application name override (e.g. to call it "redis" for example)
# # appName: redis
# replicas: 3
+
+volumes:
+ # Where to declare the pvs and pvcs (can be overridden per instance)
+ # namespace: swh
+ # Dictionary of persistent volumes specification
+ persistentVolumes: {}
+ # Dictionary of persistent volumes specification
+ # graph-20240331-persistent-local-pv:
+ # # Can be disabled, enabled by default if not provided
+ # enabled: true
+ # # namespace: swh-cassandra
+ # # The pv's specification, this is the raw yaml supported by kubernetes
+ # spec:
+ # capacity:
+ # storage: 1Gi
+ # volumeMode: Filesystem
+ # accessModes:
+ # - ReadWriteOnce
+ # persistentVolumeReclaimPolicy: Retain
+ # storageClassName: local-storage
+ # local:
+ # path: /srv/softwareheritage/ssd/graph/2024-03-31/compressed
+ # nodeAffinity:
+ # required:
+ # nodeSelectorTerms:
+ # - matchExpressions:
+ # - key: kubernetes.io/hostname
+ # operator: In
+ # values: rancher-node-highmem02
+ # Dictionary of persistent volume claims specification
+ persistentVolumeClaims: {}
+ # graph-20240331-persistent-pvc:
+ # enabled: true
+ # # optional application name to whom this pv is associated
+ # appName: graph-20240331
+ # spec:
+ # storageClassName: local-storage
+ # volumeMode: Filesystem
+ # volumeName: graph-20240331-persistent-pv
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 1Gi
+ # graph-20240331-inmemory-pvc:
+ # enabled: true
+ # spec:
+ # storageClassName: local-path
+ # volumeMode: Filesystem
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 1Gi
diff --git a/cluster-components/values/archive-staging-rke2.yaml b/cluster-components/values/archive-staging-rke2.yaml
index 5a93205d3f07667dfe6a32f1d9fbde584d5d066b..8a6ac3e663b4d046a79ff3f2469754cd87381c2a 100644
--- a/cluster-components/values/archive-staging-rke2.yaml
+++ b/cluster-components/values/archive-staging-rke2.yaml
@@ -568,3 +568,17 @@ redis:
requests:
storage: 1Gi
+volumes:
+ persistentVolumeClaims:
+ objstorage-pathslicing-pvc:
+ enabled: true
+ namespace: swh-cassandra-next-version
+ spec:
+ storageClassName: local-persistent
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteMany
+ resources:
+ requests:
+ storage: 10Gi
+
diff --git a/swh/templates/alter/deployment.yaml b/swh/templates/alter/deployment.yaml
index 67150762d37c9cda65386bae91069128a59c210c..7ef7f526304b69dcaccea31c4238f968ee3c176e 100644
--- a/swh/templates/alter/deployment.yaml
+++ b/swh/templates/alter/deployment.yaml
@@ -1,28 +1,5 @@
{{- if .Values.alter.enabled -}}
-{{- if .Values.alter.deployment.volumeClaim -}}
-{{- $volumeMode := .Values.alter.deployment.volumeClaim.volumeMode -}}
-{{- $accessModes := .Values.alter.deployment.volumeClaim.accessModes -}}
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: {{ .Values.alter.deployment.volumeClaim.name }}
- namespace: {{ .Values.namespace }}
-spec:
- storageClassName: {{ .Values.alter.deployment.volumeClaim.storageClassName }}
- {{- if $accessModes }}
- accessModes:
- {{- toYaml $accessModes | nindent 4 }}
- {{- end }}
- {{- if $volumeMode }}
- volumeMode: {{ $volumeMode }}
- {{- end }}
- resources:
- requests:
- storage: {{ .Values.alter.deployment.volumeClaim.size | default "20Gi" }}
-{{ end }}
-
{{- $service_name := "alter" -}}
{{- $configurationChecksum := include "alter.configmap" (dict "serviceType" $service_name "Values" .Values) | sha256sum -}}
{{- $affinity := .Values.alter.deployment.affinity -}}
diff --git a/swh/templates/cassandra-checks/deployment.yaml b/swh/templates/cassandra-checks/deployment.yaml
index c3c8c110577885a41a44d3aa176d6e60b2218504..ee15ad0b545d26f8fe3e59f7b96c0b78ed48b17c 100644
--- a/swh/templates/cassandra-checks/deployment.yaml
+++ b/swh/templates/cassandra-checks/deployment.yaml
@@ -1,22 +1,5 @@
{{- if .Values.cassandraChecks.enabled -}}
-{{- if and .Values.cassandraChecks.volumeClaim -}}
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: {{ .Values.cassandraChecks.volumeClaim.name }}
- namespace: swh-cassandra
-spec:
- storageClassName: {{ .Values.cassandraChecks.volumeClaim.storageClassName }}
- accessModes:
- - ReadWriteMany
- volumeMode: Filesystem
- resources:
- requests:
- storage: {{ $.Values.cassandraChecks.volumeClaim.size | default "20Gi" }}
-{{ end }}
-
{{ range $service_type, $deployment_config := .Values.cassandraChecks.deployments -}}
{{- if or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled") -}}
{{- $service_name := ( print "cassandra-checks-" $service_type ) -}}
diff --git a/swh/templates/graph/deployment.yaml b/swh/templates/graph/deployment.yaml
index 6c83cb5a7e00f6092362a8cb12ef05bd92ee38ca..0641f56020012837431388eb0bc76c0eb05caf63 100644
--- a/swh/templates/graph/deployment.yaml
+++ b/swh/templates/graph/deployment.yaml
@@ -219,9 +219,9 @@ spec:
name: backend-utils
defaultMode: 0555
{{- range $volumeName, $volumeConfig := $graphConfig.extraVolumes }}
+ {{- $pvcName := $volumeConfig.pvcName }}
- name: {{ $volumeName }}
- persistentVolumeClaim:
- claimName: {{ $volumeName }}-pvc
+ {{- toYaml $volumeConfig.volumeDefinition | nindent 8 }}
{{ end }}
{{ end }}
{{ end -}}
diff --git a/swh/templates/graph/persistent-volume-claims.yaml b/swh/templates/graph/persistent-volume-claims.yaml
deleted file mode 100644
index 78cddf6361e04b57096629a127dc0ccbe6e4c27c..0000000000000000000000000000000000000000
--- a/swh/templates/graph/persistent-volume-claims.yaml
+++ /dev/null
@@ -1,54 +0,0 @@
-{{ if .Values.graph.enabled -}}
-{{ range $graphType, $graphConfig := .Values.graph.deployments }}
-{{- if or (not (hasKey $graphConfig "enabled")) (get $graphConfig "enabled") -}}
-{{- range $volumeName, $volumeConfig := $graphConfig.extraVolumes }}
-{{- if $volumeConfig.persistentVolumeClaimDefinition -}}
-{{- include "swh.persistentVolumeClaim"
- (dict "volumeName" $volumeName
- "serviceType" ( print "graph-" $graphType )
- "volumeConfig" $volumeConfig
- "Values" $.Values) -}}
-{{- else if eq $volumeConfig.persistentVolumeType "local" -}}
----
-apiVersion: v1
-kind: PersistentVolume
-metadata:
- name: {{ $volumeName }}-local-pv
- namespace: {{ $.Values.namespace }}
-spec:
- capacity:
- storage: 1Gi
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- persistentVolumeReclaimPolicy: Retain
- storageClassName: local-storage
- local:
- path: {{ $volumeConfig.nodePath }}
- nodeAffinity:
- required:
- nodeSelectorTerms:
- - matchExpressions:
- - key: kubernetes.io/hostname
- operator: In
- values: {{ $volumeConfig.nodes | toYaml | nindent 12 }}
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: {{ $volumeName }}-pvc
- namespace: {{ $.Values.namespace }}
-spec:
- resources:
- requests:
- storage: 1Gi
- volumeMode: Filesystem
- volumeName: {{ $volumeName }}-local-pv
- storageClassName: local-storage
- accessModes:
- - ReadWriteOnce
-{{ end }}
-{{ end }}
-{{ end }}
-{{ end }}
-{{ end }}
diff --git a/swh/templates/volumes/persistent-volume-claims.yaml b/swh/templates/volumes/persistent-volume-claims.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b0c3b2ead94887e8335ad468087426569b103aa0
--- /dev/null
+++ b/swh/templates/volumes/persistent-volume-claims.yaml
@@ -0,0 +1,20 @@
+{{ if .Values.volumes.enabled -}}
+{{- range $pvcName, $pvcConfig := .Values.volumes.persistentVolumeClaims }}
+{{- if or (not (hasKey $pvcConfig "enabled")) (get $pvcConfig "enabled") -}}
+{{- $namespace := $pvcConfig.namespace | default $.Values.namespace -}}
+{{- $appName := $pvcConfig.appName -}}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ $pvcName }}
+ namespace: {{ $namespace }}
+ {{- if $appName }}
+ labels:
+ app: {{ $appName }}
+ {{- end }}
+spec:
+ {{- toYaml $pvcConfig.spec | nindent 2 }}
+{{ end }}
+{{ end }}
+{{ end }}
diff --git a/swh/templates/volumes/persistent-volume.yaml b/swh/templates/volumes/persistent-volume.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2f72f8e179fb694db03b7b09fa9da62d376b9e6f
--- /dev/null
+++ b/swh/templates/volumes/persistent-volume.yaml
@@ -0,0 +1,18 @@
+{{ if .Values.volumes.enabled -}}
+{{- range $pvName, $pvConfig := .Values.volumes.persistentVolumes }}
+{{- if or (not (hasKey $pvConfig "enabled")) (get $pvConfig "enabled") -}}
+{{- $appName := $pvConfig.appName -}}
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: {{ $pvName }}
+ {{- if $appName }}
+ labels:
+ app: {{ $appName }}
+ {{- end }}
+spec:
+ {{- toYaml $pvConfig.spec | nindent 2 }}
+{{ end }}
+{{ end }}
+{{ end }}
diff --git a/swh/values.yaml b/swh/values.yaml
index a4a17d52608c5ebf7f4dd5efccd064483486b401..d57a8026758572c66f71602de4eff7b569510f4b 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -1915,6 +1915,61 @@ rpcLocalRustWithGrpcGraphConfiguration:
grpc_server:
port: 50091
+volumes:
+ enabled: true
+ # Where to declare the pvs and pvcs (can be overridden per instance)
+ # namespace: swh
+ # Dictionary of persistent volumes specification
+ persistentVolumes: {}
+ # Dictionary of persistent volumes specification
+ # graph-20240331-persistent-local-pv:
+ # # Can be disabled, enabled by default if not provided
+ # enabled: true
+ # # namespace: swh-cassandra
+ # # The pv's specification, this is the raw yaml supported by kubernetes
+ # spec:
+ # capacity:
+ # storage: 1Gi
+ # volumeMode: Filesystem
+ # accessModes:
+ # - ReadWriteOnce
+ # persistentVolumeReclaimPolicy: Retain
+ # storageClassName: local-storage
+ # local:
+ # path: /srv/softwareheritage/ssd/graph/2024-03-31/compressed
+ # nodeAffinity:
+ # required:
+ # nodeSelectorTerms:
+ # - matchExpressions:
+ # - key: kubernetes.io/hostname
+ # operator: In
+ # values: rancher-node-highmem02
+ # Dictionary of persistent volume claims specification
+ persistentVolumeClaims: {}
+ # graph-20240331-persistent-pvc:
+ # enabled: true
+ # # optional application name to whom this pv is associated
+ # appName: graph-20240331
+ # spec:
+ # storageClassName: local-storage
+ # volumeMode: Filesystem
+ # volumeName: graph-20240331-persistent-pv
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 1Gi
+ # graph-20240331-inmemory-pvc:
+ # enabled: true
+ # spec:
+ # storageClassName: local-path
+ # volumeMode: Filesystem
+ # accessModes:
+ # - ReadWriteOnce
+ # resources:
+ # requests:
+ # storage: 1Gi
+
graph:
enabled: false
# rpc port (can be overriden per instance)
@@ -1990,26 +2045,16 @@ graph:
# # Define the volumes where to mount the data
# extraVolumes:
# graph-python3k-persistent:
- # mountPath: /srv/dataset
- # persistentVolumeClaimDefinition:
- # storageClassName: local-persistent
- # volumeMode: Filesystem
- # accessModes:
- # - ReadWriteOnce
- # resources:
- # requests:
- # storage: 1Gi
+ # enabled: true
+ # spec:
+ # mountPath: /srv/dataset
+ # pvcName: graph-python3k-persistent-pvc
# # in-memory volumes
# graph-python3k-inmemory:
- # mountPath: /srv/graph
- # persistentVolumeClaimDefinition:
- # storageClassName: local-path
- # volumeMode: Filesystem
- # accessModes:
- # - ReadWriteOnce
- # resources:
- # requests:
- # storage: 1Gi
+ # enabled: true
+ # spec:
+ # mountPath: /srv/graph
+ # pvcName: graph-python3k-inmemory-pvc
# # Deploy an ingress to access swh graph
# ingress:
# enabled: false
diff --git a/swh/values/local-cluster.yaml b/swh/values/local-cluster.yaml
index c1ac2d782caa488f8e05cd0862b092e3202a3dd9..dfc7b8077a18b7ed389aca2da9b8b1a8e1edf167 100644
--- a/swh/values/local-cluster.yaml
+++ b/swh/values/local-cluster.yaml
@@ -1292,18 +1292,8 @@ objstorage:
pathslicing-rw:
mountPath: /srv/swh/objects
volumeDefinition:
- ephemeral:
- volumeClaimTemplate:
- metadata:
- labels:
- type: persistent-volume
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
- storageClassName: standard
+ persistentVolumeClaim:
+ claimName: objstorage-pathslicing-pvc
# mandatory
# check readWriteStorageConfiguration example to configure your storage
@@ -1318,18 +1308,8 @@ objstorage:
mountPath: /srv/swh/objects
readOnly: true
volumeDefinition:
- ephemeral:
- volumeClaimTemplate:
- metadata:
- labels:
- type: persistent-volume
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
- storageClassName: standard
+ persistentVolumeClaim:
+ claimName: objstorage-pathslicing-pvc
# mandatory
# check readWriteStorageConfiguration example to configure your storage
@@ -1410,12 +1390,6 @@ alter:
volumeDefinition:
persistentVolumeClaim:
claimName: alter-recovery-bundles-pvc
- volumeClaim:
- storageClassName: standard
- name: alter-recovery-bundles-pvc
- accessModes:
- - ReadWriteOnce
- size: 1Gi
inventoryStorageConfigurationRef: rpcROStorageConfiguration
graphConfigurationRef: graphConfiguration
restorationStorageConfigurationRef: rpcRWStorageConfiguration
@@ -1536,6 +1510,74 @@ externalServices:
internalName: svix-test
target: local-cluster-control-plane
+volumes:
+ enabled: true
+ persistentVolumeClaims:
+ alter-recovery-bundles-pvc:
+ enabled: true
+ appName: alter
+ spec:
+ storageClassName: standard
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ grpc-example-dataset-example-pvc:
+ enabled: true
+ appName: graph-grpc-example
+ spec:
+ storageClassName: local-persistent
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ graph-example-inmemory-pvc:
+ enabled: true
+ appName: graph-grpc-example
+ spec:
+ storageClassName: local-path
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ graph-popular-persistent-pvc:
+ enabled: true
+ appName: graph-popular
+ spec:
+ storageClassName: local-persistent
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ # in-memory volumes
+ graph-popular-inmemory-pvc:
+ enabled: true
+ appName: graph-popular
+ spec:
+ storageClassName: local-path
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ objstorage-pathslicing-pvc:
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ storageClassName: standard
+
graph:
enabled: false
imageVersion: "20241105.2"
@@ -1572,25 +1614,15 @@ graph:
# persistent volume
grpc-example-dataset-example:
mountPath: /srv/dataset
- persistentVolumeClaimDefinition:
- storageClassName: local-persistent
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: grpc-example-dataset-example-pvc
# in-memory volumes
graph-example-inmemory:
mountPath: /srv/graph
- persistentVolumeClaimDefinition:
- storageClassName: local-path
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-example-inmemory-pvc
hosts:
- graph-grpc-example-ingress
ingress:
@@ -1643,25 +1675,15 @@ graph:
extraVolumes:
graph-popular-persistent:
mountPath: /srv/dataset
- persistentVolumeClaimDefinition:
- storageClassName: local-persistent
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-popular-persistent-pvc
# in-memory volumes
graph-popular-inmemory:
mountPath: /srv/graph
- persistentVolumeClaimDefinition:
- storageClassName: local-path
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-popular-inmemory-pvc
hosts:
- graph-grpc-popular-ingress
ingress:
diff --git a/swh/values/production/swh-cassandra.yaml b/swh/values/production/swh-cassandra.yaml
index 5d511adcd5f954f9bd621ece95bd14756a304ec3..9f7585683a32ec0b0277af6eb29b9a7e9ac006d9 100644
--- a/swh/values/production/swh-cassandra.yaml
+++ b/swh/values/production/swh-cassandra.yaml
@@ -2503,6 +2503,111 @@ rpcWithRemoteGrpc20241206GraphConfiguration:
plainGrpcGraphConfiguration:
max_ram: 500g
+volumes:
+ enabled: true
+ persistentVolumes:
+ graph-20240331-persistent-local-pv:
+ enabled: false
+ appName: graph-grpc-20240331
+ spec:
+ capacity:
+ storage: 1Gi
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ persistentVolumeReclaimPolicy: Retain
+ storageClassName: local-storage
+ local:
+ path: /srv/softwareheritage/ssd/graph/2024-03-31/compressed
+ nodeAffinity:
+ required:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - rancher-node-highmem02
+ graph-20241206-persistent-local-pv:
+ enabled: true
+ appName: graph-grpc-20241206
+ spec:
+ capacity:
+ storage: 1Gi
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ persistentVolumeReclaimPolicy: Retain
+ storageClassName: local-storage
+ local:
+ path: /srv/softwareheritage/ssd/graph/2024-12-06/compressed
+ nodeAffinity:
+ required:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - rancher-node-highmem01
+
+ persistentVolumeClaims:
+ alter-recovery-bundles-pvc:
+ enabled: true
+ appName: alter
+ spec:
+ storageClassName: cephfs
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteMany
+ resources:
+ requests:
+ storage: 30Gi
+ graph-20240331-persistent-pvc:
+ enabled: false
+ appName: graph-grpc-20240331
+ spec:
+ resources:
+ requests:
+ storage: 1Gi
+ accessModes:
+ - ReadWriteOnce
+ volumeMode: Filesystem
+ volumeName: graph-20240331-persistent-local-pv
+ storageClassName: local-storage
+ graph-20240331-inmemory-pvc:
+ enabled: false
+ appName: graph-grpc-20240331
+ spec:
+ storageClassName: local-path
+ accessModes:
+ - ReadWriteOnce
+ volumeMode: Filesystem
+ resources:
+ requests:
+ storage: 1Gi
+ graph-20241206-persistent-pvc:
+ enabled: true
+ appName: graph-grpc-20241206
+ spec:
+ resources:
+ requests:
+ storage: 1Gi
+ volumeMode: Filesystem
+ volumeName: graph-20241206-persistent-local-pv
+ storageClassName: local-storage
+ accessModes:
+ - ReadWriteOnce
+ graph-20241206-inmemory-pvc:
+ enabled: true
+ appName: graph-grpc-20241206
+ spec:
+ storageClassName: local-path
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+
graph:
enabled: true
deployments:
@@ -2519,22 +2624,16 @@ graph:
prepareMemoryVolume: true
extraVolumes:
graph-20240331-persistent:
- persistentVolumeType: local
mountPath: /srv/dataset/2024-03-31/compressed
- nodePath: /srv/softwareheritage/ssd/graph/2024-03-31/compressed
- nodes:
- - rancher-node-highmem02
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-20240331-persistent-pvc
# in-memory volumes
graph-20240331-inmemory:
mountPath: /srv/graph
- persistentVolumeClaimDefinition:
- storageClassName: local-path
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-20240331-inmemory-pvc
hosts:
- graph-grpc-20240331-ingress
ingress:
@@ -2587,22 +2686,16 @@ graph:
prepareMemoryVolume: true
extraVolumes:
graph-20241206-persistent:
- persistentVolumeType: local
mountPath: /srv/dataset/2024-12-06/compressed
- nodePath: /srv/softwareheritage/ssd/graph/2024-12-06/compressed
- nodes:
- - rancher-node-highmem01
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-20241206-persistent-pvc
# in-memory volumes
graph-20241206-inmemory:
mountPath: /srv/graph
- persistentVolumeClaimDefinition:
- storageClassName: local-path
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-20241206-inmemory-pvc
hosts:
- graph-grpc.internal.softwareheritage.org
- graph-grpc-20241206-ingress
@@ -2657,13 +2750,6 @@ alter:
volumeDefinition:
persistentVolumeClaim:
claimName: alter-recovery-bundles-pvc
- volumeClaim:
- storageClassName: cephfs
- accessModes:
- - ReadWriteMany
- volumeMode: Filesystem
- name: alter-recovery-bundles-pvc
- size: 30Gi
inventoryStorageConfigurationRef: remoteROStorageConfiguration
graphConfigurationRef: alterGraphConfiguration
diff --git a/swh/values/staging/next-version.yaml b/swh/values/staging/next-version.yaml
index 82e697fcdfa638629ba66b2340b5cd52b1d52020..2fb46700123f254727ef3aae6a37499c8ca7fe2d 100644
--- a/swh/values/staging/next-version.yaml
+++ b/swh/values/staging/next-version.yaml
@@ -648,22 +648,22 @@ objstorage:
pathslicing-rw:
mountPath: /srv/swh/objects
volumeDefinition:
- ephemeral:
- volumeClaimTemplate:
- metadata:
- labels:
- type: persistent-volume
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 10Gi
- storageClassName: local-persistent
-
+ persistentVolumeClaim:
+ claimName: objstorage-pathslicing-pvc
# mandatory
# check readWriteStorageConfiguration example to configure your storage
objstorageConfigurationRef: pathslicingRWObjstorageConfiguration
+ ro-pathslicing:
+ enabled: true
+ nodeSelector:
+ kubernetes.io/hostname: rancher-node-staging-rke2-metal01
+ extraVolumes:
+ pathslicing-rw:
+ mountPath: /srv/swh/objects
+ readOnly: true
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: objstorage-pathslicing-pvc
# mandatory
# check readWriteStorageConfiguration example to configure your storage
@@ -679,10 +679,14 @@ rwCassandraStorageWithJournalConfiguration:
journalWriterConfigurationRef: storageJournalWriterConfiguration
objstorageConfigurationRef: rpcRWObjstoragePathslicingConfiguration
+rpcROObjstoragePathslicingConfiguration:
+ cls: remote
+ url: http://objstorage-ro-pathslicing:5003
+ enable_requests_retry: true
+
roCassandraStorageConfiguration:
storageConfigurationRef: roCassandraStorage
- # TODO: Use a RO objstorage
- objstorageConfigurationRef: rpcRWObjstoragePathslicingConfiguration
+ objstorageConfigurationRef: rpcROObjstoragePathslicingConfiguration
rwCassandraStorageForReferenceTableJob:
<<: *rwCassandraStorage
@@ -1262,6 +1266,34 @@ rpcWithRemoteGrpcGraphConfiguration:
plainGrpcGraphConfiguration:
max_ram: 1g
+volumes:
+ enabled: true
+ persistentVolumeClaims:
+ swh-graph-grpc-dataset-example-pvc:
+ enabled: true
+ appName: graph-grpc-example
+ spec:
+ storageClassName: local-persistent
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ # in-memory volume (where the *.graph files and links to the files in persistent
+ # volume are)
+ swh-graph-grpc-inmemory-pvc:
+ enabled: true
+ appName: graph-grpc-example
+ spec:
+ storageClassName: local-path
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ volumeMode: Filesystem
+
graph:
enabled: true
deployments:
@@ -1284,27 +1316,16 @@ graph:
# persistent volume (where the full dataset files are stored)
swh-graph-grpc-dataset-example:
mountPath: /srv/dataset
- persistentVolumeClaimDefinition:
- storageClassName: local-persistent
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: swh-graph-grpc-dataset-example-pvc
# in-memory volume (where the *.graph files and links to the files in persistent
# volume are)
swh-graph-grpc-inmemory:
mountPath: /srv/graph
- persistentVolumeClaimDefinition:
- storageClassName: local-path
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
- volumeMode: Filesystem
-
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: swh-graph-grpc-inmemory-pvc
hosts:
- graph-grpc-next-version.internal.staging.swh.network
- graph-grpc-next-version-ingress
diff --git a/swh/values/staging/swh-cassandra.yaml b/swh/values/staging/swh-cassandra.yaml
index c1f2e6673f602df59b91e47a33e2d2d3719f8c89..a5668ed9d8789d394d4f4e0c5e2ce6dc28ac0a67 100644
--- a/swh/values/staging/swh-cassandra.yaml
+++ b/swh/values/staging/swh-cassandra.yaml
@@ -2212,10 +2212,6 @@ cassandraChecks:
# volumeDefinition:
# persistentVolumeClaim:
# claimName: cassandra-checks-pvc
- # volumeClaim:
- # storageClassName: cephfs
- # name: cassandra-checks-pvc
- # size: 20Gi
priorityClassName: swh-background-workload
nodeSelector:
@@ -2315,14 +2311,6 @@ alter:
volumeDefinition:
persistentVolumeClaim:
claimName: alter-recovery-bundles-pvc
- volumeClaim:
- storageClassName: cephfs
- accessModes:
- - ReadWriteMany
- volumeMode: Filesystem
- name: alter-recovery-bundles-pvc
- size: 30Gi
-
inventoryStorageConfigurationRef: cassandraROStorageNoMaskingConfiguration
graphConfigurationRef: alterGraphConfiguration
restorationStorageConfigurationRef: rpcRWStorageConfiguration
@@ -2423,6 +2411,44 @@ rpcWithRemoteGrpcGraphConfiguration:
plainGrpcGraphConfiguration:
max_ram: 1g
+volumes:
+ enabled: true
+ persistentVolumeClaims:
+ alter-recovery-bundles-pvc:
+ enabled: true
+ appName: alter
+ spec:
+ storageClassName: cephfs
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteMany
+ resources:
+ requests:
+ storage: 30Gi
+ graph-python3k-persistent-pvc:
+ enabled: true
+ appName: graph-grpc-python3k
+ spec:
+ storageClassName: local-persistent
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ # in-memory volumes
+ graph-python3k-inmemory-pvc:
+ enabled: true
+ appName: graph-grpc-python3k
+ spec:
+ storageClassName: local-path
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+
graph:
enabled: true
deployments:
@@ -2443,25 +2469,15 @@ graph:
extraVolumes:
graph-python3k-persistent:
mountPath: /srv/dataset
- persistentVolumeClaimDefinition:
- storageClassName: local-persistent
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-python3k-persistent-pvc
# in-memory volumes
graph-python3k-inmemory:
mountPath: /srv/graph
- persistentVolumeClaimDefinition:
- storageClassName: local-path
- volumeMode: Filesystem
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 1Gi
+ volumeDefinition:
+ persistentVolumeClaim:
+ claimName: graph-python3k-inmemory-pvc
hosts:
- graph-grpc.internal.staging.swh.network
- graph-grpc-ingress