diff --git a/swh/Chart.yaml b/swh/Chart.yaml
index 42b0715848cbd8e71ec66d1235fe0d38ab9f0711..ddd11276d6d1d6e592f1f744a0219199ce175d6f 100644
--- a/swh/Chart.yaml
+++ b/swh/Chart.yaml
@@ -1,5 +1,5 @@
apiVersion: v2
-appVersion: 438
+appVersion: 439
description: A Helm chart to deploy the softwareheritage stack
name: swh
type: application
diff --git a/swh/values/staging/next-version.yaml b/swh/values/staging/next-version.yaml
index 551c5a4437ee12762928044ff618b35478f8ba11..82e697fcdfa638629ba66b2340b5cd52b1d52020 100644
--- a/swh/values/staging/next-version.yaml
+++ b/swh/values/staging/next-version.yaml
@@ -121,14 +121,19 @@ remoteROPostgresqlStorageConfiguration:
url: http://storage-ro-postgresql:5002
enable_requests_retry: true
-remoteCassandraStorageConfiguration:
+remoteROCassandraStorageConfiguration:
+ cls: remote
+ url: http://storage-ro-cassandra:5002
+ enable_requests_retry: true
+
+remoteRWCassandraStorageConfiguration:
cls: remote
url: http://storage-rw-cassandra:5002
enable_requests_retry: true
remoteRWStorageWithPipelineConfiguration:
pipelineStepsRef: storagePipelineSteps
- storageConfigurationRef: remoteCassandraStorageConfiguration
+ storageConfigurationRef: remoteRWCassandraStorageConfiguration
rwStoragePostgresqlConfiguration:
storageConfigurationRef: postgresqlRWStorageConfiguration
@@ -299,6 +304,32 @@ loaders:
queueThreshold: 1
maxReplicaCount: 1
stopWhenNoActivity: false
+ save-code-now:
+ queues:
+ - save_code_now:swh.loader.bzr.tasks.LoadBazaar
+ - save_code_now:swh.loader.cvs.tasks.LoadCvsRepository
+ - save_code_now:swh.loader.git.tasks.UpdateGitRepository
+ - save_code_now:swh.loader.git.tasks.LoadDiskGitRepository
+ - save_code_now:swh.loader.git.tasks.UncompressAndLoadDiskGitRepository
+ - save_code_now:swh.loader.mercurial.tasks.LoadArchiveMercurial
+ - save_code_now:swh.loader.mercurial.tasks.LoadMercurial
+ - save_code_now:swh.loader.svn.tasks.LoadSvnRepository
+ - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository
+ - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository
+ - save_code_now:swh.loader.package.archive.tasks.LoadTarball
+ ackLate: true
+ autoScaling:
+ stopWhenNoActivity: true
+ queueThreshold: 1
+ minReplicaCount: 0
+ maxReplicaCount: 2
+ requestedMemory: 200Mi
+ requestedCpu: 50m
+ limitedMemory: 15Gi
+ tmpEphemeralStorage:
+ emptyDir:
+ medium: Memory
+ sizeLimit: 10Gi
svn:
enabled: true
queues:
@@ -527,7 +558,7 @@ storageReplayer:
cassandraSeeds:
- cassandra-cluster-next-version-service
-cassandraStorage: &cassandraStorage
+rwCassandraStorage: &rwCassandraStorage
cls: cassandra
cassandraSeedsRef: cassandraSeeds
keyspace: swh
@@ -547,6 +578,25 @@ cassandraStorage: &cassandraStorage
secretKeyRef: cassandra-cluster-superuser
secretKeyName: password
+roCassandraStorage:
+ cls: cassandra
+ cassandraSeedsRef: cassandraSeeds
+ keyspace: swh
+ initKeyspace: false
+ consistencyLevel: LOCAL_QUORUM
+ authProvider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ username: ${CASSANDRA_RW_USERNAME}
+ password: ${CASSANDRA_RW_PASSWORD}
+ secrets:
+ # TODO: deploy a real cassandra RO user
+ CASSANDRA_RW_USERNAME:
+ secretKeyRef: cassandra-cluster-superuser
+ secretKeyName: username
+ CASSANDRA_RW_PASSWORD:
+ secretKeyRef: cassandra-cluster-superuser
+ secretKeyName: password
+
noopObjstorageConfiguration:
cls: noop
@@ -615,29 +665,6 @@ objstorage:
# check readWriteStorageConfiguration example to configure your storage
objstorageConfigurationRef: pathslicingRWObjstorageConfiguration
- # Activate dedicated instance for read workload
- ro-pathslicing:
- enabled: true
- nodeSelector:
- kubernetes.io/hostname: rancher-node-staging-rke2-metal01
- extraVolumes:
- pathslicing-ro:
- mountPath: /srv/swh/objects
- readOnly: true
- volumeDefinition:
- ephemeral:
- volumeClaimTemplate:
- metadata:
- labels:
- type: persistent-volume
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: 10Gi
- storageClassName: local-persistent
-
# mandatory
# check readWriteStorageConfiguration example to configure your storage
objstorageConfigurationRef: pathslicingROObjstorageConfiguration
@@ -647,25 +674,24 @@ rpcRWObjstoragePathslicingConfiguration:
url: http://objstorage-rw-pathslicing:5003
enable_requests_retry: true
-rpcROObjstoragePathslicingConfiguration:
- cls: remote
- url: http://objstorage-ro-pathslicing:5003
- enable_requests_retry: true
-
-rwStorageCassandraWithJournalConfiguration:
- storageConfigurationRef: cassandraStorage
+rwCassandraStorageWithJournalConfiguration:
+ storageConfigurationRef: rwCassandraStorage
journalWriterConfigurationRef: storageJournalWriterConfiguration
objstorageConfigurationRef: rpcRWObjstoragePathslicingConfiguration
+roCassandraStorageConfiguration:
+ storageConfigurationRef: roCassandraStorage
+ # TODO: Use a RO objstorage
+ objstorageConfigurationRef: rpcRWObjstoragePathslicingConfiguration
-cassandraStorageForReferenceTableJob:
- <<: *cassandraStorage
+rwCassandraStorageForReferenceTableJob:
+ <<: *rwCassandraStorage
specificOptions:
table_options:
object_references_*: "compression = {'class': 'ZstdCompressor', 'compression_level':'19'} AND compaction = {'class': 'UnifiedCompactionStrategy'}"
-rwCassandraStorageForReferenceTableJob:
- storageConfigurationRef: cassandraStorageForReferenceTableJob
+roCassandraStorageForReferenceTableJob:
+ storageConfigurationRef: rwCassandraStorageForReferenceTableJob
objstorageConfigurationRef: noopObjstorageConfiguration
storage:
@@ -673,7 +699,7 @@ storage:
deployments:
# main storage to let the loaders write
rw-cassandra:
- storageConfigurationRef: rwStorageCassandraWithJournalConfiguration
+ storageConfigurationRef: rwCassandraStorageWithJournalConfiguration
cronjobs:
create-object-reference-partitions:
enabled: false
@@ -681,7 +707,13 @@ storage:
logLevel: INFO
cron: 5 0 * * mon
concurrencyPolicy: Forbid
- # a storage postgresql instance to replay data
+ # RO cassandra storage, for webapp
+ ro-cassandra:
+ storageConfigurationRef: roCassandraStorageConfiguration
+ autoScaling:
+ minReplicaCount: 1
+ maxReplicaCount: 1
+ # storage postgresql instance to replay data
rw-postgresql:
enabled: true
storageConfigurationRef: rwStoragePostgresqlConfiguration
@@ -728,7 +760,7 @@ web:
indexerStorageConfigurationRef: remoteIndexerStorageConfiguration
countersConfigurationRef: remoteCountersConfiguration
historyCountersUrlRef: historyCountersUrl
- storageConfigurationRef: remoteROPostgresqlStorageConfiguration
+ storageConfigurationRef: remoteROCassandraStorageConfiguration
webhooksConfigurationRef: webhooksConfiguration
provenanceConfigurationRef: provenanceConfiguration
graphConfigurationRef: webGraphConfiguration
@@ -1063,7 +1095,7 @@ indexers:
storageConfigurationRef: remoteROStorageConfiguration
schedulerConfigurationRef: remoteSchedulerConfiguration
indexerStorageConfigurationRef: remoteIndexerStorageConfiguration
- objstorageConfigurationRef: rpcROObjstoragePathslicingConfiguration
+ objstorageConfigurationRef: rpcRWObjstoragePathslicingConfiguration
journalClientConfigurationRef: indexerJournalClientConfiguration
deployments:
origin-intrinsic:
@@ -1155,7 +1187,7 @@ deposit:
# In prod this storage is read-only
storageConfigurationRef: remoteROPostgresqlStorageConfiguration
# XXX: but this one is read-write?
- storageMetadataConfigurationRef: remoteCassandraStorageConfiguration
+ storageMetadataConfigurationRef: remoteRWCassandraStorageConfiguration
djangoConfigurationRef: djangoDepositConfiguration
blobstorageConfigurationRef: azureDepositConfiguration
keycloakConfigurationRef: keycloakConfiguration
diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml
index ce10db22dc2aa93206be1a96550b4283cf5f8e22..9b6ea6dfc2e832935d33547276923a1e6ec63341 100644
--- a/values-swh-application-versions.yaml
+++ b/values-swh-application-versions.yaml
@@ -7,9 +7,9 @@ swh_alter_image_version: '20250220.1'
swh_counters_image: container-registry.softwareheritage.org/swh/infra/swh-apps/counters
swh_counters_image_version: '20250211.2'
swh_deposit_checkers_image: container-registry.softwareheritage.org/swh/infra/swh-apps/deposit_checkers
-swh_deposit_checkers_image_version: '20250227.1'
+swh_deposit_checkers_image_version: '20250304.1'
swh_deposit_image: container-registry.softwareheritage.org/swh/infra/swh-apps/deposit
-swh_deposit_image_version: '20250227.1'
+swh_deposit_image_version: '20250304.1'
swh_graph_image: container-registry.softwareheritage.org/swh/infra/swh-apps/graph
swh_graph_image_version: '20250211.3'
swh_graphql_image: container-registry.softwareheritage.org/swh/infra/swh-apps/graphql