swh/Chart.yaml

 apiVersion: v2
-appVersion: 439
+appVersion: 450
 description: A Helm chart to deploy the softwareheritage stack
 name: swh
 type: application

swh/templates/_helpers.yaml

@@ -250,7 +250,7 @@ Generate the configuration for a journal writer
 {{- end -}}
 
 {{/* Generate the init keyspace container configuration if needed */}}
-{{- define "swh.storage.cassandra.initKeyspaceContainer" -}}
+{{- define "swh.storage.cassandra.keyspaceManagementContainers" -}}
   {{- $storageDefinition := required (print "Storage definition " .configurationRef " not found") (get .Values .configurationRef) -}}
   {{- $storageConfigurationRef := required (print "storageConfigurationRef key needed in " .configurationRef) (get $storageDefinition "storageConfigurationRef") -}}
   {{- $storageConfiguration := required (print $storageConfigurationRef " declaration not found") (get .Values $storageConfigurationRef) -}}
@@ -268,6 +268,24 @@ Generate the configuration for a journal writer
   command:
   - /entrypoints/init-cassandra-keyspace.sh
   volumeMounts:
+  - name: configuration
+    mountPath: /etc/swh
+    readOnly: false
+  - name: backend-utils
+    mountPath: /entrypoints
+    readOnly: true
+    {{- end -}}
+    {{- $upgradeKeyspace := get $storageConfiguration "upgradeKeyspace" -}}
+    {{- if $upgradeKeyspace -}}
+- name: upgrade-cassandra-keyspace
+  image: {{ get .Values .imagePrefixName }}:{{ get .Values (print .imagePrefixName "_version") }}
+  imagePullPolicy: IfNotPresent
+  env:
+    - name: SWH_CONFIG_FILENAME
+      value: /etc/swh/config.yml
+  command:
+  - /entrypoints/upgrade-cassandra-keyspace.sh
+  volumeMounts:
   - name: configuration
     mountPath: /etc/swh
     readOnly: false

swh/templates/alter/_helper-configmap.yaml

@@ -12,6 +12,11 @@
                                       "service" "restoration_storage"
                                       "configurationRef" .Values.alter.restorationStorageConfigurationRef
                                       "Values" .Values) | fromYaml | mustMerge $alterConfiguration -}}
+{{- if .Values.alter.journalWriterMirrorConfigurationRef }}
+{{- $alterConfiguration = include "swh.journalWriterConfiguration" (dict
+                                      "configurationRef" .Values.alter.journalWriterMirrorConfigurationRef
+                                      "Values" .Values) | fromYaml | mustMerge $alterConfiguration -}}
+{{- end }}
 
 {{- $removalSearches := (dict) -}}
 {{- range $searchName, $searchConfig := .Values.alter.removalSearches -}}

swh/templates/storage-replayer/deployment.yaml

@@ -54,7 +54,7 @@ spec:
             mountPath: /etc/swh
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
-          {{ include "swh.storage.cassandra.initKeyspaceContainer"
+          {{ include "swh.storage.cassandra.keyspaceManagementContainers"
             (dict "Values" $.Values
                   "configurationRef" $.Values.storageReplayer.storageConfigurationRef
                   "imagePrefixName" "swh_storage_replayer_image") | nindent 8 }}

swh/templates/storage/deployment.yaml

@@ -74,25 +74,26 @@ spec:
           - name: config-utils
             mountPath: /entrypoints
             readOnly: true
-        {{ include "swh.storage.cassandra.initKeyspaceContainer" (dict "Values" $.Values
-                                                                       "configurationRef" $storageConfig.storageConfigurationRef
-                                                                       "imagePrefixName" "swh_storage_image") | nindent 8 }}
-        {{- if $initializeBackendConfig }}
-        {{ include "swh.initializeBackend" (dict "Values" $.Values
-                                                 "imagePrefixName" "swh_toolbox_image"
-                                                 "module" "storage"
-                                                 "config" $initializeBackendConfig) | nindent 8 }}
-        {{- end }}
-        {{- if $checkBackendVersion }}
-        {{ include "swh.checkDatabaseVersionContainer" (dict "Values" $.Values
-                                                             "imagePrefixName" "swh_storage_image"
-                                                             "module" "storage") | nindent 8 }}
-        {{- end }}
-        {{- if $migrateBackend }}
-        {{ include "swh.migrateBackend" (dict "Values" $.Values
-                                              "imagePrefixName" "swh_toolbox_image"
-                                              "module" "storage") | nindent 8 }}
-        {{- end }}
+{{ include "swh.storage.cassandra.keyspaceManagementContainers"
+    (dict "Values" $.Values
+          "configurationRef" $storageConfig.storageConfigurationRef
+          "imagePrefixName" "swh_storage_image") | indent 8 }}
+{{- if $initializeBackendConfig }}
+{{ include "swh.initializeBackend" (dict "Values" $.Values
+                                          "imagePrefixName" "swh_toolbox_image"
+                                          "module" "storage"
+                                          "config" $initializeBackendConfig) | indent 8 }}
+{{- end }}
+{{- if $checkBackendVersion }}
+{{ include "swh.checkDatabaseVersionContainer" (dict "Values" $.Values
+                                                      "imagePrefixName" "swh_storage_image"
+                                                      "module" "storage") | indent 8 }}
+{{- end }}
+{{- if $migrateBackend }}
+{{ include "swh.migrateBackend" (dict "Values" $.Values
+                                       "imagePrefixName" "swh_toolbox_image"
+                                       "module" "storage") | indent 8 }}
+{{- end }}
       containers:
         - name: {{ $serviceType }}
           resources:

swh/templates/utils/backend-utils.yaml

@@ -346,7 +346,45 @@ data:
     [ ! -f "${CONFIG_CASSANDRA}" ] && \
         echo "Missing <${CONFIG_CASSANDRA}> file" && exit 1
 
-    swh storage --config-file=${CONFIG_CASSANDRA} create-keyspace
+    swh storage --config-file=${CONFIG_CASSANDRA} cassandra init
+
+  upgrade-cassandra-keyspace.sh: |
+    #!/usr/bin/env bash
+
+    [ -z "${SWH_CONFIG_FILENAME}" ] && \
+        echo "<SWH_CONFIG_FILENAME> env variable must be set" && exit 1
+
+    CONFIG_CASSANDRA=/etc/swh/storage-cassandra-config.yaml
+
+    python /entrypoints/extract-storage-cassandra-config.py > \
+        ${CONFIG_CASSANDRA}
+
+    [ ! -f "${CONFIG_CASSANDRA}" ] && \
+        echo "Missing <${CONFIG_CASSANDRA}> file" && exit 1
+
+    ret=0
+
+    swh storage --config-file=${CONFIG_CASSANDRA} cassandra list-migrations
+    swh storage --config-file=${CONFIG_CASSANDRA} cassandra upgrade || ret=$?
+
+    case "$ret" in
+        0)
+          # Migrations have succeeded
+          exit 0
+          ;;
+        3)
+          # No migrations to do
+          exit 0
+          ;;
+        5)
+          # Only optional migrations are left to do, we can start the RPC service
+          exit 0
+          ;;
+        *)
+          # Unknown exit code
+          exit $ret
+          ;;
+    esac
 
   extract-storage-postgresql-config-py: |
     import yaml

swh/values/default.yaml

@@ -483,3 +483,10 @@ graph:
             operator: In
             values:
             - "true"
+
+journalWriterMirrorConfiguration:
+  brokersConfigurationRef: kafkaBrokers
+  clientId: swh.alter.removals.${HOSTNAME}
+  prefix: swh.journal.mirror-notifications
+  producerConfig:
+    message.max.bytes: 1000000000

swh/values/local-cluster.yaml

@@ -324,7 +324,9 @@ cassandraStorage:
   cls: cassandra
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
-  initKeyspace: true
+  # When bootstrapping an empty cluster, these need to be flipped
+  initKeyspace: false
+  upgradeKeyspace: true
   consistencyLevel: LOCAL_QUORUM
   specificOptions:
     directory_entries_insert_algo: batch

swh/values/production/default.yaml

@@ -96,7 +96,8 @@ cassandraStorage: &cassandraStorage
   cls: cassandra
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
-  initKeyspace: true
+  initKeyspace: false
+  upgradeKeyspace: true
   consistencyLevel: LOCAL_QUORUM
   specificOptions:
     directory_entries_insert_algo: batch

swh/values/production/swh-cassandra.yaml

@@ -5,6 +5,7 @@ cassandraROStorage:
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
   initKeyspace: false
+  upgradeKeyspace: false
   consistencyLevel: LOCAL_QUORUM
   authProvider:
     cls: cassandra.auth.PlainTextAuthProvider
@@ -20,6 +21,7 @@ cassandraChecksROStorage:
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
   initKeyspace: false
+  upgradeKeyspace: false
   consistencyLevel: LOCAL_QUORUM
   authProvider:
     cls: cassandra.auth.PlainTextAuthProvider

swh/values/staging/default.yaml

@@ -414,9 +414,11 @@ postgresqlScrubberConfiguration:
 
 kafkaBrokers:
   - journal2.internal.staging.swh.network:9092
+  - kafka3.internal.staging.swh.network:9092
 
 internalSecuredKafkaBrokers:
   - journal2.internal.staging.swh.network:9094
+  - kafka3.internal.staging.swh.network:9094
 
 indexerJournalWriterConfiguration:
   brokersConfigurationRef: kafkaBrokers

swh/values/staging/next-version.yaml

@@ -562,7 +562,8 @@ rwCassandraStorage: &rwCassandraStorage
   cls: cassandra
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
-  initKeyspace: true
+  initKeyspace: false
+  upgradeKeyspace: true
   consistencyLevel: LOCAL_QUORUM
   specificOptions:
     directory_entries_insert_algo: batch
@@ -583,6 +584,7 @@ roCassandraStorage:
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
   initKeyspace: false
+  upgradeKeyspace: false
   consistencyLevel: LOCAL_QUORUM
   authProvider:
     cls: cassandra.auth.PlainTextAuthProvider

swh/values/staging/swh-cassandra.yaml

@@ -41,7 +41,8 @@ cassandraStorage: &cassandraStorage
   cls: cassandra
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
-  initKeyspace: true
+  initKeyspace: false
+  upgradeKeyspace: true
   consistencyLevel: LOCAL_QUORUM
   specificOptions:
     directory_entries_insert_algo: batch
@@ -59,6 +60,7 @@ cassandraROStorage:
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
   initKeyspace: false
+  upgradeKeyspace: false
   consistencyLevel: LOCAL_QUORUM
   authProvider:
     cls: cassandra.auth.PlainTextAuthProvider
@@ -73,7 +75,8 @@ cassandraChecksROStorage:
   cls: cassandra
   cassandraSeedsRef: cassandraSeeds
   keyspace: swh
-  initKeyspace: true
+  initKeyspace: false
+  upgradeKeyspace: false
   consistencyLevel: LOCAL_QUORUM
   authProvider:
     cls: cassandra.auth.PlainTextAuthProvider
@@ -2368,6 +2371,7 @@ alter:
   inventoryStorageConfigurationRef: cassandraROStorageNoMaskingConfiguration
   graphConfigurationRef: alterGraphConfiguration
   restorationStorageConfigurationRef: rpcRWStorageConfiguration
+  journalWriterMirrorConfigurationRef: journalWriterMirrorConfiguration
   removalSearches:
     search:
       searchConfigurationRef: remoteSearchConfiguration

values-swh-application-versions.yaml

 cassandra_checks_image: container-registry.softwareheritage.org/swh/infra/swh-apps/cassandra_checks
-cassandra_checks_image_version: '20250211.3'
+cassandra_checks_image_version: '20250319.2'
 software_stories_image: container-registry.softwareheritage.org/swh/infra/swh-apps/software_stories
 software_stories_image_version: '20230912.1'
 swh_alter_image: container-registry.softwareheritage.org/swh/infra/swh-apps/alter
-swh_alter_image_version: '20250220.1'
+swh_alter_image_version: '20250319.1'
 swh_counters_image: container-registry.softwareheritage.org/swh/infra/swh-apps/counters
-swh_counters_image_version: '20250211.2'
+swh_counters_image_version: '20250319.1'
 swh_deposit_checkers_image: container-registry.softwareheritage.org/swh/infra/swh-apps/deposit_checkers
-swh_deposit_checkers_image_version: '20250304.1'
+swh_deposit_checkers_image_version: '20250319.2'
 swh_deposit_image: container-registry.softwareheritage.org/swh/infra/swh-apps/deposit
-swh_deposit_image_version: '20250304.1'
+swh_deposit_image_version: '20250319.1'
 swh_graph_image: container-registry.softwareheritage.org/swh/infra/swh-apps/graph
-swh_graph_image_version: '20250211.3'
+swh_graph_image_version: '20250319.1'
 swh_graphql_image: container-registry.softwareheritage.org/swh/infra/swh-apps/graphql
-swh_graphql_image_version: '20250220.1'
+swh_graphql_image_version: '20250319.1'
 swh_indexer_image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer
-swh_indexer_image_version: '20250220.1'
+swh_indexer_image_version: '20250319.1'
 swh_indexer_storage_image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage
-swh_indexer_storage_image_version: '20250220.1'
+swh_indexer_storage_image_version: '20250319.2'
 swh_lister_image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister
-swh_lister_image_version: '20250227.1'
+swh_lister_image_version: '20250319.1'
 swh_loader_bzr_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_bzr
-swh_loader_bzr_image_version: '20250227.1'
+swh_loader_bzr_image_version: '20250319.2'
 swh_loader_cvs_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_cvs
-swh_loader_cvs_image_version: '20250227.1'
+swh_loader_cvs_image_version: '20250319.1'
 swh_loader_git_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git
-swh_loader_git_image_version: '20250227.1'
+swh_loader_git_image_version: '20250319.2'
 swh_loader_mercurial_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_mercurial
-swh_loader_mercurial_image_version: '20250227.1'
+swh_loader_mercurial_image_version: '20250319.1'
 swh_loader_metadata_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata
-swh_loader_metadata_image_version: '20250227.1'
+swh_loader_metadata_image_version: '20250319.1'
 swh_loader_package_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_package
-swh_loader_package_image_version: '20250227.1'
+swh_loader_package_image_version: '20250319.1'
 swh_loader_savecodenow_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow
-swh_loader_savecodenow_image_version: '20250227.1'
+swh_loader_savecodenow_image_version: '20250319.2'
 swh_loader_svn_image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_svn
-swh_loader_svn_image_version: '20250227.1'
+swh_loader_svn_image_version: '20250319.2'
 swh_objstorage_image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage
-swh_objstorage_image_version: '20250211.3'
+swh_objstorage_image_version: '20250319.1'
 swh_objstorage_replayer_image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage_replayer
-swh_objstorage_replayer_image_version: '20250211.3'
+swh_objstorage_replayer_image_version: '20250319.1'
 swh_provenance_image: container-registry.softwareheritage.org/swh/infra/swh-apps/provenance
-swh_provenance_image_version: '20250219.1'
+swh_provenance_image_version: '20250319.1'
 swh_scheduler_image: container-registry.softwareheritage.org/swh/infra/swh-apps/scheduler
-swh_scheduler_image_version: '20250317.3'
+swh_scheduler_image_version: '20250319.1'
 swh_scrubber_image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber
-swh_scrubber_image_version: '20250227.1'
+swh_scrubber_image_version: '20250319.2'
 swh_search_image: container-registry.softwareheritage.org/swh/infra/swh-apps/search
-swh_search_image_version: '20250220.1'
+swh_search_image_version: '20250319.1'
 swh_storage_image: container-registry.softwareheritage.org/swh/infra/swh-apps/storage
-swh_storage_image_version: '20250211.3'
+swh_storage_image_version: '20250319.1'
 swh_storage_replayer_image: container-registry.softwareheritage.org/swh/infra/swh-apps/storage_replayer
-swh_storage_replayer_image_version: '20250211.3'
+swh_storage_replayer_image_version: '20250319.1'
 swh_toolbox_image: container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox
-swh_toolbox_image_version: '20250227.1'
+swh_toolbox_image_version: '20250319.3'
 swh_utils_image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils
 swh_utils_image_version: '20250211.1'
 swh_vault_cookers_image: container-registry.softwareheritage.org/swh/infra/swh-apps/vault_cookers
-swh_vault_cookers_image_version: '20250227.1'
+swh_vault_cookers_image_version: '20250319.1'
 swh_vault_image: container-registry.softwareheritage.org/swh/infra/swh-apps/vault
-swh_vault_image_version: '20250227.1'
+swh_vault_image_version: '20250319.1'
 swh_web_image: container-registry.softwareheritage.org/swh/infra/swh-apps/web
-swh_web_image_version: '20250227.1'
+swh_web_image_version: '20250319.1'
 swh_webhooks_image: container-registry.softwareheritage.org/swh/infra/swh-apps/webhooks
-swh_webhooks_image_version: '20250211.2'
+swh_webhooks_image_version: '20250319.1'