From 9ee7884718b17278f29b7b161d6e5687a6025863 Mon Sep 17 00:00:00 2001
From: Vincent SELLIER <vincent.sellier@softwareheritage.org>
Date: Wed, 19 Feb 2025 09:59:04 +0100
Subject: [PATCH 1/2] Configure reference tables options for staging
Related to --to-edit--
---
swh/values/staging/swh-cassandra.yaml | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/swh/values/staging/swh-cassandra.yaml b/swh/values/staging/swh-cassandra.yaml
index dd894c948..f3626b324 100644
--- a/swh/values/staging/swh-cassandra.yaml
+++ b/swh/values/staging/swh-cassandra.yaml
@@ -37,7 +37,7 @@ cassandraSeeds:
- cassandra2.internal.staging.swh.network
- cassandra3.internal.staging.swh.network
-cassandraStorage:
+cassandraStorage: &cassandraStorage
cls: cassandra
cassandraSeedsRef: cassandraSeeds
keyspace: swh
@@ -88,10 +88,21 @@ cassandraChecksROStorage:
secretKeyRef: common-secrets
secretKeyName: cassandra-swh-ro-password
+cassandraStorageForReferenceTableJob:
+ <<: *cassandraStorage
+ specificOptions:
+ # /!\ All the specific options of ^ are overridden
+ table_options:
+ object_references_*: "compression = {'class': 'ZstdCompressor', 'compression_level':'19'} AND compaction = {'class': 'UnifiedCompactionStrategy'}"
+
cassandraRWNoObjstorageStorageConfiguration:
storageConfigurationRef: cassandraStorage
objstorageConfigurationRef: noopObjstorageConfiguration
+cassandraRWForReferenceTableJobConfiguration:
+ storageConfigurationRef: cassandraStorageForReferenceTableJob
+ objstorageConfigurationRef: noopObjstorageConfiguration
+
postgresqlRWNoObjstorageStorageConfiguration:
storageConfigurationRef: primaryPostgresqlRWStorageConfiguration
objstorageConfigurationRef: noopObjstorageConfiguration
@@ -813,7 +824,7 @@ storage:
cronjobs:
create-object-reference-partitions:
enabled: true
- storageConfigurationRef: cassandraRWNoObjstorageStorageConfiguration
+ storageConfigurationRef: cassandraRWForReferenceTableJobConfiguration
logLevel: INFO
cron: 5 0 * * mon
concurrencyPolicy: Forbid
--
GitLab
From bc37fc1da85745ed3112bd2eaa86967b72a4cd7e Mon Sep 17 00:00:00 2001
From: Vincent SELLIER <vincent.sellier@softwareheritage.org>
Date: Wed, 19 Feb 2025 10:56:17 +0100
Subject: [PATCH 2/2] Test reference table options on next version
---
swh/values/staging/next-version.yaml | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/swh/values/staging/next-version.yaml b/swh/values/staging/next-version.yaml
index 75d03c3e5..5c0b7520b 100644
--- a/swh/values/staging/next-version.yaml
+++ b/swh/values/staging/next-version.yaml
@@ -527,7 +527,7 @@ storageReplayer:
cassandraSeeds:
- cassandra-cluster-next-version-service
-cassandraStorage:
+cassandraStorage: &cassandraStorage
cls: cassandra
cassandraSeedsRef: cassandraSeeds
keyspace: swh
@@ -657,12 +657,30 @@ rwStorageCassandraWithJournalConfiguration:
journalWriterConfigurationRef: storageJournalWriterConfiguration
objstorageConfigurationRef: rpcRWObjstoragePathslicingConfiguration
+
+cassandraStorageForReferenceTableJob:
+ <<: *cassandraStorage
+ specificOptions:
+ table_options:
+ object_references_*: "compression = {'class': 'ZstdCompressor', 'compression_level':'19'} AND compaction = {'class': 'UnifiedCompactionStrategy'}"
+
+rwCassandraStorageForReferenceTableJob:
+ storageConfigurationRef: cassandraStorageForReferenceTableJob
+ objstorageConfigurationRef: noopObjstorageConfiguration
+
storage:
enabled: true
deployments:
# main storage to let the loaders write
rw-cassandra:
storageConfigurationRef: rwStorageCassandraWithJournalConfiguration
+ cronjobs:
+ create-object-reference-partitions:
+ enabled: false
+ storageConfigurationRef: rwCassandraStorageForReferenceTableJob
+ logLevel: INFO
+ cron: 5 0 * * mon
+ concurrencyPolicy: Forbid
# a storage postgresql instance to replay data
rw-postgresql:
enabled: true
--
GitLab