From 2d6b1e1395b118ea63f8cda8401127f125d3cf96 Mon Sep 17 00:00:00 2001 From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org> Date: Wed, 13 Mar 2024 15:54:35 +0100 Subject: [PATCH] production/cassandra_checks: Activate journal client Except raw_extrinsic_metadata Refs. swh/infra/sysadm-environment#4707 --- swh/values/production/swh-cassandra.yaml | 112 +++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/swh/values/production/swh-cassandra.yaml b/swh/values/production/swh-cassandra.yaml index 81e148f1f..3fe0dbd15 100644 --- a/swh/values/production/swh-cassandra.yaml +++ b/swh/values/production/swh-cassandra.yaml @@ -44,6 +44,25 @@ cassandraROStorage: secretKeyRef: common-secrets secretKeyName: cassandra-swh-ro-password +cassandraChecksROStorage: + cls: cassandra + cassandraSeedsRef: cassandraSeeds + keyspace: swh + initKeyspace: false + consistencyLevel: LOCAL_QUORUM + authProvider: + cls: cassandra.auth.PlainTextAuthProvider + username: swh-ro + password: ${CASSANDRA_PASSWORD} + specificOptions: + directory_entries_insert_algo: batch + objstorage: + cls: noop + secrets: + CASSANDRA_PASSWORD: + secretKeyRef: common-secrets + secretKeyName: cassandra-swh-ro-password + cassandraReplayerStorageConfiguration: storageConfigurationRef: cassandraStorage objstorageConfigurationRef: noopObjectStorage @@ -481,3 +500,96 @@ scrubber: object: snapshot requestedCpu: 650m requestedMemory: 500Mi + +journalClientCassandraChecksConfiguration: + cls: kafka + brokersConfigurationRef: internalSecuredKafkaBrokers + sasl.mechanism: SCRAM-SHA-512 + security.protocol: SASL_SSL + sasl.username: ${BROKER_USER} + sasl.password: ${BROKER_USER_PASSWORD} + message.max.bytes: 524288000 + prefix: swh.journal.objects + group_id: swh-archive-prod-checks + # To override per instance + object_types: [] + secrets: + BROKER_USER: + secretKeyRef: swh-archive-broker-secret + secretKeyName: BROKER_USER + BROKER_USER_PASSWORD: + secretKeyRef: swh-archive-broker-secret + secretKeyName: BROKER_USER_PASSWORD + +cassandraChecks: + enabled: true + directoryOutput: /volume/production-cassandra-checks + volume: + name: cassandra-checks-volume + mountPath: /volume + # readOnly: true + volumeDefinition: + hostPath: + path: /volume + type: Directory + priorityClassName: swh-background-workload + nodeSelector: + kubernetes.io/hostname: rancher-node-metal01 + storagePostgresqlConfigurationRef: postgresqlROStorageConfiguration + storageCassandraConfigurationRef: cassandraChecksROStorage + journalClientConfigurationRef: journalClientCassandraChecksConfiguration + deployments: + origin: + replicas: 2 + journalClientOverrides: + object_types: + - origin + - origin_visit + - origin_visit_status + batch_size: 1000 + content: + replicas: 2 + journalClientOverrides: + object_types: + - content + - skipped_content + batch_size: 1000 + directory: + replicas: 2 + journalClientOverrides: + object_types: + - directory + batch_size: 50 + revision: + replicas: 2 + journalClientOverrides: + object_types: + - revision + batch_size: 1000 + privileged: true + release: + replicas: 2 + journalClientOverrides: + object_types: + - release + batch_size: 1000 + privileged: true + snapshot: + replicas: 2 + journalClientOverrides: + object_types: + - snapshot + batch_size: 50 + extid: + replicas: 2 + journalClientOverrides: + object_types: + - extid + batch_size: 100 + metadata: + enabled: false + replicas: 2 + journalClientOverrides: + object_types: + - raw_extrinsic_metadata + batch_size: 1000 -- GitLab