Compare revisions

Vincent Sellier · Vincent Sellier · Guillaume Samson · Guillaume Samson · Vincent Sellier · Vincent Sellier
--- a/cluster-components/templates/alerting/cassandra-alerting.yaml
+++ b/cluster-components/templates/alerting/cassandra-alerting.yaml
+{{ if .Values.alerting.enabled -}}
+{{ with .Values.alerting.cassandra -}}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    app: cassandra
+  name: critical-cassandra-service.rules
+  namespace: cattle-monitoring-system
+spec:
+  groups:
+  - name: critical-cassandra-service.rules
+    rules:
+    - alert: Cassandra_Service_Degraded_In_{{ .environment | title }}
+      annotations:
+        description: "The {{"{{"}} $labels.instance {{"}}"}} node is unreachable for more than {{ trimSuffix "m" .period.down }} minutes. This node seems down."
+        summary: "The {{"{{"}} $labels.service {{"}}"}} is degraded. Please check the {{"{{"}} $labels.instance {{"}}"}} status."
+      expr: up{service="cassandra-servers-svc"} == 0
+      for: {{ .period.down }}
+      labels:
+        severity: warning
+        namespace: cattle-monitoring-system
+    - alert: Cassandra_Table_Unrepaired_In_{{ .environment | title }}
+      annotations:
+        description: "The unrepaired bytes of table {{"{{"}} $labels.table {{"}}"}} is more than {{ div .unrepairedSize 1073741824 }} Gb."
+        summary: "Please trigger a repair on the table {{"{{"}} $labels.table {{"}}"}} in keyspace {{"{{"}} $labels.keyspace {{"}}"}}."
+      expr: sum by (keyspace, table) (cassandra_table_bytesunrepaired{table!="",job="cassandra-servers-svc"}) > {{ .unrepairedSize }}
+      for: {{ .period.unrepaired }}
+      labels:
+        severity: critical
+        namespace: cattle-monitoring-system
+{{- end }}
+{{- end }}
--- a/cluster-components/values/archive-production-rke2.yaml
+++ b/cluster-components/values/archive-production-rke2.yaml
 podPriority:
  enabled: true
+
+alerting:
+  enabled: true
+  cassandra:
+    environment: production
+    period:
+      down: 15m
+      unrepaired: 5m
+    unrepairedSize: 214748364800
--- a/cluster-components/values/archive-staging-rke2.yaml
+++ b/cluster-components/values/archive-staging-rke2.yaml
@@ -25,3 +25,12 @@ svix:
  namespace: svix-server
  redisDsn: redis-svix.svix-server
  postgresDsn: db1.internal.staging.swh.network
+
+alerting:
+  enabled: true
+  cassandra:
+    environment: staging
+    period:
+      down: 15m
+      unrepaired: 5m
+    unrepairedSize: 214748364800
--- a/swh/templates/scrubber/_journal-checker-configmap.yaml
+++ b/swh/templates/scrubber/_journal-checker-configmap.yaml
@@ -3,9 +3,7 @@
 {{- $storageConfigurationRef := (get .deployment_config "storageConfigurationRef") | default (get .Values.scrubber "storageConfigurationRef") -}}
 {{- $journalClientConfigurationRef := .Values.scrubber.journalChecker.journalClientConfigurationRef -}}
 {{- $journalClientConfiguration := required (print "journalClientConfigurationRef " .journalClientConfigurationRef " not found in scrubber.journalChecker configuration") (get .Values $journalClientConfigurationRef) -}}
-{{- $groupId := print (get $journalClientConfiguration "group_id") "-journalchecker-" .client_name -}}
-{{- $journalClientOverrides := deepCopy (get $journalClientConfiguration "journalClientOverrides" | default (dict)) -}}
-{{- $_ := set $journalClientOverrides "group_id" $groupId -}}
+{{- $journalClientOverrides := deepCopy (get .deployment_config "journalClientOverrides" | default (dict)) -}}
 ---
 apiVersion: v1
 kind: ConfigMap

--- a/swh/templates/search/_helper-journal-client-configmap.yaml
+++ b/swh/templates/search/_helper-journal-client-configmap.yaml
@@ -2,6 +2,7 @@
 Generate the configuration map for search journal client
 */}}
 {{ define "swh.journal.client.configmap" }}
+{{ $journalClientOverrides := get .journal_client_config "journalClientOverrides" | default dict }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -17,7 +18,7 @@ data:
                                             "configurationRef" .configuration.storageConfigurationRef
                                             "Values" .Values) | nindent 4 }}
    {{- include "swh.journalClientConfiguration" (dict "configurationRef" .configuration.journalConfigurationRef
-                                                       "overrides" .journal_client_config
+                                                       "overrides" $journalClientOverrides
                                                       "Values" .Values) | nindent 4 }}

 {{ end }}
--- a/swh/templates/search/journal-client-deployment.yaml
+++ b/swh/templates/search/journal-client-deployment.yaml
@@ -4,7 +4,7 @@
 {{- $configurationChecksum := include "swh.journal.client.configmap"
  (dict "journal_client_type" $journal_client_type
        "configuration" $.Values.search.journalClients
-        "configurationRef" $journal_client_config.journalConfigurationRef
+        "journal_client_config" $journal_client_config
        "Values" $.Values) -}}

 {{- $log_level := $journal_client_config.logLevel | default "INFO" -}}

--- a/swh/templates/toolbox/configmap.yaml
+++ b/swh/templates/toolbox/configmap.yaml
@@ -23,7 +23,6 @@ data:
    {{ end }}
    {{- if ( eq "scrubber-journal" $service_type ) }}
      {{- include "swh.journalClientConfiguration" (dict "configurationRef" $deployment_config.journalClientConfigurationRef
-                                        "overrides" (dict "group_id" "changeme")
                                        "Values" $.Values) | nindent 4 }}
    {{ end }}
 {{ end }}

--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -1052,6 +1052,15 @@ search:
    # deployments:
    #   objects:
    #     journalConfigurationRef: journalObjectConfiguration
+    #     # requestedCpu: 100m
+    #     # requestedMemory: 500Mi
+    #     # journalClientOverrides:
+    #     # # Declare specific journal client overrides for this consummer
+    #         # group_id: my-search-group-id
+    #         # prefix: swh.journal.objects
+    #         # object_types:
+    #         #   - object1
+    #         #   - object2
    #   indexed:
    #     journalConfigurationRef: journalIndexedConfiguration


--- a/swh/values/default.yaml
+++ b/swh/values/default.yaml
@@ -215,5 +215,18 @@ search:
            values:
            - "true"

+scrubber:
+  sentry:
+    enabled: true
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "swh/scrubber"
+            operator: In
+            values:
+            - "true"
+
 statsd_exporter:
  enabled: true
--- a/swh/values/staging/swh.yaml
+++ b/swh/values/staging/swh.yaml
@@ -63,6 +63,25 @@ postgresqlScrubberConfiguration:
      secretKeyRef: swh-scrubber-postgresql-common-secret
      secretKeyName: postgres-swh-scrubber-password

+internalSecuredKafkaBrokers:
+  - journal1.internal.staging.swh.network:9094
+  - journal2.internal.staging.swh.network:9094
+
+scrubberJournalClientConfiguration:
+  cls: kafka
+  brokersConfigurationRef: internalSecuredKafkaBrokers
+  group_id: swh-archive-stg-journalchecker
+  on_eof: restart
+  sasl.mechanism: SCRAM-SHA-512
+  security.protocol: SASL_SSL
+  sasl.username: swh-archive-stg
+  sasl.password: ${BROKER_USER_PASSWORD}
+  prefix: swh.journal.objects
+  secrets:
+    BROKER_USER_PASSWORD:
+      secretKeyRef: swh-archive-broker-secret
+      secretKeyName: BROKER_USER_PASSWORD
+
 listers:
  enabled: true
  storageConfigurationRef: remoteROStorageConfiguration
@@ -524,7 +543,7 @@ toolbox:
      storageConfigurationRef: postgresqlStorageConfiguration
    scrubber-journal:
      scrubberConfigurationRef: postgresqlScrubberConfiguration
-      journalClientConfigurationRef: journalClientConfiguration
+      journalClientConfigurationRef: scrubberJournalClientConfiguration
    vault:
      vaultConfigurationRef: postgresqlVaultConfiguration
    indexer-storage:
@@ -577,9 +596,7 @@ scheduler:
    enabled: true
    journalConfigurationRef: journalClientConfiguration
    requestedMemory: 100Mi
-    limitedMemory: 200Mi
    requestedCpu: 10m
-    limitedCpu: 20m
  extraServices:
    runner:
      enabled: true
@@ -689,19 +706,21 @@ search:
      objects:
        requestedCpu: 10m
        requestedMemory: 50Mi
-        group_id: swh.search.journal_client-v0.11
-        prefix: swh.journal.objects
-        object_types:
-        - origin
-        - origin_visit_status
+        journalClientOverrides:
+          group_id: swh.search.journal_client-v0.11
+          prefix: swh.journal.objects
+          object_types:
+          - origin
+          - origin_visit_status
      indexed:
        requestedCpu: 10m
        requestedMemory: 50Mi
-        group_id: swh.search.journal_client.indexed-v0.11
-        prefix: swh.journal.indexed
-        object_types:
-        - origin_intrinsic_metadata
-        - origin_extrinsic_metadata
+        journalClientOverrides:
+          group_id: swh.search.journal_client.indexed-v0.11
+          prefix: swh.journal.indexed
+          object_types:
+          - origin_intrinsic_metadata
+          - origin_extrinsic_metadata

 web:
  enabled: true
@@ -781,3 +800,44 @@ web:
    give:
      public_key: ${GIVE_PUBLIC_KEY}
      token: ${GIVE_PRIVATE_TOKEN}
+
+scrubber:
+  enabled: true
+  scrubberDatabaseConfigurationRef: postgresqlScrubberConfiguration
+  storageConfigurationRef: remoteStorageConfiguration
+  priorityClassName: background-workload
+  journalChecker:
+    enabled: true
+    journalClientConfigurationRef: scrubberJournalClientConfiguration
+    deployments:
+      directory:
+        configName: journal-checker-directory
+        journalClientOverrides:
+          batch_size: 100
+        object: directory
+        replicas: 1
+        requestedMemory: 200Mi
+        requestedCpu: 200m
+      release:
+        configName: journal-checker-release
+        journalClientOverrides:
+          batch_size: 200
+          privileged: true
+        object: release
+        replicas: 1
+        requestedMemory: 200Mi
+        requestedCpu: 500m
+      revision:
+        configName: journal-checker-revision
+        journalClientOverrides:
+          privileged: true
+        object: revision
+        replicas: 1
+        requestedMemory: 250Mi
+        requestedCpu: 600m
+      snapshot:
+        configName: journal-checker-snapshot
+        object: snapshot
+        replicas: 1
+        requestedCpu: 1
+        requestedMemory: 250Mi
No results found