swh: Add alert on rabbitmq when queue exceeds a given nb of messages

Refs. swh/infra/sysadm-environment#5048

swh/templates/scheduler/alert-rabbitmq.yaml

+{{- if and .Values.scheduler.enabled .Values.scheduler.alerts.enabled .Values.scheduler.alerts.tooManyMessagesInQueue }}
+{{- $environment := .Values.environment -}}
+{{- $severity := .Values.scheduler.alerts.tooManyMessagesInQueue.severity -}}
+{{- $period := .Values.scheduler.alerts.tooManyMessagesInQueue.period -}}
+{{- $threshold := .Values.scheduler.alerts.tooManyMessagesInQueue.threshold -}}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    app: swh-alerts
+  name: rabbitmq-too-many-messages-in-queue-alertmanager.rules
+  namespace: {{ .Values.namespace }}
+spec:
+  groups:
+  - name: rabbitmq-too-many-messages-in-queue.rules
+    rules:
+    - alert: RabbitmqTooManyMessagesInQueue
+      expr: |-
+        max_over_time(rabbitmq_queue_messages_ready{environment={{ $environment | quote }}}[5m]) > {{ $threshold }}
+      annotations:
+        description: "{{ $environment }}: High number of messages in rabbitmq queue <{{"{{"}} $labels.name {{"}}"}}> (server: <{{"{{"}} $labels.server {{"}}"}}>)"
+        summary: "A queue exceeds a given threshold in environment {{ $environment }}"
+      for: {{ $period }}
+      labels:
+        severity: {{ $severity }}
+        namespace: cattle-monitoring-system
+
+{{ end }}

swh/values.yaml

 namespace: swh
+environment: production
 
 sentry:
   environment: production
@@ -513,6 +514,13 @@ scheduler:
     enabled: false
     secretKeyRef: scheduler-sentry-secrets
     secretKeyName: sentry-dsn
+  alerts:
+    enabled: true
+    tooManyMessagesInQueue:
+      threshold: 100000
+      period: 30m
+      severity: warning
+
   # requestedMemory: 512Mi
   # requestedCpu: 500m
   # schedulerConfigurationRef: remoteSchedulerConfiguration
@@ -531,12 +539,6 @@ scheduler:
       opam:
       - policy: origins_without_last_update
         weight: 100
-  # runner:
-  #   enabled: false
-  # runnerPriority:
-  #   enabled: false
-  # listener:
-  #   enabled: false
   rpc:
     enabled: false
     priorityClassName: frontend-rpc

swh/values/minikube.yaml

+environment: minikube
 sentry:
   enabled: false
   environment: minikube

swh/values/production-cassandra.yaml

 namespace: swh-cassandra
+environment: production
 sentry:
   environment: production
 

swh/values/production.yaml

+environment: production
 sentry:
   environment: production
 

swh/values/staging-cassandra.yaml

 namespace: swh-cassandra
+environment: staging
 sentry:
   environment: staging
 

swh/values/staging.yaml

+environment: staging
 sentry:
   environment: staging