icinga/staging: add swh-journal-client lag monitoring

site-modules/profile/manifests/icinga2/objects/static_checks.pp

@@ -99,13 +99,23 @@ class profile::icinga2::objects::static_checks {
     },
   }
 
-  ::icinga2::object::host {'swh-journal-client':
+  ::icinga2::object::host {'swh-journal-client-production':
     check_command => 'dummy',
     address       => '127.0.0.1',
     target        => $checks_file,
     vars          => {
       dummy_state => 0,  # up
-      dummy_text  => "virtual host for journal client checks",
+      dummy_text  => "virtual host for journal client checks in production",
+    },
+  }
+
+  ::icinga2::object::host {'swh-journal-client-staging':
+    check_command => 'dummy',
+    address       => '127.0.0.1',
+    target        => $checks_file,
+    vars          => {
+      dummy_state => 0,  # up
+      dummy_text  => "virtual host for journal client checks in staging",
     },
   }
 
@@ -165,10 +175,10 @@ class profile::icinga2::objects::static_checks {
   }
 
   ['swh.scheduler.journal_client','swh.search.journal_client-v0.11','swh.counters.journal_client'].each |$consumer_group| {
-    ::icinga2::object::service {"Kafka ${consumer_group} lag":
+    ::icinga2::object::service {"Kafka ${consumer_group} lag in production":
       check_command => 'check_prometheus_metric',
       target        => $checks_file,
-      host_name     => 'swh-journal-client',
+      host_name     => 'swh-journal-client-production',
       vars          => {
         prometheus_metric_name     => "kafka ${consumer_group} lag",
         prometheus_query           => profile::icinga2::literal_var(
@@ -180,6 +190,22 @@ class profile::icinga2::objects::static_checks {
     }
 }
 
+  ['swh.scheduler.journal_client','swh.search.journal_client-v0.11','swh.counters.journal_client'].each |$consumer_group| {
+    ::icinga2::object::service {"Kafka ${consumer_group} lag in staging":
+      check_command => 'check_prometheus_metric',
+      target        => $checks_file,
+      host_name     => 'swh-journal-client-staging',
+      vars          => {
+        prometheus_metric_name     => "kafka ${consumer_group} lag",
+        prometheus_query           => profile::icinga2::literal_var(
+          join(['sum(kafka_consumer_group_lag{group=~"', $consumer_group ,'",environment="staging"})'], '')),
+        prometheus_query_type      => 'vector',
+        prometheus_metric_warning  => '10000',
+        prometheus_metric_critical => '20000',
+      },
+    }
+}
+
   ::icinga2::object::service {'Software Heritage Staging Graphql Instance':
     import        => ['generic-service'],
     host_name     => 'graphql.staging.swh.network',