add configuration for the prometheus-kafka-consumer-group-exporter

data/defaults.yaml

@@ -2871,6 +2871,10 @@ prometheus::jmx::version: 0.11.0
 prometheus::kafka::listen_network: "%{lookup('internal_network')}"
 prometheus::kafka::listen_port: 7071
 
+prometheus::kafka_consumer_group::listen_network: "%{lookup('internal_network')}"
+prometheus::kafka_consumer_group::base_port: 9208
+
+
 grafana::db::database: grafana
 grafana::db::username: grafana
 # grafana::db::password in private-data

site-modules/profile/manifests/kafka/prometheus_consumer_group_exporter.pp

+# Configure prometheus-kafka-consumer-group-exporter
+class profile::kafka::prometheus_consumer_group_exporter {
+
+  $pkg = 'prometheus-kafka-consumer-group-exporter'
+  $defaults_dir = "/etc/defaults/${pkg}"
+
+  package {$pkg:
+    ensure => 'installed',
+  }
+
+  file {$defaults_dir:
+    ensure  => 'directory',
+    purge   => true,
+    recurse => true,
+  }
+
+  $kafka_clusters = lookup('kafka::clusters', Hash)
+
+  $listen_network = lookup('prometheus::kafka_consumer_group::listen_network', Optional[String], 'first', undef)
+  $listen_address = lookup('prometheus::kafka_consumer_group::listen_address', Optional[String], 'first', undef)
+  $actual_listen_address = pick($listen_address, ip_for_network($listen_network))
+
+  $base_port = lookup('prometheus::kafka_consumer_group::base_port', Integer)
+
+  $kafka_clusters.keys.each |$index, $cluster| {
+    $defaults_file = "${defaults_dir}/${cluster}"
+    $service = "${pkg}@${cluster}"
+
+    $bootstrap_servers = $kafka_clusters[$cluster]["brokers"].keys.sort.join(',')
+    $port = $base_port + $index
+
+    file {$defaults_file:
+      ensure  => present,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+      content => template('profile/kafka/prometheus-kafka-consumer-group-exporter.default.erb'),
+    }
+
+    service {$service:
+      ensure => 'running',
+      enable => true,
+      require => [
+        File[$defaults_file],
+        Package[$pkg],
+      ],
+    }
+
+    $target = "${actual_listen_address}:${port}"
+    profile::prometheus::export_scrape_config {"kafka-consumer-group-${cluster}":
+      job    => 'kafka-consumer-group',
+      target => $target,
+      labels => {
+        cluster => $cluster,
+      }
+    }
+  }
+}

site-modules/profile/templates/kafka/prometheus-kafka-consumer-group-exporter.default.erb

+# prometheus-kafka-consumer-group-exporter (For cluster <%= @cluster %>)
+#
+# File managed by puppet (class profile::kafka::prometheus_consumer_group_exporter),
+# changes will be lost
+
+BOOTSTRAP_SERVERS=<%= @bootstrap_servers %>
+PORT=<%= @port %>

site-modules/role/manifests/swh_journal_orchestrator.pp

 class role::swh_journal_orchestrator inherits role::swh_base {
   include profile::kafka
+  include profile::kafka::prometheus_consumer_group_exporter
   include profile::swh::deploy::indexer_journal_client
 }