Ensure icinga alerts are raised if the scheduler journal client is down

That service is critical for runners to schedule correctly origins to visit. So raise an alert if something is wrong with its startup.

Related to T3502

Test Plan

octo-diff:

bin/octocatalog-diff --octocatalog-diff-args --no-truncate-details --to staging saatchi
...
+ Concat::Fragment[icinga2::object::Service::check_scheduler_journal_client] =>
   parameters =>
      "order": 60
      "target": "/etc/icinga2/zones.d/global-templates/services.conf"
      "content": >>>

apply Service "check_scheduler_journal_client" {
  import "generic-service"

  check_command = "check_systemd"
  command_endpoint = host.name
  vars.check_systemd_unit = "swh-scheduler-journal-client.service"
  assign where host.vars.os == "Linux"
  ignore where host.vars.noagent
}
<<<
*******************************************
+ Concat[/etc/icinga2/zones.d/global-templates/services.conf] =>
   parameters =>
      "backup": "puppet"
      "ensure": "present"
      "ensure_newline": false
      "force": false
      "format": "plain"
      "group": "nagios"
      "mode": "0640"
      "notify": ["Class[Icinga2::Service]"]
      "order": "alpha"
      "owner": "root"
      "path": "/etc/icinga2/zones.d/global-templates/services.conf"
      "replace": true
      "show_diff": true
      "tag": "icinga2::config::file"
      "warn": true
*******************************************
+ Concat_file[/etc/icinga2/zones.d/global-templates/services.conf] =>
   parameters =>
      "backup": "puppet"
      "ensure_newline": false
      "force": false
      "format": "plain"
      "group": "nagios"
      "mode": "0640"
      "order": "alpha"
      "owner": "root"
      "replace": true
      "show_diff": true
      "tag": "_etc_icinga2_zones.d_global-templates_services.conf"
*******************************************
+ Concat_fragment[/etc/icinga2/zones.d/global-templates/services.conf_header] =>
   parameters =>
      "order": "0"
      "tag": "_etc_icinga2_zones.d_global-templates_services.conf"
      "target": "/etc/icinga2/zones.d/global-templates/services.conf"
      "content": >>>
# This file is managed by Puppet. DO NOT EDIT.
<<<
*******************************************
+ Concat_fragment[icinga2::object::Service::check_scheduler_journal_client] =>
   parameters =>
      "order": 60
      "tag": "_etc_icinga2_zones.d_global-templates_services.conf"
      "target": "/etc/icinga2/zones.d/global-templates/services.conf"
      "content": >>>

apply Service "check_scheduler_journal_client" {
  import "generic-service"

  check_command = "check_systemd"
  command_endpoint = host.name
  vars.check_systemd_unit = "swh-scheduler-journal-client.service"
  assign where host.vars.os == "Linux"
  ignore where host.vars.noagent
}
<<<
*******************************************
+ Icinga2::Object::Service[check_scheduler_journal_client] =>
   parameters =>
      "apply": true
      "assign": ["host.vars.os == Linux"]
      "check_command": "check_systemd"
      "command_endpoint": "host.name"
      "ensure": "present"
      "ignore": ["host.vars.noagent"]
      "import": ["generic-service"]
      "name": "Check swh scheduler journal client service"
      "order": 60
      "prefix": false
      "service_name": "check_scheduler_journal_client"
      "target": "/etc/icinga2/zones.d/global-templates/services.conf"
      "template": false
      "vars": {"check_systemd_unit"=>"swh-scheduler-journal-client.service"}
*******************************************
+ Icinga2::Object[icinga2::object::Service::check_scheduler_journal_client] =>
   parameters =>
      "apply": true
      "assign": ["host.vars.os == Linux"]
      "attrs": {"check_command"=>"check_systemd", "command_endpoint"=>"host.name", "vars"=>{"check_systemd_unit"=>"swh-scheduler-journal-client.service"}}
      "attrs_list": ["display_name", "host_name", "check_command", "check_timeout", "check_interval", "check_period", "retry_interval", "max_check_attempts", "groups", "enable_notifications", "enable_active_checks", "enable_passive_checks", "enable_event_handler", "enable_flapping", "enable_perfdata", "event_command", "flapping_threshold_low", "flapping_threshold_high", "volatile", "zone", "command_endpoint", "notes", "notes_url", "action_url", "icon_image", "icon_image_alt", "vars", "Acknowledgement", "ApiBindHost", "ApiBindPort", "ApiEnvironment", "ApplicationType", "AttachDebugger", "BuildCompilerName", "BuildCompilerVersion", "BuildHostName", "Concurrency", "Critical", "Custom", "Deprecated", "Down", "DowntimeEnd", "DowntimeRemoved", "DowntimeStart", "Environment", "FlappingEnd", "FlappingStart", "HostDown", "HostUp", "IncludeConfDir", "Internal", "Json", "LocalStateDir", "LogCritical", "LogDebug", "LogInformation", "LogNotice", "LogWarning", "Math", "MaxConcurrentChecks", "ModAttrPath", "NodeName", "OK", "ObjectsPath", "PidPath", "PkgDataDir", "PlatformArchitecture", "PlatformKernel", "PlatformKernelVersion", "PlatformName", "PlatformVersion", "PrefixDir", "Problem", "Recovery", "RunAsGroup", "RunAsUser", "RunDir", "ServiceCritical", "ServiceOK", "ServiceUnknown", "ServiceWarning", "StatePath", "SysconfDir", "System", "Types", "Unknown", "Up", "UseVfork", "VarsPath", "Warning", "ZonesDir", "NodeName", "ZoneName", "TicketSalt", "PluginDir", "PluginContribDir", "ManubulonPluginDir", "name", "NodeName", "ZoneName", "TicketSalt", "PluginDir", "PluginContribDir", "ManubulonPluginDir", "name"]
      "ensure": "present"
      "ignore": ["host.vars.noagent"]
      "import": ["generic-service"]
      "object_name": "check_scheduler_journal_client"
      "object_type": "Service"
      "order": 60
      "prefix": false
      "target": "/etc/icinga2/zones.d/global-templates/services.conf"
      "template": false
*******************************************
*** End octocatalog-diff on saatchi.internal.softwareheritage.org

---

Migrated from D6156 (view on Phabricator)