Skip to content

Migrate indexer services to indexer journal client services

Antoine R. Dumont requested to merge generated-differential-D8161-source into generated-differential-D8161-target

This cleans up old indexer services not based on journal client (swh-worker@${old}.servicd for old in {indexer_content_mimetype, indexer_fossology_license}).

It installs new services swh-indexer-journal-client@${new}.service for new in {mimetype,fossology_license,intrinsic_metadata,extrinsic_metadata).

Related to T4406

Test Plan

vagrant:

root@worker0:~# puppet agent --test
...
Notice: /Stage[main]/Icinga2::Service/Service[icinga2]: Triggered 'refresh' from 1 event
Notice: /Stage[main]/Profile::Swh::Deploy::Worker::Base/Profile::Cron::D[cleanup-workers-tmp]/Profile::Cron::File[swh-worker]/File[/etc/puppet-cron.d/swh-worker]/ensure: defined content as '{md5}bfdd5f183ad8b0874184d8c58d094e7c'
Notice: /Stage[main]/Profile::Swh::Deploy::Worker::Base/Profile::Cron::D[cleanup-workers-tmp]/Profile::Cron::File[swh-worker]/File[/etc/cron.d/puppet-swh-worker]/ensure: created
Notice: Applied catalog in 4.97 seconds

root@worker0:~# systemctl list-units | grep journal-client
  swh-indexer-journal-client.service                                                        loaded active     running      Software Heritage Indexer Journal Client
  swh-indexer-journal-client@extrinsic_metadata.service                                     loaded activating auto-restart Software Heritage Indexer Journal Client extrinsic_metadata
  swh-indexer-journal-client@fossology_license.service                                      loaded activating auto-restart Software Heritage Indexer Journal Client fossology_license
  swh-indexer-journal-client@intrinsic_metadata.service                                     loaded activating auto-restart Software Heritage Indexer Journal Client intrinsic_metadata
  swh-indexer-journal-client@mimetype.service                                               loaded activating auto-restart Software Heritage Indexer Journal Client mimetype
# for each service ^, parameters.conf defines the necessary override, for example for mimetype
root@worker0:~# cat /etc/systemd/system/swh-indexer-journal-client@mimetype.service.d/parameters.conf
# Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client

[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/mimetype.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO

octo-diff:

$ $SWH_PUPPET_ENVIRONMENT_HOME/bin/octocatalog-diff worker0.internal.staging.swh.network
Found host worker0.internal.staging.swh.network
...
*******************************************
- Concat_fragment[profile::cron::swh-worker-indexer_content_mimetype-autorestart]
*******************************************
- Concat_fragment[profile::cron::swh-worker-indexer_fossology_license-autorestart]
*******************************************
+ File[/etc/softwareheritage/indexer/extrinsic_metadata.yml] =>
   parameters =>
     "content": "---\nscheduler:\n  cls: remote\n  url: http://scheduler0.interna...
     "ensure": "present",
     "group": "swhworker",
     "mode": "0640",
     "notify": "Service[swh-indexer-journal-client@extrinsic_metadata.service]",
     "owner": "root"
*******************************************
+ File[/etc/softwareheritage/indexer/fossology_license.yml] =>
   parameters =>
     "content": "---\nindexer_storage:\n  cls: remote\n  url: http://storage1.int...
     "ensure": "present",
     "group": "swhworker",
     "mode": "0640",
     "notify": "Service[swh-indexer-journal-client@fossology_license.service]",
     "owner": "root"
*******************************************
+ File[/etc/softwareheritage/indexer/intrinsic_metadata.yml] =>
   parameters =>
     "content": "---\nindexer_storage:\n  cls: remote\n  url: http://storage1.int...
     "ensure": "present",
     "group": "swhworker",
     "mode": "0640",
     "notify": "Service[swh-indexer-journal-client@intrinsic_metadata.service]",
     "owner": "root"
*******************************************
- File[/etc/softwareheritage/indexer/journal_client.yml]
*******************************************
+ File[/etc/softwareheritage/indexer/mimetype.yml] =>
   parameters =>
     "content": "---\nindexer_storage:\n  cls: remote\n  url: http://storage1.int...
     "ensure": "present",
     "group": "swhworker",
     "mode": "0640",
     "notify": "Service[swh-indexer-journal-client@mimetype.service]",
     "owner": "root"
*******************************************
  File[/etc/softwareheritage/indexer_content_mimetype.yml] =>
   parameters =>
     ensure =>
      - present
      + absent
*******************************************
  File[/etc/softwareheritage/indexer_fossology_license.yml] =>
   parameters =>
     ensure =>
      - present
      + absent
*******************************************
- File[/etc/softwareheritage/indexer_origin_intrinsic_metadata.yml]
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client.service]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@.service] =>
   parameters =>
     "content": "# Indexer Journal Client unit file\n# Managed by puppet class pr...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": "Class[Systemd::Systemctl::Daemon_reload]",
     "owner": "root",
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata.service.d/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": [
       "Class[Systemd::Systemctl::Daemon_reload]"
     ],
     "owner": "root",
     "selinux_ignore_defaults": false,
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata.service.d] =>
   parameters =>
     "ensure": "directory",
     "group": "root",
     "owner": "root",
     "purge": true,
     "recurse": true,
     "selinux_ignore_defaults": false
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@fossology_license.service.d/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": [
       "Class[Systemd::Systemctl::Daemon_reload]"
     ],
     "owner": "root",
     "selinux_ignore_defaults": false,
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@fossology_license.service.d] =>
   parameters =>
     "ensure": "directory",
     "group": "root",
     "owner": "root",
     "purge": true,
     "recurse": true,
     "selinux_ignore_defaults": false
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@intrinsic_metadata.service.d/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": [
       "Class[Systemd::Systemctl::Daemon_reload]"
     ],
     "owner": "root",
     "selinux_ignore_defaults": false,
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@intrinsic_metadata.service.d] =>
   parameters =>
     "ensure": "directory",
     "group": "root",
     "owner": "root",
     "purge": true,
     "recurse": true,
     "selinux_ignore_defaults": false
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@mimetype.service.d/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": [
       "Class[Systemd::Systemctl::Daemon_reload]"
     ],
     "owner": "root",
     "selinux_ignore_defaults": false,
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@mimetype.service.d] =>
   parameters =>
     "ensure": "directory",
     "group": "root",
     "owner": "root",
     "purge": true,
     "recurse": true,
     "selinux_ignore_defaults": false
*******************************************
  File[/etc/systemd/system/swh-worker@indexer_content_mimetype.service.d/parameters.conf] =>
   parameters =>
     ensure =>
      - file
      + absent
*******************************************
- File[/etc/systemd/system/swh-worker@indexer_content_mimetype.service.d]
*******************************************
  File[/etc/systemd/system/swh-worker@indexer_fossology_license.service.d/parameters.conf] =>
   parameters =>
     ensure =>
      - file
      + absent
*******************************************
- File[/etc/systemd/system/swh-worker@indexer_fossology_license.service.d]
*******************************************
- File[/etc/systemd/system/swh-worker@indexer_origin_intrinsic_metadata.service.d/parameters.conf]
*******************************************
  Package[python3-swh.indexer] =>
   parameters =>
     notify =>
      - ["Profile::Swh::Deploy::Worker::Instance[indexer_content_mimetype]", "Profile::Swh::Deploy::Worker::Instance[indexer_fossology_license]"]
      + ["Profile::Swh::Deploy::Indexer_journal_client[fossology_license]", "Profile::Swh::Deploy::Indexer_journal_client[mimetype]"]
*******************************************
- Profile::Cron::D[swh-worker-indexer_content_mimetype-autorestart]
*******************************************
- Profile::Cron::D[swh-worker-indexer_fossology_license-autorestart]
*******************************************
+ Profile::Swh::Deploy::Indexer_journal_client[extrinsic_metadata] =>
   parameters =>
     "ensure": "present",
     "instance_name": "extrinsic_metadata",
     "sentry_name": "indexer"
*******************************************
+ Profile::Swh::Deploy::Indexer_journal_client[fossology_license] =>
   parameters =>
     "ensure": "present",
     "instance_name": "fossology_license",
     "sentry_name": "indexer"
*******************************************
+ Profile::Swh::Deploy::Indexer_journal_client[intrinsic_metadata] =>
   parameters =>
     "ensure": "present",
     "instance_name": "intrinsic_metadata",
     "sentry_name": "indexer"
*******************************************
+ Profile::Swh::Deploy::Indexer_journal_client[mimetype] =>
   parameters =>
     "ensure": "present",
     "instance_name": "mimetype",
     "sentry_name": "indexer"
*******************************************
  Profile::Swh::Deploy::Worker::Instance[indexer_content_mimetype] =>
   parameters =>
     ensure =>
      - present
      + absent
     send_task_events =>
      - true
      + false
     sentry_name =>
      - indexer
      + indexer_content_mimetype
*******************************************
  Profile::Swh::Deploy::Worker::Instance[indexer_fossology_license] =>
   parameters =>
     ensure =>
      - present
      + absent
     send_task_events =>
      - true
      + false
     sentry_name =>
      - indexer
      + indexer_fossology_license
*******************************************
- Profile::Swh::Deploy::Worker::Instance[indexer_origin_intrinsic_metadata]
*******************************************
+ Service[swh-indexer-journal-client@extrinsic_metadata.service] =>
   parameters =>
     "enable": true,
     "ensure": "running"
*******************************************
+ Service[swh-indexer-journal-client@fossology_license.service] =>
   parameters =>
     "enable": true,
     "ensure": "running"
*******************************************
+ Service[swh-indexer-journal-client@intrinsic_metadata.service] =>
   parameters =>
     "enable": true,
     "ensure": "running"
*******************************************
+ Service[swh-indexer-journal-client@mimetype.service] =>
   parameters =>
     "enable": true,
     "ensure": "running"
*******************************************
- Service[swh-indexer-journal-client]
*******************************************
  Service[swh-worker@indexer_content_mimetype] =>
   parameters =>
     enable =>
      - true
     ensure =>
      + stopped
*******************************************
  Service[swh-worker@indexer_fossology_license] =>
   parameters =>
     enable =>
      - true
     ensure =>
      + stopped
*******************************************
- Service[swh-worker@indexer_origin_intrinsic_metadata]
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@extrinsic_metadata.service/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "daemon_reload": "lazy",
     "ensure": "present",
     "filename": "parameters.conf",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "selinux_ignore_defaults": false,
     "show_diff": true,
     "unit": "swh-indexer-journal-client@extrinsic_metadata.service"
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@fossology_license.service/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "daemon_reload": "lazy",
     "ensure": "present",
     "filename": "parameters.conf",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "selinux_ignore_defaults": false,
     "show_diff": true,
     "unit": "swh-indexer-journal-client@fossology_license.service"
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@intrinsic_metadata.service/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "daemon_reload": "lazy",
     "ensure": "present",
     "filename": "parameters.conf",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "selinux_ignore_defaults": false,
     "show_diff": true,
     "unit": "swh-indexer-journal-client@intrinsic_metadata.service"
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@mimetype.service/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "daemon_reload": "lazy",
     "ensure": "present",
     "filename": "parameters.conf",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "selinux_ignore_defaults": false,
     "show_diff": true,
     "unit": "swh-indexer-journal-client@mimetype.service"
*******************************************
  Systemd::Dropin_file[swh-worker@indexer_content_mimetype/parameters.conf] =>
   parameters =>
     content =>
      - # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::worker::instance

[Service]
Environment=CONCURRENCY=1
Environment=MAX_TASKS_PER_CHILD=5
Environment=LOGLEVEL=info
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer


Environment=SWH_WORKER_CLI_EXTRA_ARGS=--events

     ensure =>
      - present
      + absent
*******************************************
  Systemd::Dropin_file[swh-worker@indexer_fossology_license/parameters.conf] =>
   parameters =>
     content =>
      - # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::worker::instance

[Service]
Environment=CONCURRENCY=1
Environment=MAX_TASKS_PER_CHILD=5
Environment=LOGLEVEL=info
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer


Environment=SWH_WORKER_CLI_EXTRA_ARGS=--events

     ensure =>
      - present
      + absent
*******************************************
- Systemd::Dropin_file[swh-worker@indexer_origin_intrinsic_metadata/parameters.conf]
*******************************************
- Systemd::Unit_file[swh-indexer-journal-client.service]
*******************************************
+ Systemd::Unit_file[swh-indexer-journal-client@.service] =>
   parameters =>
     "content": "# Indexer Journal Client unit file\n# Managed by puppet class pr...
     "ensure": "present",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "show_diff": true
*******************************************
*** End octocatalog-diff on worker0.internal.staging.swh.network

Migrated from D8161 (view on Phabricator)

Merge request reports