worker/instance: Refactor worker systemd initialization

Related T1531

site-modules/profile/manifests/swh/deploy/worker/indexer_ctags.pp

@@ -3,12 +3,6 @@
 class profile::swh::deploy::worker::indexer_ctags {
   include ::profile::swh::deploy::indexer
 
-  $concurrency = lookup('swh::deploy::worker::indexer_ctags::concurrency')
-  $loglevel = lookup('swh::deploy::worker::indexer_ctags::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::indexer_ctags::config_file')
-  $config = lookup('swh::deploy::worker::indexer_ctags::config')
-
   $packages = ['universal-ctags']
   package {$packages:
     ensure => 'present',
@@ -16,22 +10,10 @@ class profile::swh::deploy::worker::indexer_ctags {
 
   Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_ctags':
     ensure       => 'stopped',
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Class['profile::swh::deploy::indexer'],
       Class['profile::swh::deploy::objstorage_cloud'],
-      File[$config_file],
       Package[$packages],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    # Contains passwords
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp

@@ -2,13 +2,6 @@
 
 class profile::swh::deploy::worker::indexer_fossology_license {
   include ::profile::swh::deploy::indexer
-
-  $concurrency = lookup('swh::deploy::worker::indexer_fossology_license::concurrency')
-  $loglevel = lookup('swh::deploy::worker::indexer_fossology_license::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::indexer_fossology_license::config_file')
-  $config = lookup('swh::deploy::worker::indexer_fossology_license::config')
-
   $packages = ['fossology-nomossa']
   package {$packages:
     ensure => 'present',
@@ -16,22 +9,10 @@ class profile::swh::deploy::worker::indexer_fossology_license {
 
   Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Class['profile::swh::deploy::indexer'],
       Class['profile::swh::deploy::objstorage_cloud'],
-      File[$config_file],
       Package[$packages],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    # Contains passwords
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/indexer_language.pp

@@ -2,13 +2,6 @@
 
 class profile::swh::deploy::worker::indexer_language {
   include ::profile::swh::deploy::indexer
-
-  $concurrency = lookup('swh::deploy::worker::indexer_language::concurrency')
-  $loglevel = lookup('swh::deploy::worker::indexer_language::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::indexer_language::config_file')
-  $config = lookup('swh::deploy::worker::indexer_language::config')
-
   Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_language':
     ensure       => 'stopped',
     concurrency  => $concurrency,
@@ -16,16 +9,6 @@ class profile::swh::deploy::worker::indexer_language {
     require      => [
       Class['profile::swh::deploy::indexer'],
       Class['profile::swh::deploy::objstorage_cloud'],
-      File[$config_file],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    # Contains passwords
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/indexer_mimetype.pp

@@ -3,29 +3,11 @@
 class profile::swh::deploy::worker::indexer_mimetype {
   include ::profile::swh::deploy::indexer
 
-  $concurrency = lookup('swh::deploy::worker::indexer_mimetype::concurrency')
-  $loglevel = lookup('swh::deploy::worker::indexer_mimetype::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::indexer_mimetype::config_file')
-  $config = lookup('swh::deploy::worker::indexer_mimetype::config')
-
   Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Class['profile::swh::deploy::indexer'],
       Class['profile::swh::deploy::objstorage_cloud'],
-      File[$config_file],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    # Contains passwords
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp

@@ -3,29 +3,11 @@
 class profile::swh::deploy::worker::indexer_origin_intrinsic_metadata {
   include ::profile::swh::deploy::indexer
 
-  $concurrency = lookup('swh::deploy::worker::indexer_origin_intrinsic_metadata::concurrency')
-  $loglevel = lookup('swh::deploy::worker::indexer_origin_intrinsic_metadata::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::indexer_origin_intrinsic_metadata::config_file')
-  $config = lookup('swh::deploy::worker::indexer_origin_intrinsic_metadata::config')
-
   Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Class['profile::swh::deploy::indexer'],
       Class['profile::swh::deploy::objstorage_cloud'],
-      File[$config_file],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    # Contains passwords
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp

@@ -3,29 +3,11 @@
 class profile::swh::deploy::worker::indexer_rehash {
   include ::profile::swh::deploy::indexer
 
-  $concurrency = lookup('swh::deploy::worker::indexer_rehash::concurrency')
-  $loglevel = lookup('swh::deploy::worker::indexer_rehash::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::indexer_rehash::config_file')
-  $config = lookup('swh::deploy::worker::indexer_rehash::config')
-
   Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_rehash':
     ensure       => 'stopped',
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Class['profile::swh::deploy::indexer'],
       Class['profile::swh::deploy::objstorage_cloud'],
-      File[$config_file],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    # Contains passwords
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/instance.pp

 # Instance of a worker
 define profile::swh::deploy::worker::instance (
   $ensure = present,
-  $concurrency = 10,
-  $loglevel = 'info',
   $max_tasks_per_child = 5,
   $instance_name = $title,
   $limit_no_file = undef,
@@ -12,6 +10,10 @@ define profile::swh::deploy::worker::instance (
 
   $service_basename = "swh-worker@${instance_name}"
   $service_name = "${service_basename}.service"
+  $concurrency = lookup("swh::deploy::worker::${instance_name}::concurrency")
+  $loglevel = lookup("swh::deploy::worker::${instance_name}::loglevel")
+  $config_file = lookup("swh::deploy::worker::${instance_name}::config_file")
+  $config = lookup("swh::deploy::worker::${instance_name}::config", Hash, 'deep')
 
   case $ensure {
     'present', 'running': {
@@ -29,6 +31,17 @@ define profile::swh::deploy::worker::instance (
       if $ensure == 'running' {
         service {$service_basename:
           ensure  => $ensure,
+          require => [
+            File[$config_file],
+          ]
+        }
+
+        file {$config_file:
+          ensure  => 'present',
+          owner   => 'swhworker',
+          group   => 'swhworker',
+          mode    => '0644',
+          content => inline_template("<%= @config.to_yaml %>\n"),
         }
       }
     }

site-modules/profile/manifests/swh/deploy/worker/lister_debian.pp

 # Deployment for swh-lister-debian
 class profile::swh::deploy::worker::lister_debian {
-  $concurrency = lookup('swh::deploy::worker::lister_debian::concurrency')
-  $loglevel = lookup('swh::deploy::worker::lister_debian::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::lister_debian::config_file')
-  $config = lookup('swh::deploy::worker::lister_debian::config', Hash, 'deep')
-
   include ::profile::swh::deploy::base_lister
 
   ::profile::swh::deploy::worker::instance {'lister_debian':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Package['python3-swh.lister'],
-      File[$config_file],
     ],
   }
-
-  # Contains passwords
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/lister_github.pp

 # Deployment for swh-lister-github
 class profile::swh::deploy::worker::lister_github {
-  $concurrency = lookup('swh::deploy::worker::lister_github::concurrency')
-  $loglevel = lookup('swh::deploy::worker::lister_github::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::lister_github::config_file')
-  $config = lookup('swh::deploy::worker::lister_github::config', Hash, 'deep')
-
   include ::profile::swh::deploy::base_lister
 
   ::profile::swh::deploy::worker::instance {'lister_github':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Package['python3-swh.lister'],
-      File[$config_file],
     ],
   }
-
-  # Contains passwords
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/lister_gitlab.pp

 # Deployment for swh-lister-gitlab
 class profile::swh::deploy::worker::lister_gitlab {
-  $concurrency = lookup('swh::deploy::worker::lister_gitlab::concurrency')
-  $loglevel = lookup('swh::deploy::worker::lister_gitlab::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::lister_gitlab::config_file')
-  $config = lookup('swh::deploy::worker::lister_gitlab::config', Hash, 'deep')
-
   include ::profile::swh::deploy::base_lister
 
   ::profile::swh::deploy::worker::instance {'lister_gitlab':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Package['python3-swh.lister'],
-      File[$config_file],
     ],
   }
-
-  # Contains passwords
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/lister_pypi.pp

 # Deployment for swh-lister-pypi
 class profile::swh::deploy::worker::lister_pypi {
-  $concurrency = lookup('swh::deploy::worker::lister_pypi::concurrency')
-  $loglevel = lookup('swh::deploy::worker::lister_pypi::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::lister_pypi::config_file')
-  $config = lookup('swh::deploy::worker::lister_pypi::config', Hash, 'deep')
-
   include ::profile::swh::deploy::base_lister
 
   ::profile::swh::deploy::worker::instance {'lister_pypi':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Package['python3-swh.lister'],
-      File[$config_file],
     ],
   }
-
-  # Contains passwords
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp

 # Deployment for swh-loader-debian
 class profile::swh::deploy::worker::loader_debian {
-  $concurrency = lookup('swh::deploy::worker::loader_debian::concurrency')
-  $loglevel = lookup('swh::deploy::worker::loader_debian::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::loader_debian::config_file')
-  $config = lookup('swh::deploy::worker::loader_debian::config')
-
   $packages = ['python3-swh.loader.debian']
 
   package {$packages:
@@ -14,19 +8,8 @@ class profile::swh::deploy::worker::loader_debian {
 
   ::profile::swh::deploy::worker::instance {'loader_debian':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Package[$packages],
-      File[$config_file],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhworker',
-    mode    => '0644',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp

 # Deployment for swh-loader-deposit
 class profile::swh::deploy::worker::loader_deposit {
-  $concurrency = lookup('swh::deploy::worker::loader_deposit::concurrency')
-  $loglevel = lookup('swh::deploy::worker::loader_deposit::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::loader_deposit::config_file')
-  $config = lookup('swh::deploy::worker::loader_deposit::config')
-
   $packages = ['python3-swh.deposit.loader']
   $private_tmp = lookup('swh::deploy::worker::loader_deposit::private_tmp')
 
-  $service_name = 'loader_deposit'
-
   package {$packages:
     ensure => 'latest',
-    notify => Service["swh-worker@$service_name"],
   }
 
   # This installs the swh-worker@$service_name service
-  ::profile::swh::deploy::worker::instance {$service_name:
+  ::profile::swh::deploy::worker::instance {'loader_deposit':
     ensure       => running,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     private_tmp  => $private_tmp,
     require      => [
       Package[$packages],
-      File[$config_file],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
-
 }

site-modules/profile/manifests/swh/deploy/worker/loader_git.pp

@@ -2,27 +2,11 @@
 class profile::swh::deploy::worker::loader_git {
   include ::profile::swh::deploy::base_loader_git
 
-  $concurrency = lookup('swh::deploy::worker::loader_git::concurrency')
-  $loglevel = lookup('swh::deploy::worker::loader_git::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::loader_git::config_file')
-  $config = lookup('swh::deploy::worker::loader_git::config')
-
   ::profile::swh::deploy::worker::instance {'loader_git':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Class['profile::swh::deploy::base_loader_git'],
-      File[$config_file],
     ],
   }
 
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhworker',
-    mode    => '0644',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp

 # Deployment for swh-loader-mercurial (disk)
 class profile::swh::deploy::worker::loader_mercurial {
-  include ::profile::swh::deploy::base_loader_git
-
-  $concurrency = lookup('swh::deploy::worker::loader_mercurial::concurrency')
-  $loglevel = lookup('swh::deploy::worker::loader_mercurial::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::loader_mercurial::config_file')
-  $config = lookup('swh::deploy::worker::loader_mercurial::config')
-
-  $service_name = 'loader_mercurial'
   $private_tmp = lookup('swh::deploy::worker::loader_mercurial::private_tmp')
-
   $packages = ['python3-swh.loader.mercurial']
 
   package {$packages:
     ensure => 'latest',
-    notify => Service["swh-worker@$service_name"]
   }
 
-  ::profile::swh::deploy::worker::instance {$service_name:
+  ::profile::swh::deploy::worker::instance {'loader_mercurial':
     ensure       => running,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     private_tmp  => $private_tmp,
     require      => [
-      File[$config_file],
+      Package[$packages],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhworker',
-    mode    => '0644',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp

 # Deployment for swh-loader-pypi
 class profile::swh::deploy::worker::loader_pypi {
-  $concurrency = lookup('swh::deploy::worker::loader_pypi::concurrency')
-  $loglevel = lookup('swh::deploy::worker::loader_pypi::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::loader_pypi::config_file')
-  $config = lookup('swh::deploy::worker::loader_pypi::config')
-
   $private_tmp = lookup('swh::deploy::worker::loader_pypi::private_tmp')
 
   $packages = ['python3-swh.loader.pypi']
@@ -16,20 +10,9 @@ class profile::swh::deploy::worker::loader_pypi {
 
   ::profile::swh::deploy::worker::instance {'loader_pypi':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     private_tmp  => $private_tmp,
     require      => [
       Package[$packages],
-      File[$config_file],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhworker',
-    mode    => '0644',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp

 # Deployment for swh-loader-svn
 class profile::swh::deploy::worker::loader_svn {
-  $concurrency = lookup('swh::deploy::worker::loader_svn::concurrency')
-  $loglevel = lookup('swh::deploy::worker::loader_svn::loglevel')
-
-  $config_file = '/etc/softwareheritage/loader/svn.yml'
-  $config = lookup('swh::deploy::worker::loader_svn::config')
-
   $packages = ['python3-swh.loader.svn']
   $limit_no_file = lookup('swh::deploy::worker::loader_svn::limit_no_file')
   $private_tmp = lookup('swh::deploy::worker::loader_svn::private_tmp')
@@ -16,21 +10,10 @@ class profile::swh::deploy::worker::loader_svn {
 
   ::profile::swh::deploy::worker::instance {'loader_svn':
     ensure        => present,
-    concurrency   => $concurrency,
-    loglevel      => $loglevel,
     limit_no_file => $limit_no_file,
     private_tmp   => $private_tmp,
     require       => [
       Package[$packages],
-      File[$config_file],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhworker',
-    mode    => '0644',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/storage_archiver.pp

@@ -2,29 +2,13 @@
 class profile::swh::deploy::worker::storage_archiver {
   include ::profile::swh::deploy::archiver
 
-  $concurrency = lookup('swh::deploy::worker::storage_archiver::concurrency')
   $max_tasks_per_child = lookup('swh::deploy::worker::storage_archiver::max_tasks_per_child')
-  $loglevel = lookup('swh::deploy::worker::storage_archiver::loglevel')
-
-  $config_file = lookup('swh::deploy::worker::storage_archiver::config_file')
-  $config = lookup('swh::deploy::worker::storage_archiver::config')
 
   ::profile::swh::deploy::worker::instance {'storage_archiver':
     ensure              => present,
-    concurrency         => $concurrency,
-    loglevel            => $loglevel,
     max_tasks_per_child => $max_tasks_per_child,
     require             => [
-      File[$config_file],
+      Package[$packages],
     ],
   }
-
-  file {$config_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhdev',
-    # Contains passwords
-    mode    => '0640',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp

@@ -3,27 +3,10 @@
 class profile::swh::deploy::worker::vault_cooker {
   include ::profile::swh::deploy::base_vault
 
-  $concurrency = lookup('swh::deploy::worker::vault_cooker::concurrency')
-  $loglevel = lookup('swh::deploy::worker::vault_cooker::loglevel')
-
-  $conf_file = lookup('swh::deploy::worker::vault_cooker::config_file')
-  $config = lookup('swh::deploy::worker::vault_cooker::config')
-
   ::profile::swh::deploy::worker::instance {'vault_cooker':
     ensure       => present,
-    concurrency  => $concurrency,
-    loglevel     => $loglevel,
     require      => [
       Package[$packages],
-      File[$conf_file],
     ],
   }
-
-  file {$conf_file:
-    ensure  => 'present',
-    owner   => 'swhworker',
-    group   => 'swhworker',
-    mode    => '0644',
-    content => inline_template("<%= @config.to_yaml %>\n"),
-  }
 }

[Phabricator Migration user]

Some references in the commit message have been migrated:

• T1531 is now #1531 (closed)