elastic: Expose instance yaml files to install specific loaders

Related to T4144

sysadmin/T3592-elastic-workers/worker/.gitignore

-loader-bzr.staging.values.yaml
-loader-cvs.staging.values.yaml
-loader-git.staging.values.yaml
-loader-maven.staging.values.yaml
-loader-pypi.staging.values.yaml
-loader-svn.staging.values.yaml
-loader-npm.staging.values.yaml
+*.secret.yaml
 
-loader-git-metadata-fetcher-credentials.yaml
-amqp-access-credentials.yaml
-loaders-git-sentry-secrets.yaml

sysadmin/T3592-elastic-workers/worker/README.md

@@ -98,11 +98,12 @@ Installed through:
 $ TYPE=git  # Replace mentions below in the yaml files
 $ kubectl -f $SECRET_FILE apply
 # for secret file in {
-# loader-$TYPE-metadata-fetcher-credentials.yaml
-# loader-$TYPE-sentry-secrets.yaml
-# amqp-access-credentials.yaml
+# instances/loader-$TYPE-metadata-fetcher-credentials.secret.yaml
+# ./loader-$TYPE-sentry.secret.yaml
+# ./amqp-access-credentials.secret.yaml
+# ...
 # }
-$ cat loader-$TYPE-metadata-fetcher-credentials.yaml
+$ cat instances/loader-git-metadata-fetcher-credentials.secret.yaml
 apiVersion: v1
 kind: Secret
 metadata:
@@ -117,7 +118,7 @@ stringData:
         - username: <redacted>
           password: <redacted>
         - ...
-$ cat amqp-access-credentials.yaml
+$ cat ./amqp-access-credentials.secret.yaml
 apiVersion: v1
 kind: Secret
 metadata:
@@ -126,7 +127,7 @@ metadata:
 type: Opaque
 stringData:
   host: amqp://<redacted>:<redacted>@scheduler0.internal.staging.swh.network:5672/%2f
-$ cat loaders-$TYPE-sentry-secrets.yaml
+$ cat ./loaders-$TYPE-sentry.secrets.yaml
 apiVersion: v1
 kind: Secret
 metadata:

sysadmin/T3592-elastic-workers/worker/instances/loaders-bzr.staging.values.yaml

+# Default values for worker.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+amqp:
+  host: scheduler0.internal.staging.swh.network
+  queue_threshold: 10  # spawn worker per increment of `value` messages
+  queues:
+    - swh.loader.bzr.tasks.LoadBazaar
+
+storage:
+  host: storage1.internal.staging.swh.network
+
+loader:
+  name: loaders
+  type: bzr

sysadmin/T3592-elastic-workers/worker/instances/loaders-cvs.staging.values.yaml

+# Default values for worker.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+amqp:
+  host: scheduler0.internal.staging.swh.network
+  queue_threshold: 10  # spawn worker per increment of `value` messages
+  queues:
+    - swh.loader.cvs.tasks.LoadCvsRepository
+
+storage:
+  host: storage1.internal.staging.swh.network
+
+loader:
+  name: loaders
+  type: cvs

sysadmin/T3592-elastic-workers/worker/instances/loaders-git.staging.values.yaml

+# Default values for worker.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+amqp:
+  host: scheduler0.internal.staging.swh.network
+  queue_threshold: 5  # spawn worker per increment of `value` messages
+  queues:
+    - swh.loader.git.tasks.UpdateGitRepository
+    - swh.loader.git.tasks.LoadDiskGitRepository
+    - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository
+
+storage:
+  host: storage1.internal.staging.swh.network
+
+loader:
+  name: loaders
+  type: git
+
+swh:
+  loader:
+    replicas:
+      min: 1
+      max: 10
+
+sentry:
+  swhpackage: swh.loader.git

sysadmin/T3592-elastic-workers/worker/instances/loaders-maven.staging.values.yaml

+# Default values for worker.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+amqp:
+  host: scheduler0.internal.staging.swh.network
+  queue_threshold: 10  # spawn worker per increment of `value` messages
+  queues:
+    - swh.loader.package.maven.tasks.LoadMaven
+
+storage:
+  host: storage1.internal.staging.swh.network
+
+loader:
+  name: loaders
+  type: maven

sysadmin/T3592-elastic-workers/worker/instances/loaders-npm.staging.values.yaml

+# Default values for worker.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+amqp:
+  host: scheduler0.internal.staging.swh.network
+  queue_threshold: 10  # spawn worker per increment of `value` messages
+  queues:
+    - swh.loader.package.npm.tasks.LoadNpm
+
+storage:
+  host: storage1.internal.staging.swh.network
+
+loader:
+  name: loaders
+  type: npm

sysadmin/T3592-elastic-workers/worker/instances/loaders-pypi.staging.values.yaml

+# Default values for worker.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+amqp:
+  host: scheduler0.internal.staging.swh.network
+  queue_threshold: 10  # spawn worker per increment of `value` messages
+  queues:
+    - swh.loader.package.pypi.tasks.LoadPyPI
+
+storage:
+  host: storage1.internal.staging.swh.network
+
+loader:
+  name: loaders
+  type: pypi
+

sysadmin/T3592-elastic-workers/worker/instances/loaders-svn.staging.values.yaml

+# Default values for worker.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+amqp:
+  host: scheduler0.internal.staging.swh.network
+  queue_threshold: 10  # spawn worker per increment of `value` messages
+  queues:
+    - swh.loader.svn.tasks.LoadSvnRepository
+    - swh.loader.svn.tasks.MountAndLoadSvnRepository
+    - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository
+
+storage:
+  host: storage1.internal.staging.swh.network
+
+loader:
+  name: loaders
+  type: svn

[Phabricator Migration user]

Some references in the commit message have been migrated:

• T4144 is now swh/infra/sysadm-environment#4144 (closed)