staging: Split directory loader into multiple loaders
As we don't know its consumption pattern yet, this sounds better to split it into the 4 various possible loaders.
Currently, the git-checkout loader seem to progress slowly due to some large origins. Which ends up blocking the other queues. So this might help in understanding how to properly deploy it.
This also allows to use the dedicated core, git, mercurial, or svn docker images instead of the all-in-one save-code-now one.
This creates 3 other deployments for git-checkout, hg-checkout and svn-export, adapting the initial 'directory' deployment to only ingest tarball directories:
$ cd swh && ./helm-diff.sh
Comparing changes between branches production and split-directory-loader...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
Generate config in production branch for values/default.yaml...
Generate config in production branch for values/production-cassandra.yaml...
Generate config in production branch for values/production.yaml...
Generate config in production branch for values/staging-cassandra.yaml...
Generate config in production branch for values/staging.yaml...
Switched to branch 'split-directory-loader'
Generate config in split-directory-loader branch for values/default.yaml...
Generate config in split-directory-loader branch for values/production-cassandra.yaml...
Generate config in split-directory-loader branch for values/production.yaml...
Generate config in split-directory-loader branch for values/staging-cassandra.yaml...
Generate config in split-directory-loader branch for values/staging.yaml...
------------- diff for values/default.yaml -------------
No differences
------------- diff for values/production-cassandra.yaml -------------
No differences
------------- diff for values/production.yaml -------------
No differences
------------- diff for values/staging-cassandra.yaml -------------
No differences
------------- diff for values/staging.yaml -------------
--- /tmp/swh-chart.vrlIKjI2/staging.yaml.before 2023-06-13 09:58:43.297085130 +0200
+++ /tmp/swh-chart.vrlIKjI2/staging.yaml.after 2023-06-13 09:58:43.621085378 +0200
@@ -2739,23 +2739,20 @@
- cls: filter
- cls: retry
- cls: remote
url: http://storage1.internal.staging.swh.network:5002
celery:
task_broker: amqp://${AMQP_USERNAME}:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/
task_acks_late: false
task_queues:
- swh.loader.core.tasks.LoadTarballDirectory
- - swh.loader.git.tasks.LoadGitCheckout
- - swh.loader.mercurial.tasks.LoadMercurialCheckout
- - swh.loader.svn.tasks.LoadSvnExport
sentry_settings_for_celery_tasks:
__sentry-settings-for-celery-tasks__
metadata_fetcher_credentials:
__metadata-fetcher-credentials__
init-container-entrypoint.sh: |
#!/bin/bash
set -e
CONFIG_FILE=/etc/swh/config.yml
@@ -2919,20 +2916,122 @@
root:
level: "INFO"
handlers:
- console
---
# Source: swh/templates/loaders/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: loader-git-checkout-template
+ namespace: swh
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: buffer
+ min_batch_size:
+ content: 100
+ content_bytes: 52428800
+ directory: 100
+ directory_entries: 500
+ extid: 100
+ release: 100
+ release_bytes: 52428800
+ revision: 100
+ revision_bytes: 52428800
+ revision_parents: 200
+ - cls: filter
+ - cls: retry
+ - cls: remote
+ url: http://storage1.internal.staging.swh.network:5002
+
+ celery:
+ task_broker: amqp://${AMQP_USERNAME}:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/
+ task_acks_late: false
+ task_queues:
+ - swh.loader.git.tasks.LoadGitCheckout
+ sentry_settings_for_celery_tasks:
+ __sentry-settings-for-celery-tasks__
+ metadata_fetcher_credentials:
+ __metadata-fetcher-credentials__
+ init-container-entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ CONFIG_FILE=/etc/swh/config.yml
+ CONFIG_FILE_WIP=/tmp/wip-config.yml
+
+ # substitute environment variables when creating the default config.yml
+ eval echo \""$(</etc/swh/configuration-template/config.yml.template)"\" \
+ > $CONFIG_FILE
+
+ SENTRY_SETTINGS_PATH=/etc/credentials/sentry-settings/sentry_settings_for_celery_tasks
+ if [ -f $SENTRY_SETTINGS_PATH ]; then
+ awk "/__sentry-settings-for-celery-tasks__/{system(\"sed 's/^/ /g' $SENTRY_SETTINGS_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+ mv $CONFIG_FILE_WIP $CONFIG_FILE
+ else
+ sed -i '/__sentry-settings-for-celery-tasks__//g' $CONFIG_FILE
+ fi
+
+ CREDS_LISTER_PATH=/etc/credentials/metadata-fetcher/credentials
+ if [ -f $CREDS_LISTER_PATH ]; then
+ awk "/__metadata-fetcher-credentials__/{system(\"sed 's/^/ /g' $CREDS_LISTER_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+ mv $CONFIG_FILE_WIP $CONFIG_FILE
+ else
+ sed -i '/__metadata-fetcher-credentials__//g' $CONFIG_FILE
+ fi
+
+ exit 0
+
+
+ logging-configuration.yml: |
+ version: 1
+
+ handlers:
+ console:
+ class: logging.StreamHandler
+ formatter: json
+ stream: ext://sys.stdout
+
+ formatters:
+ json:
+ class: pythonjsonlogger.jsonlogger.JsonFormatter
+ # python-json-logger parses the format argument to get the variables it actually expands into the json
+ format: "%(asctime)s:%(threadName)s:%(pathname)s:%(lineno)s:%(funcName)s:%(task_name)s:%(task_id)s:%(name)s:%(levelname)s:%(message)s"
+
+ loggers:
+ celery:
+ level: "INFO"
+ amqp:
+ level: WARNING
+ urllib3:
+ level: WARNING
+ azure.core.pipeline.policies.http_logging_policy:
+ level: WARNING
+ swh:
+ level: "INFO"
+ celery.task:
+ level: "INFO"
+
+ root:
+ level: "INFO"
+ handlers:
+ - console
+---
+# Source: swh/templates/loaders/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: loader-golang-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: buffer
min_batch_size:
content: 100
@@ -3021,20 +3120,122 @@
root:
level: "INFO"
handlers:
- console
---
# Source: swh/templates/loaders/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: loader-hg-checkout-template
+ namespace: swh
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: buffer
+ min_batch_size:
+ content: 100
+ content_bytes: 52428800
+ directory: 100
+ directory_entries: 500
+ extid: 100
+ release: 100
+ release_bytes: 52428800
+ revision: 100
+ revision_bytes: 52428800
+ revision_parents: 200
+ - cls: filter
+ - cls: retry
+ - cls: remote
+ url: http://storage1.internal.staging.swh.network:5002
+
+ celery:
+ task_broker: amqp://${AMQP_USERNAME}:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/
+ task_acks_late: false
+ task_queues:
+ - swh.loader.mercurial.tasks.LoadMercurialCheckout
+ sentry_settings_for_celery_tasks:
+ __sentry-settings-for-celery-tasks__
+ metadata_fetcher_credentials:
+ __metadata-fetcher-credentials__
+ init-container-entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ CONFIG_FILE=/etc/swh/config.yml
+ CONFIG_FILE_WIP=/tmp/wip-config.yml
+
+ # substitute environment variables when creating the default config.yml
+ eval echo \""$(</etc/swh/configuration-template/config.yml.template)"\" \
+ > $CONFIG_FILE
+
+ SENTRY_SETTINGS_PATH=/etc/credentials/sentry-settings/sentry_settings_for_celery_tasks
+ if [ -f $SENTRY_SETTINGS_PATH ]; then
+ awk "/__sentry-settings-for-celery-tasks__/{system(\"sed 's/^/ /g' $SENTRY_SETTINGS_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+ mv $CONFIG_FILE_WIP $CONFIG_FILE
+ else
+ sed -i '/__sentry-settings-for-celery-tasks__//g' $CONFIG_FILE
+ fi
+
+ CREDS_LISTER_PATH=/etc/credentials/metadata-fetcher/credentials
+ if [ -f $CREDS_LISTER_PATH ]; then
+ awk "/__metadata-fetcher-credentials__/{system(\"sed 's/^/ /g' $CREDS_LISTER_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+ mv $CONFIG_FILE_WIP $CONFIG_FILE
+ else
+ sed -i '/__metadata-fetcher-credentials__//g' $CONFIG_FILE
+ fi
+
+ exit 0
+
+
+ logging-configuration.yml: |
+ version: 1
+
+ handlers:
+ console:
+ class: logging.StreamHandler
+ formatter: json
+ stream: ext://sys.stdout
+
+ formatters:
+ json:
+ class: pythonjsonlogger.jsonlogger.JsonFormatter
+ # python-json-logger parses the format argument to get the variables it actually expands into the json
+ format: "%(asctime)s:%(threadName)s:%(pathname)s:%(lineno)s:%(funcName)s:%(task_name)s:%(task_id)s:%(name)s:%(levelname)s:%(message)s"
+
+ loggers:
+ celery:
+ level: "INFO"
+ amqp:
+ level: WARNING
+ urllib3:
+ level: WARNING
+ azure.core.pipeline.policies.http_logging_policy:
+ level: WARNING
+ swh:
+ level: "INFO"
+ celery.task:
+ level: "INFO"
+
+ root:
+ level: "INFO"
+ handlers:
+ - console
+---
+# Source: swh/templates/loaders/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: loader-maven-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: buffer
min_batch_size:
content: 100
@@ -3848,20 +4049,122 @@
swh:
level: "INFO"
celery.task:
level: "INFO"
root:
level: "INFO"
handlers:
- console
---
+# Source: swh/templates/loaders/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: loader-svn-export-template
+ namespace: swh
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: buffer
+ min_batch_size:
+ content: 100
+ content_bytes: 52428800
+ directory: 100
+ directory_entries: 500
+ extid: 100
+ release: 100
+ release_bytes: 52428800
+ revision: 100
+ revision_bytes: 52428800
+ revision_parents: 200
+ - cls: filter
+ - cls: retry
+ - cls: remote
+ url: http://storage1.internal.staging.swh.network:5002
+
+ celery:
+ task_broker: amqp://${AMQP_USERNAME}:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/
+ task_acks_late: false
+ task_queues:
+ - swh.loader.svn.tasks.LoadSvnExport
+ sentry_settings_for_celery_tasks:
+ __sentry-settings-for-celery-tasks__
+ metadata_fetcher_credentials:
+ __metadata-fetcher-credentials__
+ init-container-entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ CONFIG_FILE=/etc/swh/config.yml
+ CONFIG_FILE_WIP=/tmp/wip-config.yml
+
+ # substitute environment variables when creating the default config.yml
+ eval echo \""$(</etc/swh/configuration-template/config.yml.template)"\" \
+ > $CONFIG_FILE
+
+ SENTRY_SETTINGS_PATH=/etc/credentials/sentry-settings/sentry_settings_for_celery_tasks
+ if [ -f $SENTRY_SETTINGS_PATH ]; then
+ awk "/__sentry-settings-for-celery-tasks__/{system(\"sed 's/^/ /g' $SENTRY_SETTINGS_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+ mv $CONFIG_FILE_WIP $CONFIG_FILE
+ else
+ sed -i '/__sentry-settings-for-celery-tasks__//g' $CONFIG_FILE
+ fi
+
+ CREDS_LISTER_PATH=/etc/credentials/metadata-fetcher/credentials
+ if [ -f $CREDS_LISTER_PATH ]; then
+ awk "/__metadata-fetcher-credentials__/{system(\"sed 's/^/ /g' $CREDS_LISTER_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+ mv $CONFIG_FILE_WIP $CONFIG_FILE
+ else
+ sed -i '/__metadata-fetcher-credentials__//g' $CONFIG_FILE
+ fi
+
+ exit 0
+
+
+ logging-configuration.yml: |
+ version: 1
+
+ handlers:
+ console:
+ class: logging.StreamHandler
+ formatter: json
+ stream: ext://sys.stdout
+
+ formatters:
+ json:
+ class: pythonjsonlogger.jsonlogger.JsonFormatter
+ # python-json-logger parses the format argument to get the variables it actually expands into the json
+ format: "%(asctime)s:%(threadName)s:%(pathname)s:%(lineno)s:%(funcName)s:%(task_name)s:%(task_id)s:%(name)s:%(levelname)s:%(message)s"
+
+ loggers:
+ celery:
+ level: "INFO"
+ amqp:
+ level: WARNING
+ urllib3:
+ level: WARNING
+ azure.core.pipeline.policies.http_logging_policy:
+ level: WARNING
+ swh:
+ level: "INFO"
+ celery.task:
+ level: "INFO"
+
+ root:
+ level: "INFO"
+ handlers:
+ - console
+---
# Source: swh/templates/statsd-exporter/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-statsd-exporter
namespace: swh
data:
config.yml: |
defaults:
timer_type: histogram
@@ -7328,21 +7631,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-add-forge-now
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -7498,21 +7801,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-add-forge-now-slow
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -7668,21 +7971,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-archive
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -7836,21 +8139,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-bzr
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -8004,21 +8307,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-content
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -8172,21 +8475,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-cran
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -8340,21 +8643,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-cvs
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -8508,21 +8811,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-debian
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -8676,21 +8979,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-deposit
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -8858,21 +9161,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-directory
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -8909,29 +9212,32 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_package:20230612.1
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 768Mi
cpu: 1000m
+ limits:
+ memory: 1024Mi
+ cpu: 1500m
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
env:
- name: STATSD_HOST
value: prometheus-statsd-exporter
- name: STATSD_PORT
value: "9125"
- name: MAX_TASKS_PER_CHILD
@@ -9026,21 +9332,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-git
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -9175,40 +9481,211 @@
optional: true
# if defined at the "typed" loader level
# otherwise use the global image is defined First this needs to replace - in
# $loader_type with "" to find the proper image name.
---
# Source: swh/templates/loaders/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
+ name: loader-git-checkout
+ namespace: swh
+ labels:
+ app: loader-git-checkout
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: loader-git-checkout
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: loader-git-checkout
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/loader
+ operator: In
+ values:
+ - "true"
+ terminationGracePeriodSeconds: 3600
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: Always
+ env:
+ - name: AMQP_USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: rabbitmq-amqp-username
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: AMQP_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: rabbitmq-amqp-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ command:
+ - /entrypoint.sh
+ volumeMounts:
+ - name: configuration-template
+ mountPath: /entrypoint.sh
+ subPath: "init-container-entrypoint.sh"
+ readOnly: true
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: metadata-fetcher-credentials
+ mountPath: /etc/credentials/metadata-fetcher
+ readOnly: true
+ - name: sentry-settings-for-celery-tasks
+ mountPath: /etc/credentials/sentry-settings
+ readOnly: true
+ containers:
+ - name: loaders
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+
+ imagePullPolicy: Always
+ command:
+ - /opt/swh/entrypoint.sh
+ resources:
+ requests:
+ memory: 768Mi
+ cpu: 1000m
+ limits:
+ memory: 1024Mi
+ cpu: 1500m
+ lifecycle:
+ preStop:
+ exec:
+ command: ["/pre-stop.sh"]
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "10"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_LOG_CONFIG
+ value: /etc/swh/logging-configuration.yml
+
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "yes"
+ volumeMounts:
+ - name: loader-utils
+ mountPath: /pre-stop.sh
+ subPath: "pre-stop.sh"
+ - name: configuration
+ mountPath: /etc/swh
+ - name: localstorage
+ mountPath: /tmp
+
+ - name: configuration-template
+ mountPath: /etc/swh/logging-configuration.yml
+ subPath: "logging-configuration.yml"
+ readOnly: true
+
+ volumes:
+ - name: localstorage
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ labels:
+ type: ephemeral-volume
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 100Gi
+ storageClassName: local-path
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: loader-git-checkout-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - key: "init-container-entrypoint.sh"
+ path: "init-container-entrypoint.sh"
+
+ - key: "logging-configuration.yml"
+ path: "logging-configuration.yml"
+
+ - name: loader-utils
+ configMap:
+ name: loader-utils
+ defaultMode: 0777
+ items:
+ - key: "pre-stop-idempotent.sh"
+ path: "pre-stop.sh"
+ - name: metadata-fetcher-credentials
+ secret:
+ secretName: metadata-fetcher-credentials
+ optional: true
+ - name: sentry-settings-for-celery-tasks
+ secret:
+ secretName: sentry-settings-for-celery-tasks
+ optional: true
+
+# if defined at the "typed" loader level
+# otherwise use the global image is defined First this needs to replace - in
+# $loader_type with "" to find the proper image name.
+---
+# Source: swh/templates/loaders/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
name: loader-golang
namespace: swh
labels:
app: loader-golang
spec:
revisionHistoryLimit: 2
selector:
matchLabels:
app: loader-golang
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-golang
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -9343,40 +9820,211 @@
optional: true
# if defined at the "typed" loader level
# otherwise use the global image is defined First this needs to replace - in
# $loader_type with "" to find the proper image name.
---
# Source: swh/templates/loaders/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
+ name: loader-hg-checkout
+ namespace: swh
+ labels:
+ app: loader-hg-checkout
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: loader-hg-checkout
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: loader-hg-checkout
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/loader
+ operator: In
+ values:
+ - "true"
+ terminationGracePeriodSeconds: 3600
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: Always
+ env:
+ - name: AMQP_USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: rabbitmq-amqp-username
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: AMQP_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: rabbitmq-amqp-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ command:
+ - /entrypoint.sh
+ volumeMounts:
+ - name: configuration-template
+ mountPath: /entrypoint.sh
+ subPath: "init-container-entrypoint.sh"
+ readOnly: true
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: metadata-fetcher-credentials
+ mountPath: /etc/credentials/metadata-fetcher
+ readOnly: true
+ - name: sentry-settings-for-celery-tasks
+ mountPath: /etc/credentials/sentry-settings
+ readOnly: true
+ containers:
+ - name: loaders
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_mercurial:20230612.1
+
+ imagePullPolicy: Always
+ command:
+ - /opt/swh/entrypoint.sh
+ resources:
+ requests:
+ memory: 768Mi
+ cpu: 1000m
+ limits:
+ memory: 1024Mi
+ cpu: 1500m
+ lifecycle:
+ preStop:
+ exec:
+ command: ["/pre-stop.sh"]
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "10"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_LOG_CONFIG
+ value: /etc/swh/logging-configuration.yml
+
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "yes"
+ volumeMounts:
+ - name: loader-utils
+ mountPath: /pre-stop.sh
+ subPath: "pre-stop.sh"
+ - name: configuration
+ mountPath: /etc/swh
+ - name: localstorage
+ mountPath: /tmp
+
+ - name: configuration-template
+ mountPath: /etc/swh/logging-configuration.yml
+ subPath: "logging-configuration.yml"
+ readOnly: true
+
+ volumes:
+ - name: localstorage
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ labels:
+ type: ephemeral-volume
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 100Gi
+ storageClassName: local-path
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: loader-hg-checkout-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - key: "init-container-entrypoint.sh"
+ path: "init-container-entrypoint.sh"
+
+ - key: "logging-configuration.yml"
+ path: "logging-configuration.yml"
+
+ - name: loader-utils
+ configMap:
+ name: loader-utils
+ defaultMode: 0777
+ items:
+ - key: "pre-stop-idempotent.sh"
+ path: "pre-stop.sh"
+ - name: metadata-fetcher-credentials
+ secret:
+ secretName: metadata-fetcher-credentials
+ optional: true
+ - name: sentry-settings-for-celery-tasks
+ secret:
+ secretName: sentry-settings-for-celery-tasks
+ optional: true
+
+# if defined at the "typed" loader level
+# otherwise use the global image is defined First this needs to replace - in
+# $loader_type with "" to find the proper image name.
+---
+# Source: swh/templates/loaders/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
name: loader-maven
namespace: swh
labels:
app: loader-maven
spec:
revisionHistoryLimit: 2
selector:
matchLabels:
app: loader-maven
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-maven
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -9530,21 +10178,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-mercurial
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -9698,21 +10346,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-npm
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -9866,21 +10514,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-opam
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -10034,21 +10682,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-pubdev
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -10202,21 +10850,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-pypi
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -10370,21 +11018,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-save-code-now
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -10540,21 +11188,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loader-svn
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 411d2507a7b0795ff49e426e16909bd526cc3f651c46ce1932b71cb54fc58cd0
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/loader
operator: In
values:
- "true"
@@ -10682,20 +11330,191 @@
- key: "pre-stop-idempotent.sh"
path: "pre-stop.sh"
- name: metadata-fetcher-credentials
secret:
secretName: metadata-fetcher-credentials
optional: true
- name: sentry-settings-for-celery-tasks
secret:
secretName: sentry-settings-for-celery-tasks
optional: true
+
+# if defined at the "typed" loader level
+# otherwise use the global image is defined First this needs to replace - in
+# $loader_type with "" to find the proper image name.
+---
+# Source: swh/templates/loaders/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: loader-svn-export
+ namespace: swh
+ labels:
+ app: loader-svn-export
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: loader-svn-export
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: loader-svn-export
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 8bdb77a347dc6263f01646563177e4f7b7a182789400c265c785ef9ae0b0419e
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/loader
+ operator: In
+ values:
+ - "true"
+ terminationGracePeriodSeconds: 3600
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: Always
+ env:
+ - name: AMQP_USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: rabbitmq-amqp-username
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: AMQP_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: rabbitmq-amqp-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ command:
+ - /entrypoint.sh
+ volumeMounts:
+ - name: configuration-template
+ mountPath: /entrypoint.sh
+ subPath: "init-container-entrypoint.sh"
+ readOnly: true
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: metadata-fetcher-credentials
+ mountPath: /etc/credentials/metadata-fetcher
+ readOnly: true
+ - name: sentry-settings-for-celery-tasks
+ mountPath: /etc/credentials/sentry-settings
+ readOnly: true
+ containers:
+ - name: loaders
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_svn:20230612.1
+
+ imagePullPolicy: Always
+ command:
+ - /opt/swh/entrypoint.sh
+ resources:
+ requests:
+ memory: 768Mi
+ cpu: 1000m
+ limits:
+ memory: 1024Mi
+ cpu: 1500m
+ lifecycle:
+ preStop:
+ exec:
+ command: ["/pre-stop.sh"]
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "10"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_LOG_CONFIG
+ value: /etc/swh/logging-configuration.yml
+
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "yes"
+ volumeMounts:
+ - name: loader-utils
+ mountPath: /pre-stop.sh
+ subPath: "pre-stop.sh"
+ - name: configuration
+ mountPath: /etc/swh
+ - name: localstorage
+ mountPath: /tmp
+
+ - name: configuration-template
+ mountPath: /etc/swh/logging-configuration.yml
+ subPath: "logging-configuration.yml"
+ readOnly: true
+
+ volumes:
+ - name: localstorage
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ labels:
+ type: ephemeral-volume
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 100Gi
+ storageClassName: local-path
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: loader-svn-export-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - key: "init-container-entrypoint.sh"
+ path: "init-container-entrypoint.sh"
+
+ - key: "logging-configuration.yml"
+ path: "logging-configuration.yml"
+
+ - name: loader-utils
+ configMap:
+ name: loader-utils
+ defaultMode: 0777
+ items:
+ - key: "pre-stop-idempotent.sh"
+ path: "pre-stop.sh"
+ - name: metadata-fetcher-credentials
+ secret:
+ secretName: metadata-fetcher-credentials
+ optional: true
+ - name: sentry-settings-for-celery-tasks
+ secret:
+ secretName: sentry-settings-for-celery-tasks
+ optional: true
---
# Source: swh/templates/memcached/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: memcached
namespace: swh
labels:
chart: "swh-0.1.0"
app: memcached
@@ -13275,177 +14094,207 @@
kind: Deployment # Optional. Default: Deployment
# Mandatory. Must be in same namespace as ScaledObject
name: loader-directory
# envSourceContainerName: {container-name} # Optional. Default:
# .spec.template.spec.containers[0]
pollingInterval: 30 # Optional. Default: 30 seconds
cooldownPeriod: 300 # Optional. Default: 300 seconds
idleReplicaCount: 0 # Set to 0 to stop all the workers when there is
# no activity on the queue
minReplicaCount: 0
- maxReplicaCount: 2
+ maxReplicaCount: 1
triggers:
- type: rabbitmq
authenticationRef:
name: amqp-authentication-loader-directory
metadata:
protocol: auto # Optional. Specifies protocol to use,
# either amqp or http, or auto to
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength to trigger on number of msgs in queue
excludeUnacknowledged: "false" # QueueLength should include unacked messages
# Implies "http" protocol is used
value: "10"
queueName: swh.loader.core.tasks.LoadTarballDirectory
vhostName: / # Vhost to use for the connection
+---
+# Source: swh/templates/loaders/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: loader-git-operators
+ namespace: swh
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1 # Optional. Default: apps/v1
+ kind: Deployment # Optional. Default: Deployment
+ # Mandatory. Must be in same namespace as ScaledObject
+ name: loader-git
+ # envSourceContainerName: {container-name} # Optional. Default:
+ # .spec.template.spec.containers[0]
+ pollingInterval: 30 # Optional. Default: 30 seconds
+ cooldownPeriod: 300 # Optional. Default: 300 seconds
+ idleReplicaCount: 0 # Set to 0 to stop all the workers when there is
+ # no activity on the queue
+ minReplicaCount: 0
+ maxReplicaCount: 1
+ triggers:
- type: rabbitmq
authenticationRef:
- name: amqp-authentication-loader-directory
+ name: amqp-authentication-loader-git
metadata:
protocol: auto # Optional. Specifies protocol to use,
# either amqp or http, or auto to
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength to trigger on number of msgs in queue
excludeUnacknowledged: "false" # QueueLength should include unacked messages
# Implies "http" protocol is used
value: "10"
- queueName: swh.loader.git.tasks.LoadGitCheckout
+ queueName: swh.loader.git.tasks.UpdateGitRepository
vhostName: / # Vhost to use for the connection
- type: rabbitmq
authenticationRef:
- name: amqp-authentication-loader-directory
+ name: amqp-authentication-loader-git
metadata:
protocol: auto # Optional. Specifies protocol to use,
# either amqp or http, or auto to
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength to trigger on number of msgs in queue
excludeUnacknowledged: "false" # QueueLength should include unacked messages
# Implies "http" protocol is used
value: "10"
- queueName: swh.loader.mercurial.tasks.LoadMercurialCheckout
+ queueName: swh.loader.git.tasks.LoadDiskGitRepository
vhostName: / # Vhost to use for the connection
- type: rabbitmq
authenticationRef:
- name: amqp-authentication-loader-directory
+ name: amqp-authentication-loader-git
metadata:
protocol: auto # Optional. Specifies protocol to use,
# either amqp or http, or auto to
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength to trigger on number of msgs in queue
excludeUnacknowledged: "false" # QueueLength should include unacked messages
# Implies "http" protocol is used
value: "10"
- queueName: swh.loader.svn.tasks.LoadSvnExport
+ queueName: swh.loader.git.tasks.UncompressAndLoadDiskGitRepository
vhostName: / # Vhost to use for the connection
---
# Source: swh/templates/loaders/keda-autoscaling.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
- name: loader-git-operators
+ name: loader-git-checkout-operators
namespace: swh
spec:
scaleTargetRef:
apiVersion: apps/v1 # Optional. Default: apps/v1
kind: Deployment # Optional. Default: Deployment
# Mandatory. Must be in same namespace as ScaledObject
- name: loader-git
+ name: loader-git-checkout
# envSourceContainerName: {container-name} # Optional. Default:
# .spec.template.spec.containers[0]
pollingInterval: 30 # Optional. Default: 30 seconds
cooldownPeriod: 300 # Optional. Default: 300 seconds
idleReplicaCount: 0 # Set to 0 to stop all the workers when there is
# no activity on the queue
minReplicaCount: 0
maxReplicaCount: 1
triggers:
- type: rabbitmq
authenticationRef:
- name: amqp-authentication-loader-git
- metadata:
- protocol: auto # Optional. Specifies protocol to use,
- # either amqp or http, or auto to
- # autodetect based on the `host` value.
- # Default value is auto.
- mode: QueueLength # QueueLength to trigger on number of msgs in queue
- excludeUnacknowledged: "false" # QueueLength should include unacked messages
- # Implies "http" protocol is used
- value: "10"
- queueName: swh.loader.git.tasks.UpdateGitRepository
- vhostName: / # Vhost to use for the connection
- - type: rabbitmq
- authenticationRef:
- name: amqp-authentication-loader-git
+ name: amqp-authentication-loader-git-checkout
metadata:
protocol: auto # Optional. Specifies protocol to use,
# either amqp or http, or auto to
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength to trigger on number of msgs in queue
excludeUnacknowledged: "false" # QueueLength should include unacked messages
# Implies "http" protocol is used
value: "10"
- queueName: swh.loader.git.tasks.LoadDiskGitRepository
+ queueName: swh.loader.git.tasks.LoadGitCheckout
vhostName: / # Vhost to use for the connection
+---
+# Source: swh/templates/loaders/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: loader-golang-operators
+ namespace: swh
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1 # Optional. Default: apps/v1
+ kind: Deployment # Optional. Default: Deployment
+ # Mandatory. Must be in same namespace as ScaledObject
+ name: loader-golang
+ # envSourceContainerName: {container-name} # Optional. Default:
+ # .spec.template.spec.containers[0]
+ pollingInterval: 30 # Optional. Default: 30 seconds
+ cooldownPeriod: 300 # Optional. Default: 300 seconds
+ idleReplicaCount: 0 # Set to 0 to stop all the workers when there is
+ # no activity on the queue
+ minReplicaCount: 0
+ maxReplicaCount: 1
+ triggers:
- type: rabbitmq
authenticationRef:
- name: amqp-authentication-loader-git
+ name: amqp-authentication-loader-golang
metadata:
protocol: auto # Optional. Specifies protocol to use,
# either amqp or http, or auto to
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength to trigger on number of msgs in queue
excludeUnacknowledged: "false" # QueueLength should include unacked messages
# Implies "http" protocol is used
value: "10"
- queueName: swh.loader.git.tasks.UncompressAndLoadDiskGitRepository
+ queueName: swh.loader.package.golang.tasks.LoadGolang
vhostName: / # Vhost to use for the connection
---
# Source: swh/templates/loaders/keda-autoscaling.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
- name: loader-golang-operators
+ name: loader-hg-checkout-operators
namespace: swh
spec:
scaleTargetRef:
apiVersion: apps/v1 # Optional. Default: apps/v1
kind: Deployment # Optional. Default: Deployment
# Mandatory. Must be in same namespace as ScaledObject
- name: loader-golang
+ name: loader-hg-checkout
# envSourceContainerName: {container-name} # Optional. Default:
# .spec.template.spec.containers[0]
pollingInterval: 30 # Optional. Default: 30 seconds
cooldownPeriod: 300 # Optional. Default: 300 seconds
idleReplicaCount: 0 # Set to 0 to stop all the workers when there is
# no activity on the queue
minReplicaCount: 0
maxReplicaCount: 1
triggers:
- type: rabbitmq
authenticationRef:
- name: amqp-authentication-loader-golang
+ name: amqp-authentication-loader-hg-checkout
metadata:
protocol: auto # Optional. Specifies protocol to use,
# either amqp or http, or auto to
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength to trigger on number of msgs in queue
excludeUnacknowledged: "false" # QueueLength should include unacked messages
# Implies "http" protocol is used
value: "10"
- queueName: swh.loader.package.golang.tasks.LoadGolang
+ queueName: swh.loader.mercurial.tasks.LoadMercurialCheckout
vhostName: / # Vhost to use for the connection
---
# Source: swh/templates/loaders/keda-autoscaling.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: loader-maven-operators
namespace: swh
spec:
scaleTargetRef:
@@ -13902,20 +14751,56 @@
# either amqp or http, or auto to
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength to trigger on number of msgs in queue
excludeUnacknowledged: "false" # QueueLength should include unacked messages
# Implies "http" protocol is used
value: "10"
queueName: swh.loader.svn.tasks.DumpMountAndLoadSvnRepository
vhostName: / # Vhost to use for the connection
---
+# Source: swh/templates/loaders/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: loader-svn-export-operators
+ namespace: swh
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1 # Optional. Default: apps/v1
+ kind: Deployment # Optional. Default: Deployment
+ # Mandatory. Must be in same namespace as ScaledObject
+ name: loader-svn-export
+ # envSourceContainerName: {container-name} # Optional. Default:
+ # .spec.template.spec.containers[0]
+ pollingInterval: 30 # Optional. Default: 30 seconds
+ cooldownPeriod: 300 # Optional. Default: 300 seconds
+ idleReplicaCount: 0 # Set to 0 to stop all the workers when there is
+ # no activity on the queue
+ minReplicaCount: 0
+ maxReplicaCount: 1
+ triggers:
+ - type: rabbitmq
+ authenticationRef:
+ name: amqp-authentication-loader-svn-export
+ metadata:
+ protocol: auto # Optional. Specifies protocol to use,
+ # either amqp or http, or auto to
+ # autodetect based on the `host` value.
+ # Default value is auto.
+ mode: QueueLength # QueueLength to trigger on number of msgs in queue
+ excludeUnacknowledged: "false" # QueueLength should include unacked messages
+ # Implies "http" protocol is used
+ value: "10"
+ queueName: swh.loader.svn.tasks.LoadSvnExport
+ vhostName: / # Vhost to use for the connection
+---
# Source: swh/templates/storage-replayer/keda-autoscaling.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: storage-replayer-content-scaledobject
namespace: swh
spec:
scaleTargetRef:
name: storage-replayer-content
pollingInterval: 120
@@ -14560,32 +15445,56 @@
spec:
secretTargetRef:
- parameter: host # "host" is required by the scalerObject trigger metadata
name: common-secrets
key: rabbitmq-http-host
---
# Source: swh/templates/loaders/keda-authentication.yaml
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
+ name: amqp-authentication-loader-git-checkout
+ namespace: swh
+spec:
+ secretTargetRef:
+ - parameter: host # "host" is required by the scalerObject trigger metadata
+ name: common-secrets
+ key: rabbitmq-http-host
+---
+# Source: swh/templates/loaders/keda-authentication.yaml
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
name: amqp-authentication-loader-golang
namespace: swh
spec:
secretTargetRef:
- parameter: host # "host" is required by the scalerObject trigger metadata
name: common-secrets
key: rabbitmq-http-host
---
# Source: swh/templates/loaders/keda-authentication.yaml
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
+ name: amqp-authentication-loader-hg-checkout
+ namespace: swh
+spec:
+ secretTargetRef:
+ - parameter: host # "host" is required by the scalerObject trigger metadata
+ name: common-secrets
+ key: rabbitmq-http-host
+---
+# Source: swh/templates/loaders/keda-authentication.yaml
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
name: amqp-authentication-loader-maven
namespace: swh
spec:
secretTargetRef:
- parameter: host # "host" is required by the scalerObject trigger metadata
name: common-secrets
key: rabbitmq-http-host
---
# Source: swh/templates/loaders/keda-authentication.yaml
apiVersion: keda.sh/v1alpha1
@@ -14664,20 +15573,32 @@
kind: TriggerAuthentication
metadata:
name: amqp-authentication-loader-svn
namespace: swh
spec:
secretTargetRef:
- parameter: host # "host" is required by the scalerObject trigger metadata
name: common-secrets
key: rabbitmq-http-host
---
+# Source: swh/templates/loaders/keda-authentication.yaml
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+ name: amqp-authentication-loader-svn-export
+ namespace: swh
+spec:
+ secretTargetRef:
+ - parameter: host # "host" is required by the scalerObject trigger metadata
+ name: common-secrets
+ key: rabbitmq-http-host
+---
# Source: swh/templates/storage-replayer/keda-secrets.yaml
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: keda-storage-replayer-trigger-authentication
namespace: swh
spec:
secretTargetRef:
- parameter: sasl
name: keda-storage-replayer-kafka-secretsEdited by Antoine R. Dumont