staging: Add new pagure lister
This contains one more commit than production (since staging is 1 commit ahead for now). It's not the subject of the MR.
Refs. swh/infra/sysadm-environment#4958 (closed)
$ make helm-diff
cd swh && ./helm-diff.sh
Comparing changes between branches production and staging-add-pagure-lister...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
Generate config in production branch for values/default.yaml...
Generate config in production branch for values/production-cassandra.yaml...
Generate config in production branch for values/production.yaml...
Generate config in production branch for values/staging-cassandra.yaml...
Generate config in production branch for values/staging.yaml...
Switched to branch 'staging-add-pagure-lister'
Generate config in staging-add-pagure-lister branch for values/default.yaml...
Generate config in staging-add-pagure-lister branch for values/production-cassandra.yaml...
Generate config in staging-add-pagure-lister branch for values/production.yaml...
Generate config in staging-add-pagure-lister branch for values/staging-cassandra.yaml...
Generate config in staging-add-pagure-lister branch for values/staging.yaml...
------------- diff for values/default.yaml -------------
No differences
------------- diff for values/production-cassandra.yaml -------------
No differences
------------- diff for values/production.yaml -------------
--- /tmp/swh-chart.FGB9Si08/production.yaml.before 2023-06-26 15:34:44.457933943 +0200
+++ /tmp/swh-chart.FGB9Si08/production.yaml.after 2023-06-26 15:34:44.829934642 +0200
@@ -1299,21 +1299,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -1460,21 +1460,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -1606,21 +1606,21 @@
readOnly: true
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
containers:
- name: loader-metadata
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata:20230601.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata:20230626.1
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 350Mi
cpu: 80m
lifecycle:
preStop:
exec:
@@ -1742,21 +1742,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 1024Mi
cpu: 500m
limits:
memory: 25600Mi
@@ -1912,21 +1912,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 1024Mi
cpu: 500m
limits:
memory: 25600Mi
@@ -2254,21 +2254,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 1024Mi
cpu: 800m
lifecycle:
preStop:
@@ -2590,21 +2590,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230626.1
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 2048Mi
cpu: 200m
limits:
memory: 25600Mi
------------- diff for values/staging-cassandra.yaml -------------
No differences
------------- diff for values/staging.yaml -------------
--- /tmp/swh-chart.FGB9Si08/staging.yaml.before 2023-06-26 15:34:44.653934311 +0200
+++ /tmp/swh-chart.FGB9Si08/staging.yaml.after 2023-06-26 15:34:45.017934995 +0200
@@ -1476,20 +1476,117 @@
- console
# Set useJsonLogger to false to let the logs be plain text
---
# Source: swh/templates/listers/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: lister-pagure-template
+ namespace: swh
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: retry
+ - cls: remote
+ url: http://storage1.internal.staging.swh.network:5002
+
+ scheduler:
+ cls: remote
+ url: http://scheduler0.internal.staging.swh.network:5008/
+
+ celery:
+ task_broker: amqp://${AMQP_USERNAME}:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/
+ task_acks_late: true
+ task_queues:
+ - swh.lister.pagure.tasks.PagureListerTask
+ sentry_settings_for_celery_tasks:
+ __sentry-settings-for-celery-tasks__
+ credentials:
+ __lister-credentials__
+ init-container-entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ CONFIG_FILE=/etc/swh/config.yml
+ CONFIG_FILE_WIP=/tmp/wip-config.yml
+
+ # substitute environment variables when creating the default config.yml
+ eval echo \""$(</etc/swh/configuration-template/config.yml.template)"\" \
+ > $CONFIG_FILE
+
+ SENTRY_SETTINGS_PATH=/etc/credentials/sentry-settings/sentry_settings_for_celery_tasks
+ if [ -f $SENTRY_SETTINGS_PATH ]; then
+ awk "/__sentry-settings-for-celery-tasks__/{system(\"sed 's/^/ /g' $SENTRY_SETTINGS_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+ mv $CONFIG_FILE_WIP $CONFIG_FILE
+ else
+ sed -i '/__sentry-settings-for-celery-tasks__//g' $CONFIG_FILE
+ fi
+
+ CREDS_LISTER_PATH=/etc/credentials/listers/credentials
+ if [ -f $CREDS_LISTER_PATH ]; then
+ awk "/__lister-credentials__/{system(\"sed 's/^/ /g' $CREDS_LISTER_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+ mv $CONFIG_FILE_WIP $CONFIG_FILE
+ else
+ sed -i '/__lister-credentials__//g' $CONFIG_FILE
+ fi
+
+ exit 0
+
+
+ logging-configuration.yml: |
+ version: 1
+
+ handlers:
+ console:
+ class: logging.StreamHandler
+ formatter: json
+ stream: ext://sys.stdout
+
+ formatters:
+ json:
+ class: pythonjsonlogger.jsonlogger.JsonFormatter
+ # python-json-logger parses the format argument to get the variables it actually expands into the json
+ format: "%(asctime)s:%(threadName)s:%(pathname)s:%(lineno)s:%(funcName)s:%(task_name)s:%(task_id)s:%(name)s:%(levelname)s:%(message)s"
+
+ loggers:
+ celery:
+ level: "INFO"
+ amqp:
+ level: WARNING
+ urllib3:
+ level: WARNING
+ azure.core.pipeline.policies.http_logging_policy:
+ level: WARNING
+ swh:
+ level: "INFO"
+ celery.task:
+ level: "INFO"
+
+ root:
+ level: "INFO"
+ handlers:
+ - console
+
+
+
+# Set useJsonLogger to false to let the logs be plain text
+---
+# Source: swh/templates/listers/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: lister-pubdev-template
namespace: swh
data:
config.yml.template: |
storage:
cls: pipeline
steps:
- cls: retry
- cls: remote
url: http://storage1.internal.staging.swh.network:5002
@@ -5575,21 +5672,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-all
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -5629,21 +5726,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 512Mi
cpu: 500m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -5736,21 +5833,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-bitbucket
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -5790,21 +5887,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 600Mi
cpu: 200m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -5897,21 +5994,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-bower
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -5951,21 +6048,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -6058,21 +6155,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-gnu-full
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -6112,21 +6209,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -6219,21 +6316,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-gogs-full
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -6273,21 +6370,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -6380,21 +6477,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-golang
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -6434,21 +6531,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -6541,21 +6638,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-launchpad
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -6595,21 +6692,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -6702,21 +6799,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-maven
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -6756,21 +6853,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 6144Mi
cpu: 1000m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -6863,21 +6960,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-nixguix
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -6917,21 +7014,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -7024,21 +7121,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-opam
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -7078,21 +7175,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -7166,40 +7263,201 @@
- name: sentry-settings-for-celery-tasks
secret:
secretName: sentry-settings-for-celery-tasks
optional: true
# Set useJsonLogger to false to let the logs be plain text
---
# Source: swh/templates/listers/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
+ name: lister-pagure
+ namespace: swh
+ labels:
+ app: lister-pagure
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: lister-pagure
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: lister-pagure
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/lister
+ operator: In
+ values:
+ - "true"
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: Always
+ env:
+ - name: AMQP_USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: rabbitmq-amqp-username
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: AMQP_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: rabbitmq-amqp-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /entrypoint.sh
+ volumeMounts:
+ - name: configuration-template
+ mountPath: /entrypoint.sh
+ subPath: "init-container-entrypoint.sh"
+ readOnly: true
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: lister-credentials-secrets
+ mountPath: /etc/credentials/listers
+ readOnly: true
+ - name: sentry-settings-for-celery-tasks
+ mountPath: /etc/credentials/sentry-settings
+ readOnly: true
+ containers:
+ - name: listers
+ resources:
+ requests:
+ memory: 256Mi
+ cpu: 250m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
+ imagePullPolicy: Always
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ lifecycle:
+ preStop:
+ exec:
+ command: ["/pre-stop.sh"]
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+
+ - name: SWH_LOG_CONFIG
+ value: /etc/swh/logging-configuration.yml
+
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ volumeMounts:
+ - name: lister-utils
+ mountPath: /pre-stop.sh
+ subPath: "pre-stop.sh"
+ - name: configuration
+ mountPath: /etc/swh
+
+ - name: configuration-template
+ mountPath: /etc/swh/logging-configuration.yml
+ subPath: "logging-configuration.yml"
+ readOnly: true
+
+ volumes:
+ - name: configuration
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ labels:
+ type: ephemeral-volume
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 100Gi
+ storageClassName: local-path
+ - name: configuration-template
+ configMap:
+ name: lister-pagure-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - key: "init-container-entrypoint.sh"
+ path: "init-container-entrypoint.sh"
+
+ - key: "logging-configuration.yml"
+ path: "logging-configuration.yml"
+
+ - name: lister-utils
+ configMap:
+ name: lister-utils
+ defaultMode: 0777
+ items:
+ - key: "pre-stop-idempotent.sh"
+ path: "pre-stop.sh"
+ - name: lister-credentials-secrets
+ secret:
+ secretName: lister-credentials-secrets
+ optional: true
+ - name: sentry-settings-for-celery-tasks
+ secret:
+ secretName: sentry-settings-for-celery-tasks
+ optional: true
+# Set useJsonLogger to false to let the logs be plain text
+---
+# Source: swh/templates/listers/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
name: lister-pubdev
namespace: swh
labels:
app: lister-pubdev
spec:
revisionHistoryLimit: 2
selector:
matchLabels:
app: lister-pubdev
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-pubdev
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -7239,21 +7497,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -7346,21 +7604,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: lister-sourceforge
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+ checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/lister
operator: In
values:
@@ -7400,21 +7658,21 @@
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: listers
resources:
requests:
memory: 256Mi
cpu: 250m
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
imagePullPolicy: Always
command:
- /bin/bash
args:
- -c
- /opt/swh/entrypoint.sh
lifecycle:
preStop:
exec:
command: ["/pre-stop.sh"]
@@ -7546,21 +7804,21 @@
readOnly: true
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
containers:
- name: loader-metadata
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata:20230601.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata:20230626.1
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 350Mi
cpu: 80m
lifecycle:
preStop:
exec:
@@ -7682,21 +7940,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 256Mi
cpu: 200m
limits:
memory: 10240Mi
@@ -7853,21 +8111,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 256Mi
cpu: 200m
limits:
memory: 25600Mi
@@ -9384,21 +9642,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 512Mi
cpu: 200m
lifecycle:
preStop:
@@ -9552,21 +9810,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 768Mi
cpu: 1000m
limits:
memory: 1024Mi
@@ -11070,21 +11328,21 @@
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: metadata-fetcher-credentials
mountPath: /etc/credentials/metadata-fetcher
readOnly: true
- name: sentry-settings-for-celery-tasks
mountPath: /etc/credentials/sentry-settings
readOnly: true
containers:
- name: loaders
- image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230612.1
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230626.1
imagePullPolicy: Always
command:
- /opt/swh/entrypoint.sh
resources:
requests:
memory: 256Mi
cpu: 200m
limits:
memory: 10240Mi
@@ -13633,20 +13891,60 @@
vhostName: / # Optional. If not specified, use the vhost in the
# `host` connection string. Alternatively, you can
# use existing environment variables to read
# configuration from: See details in "Parameter
# list" section hostFromEnv: RABBITMQ_HOST%
---
# Source: swh/templates/listers/keda-autoscaling.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
+ name: lister-pagure-operators
+ namespace: swh
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1 # Optional. Default: apps/v1
+ kind: Deployment # Optional. Default: Deployment
+ # Mandatory. Must be in same namespace as ScaledObject
+ name: lister-pagure
+ # envSourceContainerName: {container-name} # Optional. Default:
+ # .spec.template.spec.containers[0]
+ pollingInterval: 30 # Optional. Default: 30 seconds
+ cooldownPeriod: 3600 # Optional. Default: 300 seconds
+ idleReplicaCount: 0 # Optional. Must be less than
+ # minReplicaCount
+ minReplicaCount: 0
+ maxReplicaCount: 1
+ triggers:
+ - type: rabbitmq
+ authenticationRef:
+ name: amqp-authentication-lister-pagure
+ metadata:
+ protocol: auto # Optional. Specifies protocol to use,
+ # either amqp or http, or auto to
+ # autodetect based on the `host` value.
+ # Default value is auto.
+ mode: QueueLength # QueueLength to trigger on number of msgs in queue
+ excludeUnacknowledged: "false" # QueueLength should include unacked messages
+ # Implies "http" protocol is used
+ value: "1"
+ queueName: swh.lister.pagure.tasks.PagureListerTask
+ vhostName: / # Optional. If not specified, use the vhost in the
+ # `host` connection string. Alternatively, you can
+ # use existing environment variables to read
+ # configuration from: See details in "Parameter
+ # list" section hostFromEnv: RABBITMQ_HOST%
+---
+# Source: swh/templates/listers/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
name: lister-pubdev-operators
namespace: swh
spec:
scaleTargetRef:
apiVersion: apps/v1 # Optional. Default: apps/v1
kind: Deployment # Optional. Default: Deployment
# Mandatory. Must be in same namespace as ScaledObject
name: lister-pubdev
# envSourceContainerName: {container-name} # Optional. Default:
# .spec.template.spec.containers[0]
@@ -15269,20 +15567,32 @@
namespace: swh
spec:
secretTargetRef:
- parameter: host # "host" is required by the scalerObject trigger metadata
name: common-secrets
key: rabbitmq-http-host
---
# Source: swh/templates/listers/keda-autoscaling.yaml
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
+metadata:
+ name: amqp-authentication-lister-pagure
+ namespace: swh
+spec:
+ secretTargetRef:
+ - parameter: host # "host" is required by the scalerObject trigger metadata
+ name: common-secrets
+ key: rabbitmq-http-host
+---
+# Source: swh/templates/listers/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
metadata:
name: amqp-authentication-lister-pubdev
namespace: swh
spec:
secretTargetRef:
- parameter: host # "host" is required by the scalerObject trigger metadata
name: common-secrets
key: rabbitmq-http-host
---
# Source: swh/templates/listers/keda-autoscaling.yaml