Skip to content

staging: Add new pagure lister

Antoine R. Dumont requested to merge staging-add-pagure-lister into staging

This contains one more commit than production (since staging is 1 commit ahead for now). It's not the subject of the MR.

Refs. swh/infra/sysadm-environment#4958 (closed)

$ make helm-diff

cd swh && ./helm-diff.sh
Comparing changes between branches production and staging-add-pagure-lister...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
Generate config in production branch for values/default.yaml...
Generate config in production branch for values/production-cassandra.yaml...
Generate config in production branch for values/production.yaml...
Generate config in production branch for values/staging-cassandra.yaml...
Generate config in production branch for values/staging.yaml...
Switched to branch 'staging-add-pagure-lister'
Generate config in staging-add-pagure-lister branch for values/default.yaml...
Generate config in staging-add-pagure-lister branch for values/production-cassandra.yaml...
Generate config in staging-add-pagure-lister branch for values/production.yaml...
Generate config in staging-add-pagure-lister branch for values/staging-cassandra.yaml...
Generate config in staging-add-pagure-lister branch for values/staging.yaml...


------------- diff for values/default.yaml -------------

No differences


------------- diff for values/production-cassandra.yaml -------------

No differences


------------- diff for values/production.yaml -------------

--- /tmp/swh-chart.FGB9Si08/production.yaml.before      2023-06-26 15:34:44.457933943 +0200
+++ /tmp/swh-chart.FGB9Si08/production.yaml.after       2023-06-26 15:34:44.829934642 +0200
@@ -1299,21 +1299,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -1460,21 +1460,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -1606,21 +1606,21 @@
             readOnly: true
           - name: configuration
             mountPath: /etc/swh
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
       containers:
       - name: loader-metadata
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata:20230601.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata:20230626.1
         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 350Mi
             cpu: 80m
         lifecycle:
           preStop:
             exec:
@@ -1742,21 +1742,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 1024Mi
             cpu: 500m
           limits:
             memory: 25600Mi
@@ -1912,21 +1912,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 1024Mi
             cpu: 500m
           limits:
             memory: 25600Mi
@@ -2254,21 +2254,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 1024Mi
             cpu: 800m
         lifecycle:
           preStop:
@@ -2590,21 +2590,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230626.1

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 2048Mi
             cpu: 200m
           limits:
             memory: 25600Mi


------------- diff for values/staging-cassandra.yaml -------------

No differences


------------- diff for values/staging.yaml -------------

--- /tmp/swh-chart.FGB9Si08/staging.yaml.before 2023-06-26 15:34:44.653934311 +0200
+++ /tmp/swh-chart.FGB9Si08/staging.yaml.after  2023-06-26 15:34:45.017934995 +0200
@@ -1476,20 +1476,117 @@
       - console



 # Set useJsonLogger to false to let the logs be plain text
 ---
 # Source: swh/templates/listers/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
 metadata:
+  name: lister-pagure-template
+  namespace: swh
+data:
+  config.yml.template: |
+    storage:
+      cls: pipeline
+      steps:
+      - cls: retry
+      - cls: remote
+        url: http://storage1.internal.staging.swh.network:5002
+
+    scheduler:
+      cls: remote
+      url: http://scheduler0.internal.staging.swh.network:5008/
+
+    celery:
+      task_broker: amqp://${AMQP_USERNAME}:${AMQP_PASSWORD}@scheduler0.internal.staging.swh.network:5672/
+      task_acks_late: true
+      task_queues:
+      - swh.lister.pagure.tasks.PagureListerTask
+      sentry_settings_for_celery_tasks:
+        __sentry-settings-for-celery-tasks__
+    credentials:
+      __lister-credentials__
+  init-container-entrypoint.sh: |
+    #!/bin/bash
+
+    set -e
+
+    CONFIG_FILE=/etc/swh/config.yml
+    CONFIG_FILE_WIP=/tmp/wip-config.yml
+
+    # substitute environment variables when creating the default config.yml
+    eval echo \""$(</etc/swh/configuration-template/config.yml.template)"\" \
+      > $CONFIG_FILE
+
+    SENTRY_SETTINGS_PATH=/etc/credentials/sentry-settings/sentry_settings_for_celery_tasks
+    if [ -f $SENTRY_SETTINGS_PATH ]; then
+      awk "/__sentry-settings-for-celery-tasks__/{system(\"sed 's/^/    /g' $SENTRY_SETTINGS_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+      mv $CONFIG_FILE_WIP $CONFIG_FILE
+    else
+      sed -i '/__sentry-settings-for-celery-tasks__//g' $CONFIG_FILE
+    fi
+
+    CREDS_LISTER_PATH=/etc/credentials/listers/credentials
+    if [ -f $CREDS_LISTER_PATH ]; then
+      awk "/__lister-credentials__/{system(\"sed 's/^/  /g' $CREDS_LISTER_PATH\");next}1" $CONFIG_FILE > $CONFIG_FILE_WIP
+      mv $CONFIG_FILE_WIP $CONFIG_FILE
+    else
+      sed -i '/__lister-credentials__//g' $CONFIG_FILE
+    fi
+
+    exit 0
+
+
+  logging-configuration.yml: |
+    version: 1
+
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: json
+        stream: ext://sys.stdout
+
+    formatters:
+      json:
+        class: pythonjsonlogger.jsonlogger.JsonFormatter
+        # python-json-logger parses the format argument to get the variables it actually expands into the json
+        format: "%(asctime)s:%(threadName)s:%(pathname)s:%(lineno)s:%(funcName)s:%(task_name)s:%(task_id)s:%(name)s:%(levelname)s:%(message)s"
+
+    loggers:
+      celery:
+        level: "INFO"
+      amqp:
+        level: WARNING
+      urllib3:
+        level: WARNING
+      azure.core.pipeline.policies.http_logging_policy:
+        level: WARNING
+      swh:
+        level: "INFO"
+      celery.task:
+        level: "INFO"
+
+    root:
+      level: "INFO"
+      handlers:
+      - console
+
+
+
+# Set useJsonLogger to false to let the logs be plain text
+---
+# Source: swh/templates/listers/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
   name: lister-pubdev-template
   namespace: swh
 data:
   config.yml.template: |
     storage:
       cls: pipeline
       steps:
       - cls: retry
       - cls: remote
         url: http://storage1.internal.staging.swh.network:5002
@@ -5575,21 +5672,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-all
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -5629,21 +5726,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 512Mi
             cpu: 500m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -5736,21 +5833,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-bitbucket
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -5790,21 +5887,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 600Mi
             cpu: 200m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -5897,21 +5994,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-bower
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -5951,21 +6048,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -6058,21 +6155,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-gnu-full
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -6112,21 +6209,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -6219,21 +6316,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-gogs-full
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -6273,21 +6370,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -6380,21 +6477,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-golang
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -6434,21 +6531,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -6541,21 +6638,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-launchpad
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -6595,21 +6692,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -6702,21 +6799,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-maven
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -6756,21 +6853,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 6144Mi
             cpu: 1000m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -6863,21 +6960,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-nixguix
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -6917,21 +7014,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -7024,21 +7121,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-opam
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -7078,21 +7175,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -7166,40 +7263,201 @@
       - name: sentry-settings-for-celery-tasks
         secret:
           secretName: sentry-settings-for-celery-tasks
           optional: true
 # Set useJsonLogger to false to let the logs be plain text
 ---
 # Source: swh/templates/listers/deployment.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  name: lister-pagure
+  namespace: swh
+  labels:
+    app: lister-pagure
+spec:
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: lister-pagure
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: lister-pagure
+      annotations:
+        # Force a rollout upgrade if the configuration changes
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
+    spec:
+      affinity:
+
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: swh/lister
+                operator: In
+                values:
+                - "true"
+      initContainers:
+        - name: prepare-configuration
+          image: debian:bullseye
+          imagePullPolicy: Always
+          env:
+          - name: AMQP_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: common-secrets
+                key: rabbitmq-amqp-username
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          - name: AMQP_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: common-secrets
+                key: rabbitmq-amqp-password
+                # 'name' secret must exist & include that ^ key
+                optional: false
+          command:
+            - /entrypoint.sh
+          volumeMounts:
+          - name: configuration-template
+            mountPath: /entrypoint.sh
+            subPath: "init-container-entrypoint.sh"
+            readOnly: true
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+          - name: lister-credentials-secrets
+            mountPath: /etc/credentials/listers
+            readOnly: true
+          - name: sentry-settings-for-celery-tasks
+            mountPath: /etc/credentials/sentry-settings
+            readOnly: true
+      containers:
+      - name: listers
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: 250m
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
+        imagePullPolicy: Always
+        command:
+        - /bin/bash
+        args:
+        - -c
+        - /opt/swh/entrypoint.sh
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/pre-stop.sh"]
+        env:
+        - name: STATSD_HOST
+          value: prometheus-statsd-exporter
+        - name: STATSD_PORT
+          value: "9125"
+        - name: MAX_TASKS_PER_CHILD
+          value: "1"
+        - name: LOGLEVEL
+          value: "INFO"
+        - name: SWH_CONFIG_FILENAME
+          value: /etc/swh/config.yml
+
+        - name: SWH_LOG_CONFIG
+          value: /etc/swh/logging-configuration.yml
+
+        - name: SWH_SENTRY_ENVIRONMENT
+          value: staging
+        volumeMounts:
+          - name: lister-utils
+            mountPath: /pre-stop.sh
+            subPath: "pre-stop.sh"
+          - name: configuration
+            mountPath: /etc/swh
+
+          - name: configuration-template
+            mountPath: /etc/swh/logging-configuration.yml
+            subPath: "logging-configuration.yml"
+            readOnly: true
+
+      volumes:
+      - name: configuration
+        ephemeral:
+          volumeClaimTemplate:
+            metadata:
+              labels:
+                type: ephemeral-volume
+            spec:
+              accessModes:
+              - ReadWriteOnce
+              resources:
+                requests:
+                  storage: 100Gi
+              storageClassName: local-path
+      - name: configuration-template
+        configMap:
+          name: lister-pagure-template
+          defaultMode: 0777
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+          - key: "init-container-entrypoint.sh"
+            path: "init-container-entrypoint.sh"
+
+          - key: "logging-configuration.yml"
+            path: "logging-configuration.yml"
+
+      - name: lister-utils
+        configMap:
+          name: lister-utils
+          defaultMode: 0777
+          items:
+          - key: "pre-stop-idempotent.sh"
+            path: "pre-stop.sh"
+      - name: lister-credentials-secrets
+        secret:
+          secretName: lister-credentials-secrets
+          optional: true
+      - name: sentry-settings-for-celery-tasks
+        secret:
+          secretName: sentry-settings-for-celery-tasks
+          optional: true
+# Set useJsonLogger to false to let the logs be plain text
+---
+# Source: swh/templates/listers/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
   name: lister-pubdev
   namespace: swh
   labels:
     app: lister-pubdev
 spec:
   revisionHistoryLimit: 2
   selector:
     matchLabels:
       app: lister-pubdev
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-pubdev
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -7239,21 +7497,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -7346,21 +7604,21 @@
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1
   template:
     metadata:
       labels:
         app: lister-sourceforge
       annotations:
         # Force a rollout upgrade if the configuration changes
-        checksum/config: 02077bfaae1ff37da6f89d5ab921df4be7105459ea16a04ab878812f626b4d59
+        checksum/config: 6e0988a73a762774a1e7eb5ae678256c07800b05099c3c472780fc2137cc6c1a
     spec:
       affinity:

         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
               - key: swh/lister
                 operator: In
                 values:
@@ -7400,21 +7658,21 @@
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: listers
         resources:
           requests:
             memory: 256Mi
             cpu: 250m
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230608.2
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/lister:20230626.1
         imagePullPolicy: Always
         command:
         - /bin/bash
         args:
         - -c
         - /opt/swh/entrypoint.sh
         lifecycle:
           preStop:
             exec:
               command: ["/pre-stop.sh"]
@@ -7546,21 +7804,21 @@
             readOnly: true
           - name: configuration
             mountPath: /etc/swh
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
       containers:
       - name: loader-metadata
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata:20230601.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_metadata:20230626.1
         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 350Mi
             cpu: 80m
         lifecycle:
           preStop:
             exec:
@@ -7682,21 +7940,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 256Mi
             cpu: 200m
           limits:
             memory: 10240Mi
@@ -7853,21 +8111,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 256Mi
             cpu: 200m
           limits:
             memory: 25600Mi
@@ -9384,21 +9642,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 512Mi
             cpu: 200m
         lifecycle:
           preStop:
@@ -9552,21 +9810,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_git:20230626.2

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 768Mi
             cpu: 1000m
           limits:
             memory: 1024Mi
@@ -11070,21 +11328,21 @@
           - name: configuration-template
             mountPath: /etc/swh/configuration-template
           - name: metadata-fetcher-credentials
             mountPath: /etc/credentials/metadata-fetcher
             readOnly: true
           - name: sentry-settings-for-celery-tasks
             mountPath: /etc/credentials/sentry-settings
             readOnly: true
       containers:
       - name: loaders
-        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230612.1
+        image: container-registry.softwareheritage.org/swh/infra/swh-apps/loader_savecodenow:20230626.1

         imagePullPolicy: Always
         command:
           - /opt/swh/entrypoint.sh
         resources:
           requests:
             memory: 256Mi
             cpu: 200m
           limits:
             memory: 10240Mi
@@ -13633,20 +13891,60 @@
       vhostName: /                   # Optional. If not specified, use the vhost in the
                                      # `host` connection string. Alternatively, you can
                                      # use existing environment variables to read
                                      # configuration from: See details in "Parameter
                                      # list" section hostFromEnv: RABBITMQ_HOST%
 ---
 # Source: swh/templates/listers/keda-autoscaling.yaml
 apiVersion: keda.sh/v1alpha1
 kind: ScaledObject
 metadata:
+  name: lister-pagure-operators
+  namespace: swh
+spec:
+  scaleTargetRef:
+    apiVersion:    apps/v1     # Optional. Default: apps/v1
+    kind:          Deployment  # Optional. Default: Deployment
+    # Mandatory. Must be in same namespace as ScaledObject
+    name:          lister-pagure
+    # envSourceContainerName: {container-name} # Optional. Default:
+                                               # .spec.template.spec.containers[0]
+  pollingInterval:  30                         # Optional. Default: 30 seconds
+  cooldownPeriod:   3600                       # Optional. Default: 300 seconds
+  idleReplicaCount: 0                          # Optional. Must be less than
+                                               # minReplicaCount
+  minReplicaCount:  0
+  maxReplicaCount:  1
+  triggers:
+  - type: rabbitmq
+    authenticationRef:
+      name: amqp-authentication-lister-pagure
+    metadata:
+      protocol: auto                 # Optional. Specifies protocol to use,
+                                     # either amqp or http, or auto to
+                                     # autodetect based on the `host` value.
+                                     # Default value is auto.
+      mode: QueueLength              # QueueLength to trigger on number of msgs in queue
+      excludeUnacknowledged: "false" # QueueLength should include unacked messages
+                                     # Implies "http" protocol is used
+      value: "1"
+      queueName: swh.lister.pagure.tasks.PagureListerTask
+      vhostName: /                   # Optional. If not specified, use the vhost in the
+                                     # `host` connection string. Alternatively, you can
+                                     # use existing environment variables to read
+                                     # configuration from: See details in "Parameter
+                                     # list" section hostFromEnv: RABBITMQ_HOST%
+---
+# Source: swh/templates/listers/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
   name: lister-pubdev-operators
   namespace: swh
 spec:
   scaleTargetRef:
     apiVersion:    apps/v1     # Optional. Default: apps/v1
     kind:          Deployment  # Optional. Default: Deployment
     # Mandatory. Must be in same namespace as ScaledObject
     name:          lister-pubdev
     # envSourceContainerName: {container-name} # Optional. Default:
                                                # .spec.template.spec.containers[0]
@@ -15269,20 +15567,32 @@
   namespace: swh
 spec:
   secretTargetRef:
   - parameter: host            # "host" is required by the scalerObject trigger metadata
     name: common-secrets
     key: rabbitmq-http-host
 ---
 # Source: swh/templates/listers/keda-autoscaling.yaml
 apiVersion: keda.sh/v1alpha1
 kind: TriggerAuthentication
+metadata:
+  name: amqp-authentication-lister-pagure
+  namespace: swh
+spec:
+  secretTargetRef:
+  - parameter: host            # "host" is required by the scalerObject trigger metadata
+    name: common-secrets
+    key: rabbitmq-http-host
+---
+# Source: swh/templates/listers/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
 metadata:
   name: amqp-authentication-lister-pubdev
   namespace: swh
 spec:
   secretTargetRef:
   - parameter: host            # "host" is required by the scalerObject trigger metadata
     name: common-secrets
     key: rabbitmq-http-host
 ---
 # Source: swh/templates/listers/keda-autoscaling.yaml 
Edited by Antoine R. Dumont

Merge request reports

Loading