bulk-load 2/2, previous mr feedbacks + preparation for prod execution

Related to swh/infra/sysadm-environment#5365 (closed)

Following the first MR, these are the adaptations and the prepartion to the execution in production

helm dyff

[swh] Comparing changes between branches production and bulk-load (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in bulk-load branch for environment staging...
[swh] Generate config in bulk-load branch for environment staging...
[swh] Generate config in bulk-load branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in bulk-load branch for environment production...
[swh] Generate config in bulk-load branch for environment production...
[swh] Generate config in bulk-load branch for environment production...


------------- diff for environment staging namespace swh -------------

     _        __  __
   _| |_   _ / _|/ _|  between /tmp/swh-chart.swh.HBV8tQpI/staging-swh.before, 135 documents
 / _' | | | | |_| |_       and /tmp/swh-chart.swh.HBV8tQpI/staging-swh.after, 135 documents
| (_| | |_| |  _|  _|
 \__,_|\__, |_| |_|   returned two differences
        |___/

data.bulk_load.sh  (v1/ConfigMap/swh/toolbox-script-utils)
  ± value change in multiline text (one insert, one deletion)
    -   --queue-name-prefix oneshot
    +   --queue-name-prefix $QUEUE_PREFIX


spec.template.metadata.annotations.checksum/configScript  (apps/v1/Deployment/swh/swh-toolbox)
  ± value change
    - 44c7ed42829afd247b54b4c28250a0d69ca2135bb706edd623fd73381a57c68f
    + 0a5424ad3f3b42e8f8a4fe646862ae5a65a30cf406dfe8269c5c3ce2def428b0



------------- diff for environment staging namespace swh-cassandra -------------

     _        __  __
   _| |_   _ / _|/ _|  between /tmp/swh-chart.swh.HBV8tQpI/staging-swh-cassandra.before, 424 documents
 / _' | | | | |_| |_       and /tmp/swh-chart.swh.HBV8tQpI/staging-swh-cassandra.after, 423 documents
| (_| | |_| |  _|  _|
 \__,_|\__, |_| |_|   returned three differences
        |___/

(file level)
  - one document removed:
    ---
    # Source: swh/templates/toolbox/bulk-load-job.yaml
    apiVersion: batch/v1
    kind: Job
    metadata:
      name: toolbox-bulk-load-mbed
      namespace: swh-cassandra
      labels:
        app: toolbox-bulk-load-mbed
    spec:
      backoffLimit: 0
      template:
        metadata:
          labels:
            app: toolbox-bulk-load-mbed
        spec:
          restartPolicy: Never
          initContainers:
          - name: prepare-configuration-scheduler
            image: "debian:bullseye"
            imagePullPolicy: IfNotPresent
            command:
            - /bin/bash
            args:
            - "-c"
            - "eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scheduler.yml"
            volumeMounts:
            - name: configuration
              mountPath: /etc/swh
            - name: configuration-template
              mountPath: /etc/swh/configuration-template
            env:
            - name: AMQP_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: amqp-secrets
                  key: swhproducer-password
                  optional: false
            - name: POSTGRESQL_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: swh-scheduler-postgresql-common-secret
                  key: postgres-swh-scheduler-password
                  optional: false
          containers:
          - name: bulk-load
            image: "container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox:20240702.2"
            imagePullPolicy: IfNotPresent
            command:
            - /opt/swh/bin/bulk_load.sh
            resources:
              requests:
                memory: 128Mi
                cpu: 100m
            env:
            - name: SWH_CONFIG_FILENAME
              value: /etc/swh/config-scheduler.yml
            - name: ORIGINS
              value: mbed.lst
            - name: ORIGINS_URL
              value: "https://gitlab.softwareheritage.org/-/project/80/uploads/44f84f66479d3365ae5e8a40f1ffb709/mbed-test.lst"
            - name: TASK_TYPE
              value: load-hg
            - name: MAX_TASKS
              value: 10
            volumeMounts:
            - name: configuration
              mountPath: /etc/swh
            - name: toolbox-script-utils
              mountPath: /opt/swh/bin
              readOnly: true
          volumes:
          - name: configuration
            emptyDir: {}
          - name: configuration-template
            configMap:
              name: toolbox-scheduler-template
              items:
              - key: config.yml.template
                path: config.yml.template
          - name: config-utils
            configMap:
              name: config-utils
              defaultMode: 0555
          - name: toolbox-script-utils
            configMap:
              name: toolbox-script-utils
              defaultMode: 0555
    
  

data.bulk_load.sh  (v1/ConfigMap/swh-cassandra/toolbox-script-utils)
  ± value change in multiline text (one insert, one deletion)
    -   --queue-name-prefix oneshot
    +   --queue-name-prefix $QUEUE_PREFIX


spec.template.metadata.annotations.checksum/configScript  (apps/v1/Deployment/swh-cassandra/swh-toolbox)
  ± value change
    - 15bd1c0b22ea86d5b8f721d0fd62e377c0df0ab66b4b5d6bd25140809a9caae8
    + c23b77094af994ace25c94bfe58070847ad8d53a772e0c96676f4a2074057152



------------- diff for environment staging namespace swh-cassandra-next-version -------------

     _        __  __
   _| |_   _ / _|/ _|  between /tmp/swh-chart.swh.HBV8tQpI/staging-swh-cassandra-next-version.before, 307 documents
 / _' | | | | |_| |_       and /tmp/swh-chart.swh.HBV8tQpI/staging-swh-cassandra-next-version.after, 307 documents
| (_| | |_| |  _|  _|
 \__,_|\__, |_| |_|   returned two differences
        |___/

data.bulk_load.sh  (v1/ConfigMap/swh-cassandra-next-version/toolbox-script-utils)
  ± value change in multiline text (one insert, one deletion)
    -   --queue-name-prefix oneshot
    +   --queue-name-prefix $QUEUE_PREFIX


spec.template.metadata.annotations.checksum/configScript  (apps/v1/Deployment/swh-cassandra-next-version/swh-toolbox)
  ± value change
    - f9e8ef30d568019ef187714d32a53f05912208d6ce078c4407a168428dc1a3de
    + a9ce0b6a8d99e84de5e67f6827f162fe07888a78f9afd808667fcff4e5dc6805



------------- diff for environment production namespace swh -------------

     _        __  __
   _| |_   _ / _|/ _|  between /tmp/swh-chart.swh.HBV8tQpI/production-swh.before, 427 documents
 / _' | | | | |_| |_       and /tmp/swh-chart.swh.HBV8tQpI/production-swh.after, 428 documents
| (_| | |_| |  _|  _|
 \__,_|\__, |_| |_|   returned three differences
        |___/

(file level)
    ---
    # Source: swh/templates/toolbox/bulk-load-job.yaml
    apiVersion: batch/v1
    kind: Job
    metadata:
      name: toolbox-bulk-load-oneshot-mbed
      namespace: swh
      labels:
        app: toolbox-bulk-load-oneshot-mbed
    spec:
      backoffLimit: 0
      template:
        metadata:
          labels:
            app: toolbox-bulk-load-oneshot-mbed
        spec:
          restartPolicy: Never
          initContainers:
          - name: prepare-configuration-scheduler
            image: "container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1"
            imagePullPolicy: IfNotPresent
            command:
            - /entrypoints/prepare-configuration.sh
            volumeMounts:
            - name: config-utils
              mountPath: /entrypoints
              readOnly: true
            - name: configuration
              mountPath: /etc/swh
            - name: configuration-template
              mountPath: /etc/swh/configuration-template
            env:
            - name: AMQP_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: amqp-secrets
                  key: swhproducer-password
                  optional: false
            - name: POSTGRESQL_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: swh-scheduler-postgresql-common-secret
                  key: postgres-swh-scheduler-password
                  optional: false
          containers:
          - name: bulk-load
            image: "container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox:20240702.2"
            imagePullPolicy: IfNotPresent
            command:
            - /opt/swh/bin/bulk_load.sh
            resources:
              requests:
                memory: 128Mi
                cpu: 100m
            env:
            - name: SWH_CONFIG_FILENAME
              value: /etc/swh/config.yml
            - name: ORIGINS
              value: mbed.lst
            - name: ORIGINS_URL
              value: "https://gitlab.softwareheritage.org/-/project/80/uploads/b93c918fb0b11fd397db8f10b4eb65e9/mbed_hg_repos_sorted.txt"
            - name: TASK_TYPE
              value: load-hg
            - name: QUEUE_PREFIX
              value: oneshot
            - name: MAX_TASKS
              value: 1000
            volumeMounts:
            - name: configuration
              mountPath: /etc/swh
            - name: toolbox-script-utils
              mountPath: /opt/swh/bin
              readOnly: true
          volumes:
          - name: configuration
            emptyDir: {}
          - name: configuration-template
            configMap:
              name: toolbox-scheduler-template
              items:
              - key: config.yml.template
                path: config.yml.template
          - name: config-utils
            configMap:
              name: config-utils
              defaultMode: 0555
          - name: toolbox-script-utils
            configMap:
              name: toolbox-script-utils
              defaultMode: 0555
    
  

data.bulk_load.sh  (v1/ConfigMap/swh/toolbox-script-utils)
  ± value change in multiline text (one insert, one deletion)
    -   --queue-name-prefix oneshot
    +   --queue-name-prefix $QUEUE_PREFIX


spec.template.metadata.annotations.checksum/configScript  (apps/v1/Deployment/swh/swh-toolbox)
  ± value change
    - 44b0188ef62d0f4b80b18a8f7e6b8a398da2e46616bab560e6a15947bb4cc88d
    + a1153f1700e14edc342602d7a1277b8da9745aef424d0eb56e97088d8648e9fe



------------- diff for environment production namespace swh-cassandra -------------

     _        __  __
   _| |_   _ / _|/ _|  between /tmp/swh-chart.swh.HBV8tQpI/production-swh-cassandra.before, 122 documents
 / _' | | | | |_| |_       and /tmp/swh-chart.swh.HBV8tQpI/production-swh-cassandra.after, 122 documents
| (_| | |_| |  _|  _|
 \__,_|\__, |_| |_|   returned two differences
        |___/

data.bulk_load.sh  (v1/ConfigMap/swh-cassandra/toolbox-script-utils)
  ± value change in multiline text (one insert, one deletion)
    -   --queue-name-prefix oneshot
    +   --queue-name-prefix $QUEUE_PREFIX


spec.template.metadata.annotations.checksum/configScript  (apps/v1/Deployment/swh-cassandra/swh-toolbox)
  ± value change
    - 875b259528e4464783a6ffa091a58e2a139f54911112ab66d20a82a61264884e
    + b34ce8c45d90bf995c19559df125c362009de9318ee7e56ca474f1cdd7a4151b

swh/templates/toolbox/bulk-load-job.yaml

@@ -2,7 +2,8 @@
 {{ $schedulerConfigRef := required "toolbox.bulkLoad.schedulerConfigurationRef is mandatory" .Values.toolbox.bulkLoad.schedulerConfigurationRef }}
 {{ $schedulerConfig := required (print $schedulerConfigRef "not found in toolbox configs") (get .Values.toolbox.configs $schedulerConfigRef) }}
 {{- range $job, $job_config := .Values.toolbox.bulkLoad.jobs -}}
-{{- $name := ( print "bulk-load-" $job ) }}
+{{- $queuePrefix := required (print "queuePrefix is mandatory for bulk load job " $job) $job_config.queuePrefix -}}
+{{- $name := ( print "bulk-load-" $queuePrefix "-" $job ) }}
 ---
 apiVersion: batch/v1
 kind: Job
@@ -28,17 +29,17 @@ spec:
       {{ end }}
       initContainers:
         - name: prepare-configuration-scheduler
-          image: debian:bullseye
+          image: {{ $.Values.swh_utils_image }}:{{ $job_config.swhUtilsImageVersion | default $.Values.swh_utils_image_version }}
           imagePullPolicy: IfNotPresent
           command:
-          - /bin/bash
-          args:
-          - -c
-          - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scheduler.yml
+          - /entrypoints/prepare-configuration.sh
           env: {{ include "swh.secrets.envFromDeploymentConfig"
                   (dict "deploymentConfig" $schedulerConfig
                         "Values" $.Values) | nindent 10 }}
           volumeMounts:
+          - name: config-utils
+            mountPath: /entrypoints
+            readOnly: true
           - name: configuration
             mountPath: /etc/swh
           - name: configuration-template
@@ -62,13 +63,15 @@ spec:
             {{- end }}
           env:
           - name: SWH_CONFIG_FILENAME
-            value: /etc/swh/config-scheduler.yml
+            value: /etc/swh/config.yml
           - name: ORIGINS
             value: {{ $job }}.lst
           - name: ORIGINS_URL
             value: {{ required (print "bulkLoad origins list url (originListUrl) is mandatory for " $job) $job_config.originListUrl }}
           - name: TASK_TYPE
             value: {{ required (print "bulkLoad taskType is mandatory for " $job) $job_config.taskType }}
+          - name: QUEUE_PREFIX
+            value: {{ $queuePrefix }}
           - name: MAX_TASKS
             value: {{ $job_config.maxTasks | default 1000 | quote }}
           command:
@@ -96,8 +99,6 @@ spec:
         configMap:
           name: toolbox-script-utils
           defaultMode: 0555
-
-
 {{ end }}
 {{- end }}