swh/staging: Ingest bitbucket code.funtoo.org origins

Related to swh/infra/add-forge-now-requests#477 (closed)

These modifications will create a job to test the bulk ingestion a list of code.funtoo.org origins in staging environment.
The test list contains only 5 origins.

Helm-diff

------------- diff for environment staging namespace swh -------------

No differences


------------- diff for environment staging namespace swh-cassandra -------------

--- /tmp/swh-chart.swh.Uz2uGgUV/staging-swh-cassandra.before	2024-08-07 15:25:04.570024883 +0200
+++ /tmp/swh-chart.swh.Uz2uGgUV/staging-swh-cassandra.after	2024-08-07 15:25:05.454040690 +0200
@@ -20841,20 +20841,110 @@
   minReplicas: 2
   maxReplicas: 4
   metrics:
   - type: Resource
     resource:
       name: cpu
       target:
         type: Utilization
         averageUtilization: 75
 ---
+# Source: swh/templates/toolbox/bulk-load-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  namespace: swh-cassandra
+  name: toolbox-bulk-load-oneshot-jobName
+  labels:
+    app: toolbox-bulk-load-oneshot-jobName
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: toolbox-bulk-load-oneshot-jobName
+    spec:
+      restartPolicy: Never
+      initContainers:
+        - name: prepare-configuration-scheduler
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+          imagePullPolicy: IfNotPresent
+          command:
+          - /entrypoints/prepare-configuration.sh
+          env: 
+          - name: AMQP_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                key: swhproducer-password
+                name: amqp-secrets
+                optional: false
+          - name: POSTGRESQL_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                key: postgres-swh-scheduler-password
+                name: swh-scheduler-postgresql-common-secret
+                optional: false
+          volumeMounts:
+          - name: config-utils
+            mountPath: /entrypoints
+            readOnly: true
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+      containers:
+        - name: bulk-load
+          image: container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox:20240718.1
+          imagePullPolicy: IfNotPresent
+          resources:
+            requests:
+              memory: 128Mi
+              cpu: 100m
+          env:
+          - name: SWH_CONFIG_FILENAME
+            value: /etc/swh/config.yml
+          - name: ORIGINS
+            value: jobName.lst
+          - name: ORIGINS_URL
+            value: https://gitlab.softwareheritage.org/-/project/474/uploads/5eea463ee190e585245a74d6f6eaf920/staging_code_funtoo_org_repos_sorted.txt
+          - name: TASK_TYPE
+            value: load-git
+          - name: QUEUE_PREFIX
+            value: oneshot
+          - name: MAX_TASKS
+            value: "10000"
+          command:
+          - /opt/swh/bin/bulk_load.sh
+          volumeMounts:
+            - name: configuration
+              mountPath: /etc/swh
+            - name: toolbox-script-utils
+              mountPath: /opt/swh/bin
+              readOnly: true
+      volumes:
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: toolbox-scheduler-template
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+      - name: config-utils
+        configMap:
+          name: config-utils
+          defaultMode: 0555
+      - name: toolbox-script-utils
+        configMap:
+          name: toolbox-script-utils
+          defaultMode: 0555
+---
 # Source: swh/templates/counters/refresh-counters-cache-cronjob.yaml
 apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: counters-refresh-counters-cache-cronjob
 spec:
   # By default, every 4h
   schedule: "0 */1 * * *"
   concurrencyPolicy: Forbid
   jobTemplate:


------------- diff for environment staging namespace swh-cassandra-next-version -------------

No differences


------------- diff for environment production namespace swh -------------

No differences


------------- diff for environment production namespace swh-cassandra -------------

No differences

swh/values/staging/swh-cassandra.yaml

@@ -1003,6 +1003,14 @@ toolbox:
       webDbConfigurationRef: postgresqlWebConfiguration
     webhooks:
       webhooksConfigurationRef: svixConfiguration
+  bulkLoad:
+    schedulerConfigurationRef: scheduler
+    jobs:
+      jobName:
+        originListUrl: https://gitlab.softwareheritage.org/-/project/474/uploads/5eea463ee190e585245a74d6f6eaf920/staging_code_funtoo_org_repos_sorted.txt
+        taskType: load-git
+        maxTasks: 5
+        queuePrefix: oneshot
 
 scrubber:
   enabled: true