swh/staging: Ingest bitbucket code.funtoo.org origins
Related to swh/infra/add-forge-now-requests#477 (closed)
These modifications will create a job to test the bulk ingestion a list of code.funtoo.org origins in staging environment.
The test list contains only 5 origins.
Helm-diff
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.Uz2uGgUV/staging-swh-cassandra.before 2024-08-07 15:25:04.570024883 +0200
+++ /tmp/swh-chart.swh.Uz2uGgUV/staging-swh-cassandra.after 2024-08-07 15:25:05.454040690 +0200
@@ -20841,20 +20841,110 @@
minReplicas: 2
maxReplicas: 4
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 75
---
+# Source: swh/templates/toolbox/bulk-load-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ namespace: swh-cassandra
+ name: toolbox-bulk-load-oneshot-jobName
+ labels:
+ app: toolbox-bulk-load-oneshot-jobName
+spec:
+ backoffLimit: 0
+ template:
+ metadata:
+ labels:
+ app: toolbox-bulk-load-oneshot-jobName
+ spec:
+ restartPolicy: Never
+ initContainers:
+ - name: prepare-configuration-scheduler
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+ - name: AMQP_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: swhproducer-password
+ name: amqp-secrets
+ optional: false
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ key: postgres-swh-scheduler-password
+ name: swh-scheduler-postgresql-common-secret
+ optional: false
+ volumeMounts:
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: bulk-load
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox:20240718.1
+ imagePullPolicy: IfNotPresent
+ resources:
+ requests:
+ memory: 128Mi
+ cpu: 100m
+ env:
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: ORIGINS
+ value: jobName.lst
+ - name: ORIGINS_URL
+ value: https://gitlab.softwareheritage.org/-/project/474/uploads/5eea463ee190e585245a74d6f6eaf920/staging_code_funtoo_org_repos_sorted.txt
+ - name: TASK_TYPE
+ value: load-git
+ - name: QUEUE_PREFIX
+ value: oneshot
+ - name: MAX_TASKS
+ value: "10000"
+ command:
+ - /opt/swh/bin/bulk_load.sh
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: toolbox-script-utils
+ mountPath: /opt/swh/bin
+ readOnly: true
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: toolbox-scheduler-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: toolbox-script-utils
+ configMap:
+ name: toolbox-script-utils
+ defaultMode: 0555
+---
# Source: swh/templates/counters/refresh-counters-cache-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
name: counters-refresh-counters-cache-cronjob
spec:
# By default, every 4h
schedule: "0 */1 * * *"
concurrencyPolicy: Forbid
jobTemplate:
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differences