Activate the storage scrubbers in production
Activate the scrubbers for:
- postgresql primary
- postgresql secondary
- cassandra
Start more replicas for cassandra because there is no real load yet
Related to swh/infra/sysadm-environment#5152 (closed)
helm diff
[swh] Comparing changes between branches production and storage-scrubber-production (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in storage-scrubber-production branch for environment staging...
[swh] Generate config in storage-scrubber-production branch for environment staging...
[swh] Generate config in storage-scrubber-production branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in storage-scrubber-production branch for environment production...
[swh] Generate config in storage-scrubber-production branch for environment production...
[swh] Generate config in storage-scrubber-production branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.O9186IZQ/production-swh.before 2023-11-14 12:28:24.434809944 +0100
+++ /tmp/swh-chart.swh.O9186IZQ/production-swh.after 2023-11-14 12:28:24.714810780 +0100
@@ -5109,20 +5109,276 @@
- kafka4.internal.softwareheritage.org:9094
cls: kafka
group_id: swh-archive-prod-journalchecker
on_eof: restart
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-prod
security.protocol: SASL_SSL
---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-primary-directory-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-primary-directory-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-primary-release-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-primary-release-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-primary-revision-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-primary-revision-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-primary-snapshot-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-primary-snapshot-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-secondary-directory-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-secondary-directory-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-secondary-release-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-secondary-release-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-secondary-revision-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-secondary-revision-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-secondary-snapshot-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: scrubber-storagechecker-secondary-snapshot-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
# Source: swh/templates/statsd-exporter/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-statsd-exporter
namespace: swh
data:
config.yml: |
defaults:
timer_type: histogram
@@ -5229,20 +5485,52 @@
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: swh-archive-prod
security.protocol: SASL_SSL
---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
+ name: toolbox-scrubber-storage-primary-template
+ namespace: swh
+data:
+ config.yml.template: |
+ storage:
+
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: toolbox-scrubber-storage-secondary-template
+ namespace: swh
+data:
+ config.yml.template: |
+ storage:
+
+ cls: postgresql
+ db: host=massmoca.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
name: toolbox-storage-template
namespace: swh
data:
config.yml.template: |
storage:
cls: postgresql
db: host=db.internal.softwareheritage.org port=5432 user=swhstorage dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
---
# Source: swh/templates/toolbox/script-utils-configmap.yaml
apiVersion: v1
@@ -5359,20 +5647,50 @@
/opt/swh/bin/check-db-version.sh scrubber-journal
migrate-scrubber-journal-db-version.sh: |
#!/bin/bash
set -eu
/opt/swh/bin/migrate-db-version.sh scrubber-journal
+ check-scrubber-storage-primary-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/check-db-version.sh scrubber-storage-primary
+
+ migrate-scrubber-storage-primary-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/migrate-db-version.sh scrubber-storage-primary
+
+
+ check-scrubber-storage-secondary-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/check-db-version.sh scrubber-storage-secondary
+
+ migrate-scrubber-storage-secondary-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/migrate-db-version.sh scrubber-storage-secondary
+
+
check-storage-db-version.sh: |
#!/bin/bash
set -eu
/opt/swh/bin/check-db-version.sh storage
migrate-storage-db-version.sh: |
#!/bin/bash
@@ -14748,20 +15066,2468 @@
name: scrubber-journalchecker-snapshot-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: database-utils
configMap:
name: database-utils
defaultMode: 0555
---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-primary-directory-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-primary-directory-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-primary-directory-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-primary-directory-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: dd45b17ad7ce43b6a67fcdccd40e9bd2907f2e468c3be995fcbff9a1febedb3b
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 100Mi
+ cpu: 100m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-primary-hashes-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-primary-directory-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-primary-directory-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-primary-directory-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-primary-directory-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-primary-directory-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: c9085c7cdf012921f427098b5a0a949139e416241c3414109065a67b5afbdf5e
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 100Mi
+ cpu: 100m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-primary-references-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-primary-directory-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-primary-release-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-primary-release-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-primary-release-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-primary-release-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 4df3ffd2a1a279b1c89cd127dd6d3754e9b931f2876c63ba79cf84c227ff6bfc
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-primary-hashes-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-primary-release-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-primary-release-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-primary-release-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-primary-release-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-primary-release-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 84709c236e038eb864dbcb9472720cf8b47706f22e74110275f039cb0e71fb42
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-primary-references-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-primary-release-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-primary-revision-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-primary-revision-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-primary-revision-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-primary-revision-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: a3b20d4e348aee9382ebe7f1d4f712b0a6eee4d38550ce2a5713ee812db51fca
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-primary-hashes-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-primary-revision-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-primary-revision-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-primary-revision-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-primary-revision-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-primary-revision-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 388c72d449233e0154a4c57c434c5be3383ffdf0f397537155ed78ad6f405ebc
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-primary-references-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-primary-revision-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-primary-snapshot-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-primary-snapshot-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-primary-snapshot-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-primary-snapshot-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 61cc4f8b2479fb29fdcc7667a7492d641b799d6af4003c380a3f7b5cb058e5cb
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 128Mi
+ cpu: 150m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-primary-hashes-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-primary-snapshot-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-primary-snapshot-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-primary-snapshot-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-primary-snapshot-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-primary-snapshot-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: d4d034838c15a681cbb33271560a83b0d928dff013b122e5a6c0970738fe343e
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 128Mi
+ cpu: 150m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-primary-references-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-primary-snapshot-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-secondary-directory-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-secondary-directory-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-secondary-directory-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-secondary-directory-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 6052fd4fb214df78d73eaf029e07078f44de85bf27179fed20fd162b8c498fce
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 100Mi
+ cpu: 100m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-secondary-hashes-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-secondary-directory-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-secondary-directory-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-secondary-directory-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-secondary-directory-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-secondary-directory-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 0a5ca9b935cec23b79593fe23c13f97fc9c93d4abd4a3465557f0d1bbf4d2227
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 100Mi
+ cpu: 100m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-secondary-references-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-secondary-directory-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-secondary-release-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-secondary-release-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-secondary-release-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-secondary-release-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: d7f2ed3cbe2739b9902a711a66fa7af63ffe7f148f9deb418208f418bda4f79e
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-secondary-hashes-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-secondary-release-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-secondary-release-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-secondary-release-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-secondary-release-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-secondary-release-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 49843c17b7544ab1d28420e03d94163295c811062c402a0b89f50bda5084aaea
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-secondary-references-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-secondary-release-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-secondary-revision-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-secondary-revision-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-secondary-revision-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-secondary-revision-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: cb773a09bfeb8fbd28dd33e6d3df82933e165a3e4220ce1a3180f881b6b29fe6
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-secondary-hashes-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-secondary-revision-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-secondary-revision-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-secondary-revision-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-secondary-revision-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-secondary-revision-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 4b518f18447dd99691e21ed9a6760afcc8f6f43113fe3bc2b5f40f5542f93d15
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-secondary-references-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-secondary-revision-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-secondary-snapshot-hashes
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-secondary-snapshot-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-secondary-snapshot-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-secondary-snapshot-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 6d52d558cc58bed2d928a0bb48fd752e20b2203f45640bf9c73cc4e575c83b2e
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 128Mi
+ cpu: 150m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-secondary-hashes-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-secondary-snapshot-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-secondary-snapshot-references
+ namespace: swh
+ labels:
+ app: scrubber-storagechecker-secondary-snapshot-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 2
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-secondary-snapshot-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-secondary-snapshot-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 5e78ad59925be5dc476ea4a6ee0aa5fe26a748bb5b245d4bbf36f003098edd78
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 128Mi
+ cpu: 150m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-secondary-references-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-secondary-snapshot-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
# Source: swh/templates/statsd-exporter/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-statsd-exporter
namespace: swh
labels:
app: prometheus-statsd-exporter
spec:
replicas: 1
@@ -14809,22 +17575,22 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: swh-toolbox
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: 0fda6c829387563faf59a9d1f01c3c511ed3cf5d06f74a7e1fef820fadf11ce5
- checksum/configScript: efeecbb356cb04184a3c2ca2f60f2d08c634f34f1ce4bc6a6ebf937b34fadcbf
+ checksum/config: 42df14c73436ec831db8ade97beda17264fc9630bcfb04ebbef792cf1bee84b3
+ checksum/configScript: b501ac689b17f791ba2e2e36970cbc651b3947ca89b59a3efd667be5ceb13a08
spec:
priorityClassName: swh-tools
initContainers:
- name: prepare-configuration-indexer-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
@@ -14912,20 +17678,92 @@
key: postgres-swh-scrubber-password
# 'name' secret must exist & include that ^ key
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-scrubber-journal-template
mountPath: /etc/swh/configuration-template
+ - name: prepare-configuration-scrubber-storage-primary
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber-storage-primary.yml
+ env:
+
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-scrubber-storage-primary-template
+ mountPath: /etc/swh/configuration-template
+ - name: prepare-configuration-scrubber-storage-secondary
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber-storage-secondary.yml
+ env:
+
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-storage-postgresql-common-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-scrubber-storage-secondary-template
+ mountPath: /etc/swh/configuration-template
- name: prepare-configuration-storage
image: debian:bullseye
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-storage.yml
env:
@@ -14984,20 +17822,36 @@
path: "config.yml.template"
- name: configuration-scrubber-journal-template
configMap:
name: toolbox-scrubber-journal-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
+ - name: configuration-scrubber-storage-primary-template
+ configMap:
+ name: toolbox-scrubber-storage-primary-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
+ - name: configuration-scrubber-storage-secondary-template
+ configMap:
+ name: toolbox-scrubber-storage-secondary-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
- name: configuration-storage-template
configMap:
name: toolbox-storage-template
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: toolbox-script-utils
configMap:
------------- diff for environment production namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.O9186IZQ/production-swh-cassandra.before 2023-11-14 12:28:24.506810159 +0100
+++ /tmp/swh-chart.swh.O9186IZQ/production-swh-cassandra.after 2023-11-14 12:28:24.810811068 +0100
@@ -140,20 +140,268 @@
debug: false
introspection: true
max_raw_content_size: 10000
max_query_cost:
anonymous: 50
user: 500
---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-directory-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-directory-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-release-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-release-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-revision-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-revision-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-snapshot-hashes-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
+# Source: swh/templates/scrubber/storage-checker-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: scrubber-storagechecker-snapshot-references-template
+data:
+ config.yml.template: |
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+---
# Source: swh/templates/statsd-exporter/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-statsd-exporter
namespace: swh-cassandra
data:
config.yml: |
defaults:
timer_type: histogram
@@ -898,20 +1146,143 @@
auth_provider:
cls: cassandra.auth.PlainTextAuthProvider
password: ${CASSANDRA_PASSWORD}
username: swh-rw
directory_entries_insert_algo: batch
objstorage:
cls: multiplexer
objstorages:
${OBJSTORAGECONFIG}
---
+# Source: swh/templates/toolbox/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: toolbox-scrubber-storage-template
+ namespace: swh-cassandra
+data:
+ config.yml.template: |
+ storage:
+ cls: cassandra
+ hosts:
+ - cassandra01.internal.softwareheritage.org
+ - cassandra02.internal.softwareheritage.org
+ - cassandra03.internal.softwareheritage.org
+ - cassandra04.internal.softwareheritage.org
+ - cassandra05.internal.softwareheritage.org
+ - cassandra06.internal.softwareheritage.org
+ - cassandra07.internal.softwareheritage.org
+ - cassandra08.internal.softwareheritage.org
+ - cassandra09.internal.softwareheritage.org
+ - cassandra10.internal.softwareheritage.org
+ keyspace: swh
+ consistency_level: LOCAL_QUORUM
+ auth_provider:
+ cls: cassandra.auth.PlainTextAuthProvider
+ password: ${CASSANDRA_PASSWORD}
+ username: swh-ro
+
+ scrubber:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=swh-scrubber dbname=swh-scrubber password=${SCRUBBER_POSTGRESQL_PASSWORD}
+---
+# Source: swh/templates/toolbox/script-utils-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: toolbox-script-utils
+ namespace: swh-cassandra
+data:
+ register-task-types.sh: |
+ #!/bin/bash
+
+ set -eux
+
+ export SWH_CONFIG_FILENAME=/etc/swh/config-scheduler.yml
+
+ swh scheduler -C $SWH_CONFIG_FILENAME task-type register
+ check-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ MODULE=$1
+ CODE_VERSION=${2-""}
+ DB_VERSION=${3-""}
+
+ if [ -z ${MODULE} ]; then
+ echo The environment variable must be defined with the module to check
+ echo for example "storage"
+ exit 1
+ fi
+
+ # checking the database status
+ swh db --config-file=/etc/swh/config-${MODULE}.yml version "${MODULE}" | \
+ tee "${TEMP_FILE}"
+
+ CODE_VERSION=$(awk -F':' '/code/ {print $2}' ${TEMP_FILE})
+ DB_VERSION=$(awk -F':' '/^version/ {print $2}' ${TEMP_FILE})
+
+ if [ -e "${CODE_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ -e "${DB_VERSION}" ]; then
+ echo "Unable to find the code version"
+ exit 1
+ fi
+
+ if [ "$DB_VERSION" -eq "$CODE_VERSION" ]; then
+ echo "Database already configured at the latest version"
+ else
+ echo "Migration required from $DB_VERSION to $CODE_VERSION."
+ fi
+
+ migrate-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ TEMP_FILE=/tmp/db-version.txt
+
+ MODULE=$1
+ CODE_VERSION=${2-""}
+
+ if [ -z "${MODULE}" ]; then
+ echo The environment variable must be defined with the module to check
+ echo for example "storage"
+ exit 1
+ fi
+
+ if [ ! -z "${CODE_VERSION}" ]; then
+ swh db --config-file=/etc/swh/config-${MODULE}.yml upgrade "${MODULE}" \
+ --to-version ${CODE_VERSION}
+ else
+ swh db --config-file=/etc/swh/config-${MODULE}.yml upgrade "${MODULE}"
+ fi
+ check-scrubber-storage-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/check-db-version.sh scrubber-storage
+
+ migrate-scrubber-storage-db-version.sh: |
+ #!/bin/bash
+
+ set -eu
+
+ /opt/swh/bin/migrate-db-version.sh scrubber-storage
+---
# Source: swh/templates/utils/database-utils.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: database-utils
namespace: swh-cassandra
data:
init-keyspace.py: |
from swh.core import config
from swh.storage.cassandra import create_keyspace
@@ -1409,20 +1780,1244 @@
port: 9150
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /metrics
port: 9150
initialDelaySeconds: 5
periodSeconds: 10
---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-directory-hashes
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-directory-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 4
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-directory-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-directory-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 9eb6e6d23135007b4b2fa7479700bdcc15f80a5d55721c0423cc63fd7444ed47
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 200Mi
+ cpu: 400m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-hashes-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-directory-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-directory-references
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-directory-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 4
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-directory-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-directory-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 7f63436dc47803a97b1b03ccea316a8dcbe387654568cc1876cf561ee9618b6b
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 200Mi
+ cpu: 400m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-references-directory
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-directory-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-release-hashes
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-release-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 4
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-release-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-release-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 0ee498dbd291c3185128ac9d76285be000c39be941414473697df0e1c2667ab9
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-hashes-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-release-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-release-references
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-release-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 4
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-release-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-release-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 65fdad479bfea9d61d3dc6ec4de36204eeee25c11d76d5f77da1510580a21764
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-references-release
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-release-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-revision-hashes
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-revision-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-revision-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-revision-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 1a7f92884b1875ed9d35b10371051fd7c2dc28b2c40e211e4523390f878acd55
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 200Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-hashes-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-revision-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-revision-references
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-revision-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 1
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-revision-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-revision-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: b82f0daf2e4679579d2ef5d93466cc8b7ed9b122c6c2a9d56a0191a0982729c3
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 200Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-references-revision
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-revision-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-snapshot-hashes
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-snapshot-hashes
+spec:
+ revisionHistoryLimit: 2
+ replicas: 4
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-snapshot-hashes
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-snapshot-hashes
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: fcdbf1d4a38c0ae83441f1c59b158f52eebe6797635c9137488ff3c04b19e83a
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 500Mi
+ cpu: 650m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-hashes-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-snapshot-hashes-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
+# Source: swh/templates/scrubber/storage-checker-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: scrubber-storagechecker-snapshot-references
+ namespace: swh-cassandra
+ labels:
+ app: scrubber-storagechecker-snapshot-references
+spec:
+ revisionHistoryLimit: 2
+ replicas: 4
+ selector:
+ matchLabels:
+ app: scrubber-storagechecker-snapshot-references
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: scrubber-storagechecker-snapshot-references
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: a793e40df0132b81f124926400d0560dd0ea85d47fc33e9e1f222b29efbfad50
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/scrubber
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-background-workload
+
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ env:
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ # TODO: Add the "datastore" registration
+ # A workaround is needed as the registration is not idempotent
+ # and can't be launched each time a scrubber is launched
+ - name: check-scrubber-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-scrubber-db-version.sh
+ env:
+ - name: MODULE
+ value: scrubber
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ - name: check-storage-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ command:
+ - /entrypoints/check-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: strorage-checker
+ resources:
+ requests:
+ memory: 500Mi
+ cpu: 650m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber:20231107.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ - swh
+ - scrubber
+ - check
+ - storage
+ - storage-cassandra-references-snapshot
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.scrubber
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: scrubber-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: scrubber-storagechecker-snapshot-references-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+---
# Source: swh/templates/statsd-exporter/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-statsd-exporter
namespace: swh-cassandra
labels:
app: prometheus-statsd-exporter
spec:
replicas: 1
@@ -2910,20 +4505,121 @@
configMap:
name: storage-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: database-utils
configMap:
name: database-utils
defaultMode: 0555
---
+# Source: swh/templates/toolbox/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: swh-toolbox
+ namespace: swh-cassandra
+ labels:
+ app: swh-toolbox
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: swh-toolbox
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: swh-toolbox
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: f6537f61d275a3ce2b44ce0d64a317877f2fa4c65cd9395e4390563867c9a266
+ checksum/configScript: e524fc0d85bca4929459b068d030f3d7dd3680cd450d39c51791420840539736
+ spec:
+ priorityClassName: swh-cassandra-tools
+
+ initContainers:
+ - name: prepare-configuration-scrubber-storage
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config-scrubber-storage.yml
+ env:
+
+
+ - name: SCRUBBER_POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-scrubber-postgresql-common-secret
+ key: postgres-swh-scrubber-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+
+
+ - name: CASSANDRA_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: cassandra-swh-ro-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-scrubber-storage-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: swh-toolbox
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/toolbox:20231110.1
+ imagePullPolicy: IfNotPresent
+ resources:
+ requests:
+ memory: 256Mi
+ cpu: 250m
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: toolbox-script-utils
+ mountPath: /opt/swh/bin
+ readOnly: true
+ volumes:
+ - name: configuration
+ emptyDir: {}
+
+ - name: configuration-scrubber-storage-template
+ configMap:
+ name: toolbox-scrubber-storage-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+
+ - name: toolbox-script-utils
+ configMap:
+ name: toolbox-script-utils
+ defaultMode: 0555
+---
# Source: swh/templates/web/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh-cassandra
name: web
labels:
app: web
spec:
revisionHistoryLimit: 2
Edited by Vincent Sellier