create-object-reference-partitions: Fix cronjob
It used to run (and failing) using the complicated (and unneeded in that context) storage configmap (the one needed for the storage rpc).
With the following, it's using a dedicated simple configmap with the only required postgresql credentials.
helm diff
[swh] Comparing changes between branches production and fix-cronjob-create-object-ref-partition (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in fix-cronjob-create-object-ref-partition branch for environment staging...
[swh] Generate config in fix-cronjob-create-object-ref-partition branch for environment staging...
[swh] Generate config in fix-cronjob-create-object-ref-partition branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in fix-cronjob-create-object-ref-partition branch for environment production...
[swh] Generate config in fix-cronjob-create-object-ref-partition branch for environment production...
[swh] Generate config in fix-cronjob-create-object-ref-partition branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.QWxoY9tb/staging-swh.before 2024-03-25 16:18:48.681346751 +0100
+++ /tmp/swh-chart.swh.QWxoY9tb/staging-swh.after 2024-03-25 16:18:49.401345582 +0100
@@ -1033,20 +1033,34 @@
journal_writer:
cls: kafka
brokers:
- journal2.internal.staging.swh.network
prefix: swh.journal.objects
client_id: swh.storage.journal_writer.${HOSTNAME}
anonymize: true
producer_config:
message.max.bytes: 1000000000
---
+# Source: swh/templates/storage/cronjob-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: cronjob-create-object-reference-partitions-configuration-template
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: postgresql
+ db: host=db1.internal.staging.swh.network port=5432 user=swh dbname=swh password=${POSTGRESQL_PASSWORD}
+---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-scrubber-journal-template
namespace: swh
data:
config.yml.template: |
scrubber:
cls: postgresql
@@ -15084,21 +15098,21 @@
containers:
- name: create-object-reference-partitions
resources:
requests:
memory: 512Mi
cpu: 500m
image: container-registry.softwareheritage.org/swh/infra/swh-apps/storage:20240322.2
command:
- /opt/swh/entrypoint.sh
args:
- - create-object-references-partitions
+ - create-object-reference-partitions
env:
- name: STATSD_HOST
value: prometheus-statsd-exporter
- name: STATSD_PORT
value: "9125"
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
- name: LOG_LEVEL
value: INFO
- name: SWH_SENTRY_ENVIRONMENT
@@ -15116,21 +15130,21 @@
value: "true"
imagePullPolicy: IfNotPresent
volumeMounts:
- name: configuration
mountPath: /etc/swh
volumes:
- name: configuration
emptyDir: {}
- name: configuration-template
configMap:
- name: storage-postgresql-read-write-configuration-template
+ name: cronjob-create-object-reference-partitions-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: config-utils
configMap:
name: config-utils
defaultMode: 0555
restartPolicy: OnFailure
---
# Source: swh/templates/graphql/ingress.yaml
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.QWxoY9tb/production-swh.before 2024-03-25 16:18:50.313344102 +0100
+++ /tmp/swh-chart.swh.QWxoY9tb/production-swh.after 2024-03-25 16:18:50.909343134 +0100
@@ -7247,20 +7247,34 @@
- kafka1.internal.softwareheritage.org
- kafka2.internal.softwareheritage.org
- kafka3.internal.softwareheritage.org
- kafka4.internal.softwareheritage.org
prefix: swh.journal.objects
client_id: swh.storage.journal_writer.${HOSTNAME}
anonymize: true
producer_config:
message.max.bytes: 1000000000
---
+# Source: swh/templates/storage/cronjob-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: cronjob-create-object-reference-partitions-configuration-template
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: postgresql
+ db: host=postgresql-storage-rw.internal.softwareheritage.org port=5432 user=swhstorage dbname=softwareheritage password=${POSTGRESQL_PASSWORD}
+---
# Source: swh/templates/toolbox/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: toolbox-indexer-storage-template
namespace: swh
data:
config.yml.template: |
indexer_storage:
cls: postgresql
@@ -35345,244 +35359,20 @@
- "true"
initContainers:
- name: prepare-configuration
image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
- /entrypoints/prepare-configuration.sh
env:
- - name: ACCOUNT_NAME_0
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 0_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_1
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 1_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_10
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 10_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_11
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 11_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_12
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 12_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_13
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 13_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_14
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 14_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_15
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 15_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_2
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 2_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_3
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 3_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_4
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 4_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_5
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 5_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_6
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 6_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_7
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 7_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_8
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 8_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: ACCOUNT_NAME_9
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 9_account_name
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_0
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 0_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_1
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 1_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_10
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 10_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_11
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 11_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_12
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 12_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_13
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 13_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_14
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 14_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_15
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 15_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_2
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 2_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_3
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 3_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_4
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 4_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_5
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 5_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_6
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 6_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_7
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 7_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_8
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 8_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- - name: API_SECRET_KEY_9
- valueFrom:
- secretKeyRef:
- name: swh-objstorage-config
- key: 9_api_secret_key
- # 'name' secret must exist & include that ^ key
- optional: false
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-storage-postgresql-common-secret
key: postgres-swh-storage-password
# 'name' secret must exist & include that ^ key
optional: false
volumeMounts:
- name: configuration
@@ -35595,21 +35385,21 @@
containers:
- name: create-object-reference-partitions
resources:
requests:
memory: 512Mi
cpu: 500m
image: container-registry.softwareheritage.org/swh/infra/swh-apps/storage:20240322.2
command:
- /opt/swh/entrypoint.sh
args:
- - create-object-references-partitions
+ - create-object-reference-partitions
env:
- name: STATSD_HOST
value: prometheus-statsd-exporter
- name: STATSD_PORT
value: "9125"
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
- name: LOG_LEVEL
value: INFO
- name: SWH_SENTRY_ENVIRONMENT
@@ -35627,21 +35417,21 @@
value: "true"
imagePullPolicy: IfNotPresent
volumeMounts:
- name: configuration
mountPath: /etc/swh
volumes:
- name: configuration
emptyDir: {}
- name: configuration-template
configMap:
- name: storage-postgresql-saam-zfs-configuration-template
+ name: cronjob-create-object-reference-partitions-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: config-utils
configMap:
name: config-utils
defaultMode: 0555
restartPolicy: OnFailure
---
# Source: swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml
------------- diff for environment production namespace swh-cassandra -------------
No differences