From 82f071b29678a6dab755740c1fe732be262a016f Mon Sep 17 00:00:00 2001
From: Vincent SELLIER <vincent.sellier@softwareheritage.org>
Date: Tue, 27 Aug 2024 11:38:03 +0200
Subject: [PATCH] swh/web: Allow to launch generic django admin commands
Refactor the sync mailmaps admin command to make it generic
Related to swh/infra/sysadm-environment#5391
---
swh/templates/web/_helper-cronjob.yaml | 11 +++++++---
swh/templates/web/configmap-pgservice.yaml | 20 -----------------
swh/templates/web/cronjob-configmap.yaml | 22 +++++++++++++++++++
swh/templates/web/cronjob.yaml | 18 +++++++++++++++
swh/templates/web/sync-mailmaps-cronjob.yaml | 16 --------------
swh/values/production/swh.yaml | 16 +++++++++-----
.../overrides/swh-cassandra-next-version.yaml | 5 +++--
swh/values/staging/swh-cassandra.yaml | 16 +++++++++-----
8 files changed, 71 insertions(+), 53 deletions(-)
delete mode 100644 swh/templates/web/configmap-pgservice.yaml
create mode 100644 swh/templates/web/cronjob-configmap.yaml
create mode 100644 swh/templates/web/cronjob.yaml
delete mode 100644 swh/templates/web/sync-mailmaps-cronjob.yaml
diff --git a/swh/templates/web/_helper-cronjob.yaml b/swh/templates/web/_helper-cronjob.yaml
index 258672673..34be68444 100644
--- a/swh/templates/web/_helper-cronjob.yaml
+++ b/swh/templates/web/_helper-cronjob.yaml
@@ -4,6 +4,7 @@
{{- define "swh.web.cronjob" -}}
{{- with .configuration -}}
{{- $log_level := .logLevel -}}
+---
apiVersion: batch/v1
kind: CronJob
metadata:
@@ -28,13 +29,16 @@ spec:
initContainers:
{{- if $.pgService }}
- name: prepare-pgservice-configuration
- image: debian:bullseye
+ image: {{ $.Values.swh_utils_image }}:{{ $.Values.swh_utils_image_version }}
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- - eval "cp /etc/swh/config/pg_service.conf /etc/swh/.pg_service.conf"
+ - eval "cat /etc/swh/config/pg_service.conf | envsubst > /etc/swh/.pg_service.conf"
+ env:
+ {{- include "swh.secrets.envFromDeploymentConfig" (dict "deploymentConfig" $.webConfiguration
+ "Values" $.Values) | nindent 16 }}
volumeMounts:
- name: configuration
mountPath: /etc/swh
@@ -86,6 +90,7 @@ spec:
command:
- /opt/swh/entrypoint.sh
args:
+ - django-admin
{{- range $cmd := $.command }}
- {{ $cmd }}
{{- end }}
@@ -140,7 +145,7 @@ spec:
{{- if $.pgService }}
- name: pgservice-configuration-template
configMap:
- name: pgservice-{{ $.webType }}-configuration-template
+ name: {{ lower $.serviceType }}-config-template
items:
- key: "pg-service-conf"
path: "pg_service.conf"
diff --git a/swh/templates/web/configmap-pgservice.yaml b/swh/templates/web/configmap-pgservice.yaml
deleted file mode 100644
index d65e614f2..000000000
--- a/swh/templates/web/configmap-pgservice.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-{{- if .Values.web.enabled -}}
-{{ range $web_type, $web_config := .Values.web.deployments }}
-{{- if and (or (not (hasKey $web_config "enabled"))
- (get $web_config "enabled"))
- (hasKey $web_config "syncMailmaps")
- $web_config.syncMailmaps.enabled -}}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: pgservice-{{ $web_type }}-configuration-template
- namespace: {{ $.Values.namespace }}
-data:
- pg-service-conf: |
- {{- include "swh.web.pgService" (dict "serviceType" "syncmailmaps"
- "configurationRef" $web_config.syncMailmaps.configurationRef
- "Values" $.Values) | nindent 4 }}
-{{- end -}}
-{{ end -}}
-{{- end -}}
diff --git a/swh/templates/web/cronjob-configmap.yaml b/swh/templates/web/cronjob-configmap.yaml
new file mode 100644
index 000000000..1b832ae41
--- /dev/null
+++ b/swh/templates/web/cronjob-configmap.yaml
@@ -0,0 +1,22 @@
+{{ if .Values.web.enabled -}}
+{{- range $webType, $webConfig := .Values.web.deployments -}}
+{{- if and (or (not (hasKey $webConfig "enabled"))
+ (get $webConfig "enabled")) -}}
+{{- range $cronJobType, $cronJobConfig := $webConfig.cronJobs -}}
+{{- if $cronJobConfig.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: web-{{ $webType }}-{{ $cronJobType | lower }}-config-template
+ namespace: {{ $.Values.namespace }}
+data:
+ pg-service-conf: |
+ {{- include "swh.web.pgService" (dict "serviceType" (lower $cronJobType)
+ "configurationRef" $cronJobConfig.configurationRef
+ "Values" $.Values) | nindent 4 }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end }}
diff --git a/swh/templates/web/cronjob.yaml b/swh/templates/web/cronjob.yaml
new file mode 100644
index 000000000..95d15ebeb
--- /dev/null
+++ b/swh/templates/web/cronjob.yaml
@@ -0,0 +1,18 @@
+{{ if .Values.web.enabled -}}
+{{ range $webType, $webConfig := .Values.web.deployments }}
+{{- if and (or (not (hasKey $webConfig "enabled"))
+ (get $webConfig "enabled")) -}}
+{{- range $cronJobType, $cronJobConfig := $webConfig.cronJobs -}}
+{{- if $cronJobConfig.enabled -}}
+{{- include "swh.web.cronjob" (dict "Values" $.Values
+ "serviceType" (print "web-" $webType "-" (lower $cronJobType) )
+ "configuration" $cronJobConfig
+ "webConfiguration" $webConfig
+ "pgService" true
+ "webType" $webType
+ "command" $cronJobConfig.command) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{ end -}}
+{{- end -}}
diff --git a/swh/templates/web/sync-mailmaps-cronjob.yaml b/swh/templates/web/sync-mailmaps-cronjob.yaml
deleted file mode 100644
index d0122b81c..000000000
--- a/swh/templates/web/sync-mailmaps-cronjob.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-{{ if .Values.web.enabled -}}
-{{ range $web_type, $web_config := .Values.web.deployments }}
-{{- if and (or (not (hasKey $web_config "enabled"))
- (get $web_config "enabled"))
- (hasKey $web_config "syncMailmaps")
- $web_config.syncMailmaps.enabled -}}
-{{- include "swh.web.cronjob" (dict "Values" $.Values
- "serviceType" (print "web-" $web_type "-sync-mailmaps")
- "configuration" $web_config.syncMailmaps
- "webConfiguration" $web_config
- "pgService" true
- "webType" $web_type
- "command" (list "sync-mailmaps" "service=syncmailmaps")) -}}
-{{- end -}}
-{{ end -}}
-{{- end -}}
diff --git a/swh/values/production/swh.yaml b/swh/values/production/swh.yaml
index 268416d67..db655fa82 100644
--- a/swh/values/production/swh.yaml
+++ b/swh/values/production/swh.yaml
@@ -1429,12 +1429,16 @@ web:
concurrencyPolicy: Forbid
priorityClassName: frontend-rpc-workload
cron: "*/2 * * * *"
- syncMailmaps:
- enabled: true
- concurrencyPolicy: Forbid
- configurationRef: postgresqlSyncmailmapsConfiguration
- priorityClassName: frontend-rpc-workload
- cron: "15 * * * *"
+ cronJobs:
+ syncMailmaps:
+ enabled: true
+ command:
+ - "sync_mailmaps"
+ - "service=syncmailmaps"
+ concurrencyPolicy: Forbid
+ configurationRef: postgresqlSyncmailmapsConfiguration
+ priorityClassName: frontend-rpc-workload
+ cron: "15 * * * *"
hosts:
- archive.softwareheritage.org
- base.softwareheritage.org
diff --git a/swh/values/staging/overrides/swh-cassandra-next-version.yaml b/swh/values/staging/overrides/swh-cassandra-next-version.yaml
index f79b6a5e3..30aa1e4f4 100644
--- a/swh/values/staging/overrides/swh-cassandra-next-version.yaml
+++ b/swh/values/staging/overrides/swh-cassandra-next-version.yaml
@@ -647,8 +647,9 @@ web:
- webapp-cassandra-next-version.internal.staging.swh.network
refreshSavecodenowStatus:
enabled: false
- syncMailmaps:
- enabled: false
+ cronJobs:
+ syncMailmaps:
+ enabled: false
ingress:
whitelistSourceRangeRef: stagingNetworkRanges
endpoints:
diff --git a/swh/values/staging/swh-cassandra.yaml b/swh/values/staging/swh-cassandra.yaml
index f7425b2c3..c8d7da20a 100644
--- a/swh/values/staging/swh-cassandra.yaml
+++ b/swh/values/staging/swh-cassandra.yaml
@@ -714,12 +714,16 @@ web:
hosts:
- webapp.staging.swh.network
- webapp-cassandra.internal.staging.swh.network
- syncMailmaps:
- enabled: true
- concurrencyPolicy: Forbid
- configurationRef: postgresqlSyncmailmapsConfiguration
- priorityClassName: frontend-rpc-workload
- cron: "15 * * * *"
+ cronJobs:
+ syncMailmaps:
+ enabled: true
+ command:
+ - "sync_mailmaps"
+ - "service=syncmailmaps"
+ concurrencyPolicy: Forbid
+ configurationRef: postgresqlSyncmailmapsConfiguration
+ priorityClassName: frontend-rpc-workload
+ cron: "15 * * * *"
ingress:
enabled: true
secretName: swh-web-crt
--
GitLab