From 455c492640507583339964caf27feada53ce648a Mon Sep 17 00:00:00 2001
From: Vincent SELLIER <vincent.sellier@softwareheritage.org>
Date: Tue, 27 Aug 2024 11:38:03 +0200
Subject: [PATCH] swh/web: Allow to launch generic django admin commands
Refactor the sync mailmaps admin command to make it generic
Related to swh/infra/sysadm-environment#5391
---
swh/templates/web/_helper-cronjob.yaml | 17 +++---
swh/templates/web/configmap-pgservice.yaml | 20 -------
swh/templates/web/cronjob-configmap.yaml | 22 ++++++++
swh/templates/web/cronjob.yaml | 17 ++++++
.../refresh-savecodenow-statuses-cronjob.yaml | 15 ------
swh/templates/web/sync-mailmaps-cronjob.yaml | 16 ------
swh/values.yaml | 54 ++++++++++++-------
swh/values/local-cluster.yaml | 25 ++++++---
swh/values/production/swh.yaml | 34 ++++++------
.../overrides/swh-cassandra-next-version.yaml | 11 ++--
swh/values/staging/swh-cassandra.yaml | 25 ++++++---
values-swh-application-versions.yaml | 2 +-
12 files changed, 149 insertions(+), 109 deletions(-)
delete mode 100644 swh/templates/web/configmap-pgservice.yaml
create mode 100644 swh/templates/web/cronjob-configmap.yaml
create mode 100644 swh/templates/web/cronjob.yaml
delete mode 100644 swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml
delete mode 100644 swh/templates/web/sync-mailmaps-cronjob.yaml
diff --git a/swh/templates/web/_helper-cronjob.yaml b/swh/templates/web/_helper-cronjob.yaml
index 258672673..1e7318c31 100644
--- a/swh/templates/web/_helper-cronjob.yaml
+++ b/swh/templates/web/_helper-cronjob.yaml
@@ -4,6 +4,7 @@
{{- define "swh.web.cronjob" -}}
{{- with .configuration -}}
{{- $log_level := .logLevel -}}
+---
apiVersion: batch/v1
kind: CronJob
metadata:
@@ -26,15 +27,18 @@ spec:
priorityClassName: {{ $.Values.namespace }}-{{ .priorityClassName }}
{{ end }}
initContainers:
- {{- if $.pgService }}
+ {{- if .pgService }}
- name: prepare-pgservice-configuration
- image: debian:bullseye
+ image: {{ $.Values.swh_utils_image }}:{{ $.Values.swh_utils_image_version }}
imagePullPolicy: IfNotPresent
command:
- /bin/bash
args:
- -c
- - eval "cp /etc/swh/config/pg_service.conf /etc/swh/.pg_service.conf"
+ - eval "cat /etc/swh/config/pg_service.conf | envsubst > /etc/swh/.pg_service.conf"
+ env:
+ {{- include "swh.secrets.envFromDeploymentConfig" (dict "deploymentConfig" $.webConfiguration
+ "Values" $.Values) | nindent 16 }}
volumeMounts:
- name: configuration
mountPath: /etc/swh
@@ -86,6 +90,7 @@ spec:
command:
- /opt/swh/entrypoint.sh
args:
+ - django-admin
{{- range $cmd := $.command }}
- {{ $cmd }}
{{- end }}
@@ -115,7 +120,7 @@ spec:
- name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
value: "true"
{{- end }}
- {{- if $.pgService }}
+ {{- if .pgService }}
- name: PGSERVICEFILE
value: /etc/swh/.pg_service.conf
{{- include "swh.secrets.envFromDeploymentConfig" (dict "deploymentConfig" $.configuration "Values" $.Values) | nindent 16 }}
@@ -137,10 +142,10 @@ spec:
items:
- key: "config.yml.template"
path: "config.yml.template"
- {{- if $.pgService }}
+ {{- if .pgService }}
- name: pgservice-configuration-template
configMap:
- name: pgservice-{{ $.webType }}-configuration-template
+ name: {{ lower $.serviceType }}-config-template
items:
- key: "pg-service-conf"
path: "pg_service.conf"
diff --git a/swh/templates/web/configmap-pgservice.yaml b/swh/templates/web/configmap-pgservice.yaml
deleted file mode 100644
index d65e614f2..000000000
--- a/swh/templates/web/configmap-pgservice.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
-{{- if .Values.web.enabled -}}
-{{ range $web_type, $web_config := .Values.web.deployments }}
-{{- if and (or (not (hasKey $web_config "enabled"))
- (get $web_config "enabled"))
- (hasKey $web_config "syncMailmaps")
- $web_config.syncMailmaps.enabled -}}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: pgservice-{{ $web_type }}-configuration-template
- namespace: {{ $.Values.namespace }}
-data:
- pg-service-conf: |
- {{- include "swh.web.pgService" (dict "serviceType" "syncmailmaps"
- "configurationRef" $web_config.syncMailmaps.configurationRef
- "Values" $.Values) | nindent 4 }}
-{{- end -}}
-{{ end -}}
-{{- end -}}
diff --git a/swh/templates/web/cronjob-configmap.yaml b/swh/templates/web/cronjob-configmap.yaml
new file mode 100644
index 000000000..9b591a4ed
--- /dev/null
+++ b/swh/templates/web/cronjob-configmap.yaml
@@ -0,0 +1,22 @@
+{{ if .Values.web.enabled -}}
+{{- range $webType, $webConfig := .Values.web.deployments -}}
+{{- if and (or (not (hasKey $webConfig "enabled"))
+ (get $webConfig "enabled")) -}}
+{{- range $cronJobType, $cronJobConfig := $webConfig.cronJobs -}}
+{{- if and $cronJobConfig.enabled $cronJobConfig.pgService }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: web-{{ $webType }}-{{ $cronJobType | lower }}-config-template
+ namespace: {{ $.Values.namespace }}
+data:
+ pg-service-conf: |
+ {{- include "swh.web.pgService" (dict "serviceType" (lower $cronJobType)
+ "configurationRef" $cronJobConfig.configurationRef
+ "Values" $.Values) | nindent 4 }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end }}
diff --git a/swh/templates/web/cronjob.yaml b/swh/templates/web/cronjob.yaml
new file mode 100644
index 000000000..df045926e
--- /dev/null
+++ b/swh/templates/web/cronjob.yaml
@@ -0,0 +1,17 @@
+{{ if .Values.web.enabled -}}
+{{ range $webType, $webConfig := .Values.web.deployments }}
+{{- if and (or (not (hasKey $webConfig "enabled"))
+ (get $webConfig "enabled")) -}}
+{{- range $cronJobType, $cronJobConfig := $webConfig.cronJobs -}}
+{{- if $cronJobConfig.enabled -}}
+{{- include "swh.web.cronjob" (dict "Values" $.Values
+ "serviceType" (print "web-" $webType "-" (lower $cronJobType) )
+ "configuration" $cronJobConfig
+ "webConfiguration" $webConfig
+ "webType" $webType
+ "command" $cronJobConfig.command) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{ end -}}
+{{- end -}}
diff --git a/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml b/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml
deleted file mode 100644
index c80030000..000000000
--- a/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-{{ if .Values.web.enabled -}}
-{{ range $web_type, $web_config := .Values.web.deployments }}
-{{- if and (or (not (hasKey $web_config "enabled"))
- (get $web_config "enabled"))
- (hasKey $web_config "refreshSavecodenowStatus")
- $web_config.refreshSavecodenowStatus.enabled -}}
-{{- include "swh.web.cronjob" (dict "Values" $.Values
- "serviceType" (print "web-" $web_type "-refresh-savecodenow-statuses")
- "configuration" $web_config.refreshSavecodenowStatus
- "webConfiguration" $web_config
- "webType" $web_type
- "command" (list "refresh")) -}}
-{{- end -}}
-{{ end -}}
-{{- end -}}
diff --git a/swh/templates/web/sync-mailmaps-cronjob.yaml b/swh/templates/web/sync-mailmaps-cronjob.yaml
deleted file mode 100644
index d0122b81c..000000000
--- a/swh/templates/web/sync-mailmaps-cronjob.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-{{ if .Values.web.enabled -}}
-{{ range $web_type, $web_config := .Values.web.deployments }}
-{{- if and (or (not (hasKey $web_config "enabled"))
- (get $web_config "enabled"))
- (hasKey $web_config "syncMailmaps")
- $web_config.syncMailmaps.enabled -}}
-{{- include "swh.web.cronjob" (dict "Values" $.Values
- "serviceType" (print "web-" $web_type "-sync-mailmaps")
- "configuration" $web_config.syncMailmaps
- "webConfiguration" $web_config
- "pgService" true
- "webType" $web_type
- "command" (list "sync-mailmaps" "service=syncmailmaps")) -}}
-{{- end -}}
-{{ end -}}
-{{- end -}}
diff --git a/swh/values.yaml b/swh/values.yaml
index 959d5009c..e7c2f674c 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -835,24 +835,42 @@ web:
# minReplicaCount: 2
# maxReplicaCount: 10
# cpuPercentageUsage: 50
- refreshSavecodenowStatus:
- enabled: false
- priorityClassName: frontend-rpc-workload
- logLevel: INFO
- # concurrencyPolicy: Forbid
- # Every 2 minute
- cron: "*/2 * * * *"
- syncMailmaps:
- enabled: false
- priorityClassName: frontend-rpc-workload
- logLevel: INFO
- # concurrencyPolicy: Forbid
- # At minute 15 every hour
- cron: "15 * * * *"
- # gunicorn:
- # threads: 5
- # workers: 2
- # timeout: 60
+ # cronJobs:
+ # refreshSCNStatus:
+ # enabled: false
+ # command:
+ # - "refresh_savecodenow_statuses"
+ # priorityClassName: frontend-rpc-workload
+ # logLevel: INFO
+ # # concurrencyPolicy: Forbid
+ # # Every 2 minute
+ # cron: "*/2 * * * *"
+ # syncMailmaps:
+ # enabled: false
+ # command:
+ # - "sync_mailmaps"
+ # - "--perform"
+ # - "service=syncmailmaps"
+ # cron: "0 */1 * * *"
+ # concurrencyPolicy: Forbid
+ # configurationRef: postgresqlSyncmailmapsConfiguration
+ # pgService: true
+ # # priorityClassName: frontend-rpc-workload
+ # syncMaskingMailmaps:
+ # enabled: false
+ # command:
+ # - "sync_masking_mailmaps"
+ # - "--perform"
+ # - "service=syncmaskingmailmaps"
+ # cron: "0 */1 * * *"
+ # concurrencyPolicy: Forbid
+ # configurationRef: postgresqlSyncMaskinProxyMailmapsConfiguration
+ # pgService: true
+ # # priorityClassName: frontend-rpc-workload
+ # gunicorn:
+ # threads: 5
+ # workers: 2
+ # timeout: 60
# host: webapp
ingress:
enabled: false
diff --git a/swh/values/local-cluster.yaml b/swh/values/local-cluster.yaml
index 51ddb701a..5af5efd2b 100644
--- a/swh/values/local-cluster.yaml
+++ b/swh/values/local-cluster.yaml
@@ -430,6 +430,23 @@ web:
enabled: true
migrations:
enabled: true
+ cronJobs:
+ syncMailmaps:
+ enabled: false
+ command:
+ - "sync_mailmaps"
+ - "--perform"
+ - "service=syncmailmaps"
+ cron: "0 */1 * * *"
+ concurrencyPolicy: Forbid
+ configurationRef: fakeSyncmailmapsConfiguration
+ pgService: true
+ refreshSCNStatus:
+ enabled: false
+ cron: "* 5 * * *"
+ concurrencyPolicy: Forbid
+ command:
+ - "refresh_savecodenow_statuses"
metricsScrapingEnabled: false
replicas: 1
# autoScaling:
@@ -483,14 +500,6 @@ web:
# - path: /api/1/content/[^/]+/symbol/
# # auth-file with authentication
# authentication: basic-auth
- # (either this cron of the webhookConfigurationRef entry key)
- # refreshSavecodenowStatus:
- # enabled: false
- # cron: "* 5 * * *"
- syncMailmaps:
- enabled: false
- cron: "30 * * * *"
- configurationRef: fakeSyncmailmapsConfiguration
sentry:
enabled: false
extraConfig:
diff --git a/swh/values/production/swh.yaml b/swh/values/production/swh.yaml
index d634170a8..d0d88ca08 100644
--- a/swh/values/production/swh.yaml
+++ b/swh/values/production/swh.yaml
@@ -1424,17 +1424,25 @@ web:
threshold: 10
requestedCpu: 500m
requestedMemory: 6Gi
- refreshSavecodenowStatus:
- enabled: false
- concurrencyPolicy: Forbid
- priorityClassName: frontend-rpc-workload
- cron: "*/2 * * * *"
- syncMailmaps:
- enabled: true
- concurrencyPolicy: Forbid
- configurationRef: postgresqlSyncmailmapsConfiguration
- priorityClassName: frontend-rpc-workload
- cron: "15 * * * *"
+ cronJobs:
+ refreshSCNStatus:
+ enabled: false
+ cron: "*/2 * * * *"
+ command:
+ - "refresh_savecodenow_statuses"
+ concurrencyPolicy: Forbid
+ priorityClassName: frontend-rpc-workload
+ syncMailmaps:
+ enabled: true
+ cron: "15 * * * *"
+ command:
+ - "sync_mailmaps"
+ - "--perform"
+ - "service=syncmailmaps"
+ concurrencyPolicy: Forbid
+ configurationRef: postgresqlSyncmailmapsConfiguration
+ priorityClassName: frontend-rpc-workload
+ pgService: true
hosts:
- archive.softwareheritage.org
- base.softwareheritage.org
@@ -1520,10 +1528,6 @@ web:
metricsScrapingEnabled: false
requestedCpu: 500m
requestedMemory: 1Gi
- refreshSavecodenowStatus:
- enabled: false
- syncMailmaps:
- enabled: false
replicas: 2
# autoScaling:
# minReplicaCount: 2
diff --git a/swh/values/staging/overrides/swh-cassandra-next-version.yaml b/swh/values/staging/overrides/swh-cassandra-next-version.yaml
index f79b6a5e3..d1294ba12 100644
--- a/swh/values/staging/overrides/swh-cassandra-next-version.yaml
+++ b/swh/values/staging/overrides/swh-cassandra-next-version.yaml
@@ -645,10 +645,13 @@ web:
history_counters_url: http://counters-rpc:5011/counters_history/history.json#
hosts:
- webapp-cassandra-next-version.internal.staging.swh.network
- refreshSavecodenowStatus:
- enabled: false
- syncMailmaps:
- enabled: false
+ cronJobs:
+ refreshSCNStatus:
+ enabled: false
+ syncMailmaps:
+ enabled: false
+ syncMaskingMailmaps:
+ enabled: false
ingress:
whitelistSourceRangeRef: stagingNetworkRanges
endpoints:
diff --git a/swh/values/staging/swh-cassandra.yaml b/swh/values/staging/swh-cassandra.yaml
index f7425b2c3..756870b97 100644
--- a/swh/values/staging/swh-cassandra.yaml
+++ b/swh/values/staging/swh-cassandra.yaml
@@ -714,12 +714,25 @@ web:
hosts:
- webapp.staging.swh.network
- webapp-cassandra.internal.staging.swh.network
- syncMailmaps:
- enabled: true
- concurrencyPolicy: Forbid
- configurationRef: postgresqlSyncmailmapsConfiguration
- priorityClassName: frontend-rpc-workload
- cron: "15 * * * *"
+ cronJobs:
+ refreshSCNStatus:
+ enabled: false
+ cron: "* 5 * * *"
+ concurrencyPolicy: Forbid
+ priorityClassName: frontend-rpc-workload
+ command:
+ - "refresh_savecodenow_statuses"
+ syncMailmaps:
+ enabled: true
+ command:
+ - "sync_mailmaps"
+ - "--perform"
+ - "service=syncmailmaps"
+ cron: "0 */1 * * *"
+ concurrencyPolicy: Forbid
+ priorityClassName: frontend-rpc-workload
+ configurationRef: postgresqlSyncmailmapsConfiguration
+ pgService: true
ingress:
enabled: true
secretName: swh-web-crt
diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml
index a4b04b802..cad842cd8 100644
--- a/values-swh-application-versions.yaml
+++ b/values-swh-application-versions.yaml
@@ -57,6 +57,6 @@ swh_vault_cookers_image_version: '20240822.1'
swh_vault_image: container-registry.softwareheritage.org/swh/infra/swh-apps/vault
swh_vault_image_version: '20240822.1'
swh_web_image: container-registry.softwareheritage.org/swh/infra/swh-apps/web
-swh_web_image_version: '20240822.2'
+swh_web_image_version: '20240828.1'
swh_webhooks_image: container-registry.softwareheritage.org/swh/infra/swh-apps/webhooks
swh_webhooks_image_version: '20240722.1'
--
GitLab