From c53032386930461bff6fb6ebe13adedfedc99bbf Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Tue, 17 Oct 2023 16:59:38 +0200
Subject: [PATCH] web: Add syncmail-maps cronjob
Refs. swh/infra/sysadm-environment#5095
---
swh/templates/web/_helper-cronjob.tpl | 130 ++++++++++++++++++
.../refresh-savecodenow-statuses-cronjob.yaml | 126 +----------------
swh/templates/web/sync-mailmaps-cronjob.yaml | 5 +
swh/values.yaml | 7 +
swh/values/minikube.yaml | 8 +-
swh/values/staging/swh.yaml | 4 +
6 files changed, 154 insertions(+), 126 deletions(-)
create mode 100644 swh/templates/web/_helper-cronjob.tpl
create mode 100644 swh/templates/web/sync-mailmaps-cronjob.yaml
diff --git a/swh/templates/web/_helper-cronjob.tpl b/swh/templates/web/_helper-cronjob.tpl
new file mode 100644
index 000000000..f61f0a2fc
--- /dev/null
+++ b/swh/templates/web/_helper-cronjob.tpl
@@ -0,0 +1,130 @@
+# -*- yaml -*-
+
+{{/*
+Create a Kind CronJob for service .serviceType
+*/}}
+{{- define "swh.web.cronjob" -}}
+{{- with .configuration -}}
+{{- $log_level := .logLevel -}}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: {{ $.serviceType }}-cronjob
+spec:
+ # By default, hourly at minute 15
+ schedule: {{ .cron | quote}}
+ jobTemplate:
+ spec:
+ {{- if .concurrencyPolicy }}
+ concurrencyPolicy: {{ .concurrencyPolicy }}
+ {{- end }}
+ template:
+ spec:
+ {{- if and $.Values.podPriority.enabled .priorityClassName }}
+ priorityClassName: {{ $.Values.namespace }}-{{ .priorityClassName }}
+ {{ end }}
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ env:
+ {{- if $.Values.web.databaseConfigurationRef }}
+ {{- include "swh.secrets.environment" (dict "Values" $.Values
+ "configurationRef" $.Values.web.databaseConfigurationRef) | nindent 16 -}}
+ {{ end }}
+ {{- if $.Values.web.djangoConfigurationRef }}
+ {{- include "swh.secrets.environment" (dict "Values" $.Values
+ "configurationRef" $.Values.web.djangoConfigurationRef) | nindent 16 }}
+ {{ end }}
+ {{- if $.Values.web.depositConfigurationRef -}}
+ {{- include "swh.secrets.environment" (dict "Values" $.Values
+ "configurationRef" $.Values.web.depositConfigurationRef) | nindent 16 }}
+ {{ end }}
+ {{- if $.Values.web.giveConfigurationRef -}}
+ {{- include "swh.secrets.environment" (dict "Values" $.Values
+ "configurationRef" $.Values.web.giveConfigurationRef) | nindent 16 }}
+ {{ end }}
+ {{- if $.Values.web.sentry.enabled }}
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: {{ $.Values.web.sentry.secretKeyRef }}
+ key: {{ $.Values.web.sentry.secretKeyName }}
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ {{ end }}
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: sync-mailmaps
+ resources:
+ requests:
+ memory: {{ .requestedMemory | default "512Mi" }}
+ cpu: {{ .requestedCpu | default "500m" }}
+ {{- if or .limitedMemory .limitedCpu }}
+ limits:
+ {{- if .limitedMemory }}
+ memory: {{ .limitedMemory }}
+ {{- end }}
+ {{- if .limitedCpu }}
+ cpu: {{ .limitedCpu }}
+ {{- end }}
+ {{ end }}
+ image: {{ $.Values.swh_web_image }}:{{ $.Values.swh_web_image_version }}
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ # - shell
+ # - sleep
+ # - infinity
+ - refresh
+ env:
+ - name: STATSD_HOST
+ value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ - name: STATSD_PORT
+ value: {{ $.Values.statsdPort | default "9125" | quote }}
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: LOG_LEVEL
+ value: {{ $log_level | default "INFO" }}
+ {{- if $.Values.web.sentry.enabled }}
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: {{ $.Values.sentry.environment }}
+ - name: SWH_MAIN_PACKAGE
+ value: swh.web
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: {{ $.Values.web.sentry.secretKeyRef }}
+ key: {{ $.Values.web.sentry.secretKeyName }}
+ # if the setting doesn't exist, sentry issue pushes will be disabled
+ optional: false
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ {{- end }}
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: web-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ restartPolicy: OnFailure
+
+{{ end }}
+{{- end -}}
diff --git a/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml b/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml
index 56578d59b..fd95ade9b 100644
--- a/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml
+++ b/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml
@@ -1,125 +1,5 @@
{{ if and .Values.web.enabled .Values.web.refreshSavecodenowStatus.enabled -}}
-{{- with .Values.web.refreshSavecodenowStatus -}}
-{{- $log_level := .logLevel -}}
-apiVersion: batch/v1
-kind: CronJob
-metadata:
- name: refresh-savecodenow-statuses-cronjob
-spec:
- # By default, every 4h at midnight
- schedule: {{ .cron | default "* * * * *" | quote}}
- jobTemplate:
- spec:
- {{- if .concurrencyPolicy }}
- concurrencyPolicy: {{ .concurrencyPolicy }}
- {{- end }}
- template:
- spec:
- {{- if and $.Values.podPriority.enabled .priorityClassName }}
- priorityClassName: {{ $.Values.namespace }}-{{ .priorityClassName }}
- {{ end }}
- initContainers:
- - name: prepare-configuration
- image: debian:bullseye
- imagePullPolicy: IfNotPresent
- command:
- - /bin/bash
- args:
- - -c
- - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
- env:
- {{- if $.Values.web.databaseConfigurationRef }}
- {{- include "swh.secrets.environment" (dict "Values" $.Values
- "configurationRef" $.Values.web.databaseConfigurationRef) | nindent 16 -}}
- {{ end }}
- {{- if $.Values.web.djangoConfigurationRef }}
- {{- include "swh.secrets.environment" (dict "Values" $.Values
- "configurationRef" $.Values.web.djangoConfigurationRef) | nindent 16 }}
- {{ end }}
- {{- if $.Values.web.depositConfigurationRef -}}
- {{- include "swh.secrets.environment" (dict "Values" $.Values
- "configurationRef" $.Values.web.depositConfigurationRef) | nindent 16 }}
- {{ end }}
- {{- if $.Values.web.giveConfigurationRef -}}
- {{- include "swh.secrets.environment" (dict "Values" $.Values
- "configurationRef" $.Values.web.giveConfigurationRef) | nindent 16 }}
- {{ end }}
- {{- if $.Values.web.sentry.enabled }}
- - name: SWH_SENTRY_DSN
- valueFrom:
- secretKeyRef:
- name: {{ $.Values.web.sentry.secretKeyRef }}
- key: {{ $.Values.web.sentry.secretKeyName }}
- # 'name' secret should exist & include key
- # if the setting doesn't exist, sentry pushes will be disabled
- optional: true
- {{ end }}
- volumeMounts:
- - name: configuration
- mountPath: /etc/swh
- - name: configuration-template
- mountPath: /etc/swh/configuration-template
- containers:
- - name: refresh-savecodenow-statuses
- resources:
- requests:
- memory: {{ .requestedMemory | default "512Mi" }}
- cpu: {{ .requestedCpu | default "500m" }}
- {{- if or .limitedMemory .limitedCpu }}
- limits:
- {{- if .limitedMemory }}
- memory: {{ .limitedMemory }}
- {{- end }}
- {{- if .limitedCpu }}
- cpu: {{ .limitedCpu }}
- {{- end }}
- {{ end }}
- image: {{ $.Values.swh_web_image }}:{{ $.Values.swh_web_image_version }}
- command:
- - /opt/swh/entrypoint.sh
- args:
- # - shell
- # - sleep
- # - infinity
- - refresh
- env:
- - name: STATSD_HOST
- value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
- - name: STATSD_PORT
- value: {{ $.Values.statsdPort | default "9125" | quote }}
- - name: SWH_CONFIG_FILENAME
- value: /etc/swh/config.yml
- - name: LOG_LEVEL
- value: {{ $log_level | default "INFO" }}
- {{- if $.Values.web.sentry.enabled }}
- - name: SWH_SENTRY_ENVIRONMENT
- value: {{ $.Values.sentry.environment }}
- - name: SWH_MAIN_PACKAGE
- value: swh.web
- - name: SWH_SENTRY_DSN
- valueFrom:
- secretKeyRef:
- name: {{ $.Values.web.sentry.secretKeyRef }}
- key: {{ $.Values.web.sentry.secretKeyName }}
- # if the setting doesn't exist, sentry issue pushes will be disabled
- optional: false
- - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
- value: "true"
- {{- end }}
- imagePullPolicy: IfNotPresent
- volumeMounts:
- - name: configuration
- mountPath: /etc/swh
- volumes:
- - name: configuration
- emptyDir: {}
- - name: configuration-template
- configMap:
- name: web-configuration-template
- items:
- - key: "config.yml.template"
- path: "config.yml.template"
- restartPolicy: OnFailure
-
-{{ end }}
+{{- include "swh.web.cronjob" (dict "Values" .Values
+ "serviceType" "refresh-savecodenow-statuses"
+ "configuration" .Values.web.refreshSavecodenowStatus) -}}
{{- end -}}
diff --git a/swh/templates/web/sync-mailmaps-cronjob.yaml b/swh/templates/web/sync-mailmaps-cronjob.yaml
new file mode 100644
index 000000000..517d370ab
--- /dev/null
+++ b/swh/templates/web/sync-mailmaps-cronjob.yaml
@@ -0,0 +1,5 @@
+{{ if and .Values.web.enabled .Values.web.syncMailmaps.enabled -}}
+{{- include "swh.web.cronjob" (dict "Values" .Values
+ "serviceType" "sync-mailmaps"
+ "configuration" .Values.web.syncMailmaps) -}}
+{{- end -}}
diff --git a/swh/values.yaml b/swh/values.yaml
index 1079ffd7f..ce49026ac 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -664,6 +664,13 @@ web:
# concurrencyPolicy: Forbid
# At every minute
cron: "* * * * *"
+ syncMailmaps:
+ enabled: false
+ priorityClassName: frontend-rpc-workload
+ logLevel: INFO
+ # concurrencyPolicy: Forbid
+ # At minute 15 every hour
+ cron: "15 * * * *"
# gunicorn:
# threads: 5
# workers: 2
diff --git a/swh/values/minikube.yaml b/swh/values/minikube.yaml
index dfc3e8c79..b54ddc9bc 100644
--- a/swh/values/minikube.yaml
+++ b/swh/values/minikube.yaml
@@ -208,8 +208,7 @@ web:
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
nginx.ingress.kubernetes.io/proxy-body-size: "4G"
- whitelistSourceRange:
- - 192.168.100.0/24
+ whitelistSourceRangeRef: internalNetworkRanges
endpoints:
default:
paths:
@@ -229,7 +228,10 @@ web:
cpuPercentageUsage: 50
refreshSavecodenowStatus:
enabled: false
- cron: "* 9 * * *"
+ cron: "* 5 * * *"
+ syncMailmaps:
+ enabled: false
+ cron: "* 30 * * *"
sentry:
enabled: false
secretKeyRef: sentry-secrets
diff --git a/swh/values/staging/swh.yaml b/swh/values/staging/swh.yaml
index 88751cc5c..443dbba63 100644
--- a/swh/values/staging/swh.yaml
+++ b/swh/values/staging/swh.yaml
@@ -663,6 +663,10 @@ web:
extraWhitelistSourceRange:
# vpn network
- 192.168.101.0/24
+ refreshSavecodenowStatus:
+ enabled: true
+ syncMailmaps:
+ enabled: true
databaseConfigurationRef: postgresqlWebConfiguration
searchConfigurationRef: remoteSearchConfiguration
schedulerConfigurationRef: remoteSchedulerConfiguration
--
GitLab