From 396d9a74f206c1cb707f6712aeda89a7fc764ec6 Mon Sep 17 00:00:00 2001 From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org> Date: Tue, 19 Dec 2023 14:54:05 +0100 Subject: [PATCH] web: Make template multi-instance This adapts the configuration accordingly. Refs. swh/infra/sysadm-environment#5194 --- swh/templates/web/_helper-cronjob.yaml | 23 +- swh/templates/web/_helper_configmap.yaml | 83 +++++++ swh/templates/web/autoscaling.yaml | 14 +- swh/templates/web/configmap-pgservice.yaml | 17 +- swh/templates/web/configmap.yaml | 76 +------ swh/templates/web/deployment.yaml | 128 ++++++----- swh/templates/web/ingress.yaml | 15 +- .../refresh-savecodenow-statuses-cronjob.yaml | 19 +- swh/templates/web/service.yaml | 16 +- swh/templates/web/sync-mailmaps-cronjob.yaml | 21 +- swh/values.yaml | 214 +++++++++--------- swh/values/minikube.yaml | 171 +++++++------- swh/values/production/swh-cassandra.yaml | 130 +++++------ swh/values/production/swh.yaml | 168 +++++++------- .../overrides/swh-cassandra-next-version.yaml | 65 +++--- swh/values/staging/swh-cassandra.yaml | 168 +++++++------- swh/values/staging/swh.yaml | 151 ++++++------ 17 files changed, 791 insertions(+), 688 deletions(-) create mode 100644 swh/templates/web/_helper_configmap.yaml diff --git a/swh/templates/web/_helper-cronjob.yaml b/swh/templates/web/_helper-cronjob.yaml index 11f50f956..f0f2d86de 100644 --- a/swh/templates/web/_helper-cronjob.yaml +++ b/swh/templates/web/_helper-cronjob.yaml @@ -1,6 +1,6 @@ {{/* -Create a Kind CronJob for service .serviceType -*/}} + * Create a Kind CronJob for service .serviceType + */}} {{- define "swh.web.cronjob" -}} {{- with .configuration -}} {{- $log_level := .logLevel -}} @@ -8,6 +8,7 @@ apiVersion: batch/v1 kind: CronJob metadata: name: {{ $.serviceType }}-cronjob + namespace: {{ $.Values.namespace }} spec: schedule: {{ .cron | quote}} {{- if .concurrencyPolicy }} @@ -49,21 +50,21 @@ spec: - -c - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml env: - {{- if $.Values.web.databaseConfigurationRef }} + {{- if $.webConfiguration.databaseConfigurationRef }} {{- include "swh.secrets.environment" (dict "Values" $.Values - "configurationRef" $.Values.web.databaseConfigurationRef) | nindent 16 -}} + "configurationRef" $.webConfiguration.databaseConfigurationRef) | nindent 16 -}} {{ end }} - {{- if $.Values.web.djangoConfigurationRef }} + {{- if $.webConfiguration.djangoConfigurationRef }} {{- include "swh.secrets.environment" (dict "Values" $.Values - "configurationRef" $.Values.web.djangoConfigurationRef) | nindent 16 }} + "configurationRef" $.webConfiguration.djangoConfigurationRef) | nindent 16 }} {{ end }} - {{- if $.Values.web.depositConfigurationRef -}} + {{- if $.webConfiguration.depositConfigurationRef -}} {{- include "swh.secrets.environment" (dict "Values" $.Values - "configurationRef" $.Values.web.depositConfigurationRef) | nindent 16 }} + "configurationRef" $.webConfiguration.depositConfigurationRef) | nindent 16 }} {{ end }} - {{- if $.Values.web.giveConfigurationRef -}} + {{- if $.webConfiguration.giveConfigurationRef -}} {{- include "swh.secrets.environment" (dict "Values" $.Values - "configurationRef" $.Values.web.giveConfigurationRef) | nindent 16 }} + "configurationRef" $.webConfiguration.giveConfigurationRef) | nindent 16 }} {{ end }} {{- if $.Values.web.sentry.enabled }} - name: SWH_SENTRY_DSN @@ -149,7 +150,7 @@ spec: {{- if $.pgService }} - name: pgservice-configuration-template configMap: - name: pgservice-configuration-template + name: pgservice-{{ $.webType }}-configuration-template items: - key: "pg-service-conf" path: "pg_service.conf" diff --git a/swh/templates/web/_helper_configmap.yaml b/swh/templates/web/_helper_configmap.yaml new file mode 100644 index 000000000..af57549b0 --- /dev/null +++ b/swh/templates/web/_helper_configmap.yaml @@ -0,0 +1,83 @@ +{{/* + * Create a web configmap for service .serviceType + */}} +{{ define "swh.web.configmap" }} +{{- $hosts := .configuration.hosts }} +{{- $allowed_instance := first $hosts }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.namespace }} + name: {{ .serviceType }}-configuration-template +data: + config.yml.template: | + instance_name: {{ $allowed_instance }} + allowed_hosts: + {{- range $host := $hosts }} + - {{ $host }} + {{- end }} + {{- if and .Values.environment (eq .Values.environment "production") }} + production_server_names: + {{- range $host := $hosts }} + - {{ $host }} + {{- end }} + {{- end }} + {{- include "swh.service.fromYaml" (dict "service" "storage" + "configurationRef" .configuration.storageConfigurationRef + "Values" .Values) | nindent 4 }} + {{- if .configuration.searchConfigurationRef }} + {{- include "swh.service.fromYaml" (dict "service" "search" + "configurationRef" .configuration.searchConfigurationRef + "Values" .Values) | nindent 4 }} + {{- end -}} + {{- if .configuration.schedulerConfigurationRef }} + {{- include "swh.service.fromYaml" (dict "service" "scheduler" + "configurationRef" .configuration.schedulerConfigurationRef + "Values" .Values) | nindent 4 }} + {{- end -}} + {{- if .configuration.vaultConfigurationRef }} + {{- include "swh.service.fromYaml" (dict "service" "vault" + "configurationRef" .configuration.vaultConfigurationRef + "Values" .Values) | nindent 4 }} + {{- end -}} + {{- if .configuration.indexerStorageConfigurationRef }} + {{- include "swh.service.fromYaml" (dict "service" "indexer_storage" + "configurationRef" .configuration.indexerStorageConfigurationRef + "Values" .Values) | nindent 4 }} + {{- end -}} + {{- if .configuration.countersConfigurationRef }} + counters_backend: swh-counters + {{- include "swh.service.fromYaml" (dict "service" "counters" + "configurationRef" .configuration.countersConfigurationRef + "Values" .Values) | nindent 4 }} + {{- end -}} + {{- if .configuration.depositConfigurationRef }} + {{- include "deposit.configuration.api.private" (dict "configurationRef" .configuration.depositConfigurationRef + "Values" .Values) | nindent 4 }} + {{- end -}} + {{- if .configuration.addForgeNowConfigurationRef }} + {{- include "addforgenow.configuration" (dict "configurationRef" .configuration.addForgeNowConfigurationRef + "Values" .Values) | nindent 4 }} + {{- end -}} +{{/* TODO: Manage the webapp logging */}} +{{/* log_dir: */}} + secret_key: ${DJANGO_SECRET_KEY} + {{- if .configuration.databaseConfigurationRef }} + production_db: + {{- include "django.postgresql" (dict "configurationRef" .configuration.databaseConfigurationRef + "Values" .Values) | nindent 4 -}} + {{ end }} + {{- if $.Values.web.sentry.enabled }} + client_config: + sentry_dsn: ${SWH_SENTRY_DSN} + {{- end }} + {{- if .configuration.throttlingConfigurationRef -}} + {{- include "swh.web.throttling" (dict "configurationRef" .configuration.throttlingConfigurationRef + "Values" .Values) | nindent 4 -}} + {{- end }} + {{- if .configuration.extraConfig -}} + {{ toYaml .configuration.extraConfig | nindent 4 }} + {{- end }} + +{{- end -}} diff --git a/swh/templates/web/autoscaling.yaml b/swh/templates/web/autoscaling.yaml index fcbee1bc7..aafacf032 100644 --- a/swh/templates/web/autoscaling.yaml +++ b/swh/templates/web/autoscaling.yaml @@ -1,5 +1,11 @@ -{{- if and .Values.web.enabled .Values.web.autoScaling -}} -{{- include "swh.autoscale" (dict "Values" .Values - "serviceType" "web" - "configuration" .Values.web) -}} +{{- if and .Values.web.enabled -}} +{{ range $web_type, $web_config := .Values.web.deployments }} +{{- if and (or (not (hasKey $web_config "enabled")) (get $web_config "enabled")) + (hasKey $web_config "autoScaling") -}} +{{- include "swh.autoscale" (dict "Values" $.Values + "serviceType" ( print "web-" $web_type ) + "configuration" $web_config) -}} {{- end -}} +{{ end -}} +{{- end -}} + diff --git a/swh/templates/web/configmap-pgservice.yaml b/swh/templates/web/configmap-pgservice.yaml index 53a29dca6..d65e614f2 100644 --- a/swh/templates/web/configmap-pgservice.yaml +++ b/swh/templates/web/configmap-pgservice.yaml @@ -1,13 +1,20 @@ -{{- if and .Values.web.enabled .Values.web.syncMailmaps.enabled -}} +{{- if .Values.web.enabled -}} +{{ range $web_type, $web_config := .Values.web.deployments }} +{{- if and (or (not (hasKey $web_config "enabled")) + (get $web_config "enabled")) + (hasKey $web_config "syncMailmaps") + $web_config.syncMailmaps.enabled -}} --- apiVersion: v1 kind: ConfigMap metadata: - namespace: {{ .Values.namespace }} - name: pgservice-configuration-template + name: pgservice-{{ $web_type }}-configuration-template + namespace: {{ $.Values.namespace }} data: pg-service-conf: | {{- include "swh.web.pgService" (dict "serviceType" "syncmailmaps" - "configurationRef" .Values.web.syncMailmaps.configurationRef - "Values" .Values) | nindent 4 }} + "configurationRef" $web_config.syncMailmaps.configurationRef + "Values" $.Values) | nindent 4 }} +{{- end -}} +{{ end -}} {{- end -}} diff --git a/swh/templates/web/configmap.yaml b/swh/templates/web/configmap.yaml index 93a820b17..09c1fd889 100644 --- a/swh/templates/web/configmap.yaml +++ b/swh/templates/web/configmap.yaml @@ -1,69 +1,9 @@ -{{- if .Values.web.enabled -}} -{{- $hosts := .Values.web.hosts }} -{{- $allowed_instance := first $hosts }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ .Values.namespace }} - name: web-configuration-template -data: - config.yml.template: | - instance_name: {{ $allowed_instance }} - allowed_hosts: - {{- range $host := $hosts }} - - {{ $host }} - {{- end }} - {{- if and .Values.environment (eq .Values.environment "production") }} - production_server_names: - {{- range $host := $hosts }} - - {{ $host }} - {{- end }} - {{- end }} - {{- include "swh.service.fromYaml" (dict "service" "storage" - "configurationRef" .Values.web.storageConfigurationRef - "Values" .Values) | nindent 4 }} - {{- if .Values.web.searchConfigurationRef }} - {{- include "swh.service.fromYaml" (dict "service" "search" "configurationRef" .Values.web.searchConfigurationRef "Values" .Values) | nindent 4 }} - {{- end -}} - {{- if .Values.web.schedulerConfigurationRef }} - {{- include "swh.service.fromYaml" (dict "service" "scheduler" "configurationRef" .Values.web.schedulerConfigurationRef "Values" .Values) | nindent 4 }} - {{- end -}} - {{- if .Values.web.vaultConfigurationRef }} - {{- include "swh.service.fromYaml" (dict "service" "vault" "configurationRef" .Values.web.vaultConfigurationRef "Values" .Values) | nindent 4 }} - {{- end -}} - {{- if .Values.web.indexerStorageConfigurationRef }} - {{- include "swh.service.fromYaml" (dict "service" "indexer_storage" "configurationRef" .Values.web.indexerStorageConfigurationRef "Values" .Values) | nindent 4 }} - {{- end -}} - {{- if .Values.web.countersConfigurationRef }} - counters_backend: swh-counters - {{- include "swh.service.fromYaml" (dict "service" "counters" "configurationRef" .Values.web.countersConfigurationRef "Values" .Values) | nindent 4 }} - {{- end -}} - {{- if .Values.web.depositConfigurationRef }} - {{- include "deposit.configuration.api.private" (dict "configurationRef" .Values.web.depositConfigurationRef - "Values" .Values) | nindent 4 }} - {{- end -}} - {{- if .Values.web.addForgeNowConfigurationRef }} - {{- include "addforgenow.configuration" (dict "configurationRef" .Values.web.addForgeNowConfigurationRef "Values" .Values) | nindent 4 }} - {{- end -}} -{{/* TODO: Manage the webapp logging */}} -{{/* log_dir: */}} - secret_key: ${DJANGO_SECRET_KEY} - {{- if .Values.web.databaseConfigurationRef }} - production_db: - {{- include "django.postgresql" (dict "configurationRef" .Values.web.databaseConfigurationRef - "Values" .Values) | nindent 4 -}} - {{ end }} - {{- if .Values.web.sentry.enabled }} - client_config: - sentry_dsn: ${SWH_SENTRY_DSN} - {{- end }} - {{- if .Values.web.throttlingConfigurationRef -}} - {{- include "swh.web.throttling" (dict "configurationRef" .Values.web.throttlingConfigurationRef - "Values" .Values) | nindent 4 -}} - {{- end }} - {{- if .Values.web.extraConfig -}} - {{ toYaml .Values.web.extraConfig | nindent 4 }} - {{- end }} - +{{ if .Values.web.enabled -}} +{{ range $web_type, $web_config := .Values.web.deployments }} +{{- if or (not (hasKey $web_config "enabled")) (get $web_config "enabled") -}} +{{ include "swh.web.configmap" (dict "serviceType" (print "web-" $web_type) + "configuration" $web_config + "Values" $.Values) }} +{{- end -}} +{{ end -}} {{- end -}} diff --git a/swh/templates/web/deployment.yaml b/swh/templates/web/deployment.yaml index b582b2a99..e45c951f5 100644 --- a/swh/templates/web/deployment.yaml +++ b/swh/templates/web/deployment.yaml @@ -1,20 +1,25 @@ {{ if .Values.web.enabled -}} +{{ range $web_type, $web_config := .Values.web.deployments }} +{{- if or (not (hasKey $web_config "enabled")) (get $web_config "enabled") -}} +{{- $serviceType := ( print "web-" $web_type ) -}} +{{- $do_migrations := or (and $web_config.migrations $web_config.migrations.enabled) + (and $.Values.web.migrations $.Values.web.migrations.enabled) -}} --- apiVersion: apps/v1 kind: Deployment metadata: - namespace: {{ .Values.namespace }} - name: web + namespace: {{ $.Values.namespace }} + name: {{ $serviceType }} labels: - app: web + app: {{ $serviceType }} spec: revisionHistoryLimit: 2 - {{ if .Values.web.replicas -}} - replicas: {{ .Values.web.replicas }} + {{ if $web_config.replicas -}} + replicas: {{ $web_config.replicas }} {{ end -}} selector: matchLabels: - app: web + app: {{ $serviceType }} strategy: type: RollingUpdate rollingUpdate: @@ -22,13 +27,15 @@ spec: template: metadata: labels: - app: web + app: {{ $serviceType }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/web/configmap.yaml") . | sha256sum }} + checksum/config: {{ include "swh.web.configmap" (dict "serviceType" $serviceType + "configuration" $web_config + "Values" $.Values) | sha256sum }} spec: - {{- if .Values.web.affinity }} + {{- if $.Values.web.affinity }} affinity: - {{- toYaml .Values.web.affinity | nindent 8 }} + {{- toYaml $.Values.web.affinity | nindent 8 }} {{- end }} {{- if and $.Values.podPriority.enabled $.Values.web.priorityClassName }} priorityClassName: {{ $.Values.namespace }}-{{ $.Values.web.priorityClassName }} @@ -43,36 +50,36 @@ spec: - -c - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml env: - {{- if .Values.web.databaseConfigurationRef }} - {{- include "swh.secrets.environment" (dict "Values" .Values - "configurationRef" .Values.web.databaseConfigurationRef) | nindent 12 -}} + {{- if $web_config.databaseConfigurationRef }} + {{- include "swh.secrets.environment" (dict "Values" $.Values + "configurationRef" $web_config.databaseConfigurationRef) | nindent 12 -}} {{ end }} - {{- if .Values.web.djangoConfigurationRef }} - {{- include "swh.secrets.environment" (dict "Values" .Values - "configurationRef" .Values.web.djangoConfigurationRef) | nindent 12 }} + {{- if $web_config.djangoConfigurationRef }} + {{- include "swh.secrets.environment" (dict "Values" $.Values + "configurationRef" $web_config.djangoConfigurationRef) | nindent 12 }} {{ end }} - {{- if .Values.web.depositConfigurationRef -}} + {{- if $web_config.depositConfigurationRef -}} {{- include "swh.secrets.environment" (dict "Values" $.Values - "configurationRef" .Values.web.depositConfigurationRef) | nindent 12 }} + "configurationRef" $web_config.depositConfigurationRef) | nindent 12 }} {{ end }} - {{- if .Values.web.giveConfigurationRef -}} + {{- if $web_config.giveConfigurationRef -}} {{- include "swh.secrets.environment" (dict "Values" $.Values - "configurationRef" .Values.web.giveConfigurationRef) | nindent 12 }} + "configurationRef" $web_config.giveConfigurationRef) | nindent 12 }} {{ end }} - {{- if .Values.web.addForgeNowConfigurationRef -}} + {{- if $web_config.addForgeNowConfigurationRef -}} {{- include "swh.secrets.environment" (dict "Values" $.Values - "configurationRef" .Values.web.addForgeNowConfigurationRef) | nindent 12 }} + "configurationRef" $web_config.addForgeNowConfigurationRef) | nindent 12 }} {{- end -}} - {{- if .Values.web.webhooksConfigurationRef -}} + {{- if $web_config.webhooksConfigurationRef -}} {{- include "swh.secrets.environment" (dict "Values" $.Values - "configurationRef" .Values.web.webhooksConfigurationRef) | nindent 12 }} + "configurationRef" $web_config.webhooksConfigurationRef) | nindent 12 }} {{- end -}} - {{- if .Values.web.sentry.enabled }} + {{- if $.Values.web.sentry.enabled }} - name: SWH_SENTRY_DSN valueFrom: secretKeyRef: - name: {{ .Values.web.sentry.secretKeyRef }} - key: {{ .Values.web.sentry.secretKeyName }} + name: {{ $.Values.web.sentry.secretKeyRef }} + key: {{ $.Values.web.sentry.secretKeyName }} # 'name' secret should exist & include key # if the setting doesn't exist, sentry pushes will be disabled optional: false @@ -82,9 +89,9 @@ spec: mountPath: /etc/swh - name: configuration-template mountPath: /etc/swh/configuration-template - {{- if .Values.web.migrations.enabled }} + {{- if $do_migrations }} - name: do-migration - image: {{ .Values.swh_web_image }}:{{ .Values.swh_web_image_version }} + image: {{ $.Values.swh_web_image }}:{{ $.Values.swh_web_image_version }} imagePullPolicy: IfNotPresent env: - name: SWH_CONFIG_FILENAME @@ -99,7 +106,7 @@ spec: mountPath: /etc/swh {{ end }} - name: prepare-static - image: {{ .Values.swh_web_image }}:{{ .Values.swh_web_image_version }} + image: {{ $.Values.swh_web_image }}:{{ $.Values.swh_web_image_version }} imagePullPolicy: IfNotPresent command: - /bin/bash @@ -110,24 +117,24 @@ spec: - name: static mountPath: /usr/share/swh/web/static containers: - - name: web + - name: {{ $serviceType }} resources: requests: - memory: {{ .Values.web.requestedMemory | default "512Mi" }} - cpu: {{ .Values.web.requestedCpu | default "500m" }} - {{- if or .Values.web.limitedMemory .Values.web.limitedCpu }} + memory: {{ $web_config.requestedMemory | default "512Mi" }} + cpu: {{ $web_config.requestedCpu | default "500m" }} + {{- if or $web_config.limitedMemory $web_config.limitedCpu }} limits: - {{- if .Values.web.limitedMemory }} - memory: {{ .Values.web.limitedMemory }} + {{- if $web_config.limitedMemory }} + memory: {{ $web_config.limitedMemory }} {{- end }} - {{- if .Values.web.limitedCpu }} - cpu: {{ .Values.web.limitedCpu }} + {{- if $web_config.limitedCpu }} + cpu: {{ $web_config.limitedCpu }} {{- end }} {{ end }} - image: {{ .Values.swh_web_image }}:{{ .Values.swh_web_image_version }} + image: {{ $.Values.swh_web_image }}:{{ $.Values.swh_web_image_version }} imagePullPolicy: IfNotPresent ports: - - containerPort: {{ .Values.web.port }} + - containerPort: {{ $web_config.port | default $.Values.web.port }} name: webapp readinessProbe: httpGet: @@ -135,7 +142,7 @@ spec: port: webapp httpHeaders: - name: Host - value: {{ first .Values.web.hosts }} + value: {{ first $web_config.hosts }} initialDelaySeconds: 5 failureThreshold: 30 periodSeconds: 10 @@ -146,7 +153,7 @@ spec: port: webapp httpHeaders: - name: Host - value: {{ first .Values.web.hosts }} + value: {{ first $web_config.hosts }} initialDelaySeconds: 3 periodSeconds: 10 timeoutSeconds: 30 @@ -156,41 +163,41 @@ spec: - -c - /opt/swh/entrypoint.sh env: - {{ if .Values.web.gunicorn -}} + {{ if $web_config.gunicorn -}} - name: THREADS - value: {{ .Values.web.gunicorn.threads | default 5 | quote }} + value: {{ $web_config.gunicorn.threads | default 5 | quote }} - name: WORKERS - value: {{ .Values.web.gunicorn.workers | default 2 | quote }} + value: {{ $web_config.gunicorn.workers | default 2 | quote }} - name: TIMEOUT - value: {{ .Values.web.gunicorn.timeout | default 60 | quote }} + value: {{ $web_config.gunicorn.timeout | default 60 | quote }} {{ end -}} - name: STATSD_HOST - value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }} + value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }} - name: STATSD_PORT - value: {{ .Values.statsdPort | default "9125" | quote }} + value: {{ $.Values.statsdPort | default "9125" | quote }} - name: LOG_LEVEL - value: {{ .Values.web.logLevel | quote }} + value: {{ $web_config.logLevel | default $.Values.web.logLevel | quote }} - name: SWH_CONFIG_FILENAME value: /etc/swh/config.yml - {{- if .Values.web.sentry.enabled }} + {{- if $.Values.web.sentry.enabled }} - name: SWH_SENTRY_ENVIRONMENT - value: {{ .Values.sentry.environment }} + value: {{ $.Values.sentry.environment }} - name: SWH_MAIN_PACKAGE value: swh.web - name: SWH_SENTRY_DSN valueFrom: secretKeyRef: - name: {{ .Values.web.sentry.secretKeyRef }} - key: {{ .Values.web.sentry.secretKeyName }} + name: {{ $.Values.web.sentry.secretKeyRef }} + key: {{ $.Values.web.sentry.secretKeyName }} # 'name' secret should exist & include key # if the setting doesn't exist, sentry pushes will be disabled optional: true - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS value: "true" {{- end }} - {{- if .Values.web.djangoConfigurationRef }} - {{- include "swh.secrets.environment" (dict "Values" .Values - "configurationRef" .Values.web.djangoConfigurationRef) | nindent 12 }} + {{- if $web_config.djangoConfigurationRef }} + {{- include "swh.secrets.environment" (dict "Values" $.Values + "configurationRef" $web_config.djangoConfigurationRef) | nindent 12 }} {{ end }} volumeMounts: - name: configuration @@ -199,8 +206,8 @@ spec: - name: nginx resources: requests: - memory: {{ .Values.web.nginxRequestedMemory | default "50Mi" }} - cpu: {{ .Values.web.nginxRequestedCpu | default "10m" }} + memory: {{ $web_config.nginxRequestedMemory | default "50Mi" }} + cpu: {{ $web_config.nginxRequestedCpu | default "10m" }} image: nginx:bullseye imagePullPolicy: IfNotPresent ports: @@ -227,10 +234,13 @@ spec: emptyDir: {} - name: configuration-template configMap: - name: web-configuration-template + name: {{ $serviceType }}-configuration-template items: - key: "config.yml.template" path: "config.yml.template" - name: static emptyDir: {} {{ end }} +{{ end -}} +{{- end -}} + diff --git a/swh/templates/web/ingress.yaml b/swh/templates/web/ingress.yaml index 09b5b9aa0..18b3d5570 100644 --- a/swh/templates/web/ingress.yaml +++ b/swh/templates/web/ingress.yaml @@ -1,5 +1,10 @@ -{{ if and .Values.web.enabled .Values.web.ingress.enabled -}} -{{- include "swh.ingress" (dict "Values" .Values - "serviceType" "web" - "configuration" .Values.web) -}} -{{ end }} +{{ if .Values.web.enabled -}} +{{ range $web_type, $web_config := .Values.web.deployments }} +{{- if and (or (not (hasKey $web_config "enabled")) (get $web_config "enabled")) + (and (hasKey $web_config "ingress") $web_config.ingress.enabled) -}} +{{- include "swh.ingress" (dict "serviceType" ( print "web-" $web_type ) + "configuration" $web_config + "Values" $.Values) -}} +{{- end -}} +{{ end -}} +{{- end -}} diff --git a/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml b/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml index df031bf72..c80030000 100644 --- a/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml +++ b/swh/templates/web/refresh-savecodenow-statuses-cronjob.yaml @@ -1,6 +1,15 @@ -{{ if and .Values.web.enabled .Values.web.refreshSavecodenowStatus.enabled -}} -{{- include "swh.web.cronjob" (dict "Values" .Values - "serviceType" "refresh-savecodenow-statuses" - "configuration" .Values.web.refreshSavecodenowStatus - "command" (list "refresh")) -}} +{{ if .Values.web.enabled -}} +{{ range $web_type, $web_config := .Values.web.deployments }} +{{- if and (or (not (hasKey $web_config "enabled")) + (get $web_config "enabled")) + (hasKey $web_config "refreshSavecodenowStatus") + $web_config.refreshSavecodenowStatus.enabled -}} +{{- include "swh.web.cronjob" (dict "Values" $.Values + "serviceType" (print "web-" $web_type "-refresh-savecodenow-statuses") + "configuration" $web_config.refreshSavecodenowStatus + "webConfiguration" $web_config + "webType" $web_type + "command" (list "refresh")) -}} +{{- end -}} +{{ end -}} {{- end -}} diff --git a/swh/templates/web/service.yaml b/swh/templates/web/service.yaml index 3d484447b..e4c71ad50 100644 --- a/swh/templates/web/service.yaml +++ b/swh/templates/web/service.yaml @@ -1,5 +1,15 @@ {{ if .Values.web.enabled -}} -{{- include "swh.service" (dict "Values" .Values - "serviceType" "web" - "configuration" .Values.web) -}} +{{ range $web_type, $web_config := .Values.web.deployments }} +{{- if or (not (hasKey $web_config "enabled")) (get $web_config "enabled") -}} +{{- if not (hasKey $web_config "port") }} +{{- $web_config := set $web_config "port" $.Values.web.port -}} +{{- end -}} +{{- if not (hasKey $web_config "extraPorts") }} +{{- $web_config := set $web_config "extraPorts" $.Values.web.extraPorts -}} +{{- end -}} +{{- include "swh.service" (dict "serviceType" ( print "web-" $web_type ) + "configuration" $web_config + "Values" $.Values) -}} +{{- end -}} +{{ end -}} {{- end -}} diff --git a/swh/templates/web/sync-mailmaps-cronjob.yaml b/swh/templates/web/sync-mailmaps-cronjob.yaml index 0b4b6d6f1..d0122b81c 100644 --- a/swh/templates/web/sync-mailmaps-cronjob.yaml +++ b/swh/templates/web/sync-mailmaps-cronjob.yaml @@ -1,7 +1,16 @@ -{{ if and .Values.web.enabled .Values.web.syncMailmaps.enabled -}} -{{- include "swh.web.cronjob" (dict "Values" .Values - "serviceType" "sync-mailmaps" - "configuration" .Values.web.syncMailmaps - "pgService" true - "command" (list "sync-mailmaps" "service=syncmailmaps")) -}} +{{ if .Values.web.enabled -}} +{{ range $web_type, $web_config := .Values.web.deployments }} +{{- if and (or (not (hasKey $web_config "enabled")) + (get $web_config "enabled")) + (hasKey $web_config "syncMailmaps") + $web_config.syncMailmaps.enabled -}} +{{- include "swh.web.cronjob" (dict "Values" $.Values + "serviceType" (print "web-" $web_type "-sync-mailmaps") + "configuration" $web_config.syncMailmaps + "webConfiguration" $web_config + "pgService" true + "webType" $web_type + "command" (list "sync-mailmaps" "service=syncmailmaps")) -}} +{{- end -}} +{{ end -}} {{- end -}} diff --git a/swh/values.yaml b/swh/values.yaml index 4d3a53c17..9e87c4f43 100644 --- a/swh/values.yaml +++ b/swh/values.yaml @@ -719,40 +719,12 @@ storage: web: enabled: false port: 5004 - extraPorts: - webstatic: 80 + logLevel: INFO migrations: enabled: false priorityClassName: frontend-rpc - # debug: false - logLevel: INFO - requestedCpu: 50m - requestedMemory: 100Mi - # nginxRequestedCpu: 10m - # nginxRequestedMemory: 50m - # replicas: 1 - # autoScaling: - # minReplicaCount: 2 - # maxReplicaCount: 10 - # cpuPercentageUsage: 50 - refreshSavecodenowStatus: - enabled: false - priorityClassName: frontend-rpc-workload - logLevel: INFO - # concurrencyPolicy: Forbid - # Every 2 minute - cron: "*/2 * * * *" - syncMailmaps: - enabled: false - priorityClassName: frontend-rpc-workload - logLevel: INFO - # concurrencyPolicy: Forbid - # At minute 15 every hour - cron: "15 * * * *" -# gunicorn: -# threads: 5 -# workers: 2 -# timeout: 60 + extraPorts: + webstatic: 80 sentry: enabled: false # name of the secret containing the $secretKeyName value @@ -760,77 +732,115 @@ web: # like https://token@sentry.host/id secretKeyRef: common-secrets secretKeyName: web-sentry-dsn - # host: webapp - ingress: - enabled: false - # extraAnnotations: - # cert-manager.io/cluster-issuer: letsencrypt-production-gandi - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - # nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - # tlsEnabled: true - # whitelistSourceRangeRef: internalNetworkRanges - endpoints: - default: - paths: - - path: / - - path: /static - port: 80 - # authenticated: - # paths: - # - path: /api/1/provenance/ - # - path: /api/1/entity/ - # - path: /api/1/content/[^/]+/symbol/ - # # auth-file with authentication - # authentication: swh/ingress-auth - # searchConfigurationRef: searchConfiguration - # Configure the scheduler instance used for save code now requests - # schedulerConfigurationRef: schedulerConfiguration - # storageConfigurationRef: globalROStorageConfiguration - # vaultConfigurationRef: remoteVaultConfiguration - # indexerStorageConfigurationRef: remoteIndexerStorageConfiguration - # countersConfigurationRef: remoteCountersConfiguration - # djangoConfigurationRef: djangoWebConfiguration - # If not specified, this will use a sqlite db which is not performant enough for - # production use. - # databaseConfigurationRef: postgresqlWebConfiguration - # depositConfigurationRef: depositConfiguration - # giveConfigurationRef: giveConfiguration - # throttlingConfigurationRef: throttlingConfiguration - # extraConfig: - # debug: false - # history_counters_url: http://elastichost.swh.org:5011/counters_history/history.json - # es_workers_index_url: http://elastichost.swh.org:9200/swh_workers-* - # keycloak: - # server_url: https://auth.swh.org/auth/ - # realm_name: SWHRealm - # search_config: - # # swh-indexer-storage or swh-search - # metadata_backend: swh-indexer-storage - # # max content size in bytes - # content_display_max_size: 5242880 - # swh_extra_django_apps: - # - swh.web.add_forge_now - # - swh.web.archive_coverage - # - swh.web.badges - # - swh.web.banners - # - swh.web.deposit - # - swh.web.inbound_email - # - swh.web.jslicenses - # - swh.web.mailmap - # - swh.web.metrics - # - swh.web.save_code_now - # - swh.web.save_origin_webhooks - # - swh.web.vault - # add_forge_now: - # email_address: add-forge-now@archive.swh.org - # deposit: - # private_api_url: "https://deposit-rp.i.s.s.n/1/private/" - # private_api_user: "${DEPOSIT_USERNAME}" - # private_api_password: "${DEPOSIT_PASSWORD}" - # give: - # public_key: ${GIVE_PUBLIC_KEY} - # token: ${GIVE_PRIVATE_TOKEN} + deployments: + instance: + enabled: false + # can be overriden per instance + # port: 5004 + # extraPorts: + # webstatic: 80 + # migrations: + # enabled: truew + # logLevel: INFO + # # debug: false + # requestedCpu: 50m + # requestedMemory: 100Mi + # nginxRequestedCpu: 10m + # nginxRequestedMemory: 50m + # replicas: 1 + # autoScaling: + # minReplicaCount: 2 + # maxReplicaCount: 10 + # cpuPercentageUsage: 50 + refreshSavecodenowStatus: + enabled: false + priorityClassName: frontend-rpc-workload + logLevel: INFO + # concurrencyPolicy: Forbid + # Every 2 minute + cron: "*/2 * * * *" + syncMailmaps: + enabled: false + priorityClassName: frontend-rpc-workload + logLevel: INFO + # concurrencyPolicy: Forbid + # At minute 15 every hour + cron: "15 * * * *" + # gunicorn: + # threads: 5 + # workers: 2 + # timeout: 60 + # host: webapp + ingress: + enabled: false + # extraAnnotations: + # cert-manager.io/cluster-issuer: letsencrypt-production-gandi + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + # tlsEnabled: true + # whitelistSourceRangeRef: internalNetworkRanges + endpoints: + default: + paths: + - path: / + - path: /static + port: 80 + # authenticated: + # paths: + # - path: /api/1/provenance/ + # - path: /api/1/entity/ + # - path: /api/1/content/[^/]+/symbol/ + # # auth-file with authentication + # authentication: swh/ingress-auth + # searchConfigurationRef: searchConfiguration + # # Configure the scheduler instance used for save code now requests + # schedulerConfigurationRef: schedulerConfiguration + # storageConfigurationRef: globalROStorageConfiguration + # vaultConfigurationRef: remoteVaultConfiguration + # indexerStorageConfigurationRef: remoteIndexerStorageConfiguration + # countersConfigurationRef: remoteCountersConfiguration + # djangoConfigurationRef: djangoWebConfiguration + # # If not specified, this will use a sqlite db which is not performant enough for + # # production use. + # databaseConfigurationRef: postgresqlWebConfiguration + # depositConfigurationRef: depositConfiguration + # giveConfigurationRef: giveConfiguration + # throttlingConfigurationRef: throttlingConfiguration + # extraConfig: + # debug: false + # history_counters_url: http://elastichost.swh.org:5011/counters_history/history.json + # es_workers_index_url: http://elastichost.swh.org:9200/swh_workers-* + # keycloak: + # server_url: https://auth.swh.org/auth/ + # realm_name: SWHRealm + # search_config: + # # swh-indexer-storage or swh-search + # metadata_backend: swh-indexer-storage + # # max content size in bytes + # content_display_max_size: 5242880 + # swh_extra_django_apps: + # - swh.web.add_forge_now + # - swh.web.archive_coverage + # - swh.web.badges + # - swh.web.banners + # - swh.web.deposit + # - swh.web.inbound_email + # - swh.web.jslicenses + # - swh.web.mailmap + # - swh.web.metrics + # - swh.web.save_code_now + # - swh.web.save_origin_webhooks + # - swh.web.vault + # add_forge_now: + # email_address: add-forge-now@archive.swh.org + # deposit: + # private_api_url: "https://deposit-rp.i.s.s.n/1/private/" + # private_api_user: "${DEPOSIT_USERNAME}" + # private_api_password: "${DEPOSIT_PASSWORD}" + # give: + # public_key: ${GIVE_PUBLIC_KEY} + # token: ${GIVE_PRIVATE_TOKEN} statsd_exporter: enabled: false diff --git a/swh/values/minikube.yaml b/swh/values/minikube.yaml index 06cab9e37..93d293f51 100644 --- a/swh/values/minikube.yaml +++ b/swh/values/minikube.yaml @@ -263,91 +263,94 @@ webThrottling: web: enabled: false - requestedMemory: 50Mi - requestedCpu: 50m - storageConfigurationRef: fakeRemoteStorageConfiguration + deployments: + minikube: + enabled: true + hosts: + - webapp.internal.minikube + requestedMemory: 50Mi + requestedCpu: 50m + storageConfigurationRef: fakeRemoteStorageConfiguration - searchConfigurationRef: fakeRemoteSearchConfiguration - schedulerConfigurationRef: fakeRemoteSchedulerConfiguration - vaultConfigurationRef: fakeRemoteVaultConfiguration - countersConfigurationRef: fakeRemoteCountersConfiguration - databaseConfigurationRef: fakePostgresqlWebConfiguration - djangoConfigurationRef: djangoWebConfiguration - depositConfigurationRef: fakeDepositConfiguration - giveConfigurationRef: giveConfiguration - throttlingConfigurationRef: webThrottling - hosts: - - webapp.internal.minikube - ingress: - enabled: true - tlsEnabled: true - extraAnnotations: - nginx.ingress.kubernetes.io/proxy-connect-timeout: "90" - nginx.ingress.kubernetes.io/proxy-send-timeout: "90" - nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" - nginx.ingress.kubernetes.io/proxy-request-buffering: "on" - nginx.ingress.kubernetes.io/proxy-body-size: "4G" - whitelistSourceRangeRef: internalNetworkRanges - endpoints: - default: - paths: - - path: / - - path: /static - port: 80 - authenticated: - paths: - - path: /api/1/provenance/ - - path: /api/1/entity/ - - path: /api/1/content/[^/]+/symbol/ - # auth-file with authentication - authentication: basic-auth - autoScaling: - minReplicaCount: 1 - maxReplicaCount: 2 - cpuPercentageUsage: 50 - refreshSavecodenowStatus: - enabled: false - cron: "* 5 * * *" - syncMailmaps: - enabled: false - cron: "30 * * * *" - configurationRef: fakeSyncmailmapsConfiguration - sentry: - enabled: false - secretKeyRef: sentry-secrets - secretKeyName: web-sentry-dsn - extraConfig: - debug: false - keycloak: - server_url: https://auth.s.o/auth/ - realm_name: SwhMinikube - search_config: - metadata_backend: swh-indexer-storage - content_display_max_size: 5242880 - history_counters_url: http://counters0.i.s.s.n:5011/counters_history/history.json - add_forge_now: - email_address: add-forge-now@webapp.s.s.n - swh_extra_django_apps: - - swh.web.add_forge_now - - swh.web.archive_coverage - - swh.web.badges - - swh.web.banners - - swh.web.deposit - - swh.web.inbound_email - - swh.web.jslicenses - - swh.web.mailmap - - swh.web.metrics - - swh.web.save_code_now - - swh.web.save_origin_webhooks - - swh.web.vault - matomo: {} - deposit: - private_api_url: "https://deposit-rp.i.s.s.n/1/private/" - private_api_user: "${DEPOSIT_USERNAME}" - private_api_password: "${DEPOSIT_PASSWORD}" - give: - public_key: ${GIVE_PUBLIC_KEY} - token: ${GIVE_PRIVATE_TOKEN} + searchConfigurationRef: fakeRemoteSearchConfiguration + schedulerConfigurationRef: fakeRemoteSchedulerConfiguration + vaultConfigurationRef: fakeRemoteVaultConfiguration + countersConfigurationRef: fakeRemoteCountersConfiguration + databaseConfigurationRef: fakePostgresqlWebConfiguration + djangoConfigurationRef: djangoWebConfiguration + depositConfigurationRef: fakeDepositConfiguration + giveConfigurationRef: giveConfiguration + throttlingConfigurationRef: webThrottling + ingress: + enabled: true + tlsEnabled: true + extraAnnotations: + nginx.ingress.kubernetes.io/proxy-connect-timeout: "90" + nginx.ingress.kubernetes.io/proxy-send-timeout: "90" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-request-buffering: "on" + nginx.ingress.kubernetes.io/proxy-body-size: "4G" + whitelistSourceRangeRef: internalNetworkRanges + endpoints: + default: + paths: + - path: / + - path: /static + port: 80 + authenticated: + paths: + - path: /api/1/provenance/ + - path: /api/1/entity/ + - path: /api/1/content/[^/]+/symbol/ + # auth-file with authentication + authentication: basic-auth + autoScaling: + minReplicaCount: 1 + maxReplicaCount: 2 + cpuPercentageUsage: 50 + refreshSavecodenowStatus: + enabled: false + cron: "* 5 * * *" + syncMailmaps: + enabled: false + cron: "30 * * * *" + configurationRef: fakeSyncmailmapsConfiguration + sentry: + enabled: false + secretKeyRef: sentry-secrets + secretKeyName: web-sentry-dsn + extraConfig: + debug: false + keycloak: + server_url: https://auth.s.o/auth/ + realm_name: SwhMinikube + search_config: + metadata_backend: swh-indexer-storage + content_display_max_size: 5242880 + history_counters_url: http://counters0.i.s.s.n:5011/counters_history/history.json + add_forge_now: + email_address: add-forge-now@webapp.s.s.n + swh_extra_django_apps: + - swh.web.add_forge_now + - swh.web.archive_coverage + - swh.web.badges + - swh.web.banners + - swh.web.deposit + - swh.web.inbound_email + - swh.web.jslicenses + - swh.web.mailmap + - swh.web.metrics + - swh.web.save_code_now + - swh.web.save_origin_webhooks + - swh.web.vault + matomo: {} + deposit: + private_api_url: "https://deposit-rp.i.s.s.n/1/private/" + private_api_user: "${DEPOSIT_USERNAME}" + private_api_password: "${DEPOSIT_PASSWORD}" + give: + public_key: ${GIVE_PUBLIC_KEY} + token: ${GIVE_PRIVATE_TOKEN} fakeAzureDepositConfiguration: connection_string: "DefaultEndpointsProtocol=https;AccountName=swhdepositstoragestaging;AccountKey=${ACCOUNT_KEY};EndpointSuffix=core.windows.net" diff --git a/swh/values/production/swh-cassandra.yaml b/swh/values/production/swh-cassandra.yaml index 966943671..61e06b5de 100644 --- a/swh/values/production/swh-cassandra.yaml +++ b/swh/values/production/swh-cassandra.yaml @@ -260,71 +260,73 @@ postgresqlWebConfiguration: web: enabled: true logLevel: INFO - requestedCpu: 500m - requestedMemory: 500Mi - autoScaling: - minReplicaCount: 2 - maxReplicaCount: 4 - cpuPercentageUsage: 50 - hosts: - - webapp-cassandra.internal.softwareheritage.org - ingress: - enabled: true - secretName: swh-web-crt - extraAnnotations: - cert-manager.io/cluster-issuer: letsencrypt-production-gandi - kubernetes.io/ingress.class: nginx - kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - tlsEnabled: true - whitelistSourceRangeRef: internalNetworkRanges - endpoints: - default: - paths: - - path: / - - path: /static - port: 80 - extraWhitelistSourceRange: - - 192.168.50.0/24 # for blackbox monitoring - - 192.168.101.0/24 # vpn network + deployments: + cassandra: + requestedCpu: 500m + requestedMemory: 500Mi + autoScaling: + minReplicaCount: 2 + maxReplicaCount: 4 + cpuPercentageUsage: 50 + hosts: + - webapp-cassandra.internal.softwareheritage.org + ingress: + enabled: true + secretName: swh-web-crt + extraAnnotations: + cert-manager.io/cluster-issuer: letsencrypt-production-gandi + kubernetes.io/ingress.class: nginx + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + tlsEnabled: true + whitelistSourceRangeRef: internalNetworkRanges + endpoints: + default: + paths: + - path: / + - path: /static + port: 80 + extraWhitelistSourceRange: + - 192.168.50.0/24 # for blackbox monitoring + - 192.168.101.0/24 # vpn network - databaseConfigurationRef: postgresqlWebConfiguration - searchConfigurationRef: remoteSearchConfiguration - schedulerConfigurationRef: remoteSchedulerConfiguration - storageConfigurationRef: remoteStorageConfiguration - vaultConfigurationRef: remoteVaultConfiguration - indexerStorageConfigurationRef: remoteIndexerStorageConfiguration - countersConfigurationRef: remoteCountersConfiguration - djangoConfigurationRef: djangoWebConfiguration - giveConfigurationRef: giveConfiguration - throttlingConfigurationRef: webThrottling - addForgeNowConfigurationRef: addForgeNowConfiguration - depositConfigurationRef: depositConfiguration - extraConfig: - keycloak: - server_url: https://auth.softwareheritage.org/auth/ - realm_name: SoftwareHeritage - search_config: - metadata_backend: swh-search - content_display_max_size: 5242880 - history_counters_url: http://counters1.internal.softwareheritage.org:5011/counters_history/history.json# - es_workers_index_url: http://esnode1.internal.softwareheritage.org:9200/swh_workers-* - swh_extra_django_apps: - - swh.web.add_forge_now - - swh.web.archive_coverage - - swh.web.badges - - swh.web.banners - - swh.web.deposit - - swh.web.inbound_email - - swh.web.jslicenses - - swh.web.mailmap - - swh.web.metrics - - swh.web.save_code_now - - swh.web.save_origin_webhooks - - swh.web.vault - give: - public_key: ${GIVE_PUBLIC_KEY} - token: ${GIVE_PRIVATE_TOKEN} + databaseConfigurationRef: postgresqlWebConfiguration + searchConfigurationRef: remoteSearchConfiguration + schedulerConfigurationRef: remoteSchedulerConfiguration + storageConfigurationRef: remoteStorageConfiguration + vaultConfigurationRef: remoteVaultConfiguration + indexerStorageConfigurationRef: remoteIndexerStorageConfiguration + countersConfigurationRef: remoteCountersConfiguration + djangoConfigurationRef: djangoWebConfiguration + giveConfigurationRef: giveConfiguration + throttlingConfigurationRef: webThrottling + addForgeNowConfigurationRef: addForgeNowConfiguration + depositConfigurationRef: depositConfiguration + extraConfig: + keycloak: + server_url: https://auth.softwareheritage.org/auth/ + realm_name: SoftwareHeritage + search_config: + metadata_backend: swh-search + content_display_max_size: 5242880 + history_counters_url: http://counters1.internal.softwareheritage.org:5011/counters_history/history.json# + es_workers_index_url: http://esnode1.internal.softwareheritage.org:9200/swh_workers-* + swh_extra_django_apps: + - swh.web.add_forge_now + - swh.web.archive_coverage + - swh.web.badges + - swh.web.banners + - swh.web.deposit + - swh.web.inbound_email + - swh.web.jslicenses + - swh.web.mailmap + - swh.web.metrics + - swh.web.save_code_now + - swh.web.save_origin_webhooks + - swh.web.vault + give: + public_key: ${GIVE_PUBLIC_KEY} + token: ${GIVE_PRIVATE_TOKEN} graphql: enabled: true diff --git a/swh/values/production/swh.yaml b/swh/values/production/swh.yaml index fc0475be2..01d09a1d3 100644 --- a/swh/values/production/swh.yaml +++ b/swh/values/production/swh.yaml @@ -1063,87 +1063,93 @@ storage: web: enabled: true logLevel: INFO - requestedCpu: 500m - requestedMemory: 1024Mi - refreshSavecodenowStatus: - enabled: true - concurrencyPolicy: Forbid - syncMailmaps: - enabled: true - concurrencyPolicy: Forbid - configurationRef: postgresqlSyncmailmapsConfiguration - replicas: 2 - # autoScaling: - # minReplicaCount: 2 - # maxReplicaCount: 4 - # cpuPercentageUsage: 100 - hosts: - - webapp1.internal.softwareheritage.org - - archive.softwareheritage.org - - base.softwareheritage.org - - archive.internal.softwareheritage.org - ingress: - enabled: true - secretName: swh-web-crt - extraAnnotations: - cert-manager.io/cluster-issuer: letsencrypt-production-gandi - kubernetes.io/ingress.class: nginx - kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - tlsEnabled: true - endpoints: - default: - paths: - - path: / - - path: /static - port: 80 - authenticated: - paths: - - path: /api/1/provenance/ - - path: /api/1/entity/ - - path: /api/1/content/[^/]+/symbol/ - # auth-file with authentication - authentication: swh/web-auth-secrets - databaseConfigurationRef: postgresqlWebConfiguration - searchConfigurationRef: remoteSearchConfiguration - schedulerConfigurationRef: remoteSchedulerConfiguration - storageConfigurationRef: remoteStorageConfiguration - vaultConfigurationRef: remoteVaultConfiguration - indexerStorageConfigurationRef: remoteReadOnlyIndexerStorageConfiguration - countersConfigurationRef: remoteCountersConfiguration - djangoConfigurationRef: djangoWebConfiguration - giveConfigurationRef: giveConfiguration - throttlingConfigurationRef: webThrottling - addForgeNowConfigurationRef: addForgeNowConfiguration - depositConfigurationRef: depositConfiguration - extraConfig: - keycloak: - server_url: https://auth.softwareheritage.org/auth/ - realm_name: SoftwareHeritage - search_config: - metadata_backend: swh-search - content_display_max_size: 5242880 - history_counters_url: http://counters1.internal.softwareheritage.org:5011/counters_history/history.json# - es_workers_index_url: http://esnode1.internal.softwareheritage.org:9200/swh_workers-* - swh_extra_django_apps: - - swh.web.add_forge_now - - swh.web.archive_coverage - - swh.web.badges - - swh.web.banners - - swh.web.deposit - - swh.web.inbound_email - - swh.web.jslicenses - - swh.web.mailmap - - swh.web.metrics - - swh.web.save_code_now - - swh.web.save_origin_webhooks - - swh.web.vault - give: - public_key: ${GIVE_PUBLIC_KEY} - token: ${GIVE_PRIVATE_TOKEN} - matomo: - url: https://piwik.inria.fr/ - site_id: 59 + deployments: + app1: + requestedCpu: 500m + requestedMemory: 1024Mi + refreshSavecodenowStatus: + enabled: true + concurrencyPolicy: Forbid + priorityClassName: frontend-rpc-workload + cron: "*/2 * * * *" + syncMailmaps: + enabled: true + concurrencyPolicy: Forbid + configurationRef: postgresqlSyncmailmapsConfiguration + priorityClassName: frontend-rpc-workload + cron: "15 * * * *" + replicas: 2 + # autoScaling: + # minReplicaCount: 2 + # maxReplicaCount: 4 + # cpuPercentageUsage: 100 + hosts: + - webapp1.internal.softwareheritage.org + - archive.softwareheritage.org + - base.softwareheritage.org + - archive.internal.softwareheritage.org + ingress: + enabled: true + secretName: swh-web-crt + extraAnnotations: + cert-manager.io/cluster-issuer: letsencrypt-production-gandi + kubernetes.io/ingress.class: nginx + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + tlsEnabled: true + endpoints: + default: + paths: + - path: / + - path: /static + port: 80 + authenticated: + paths: + - path: /api/1/provenance/ + - path: /api/1/entity/ + - path: /api/1/content/[^/]+/symbol/ + # auth-file with authentication + authentication: swh/web-auth-secrets + databaseConfigurationRef: postgresqlWebConfiguration + searchConfigurationRef: remoteSearchConfiguration + schedulerConfigurationRef: remoteSchedulerConfiguration + storageConfigurationRef: remoteStorageConfiguration + vaultConfigurationRef: remoteVaultConfiguration + indexerStorageConfigurationRef: remoteReadOnlyIndexerStorageConfiguration + countersConfigurationRef: remoteCountersConfiguration + djangoConfigurationRef: djangoWebConfiguration + giveConfigurationRef: giveConfiguration + throttlingConfigurationRef: webThrottling + addForgeNowConfigurationRef: addForgeNowConfiguration + depositConfigurationRef: depositConfiguration + extraConfig: + keycloak: + server_url: https://auth.softwareheritage.org/auth/ + realm_name: SoftwareHeritage + search_config: + metadata_backend: swh-search + content_display_max_size: 5242880 + history_counters_url: http://counters1.internal.softwareheritage.org:5011/counters_history/history.json# + es_workers_index_url: http://esnode1.internal.softwareheritage.org:9200/swh_workers-* + swh_extra_django_apps: + - swh.web.add_forge_now + - swh.web.archive_coverage + - swh.web.badges + - swh.web.banners + - swh.web.deposit + - swh.web.inbound_email + - swh.web.jslicenses + - swh.web.mailmap + - swh.web.metrics + - swh.web.save_code_now + - swh.web.save_origin_webhooks + - swh.web.vault + give: + public_key: ${GIVE_PUBLIC_KEY} + token: ${GIVE_PRIVATE_TOKEN} + matomo: + url: https://piwik.inria.fr/ + site_id: 59 objstorage: enabled: true diff --git a/swh/values/staging/overrides/swh-cassandra-next-version.yaml b/swh/values/staging/overrides/swh-cassandra-next-version.yaml index 91618f9cc..092f8d8e6 100644 --- a/swh/values/staging/overrides/swh-cassandra-next-version.yaml +++ b/swh/values/staging/overrides/swh-cassandra-next-version.yaml @@ -234,37 +234,40 @@ postgresqlWebConfiguration: secretKeyName: postgres-swh-web-password web: - replicas: 1 - autoScaling: - minReplicaCount: 1 - maxReplicaCount: 1 - hosts: - - webapp-cassandra-next-version.internal.staging.swh.network - refreshSavecodenowStatus: - enabled: false - syncMailmaps: - enabled: false - ingress: - whitelistSourceRangeRef: stagingNetworkRanges - endpoints: - default: - paths: - - path: / - - path: /static - port: 80 - extraWhitelistSourceRange: - # vpn network - - 192.168.101.0/24 - authenticated: - paths: - - path: /api/1/provenance/ - - path: /api/1/entity/ - - path: /api/1/content/[^/]+/symbol/ - # auth-file with authentication - authentication: swh-cassandra/web-auth-secrets - extraWhitelistSourceRange: - # vpn network - - 192.168.101.0/24 + deployments: + cassandra: + replicas: 1 + autoScaling: + minReplicaCount: 1 + maxReplicaCount: 1 + hosts: + - webapp-cassandra-next-version.internal.staging.swh.network + refreshSavecodenowStatus: + enabled: false + syncMailmaps: + enabled: false + ingress: + whitelistSourceRangeRef: stagingNetworkRanges + endpoints: + default: + paths: + - path: / + - path: /static + port: 80 + extraWhitelistSourceRange: + # vpn network + - 192.168.101.0/24 + authenticated: + paths: + - path: /api/1/provenance/ + - path: /api/1/entity/ + - path: /api/1/content/[^/]+/symbol/ + # auth-file with authentication + authentication: swh-cassandra/web-auth-secrets + extraWhitelistSourceRange: + # vpn network + - 192.168.101.0/24 + deposit: enabled: false diff --git a/swh/values/staging/swh-cassandra.yaml b/swh/values/staging/swh-cassandra.yaml index 2d9c0cdee..db300852c 100644 --- a/swh/values/staging/swh-cassandra.yaml +++ b/swh/values/staging/swh-cassandra.yaml @@ -548,88 +548,92 @@ storage: web: enabled: true logLevel: INFO - # Usual consumption with visits webhook is 200m cpu (/2 replicas) - # Scale up if a real burst happen (>1 cpu per pod) - requestedCpu: 100m - requestedMemory: 300Mi - autoScaling: - minReplicaCount: 2 - maxReplicaCount: 4 # 4 to test the autoscaling in extreme scenarios - cpuPercentageUsage: 1000 - hosts: - - webapp.staging.swh.network - - webapp-cassandra.internal.staging.swh.network - syncMailmaps: - enabled: true - concurrencyPolicy: Forbid - configurationRef: postgresqlSyncmailmapsConfiguration - ingress: - enabled: true - secretName: swh-web-crt - extraAnnotations: - cert-manager.io/cluster-issuer: letsencrypt-production-gandi - kubernetes.io/ingress.class: nginx - kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - tlsEnabled: true - endpoints: - default: - paths: - - path: / - - path: /static - port: 80 - authenticated: - paths: - - path: /api/1/provenance/ - - path: /api/1/entity/ - - path: /api/1/content/[^/]+/symbol/ - # auth-file with authentication - authentication: swh-cassandra/web-auth-secrets - webhooks: - paths: - - path: /save/origin/visit/webhook - extraWhitelistSourceRange: - - 192.168.130.1 - - 192.168.130.2 - databaseConfigurationRef: postgresqlWebConfiguration - searchConfigurationRef: remoteSearchConfiguration - schedulerConfigurationRef: remoteSchedulerConfiguration - storageConfigurationRef: remoteStorageConfiguration - vaultConfigurationRef: remoteVaultConfiguration - indexerStorageConfigurationRef: remoteIndexerStorageConfiguration - countersConfigurationRef: remoteCountersConfiguration - djangoConfigurationRef: djangoWebConfiguration - depositConfigurationRef: depositConfiguration - giveConfigurationRef: giveConfiguration - throttlingConfigurationRef: webThrottling - addForgeNowConfigurationRef: addForgeNowConfiguration - webhooksConfigurationRef: webhooksConfiguration - extraConfig: - keycloak: - server_url: https://auth.softwareheritage.org/auth/ - realm_name: SoftwareHeritageStaging - save_code_now_webhook_secret: ${WEBHOOKS_SECRET} - search_config: - metadata_backend: swh-search - content_display_max_size: 5242880 - history_counters_url: http://counters-rpc-ingress/counters_history/history.json - swh_extra_django_apps: - - swh.web.add_forge_now - - swh.web.archive_coverage - - swh.web.badges - - swh.web.banners - - swh.web.deposit - - swh.web.inbound_email - - swh.web.jslicenses - - swh.web.mailmap - - swh.web.metrics - - swh.web.save_code_now - - swh.web.save_origin_webhooks - - swh.web.vault - matomo: {} - give: - public_key: ${GIVE_PUBLIC_KEY} - token: ${GIVE_PRIVATE_TOKEN} + deployments: + cassandra: + # Usual consumption with visits webhook is 200m cpu (/2 replicas) + # Scale up if a real burst happen (>1 cpu per pod) + requestedCpu: 100m + requestedMemory: 300Mi + autoScaling: + minReplicaCount: 2 + maxReplicaCount: 4 # 4 to test the autoscaling in extreme scenarios + cpuPercentageUsage: 1000 + hosts: + - webapp.staging.swh.network + - webapp-cassandra.internal.staging.swh.network + syncMailmaps: + enabled: true + concurrencyPolicy: Forbid + configurationRef: postgresqlSyncmailmapsConfiguration + priorityClassName: frontend-rpc-workload + cron: "15 * * * *" + ingress: + enabled: true + secretName: swh-web-crt + extraAnnotations: + cert-manager.io/cluster-issuer: letsencrypt-production-gandi + kubernetes.io/ingress.class: nginx + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + tlsEnabled: true + endpoints: + default: + paths: + - path: / + - path: /static + port: 80 + authenticated: + paths: + - path: /api/1/provenance/ + - path: /api/1/entity/ + - path: /api/1/content/[^/]+/symbol/ + # auth-file with authentication + authentication: swh-cassandra/web-auth-secrets + webhooks: + paths: + - path: /save/origin/visit/webhook + extraWhitelistSourceRange: + - 192.168.130.1 + - 192.168.130.2 + databaseConfigurationRef: postgresqlWebConfiguration + searchConfigurationRef: remoteSearchConfiguration + schedulerConfigurationRef: remoteSchedulerConfiguration + storageConfigurationRef: remoteStorageConfiguration + vaultConfigurationRef: remoteVaultConfiguration + indexerStorageConfigurationRef: remoteIndexerStorageConfiguration + countersConfigurationRef: remoteCountersConfiguration + djangoConfigurationRef: djangoWebConfiguration + depositConfigurationRef: depositConfiguration + giveConfigurationRef: giveConfiguration + throttlingConfigurationRef: webThrottling + addForgeNowConfigurationRef: addForgeNowConfiguration + webhooksConfigurationRef: webhooksConfiguration + extraConfig: + keycloak: + server_url: https://auth.softwareheritage.org/auth/ + realm_name: SoftwareHeritageStaging + save_code_now_webhook_secret: ${WEBHOOKS_SECRET} + search_config: + metadata_backend: swh-search + content_display_max_size: 5242880 + history_counters_url: http://counters-rpc-ingress/counters_history/history.json + swh_extra_django_apps: + - swh.web.add_forge_now + - swh.web.archive_coverage + - swh.web.badges + - swh.web.banners + - swh.web.deposit + - swh.web.inbound_email + - swh.web.jslicenses + - swh.web.mailmap + - swh.web.metrics + - swh.web.save_code_now + - swh.web.save_origin_webhooks + - swh.web.vault + matomo: {} + give: + public_key: ${GIVE_PUBLIC_KEY} + token: ${GIVE_PRIVATE_TOKEN} graphql: enabled: true diff --git a/swh/values/staging/swh.yaml b/swh/values/staging/swh.yaml index bc5d86124..8e5bc479a 100644 --- a/swh/values/staging/swh.yaml +++ b/swh/values/staging/swh.yaml @@ -753,84 +753,79 @@ search: web: enabled: true logLevel: INFO - # small footprint because this webapp is almost not used, only for tests - replicas: 1 - requestedCpu: 50m - requestedMemory: 250Mi - hosts: - - webapp-postgresql.internal.staging.swh.network - refreshSavecodenowStatus: - enabled: false - concurrencyPolicy: Forbid - syncMailmaps: - enabled: false - concurrencyPolicy: Forbid - configurationRef: postgresqlSyncmailmapsConfiguration - ingress: - enabled: true - secretName: swh-web-crt - extraAnnotations: - cert-manager.io/cluster-issuer: letsencrypt-production-gandi - kubernetes.io/ingress.class: nginx - kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/ssl-redirect: "true" - tlsEnabled: true - endpoints: - default: - paths: - - path: / - - path: /static - port: 80 - authenticated: - paths: - - path: /api/1/provenance/ - - path: /api/1/entity/ - - path: /api/1/content/[^/]+/symbol/ - # auth-file with authentication - authentication: swh-cassandra/web-auth-secrets - webhooks: - paths: - - path: /save/origin/visit/webhook - extraWhitelistSourceRange: - - 192.168.130.1 - - 192.168.130.2 - databaseConfigurationRef: postgresqlWebConfiguration - searchConfigurationRef: remoteSearchConfiguration - schedulerConfigurationRef: remoteSchedulerConfiguration - storageConfigurationRef: remoteReadOnlyStorageConfiguration - vaultConfigurationRef: remoteVaultConfiguration - indexerStorageConfigurationRef: remoteIndexerStorageConfiguration - countersConfigurationRef: remoteCountersConfiguration - djangoConfigurationRef: djangoWebConfiguration - depositConfigurationRef: depositConfiguration - giveConfigurationRef: giveConfiguration - throttlingConfigurationRef: webThrottling - addForgeNowConfigurationRef: addForgeNowConfiguration - extraConfig: - keycloak: - server_url: https://auth.softwareheritage.org/auth/ - realm_name: SoftwareHeritageStaging - search_config: - metadata_backend: swh-search - content_display_max_size: 5242880 - history_counters_url: http://counters-rpc-ingress/counters_history/history.json - swh_extra_django_apps: - - swh.web.add_forge_now - - swh.web.archive_coverage - - swh.web.badges - - swh.web.banners - - swh.web.deposit - - swh.web.inbound_email - - swh.web.jslicenses - - swh.web.mailmap - - swh.web.metrics - - swh.web.save_code_now - - swh.web.save_origin_webhooks - - swh.web.vault - matomo: {} - give: - public_key: ${GIVE_PUBLIC_KEY} - token: ${GIVE_PRIVATE_TOKEN} + deployments: + postgresql: + # small footprint because this webapp is almost not used, only for tests + replicas: 1 + requestedCpu: 50m + requestedMemory: 250Mi + hosts: + - webapp-postgresql.internal.staging.swh.network + ingress: + enabled: true + secretName: swh-web-crt + extraAnnotations: + cert-manager.io/cluster-issuer: letsencrypt-production-gandi + kubernetes.io/ingress.class: nginx + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + tlsEnabled: true + endpoints: + default: + paths: + - path: / + - path: /static + port: 80 + authenticated: + paths: + - path: /api/1/provenance/ + - path: /api/1/entity/ + - path: /api/1/content/[^/]+/symbol/ + # auth-file with authentication + authentication: swh-cassandra/web-auth-secrets + webhooks: + paths: + - path: /save/origin/visit/webhook + extraWhitelistSourceRange: + - 192.168.130.1 + - 192.168.130.2 + databaseConfigurationRef: postgresqlWebConfiguration + searchConfigurationRef: remoteSearchConfiguration + schedulerConfigurationRef: remoteSchedulerConfiguration + storageConfigurationRef: remoteReadOnlyStorageConfiguration + vaultConfigurationRef: remoteVaultConfiguration + indexerStorageConfigurationRef: remoteIndexerStorageConfiguration + countersConfigurationRef: remoteCountersConfiguration + djangoConfigurationRef: djangoWebConfiguration + depositConfigurationRef: depositConfiguration + giveConfigurationRef: giveConfiguration + throttlingConfigurationRef: webThrottling + addForgeNowConfigurationRef: addForgeNowConfiguration + extraConfig: + keycloak: + server_url: https://auth.softwareheritage.org/auth/ + realm_name: SoftwareHeritageStaging + search_config: + metadata_backend: swh-search + content_display_max_size: 5242880 + history_counters_url: http://counters-rpc-ingress/counters_history/history.json + swh_extra_django_apps: + - swh.web.add_forge_now + - swh.web.archive_coverage + - swh.web.badges + - swh.web.banners + - swh.web.deposit + - swh.web.inbound_email + - swh.web.jslicenses + - swh.web.mailmap + - swh.web.metrics + - swh.web.save_code_now + - swh.web.save_origin_webhooks + - swh.web.vault + matomo: {} + give: + public_key: ${GIVE_PUBLIC_KEY} + token: ${GIVE_PRIVATE_TOKEN} scrubber: enabled: true -- GitLab