From 29307d0d16b8c0d0560669cd7294cf694433506c Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Wed, 19 Mar 2025 16:23:42 +0100
Subject: [PATCH] provenance/deployment: Compute mandatory configuration per
type
And set environment variables per server type.
Refs. swh/infra/sysadm-environment#5608
---
swh/templates/provenance/_helper-config.yaml | 34 ++++++++++++++++----
swh/templates/provenance/deployment.yaml | 20 +++++++++---
2 files changed, 43 insertions(+), 11 deletions(-)
diff --git a/swh/templates/provenance/_helper-config.yaml b/swh/templates/provenance/_helper-config.yaml
index d076c4185..92b44d587 100644
--- a/swh/templates/provenance/_helper-config.yaml
+++ b/swh/templates/provenance/_helper-config.yaml
@@ -2,7 +2,8 @@
* Create a provenance configmap for service .serviceType
*/}}
{{ define "swh.provenance.configmap" }}
-{{- $graphConfigurationRef := required (print "<.configuration.graphConfigurationRef> key is mandatory" ) .configuration.graphConfigurationRef -}}
+{{- $graphConfigurationRef := .configuration.graphConfigurationRef -}}
+{{- if $graphConfigurationRef -}}
{{- $graphConfiguration := get .Values $graphConfigurationRef -}}
---
apiVersion: v1
@@ -14,6 +15,7 @@ data:
config.yml.template: |
provenance:
{{- toYaml $graphConfiguration | nindent 6 -}}
+{{ end }}
{{- end -}}
{{/*
@@ -32,7 +34,8 @@ data:
{{/*
- * Read default provenance configuration
+ * Configure provenance configuration with default values.
+ * This also raises when mandatory configuration is missing.
*/}}
{{ define "swh.provenance.config" }}
{{- $configuration := .configuration -}}
@@ -45,13 +48,30 @@ data:
{{- $configuration := set $configuration "port" $port -}}
{{- end -}}
-{{- $logLevel := .configuration.logLevel | default $.Values.provenance.logLevel | default "INFO" | upper -}}
+{{- $logLevel := $configuration.logLevel | default $.Values.provenance.logLevel | default "INFO" | upper -}}
{{- $configuration := set $configuration "logLevel" $logLevel -}}
-{{ $gunicornConfig := eq $type "rpc" | ternary
- (fromYaml (include "swh.gunicorn.config"
- (dict "configuration" $provenanceConfig.gunicorn
- "defaultConfiguration" $.Values.provenance.gunicorn)) dict -}}
+
+{{- if eq $type "rpc" -}}
+{{- $graphConfigurationRef := required (print "<.configuration.graphConfigurationRef> key is mandatory" ) .configuration.graphConfigurationRef -}}
+{{- $gunicornConfig := fromYaml
+ (include "swh.gunicorn.config"
+ (dict "defaultConfiguration" $.Values.provenance.gunicorn
+ "configuration" $configuration.gunicorn)) -}}
{{- $configuration := set $configuration "gunicornConfig" $gunicornConfig -}}
+{{- else -}}
+{{- $provenanceIndexDataset := dig "dataset" "provenance" "index" false $configuration -}}
+{{- $configuration := set $configuration "provenanceIndexDataset" $provenanceIndexDataset -}}
+{{- $provenanceFetchDataset := dig "dataset" "provenance" "fetch" false $configuration -}}
+{{- $configuration := set $configuration "provenanceFetchDataset" $provenanceFetchDataset -}}
+{{- if or $provenanceIndexDataset $provenanceFetchDataset -}}
+{{- $datasetName := required (print "<.configuration.dataset.name> key is mandatory") (dig "dataset" "name" nil $configuration) -}}
+{{- $configuration := set $configuration "datasetName" $datasetName -}}
+{{- end -}}
+{{- $graphPath := required (print "<.configuration.dataset.graph.path> key is mandatory") $configuration.dataset.graph.path -}}
+{{- $configuration := set $configuration "graphPath" $graphPath -}}
+{{- $provenancePath := $configuration.dataset.provenance.path -}}
+{{- $graphPath := required (print "<.configuration.dataset.provenance.path> key is mandatory") $configuration.dataset.provenance.path -}}
+{{- $configuration := set $configuration "provenancePath" $provenancePath -}}
{{- end -}}
{{- toYaml $configuration -}}
{{- end }}
diff --git a/swh/templates/provenance/deployment.yaml b/swh/templates/provenance/deployment.yaml
index 7fbbe7450..8146a63e0 100644
--- a/swh/templates/provenance/deployment.yaml
+++ b/swh/templates/provenance/deployment.yaml
@@ -16,7 +16,10 @@
{{- $type := $provenanceConfig.type -}}
{{- $port := $provenanceConfig.port -}}
{{- $logLevel := $provenanceConfig.logLevel -}}
-{{- $gunicornConfig := .configuration.gunicornConfig -}}
+{{- $datasetName := $provenanceConfig.datasetName | default "" -}}
+{{- $graphPath := $provenanceConfig.graphPath | default "" -}}
+{{- $provenancePath := $provenanceConfig.provenancePath | default "" -}}
+{{- $gunicornConfig := $provenanceConfig.gunicornConfig | default dict -}}
---
apiVersion: apps/v1
kind: Deployment
@@ -123,7 +126,11 @@ spec:
- -c
- /opt/swh/entrypoint.sh
env:
- {{ if $gunicornConfig -}}
+ - name: PROVENANCE_TYPE
+ value: {{ $type }}
+ - name: PORT
+ value: {{ $port }}
+ {{ if eq $type "rpc" -}}
- name: WORKERS
value: {{ $gunicornConfig.workers | quote }}
- name: THREADS
@@ -132,6 +139,13 @@ spec:
value: {{ $gunicornConfig.timeout | quote }}
- name: SWH_LOG_CONFIG_JSON
value: /etc/swh/logging/logging-gunicorn.json
+ - name: STATSD_SERVICE_TYPE
+ value: {{ $serviceType }}
+ {{ else -}}
+ - name: PROVENANCE_PATH
+ value: {{ $provenancePath }}
+ - name: GRAPH_PATH
+ value: {{ $graphPath }}
{{ end -}}
- name: STATSD_HOST
value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
@@ -139,8 +153,6 @@ spec:
value: {{ $.Values.statsdPort | default "9125" | quote }}
- name: STATSD_TAGS
value: deployment:{{ $serviceType }}
- - name: STATSD_SERVICE_TYPE
- value: {{ $serviceType }}
- name: SWH_LOG_LEVEL
value: {{ $logLevel }}
{{- if $.Values.provenance.sentry.enabled }}
--
GitLab