From d78cff5facdcbf7ac0d1e98ec6d4a1436efd332f Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Tue, 5 Dec 2023 15:32:23 +0100
Subject: [PATCH] swh-counters: Add template to deploy associated rpc & journal
client
Refs. swh/infra/sysadm-environment#5179
---
.../counters/journal-client-configmap.yaml | 21 +++
.../counters/journal-client-deployment.yaml | 115 +++++++++++++++
swh/templates/counters/rpc-autoscale.yaml | 5 +
swh/templates/counters/rpc-configmap.yaml | 22 +++
swh/templates/counters/rpc-deployment.yaml | 133 ++++++++++++++++++
swh/templates/counters/rpc-ingress.yaml | 5 +
swh/templates/counters/rpc-service.yaml | 5 +
swh/values.yaml | 59 ++++++++
swh/values/minikube.yaml | 52 +++++++
9 files changed, 417 insertions(+)
create mode 100644 swh/templates/counters/journal-client-configmap.yaml
create mode 100644 swh/templates/counters/journal-client-deployment.yaml
create mode 100644 swh/templates/counters/rpc-autoscale.yaml
create mode 100644 swh/templates/counters/rpc-configmap.yaml
create mode 100644 swh/templates/counters/rpc-deployment.yaml
create mode 100644 swh/templates/counters/rpc-ingress.yaml
create mode 100644 swh/templates/counters/rpc-service.yaml
diff --git a/swh/templates/counters/journal-client-configmap.yaml b/swh/templates/counters/journal-client-configmap.yaml
new file mode 100644
index 000000000..1654ad661
--- /dev/null
+++ b/swh/templates/counters/journal-client-configmap.yaml
@@ -0,0 +1,21 @@
+{{ if and .Values.counters.enabled .Values.counters.journalClient.enabled -}}
+{{- with .Values.counters.journalClient }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: counters-journal-client-configuration-template
+ namespace: {{ $.Values.namespace }}
+data:
+ config.yml.template: |
+ {{- include "swh.service.fromYaml"
+ (dict "service" "counters"
+ "configurationRef" .countersConfigurationRef
+ "Values" $.Values) | nindent 4 }}
+
+ {{- include "swh.journalClientConfiguration"
+ (dict "configurationRef" .journalConfigurationRef
+ "Values" $.Values) | nindent 4 }}
+
+{{- end -}}
+{{- end -}}
diff --git a/swh/templates/counters/journal-client-deployment.yaml b/swh/templates/counters/journal-client-deployment.yaml
new file mode 100644
index 000000000..9dc69791e
--- /dev/null
+++ b/swh/templates/counters/journal-client-deployment.yaml
@@ -0,0 +1,115 @@
+{{ if and .Values.counters.enabled .Values.counters.journalClient.enabled -}}
+{{- $configurationChecksum := include (print .Template.BasePath "/scheduler/journal-client-configmap.yaml") . -}}
+{{- $log_level := .Values.counters.journalClient.logLevel }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: {{ .Values.namespace }}
+ name: counters-journal-client
+ labels:
+ app: counters-journal-client
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: counters-journal-client
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: counters-journal-client
+ annotations:
+ checksum/config: {{ $configurationChecksum | sha256sum }}
+ spec:
+ {{- if .Values.counters.affinity }}
+ affinity:
+ {{- toYaml .Values.counters.affinity | nindent 8 }}
+ {{- end }}
+ {{- if and $.Values.podPriority.enabled $.Values.counters.priorityClassName }}
+ priorityClassName: {{ $.Values.namespace }}-{{ $.Values.counters.priorityClassName }}
+ {{ end }}
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: counters-journal-client
+ resources:
+ requests:
+ memory: {{ .Values.counters.journalClient.requestedMemory | default "512Mi" }}
+ cpu: {{ .Values.counters.journalClient.requestedCpu | default "500m" }}
+ {{- if or .Values.counters.journalClient.limitedMemory .Values.counters.journalClient.limitedCpu }}
+ limits:
+ {{- if .Values.counters.journalClient.limitedMemory }}
+ memory: {{ .Values.counters.journalClient.limitedMemory }}
+ {{- end }}
+ {{- if .Values.counters.journalClient.limitedCpu }}
+ cpu: {{ .Values.counters.journalClient.limitedCpu }}
+ {{- end }}
+ {{ end }}
+ image: {{ .Values.swh_scheduler_image }}:{{ .Values.swh_scheduler_image_version }}
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ # - shell
+ # - sleep
+ # - infinity
+ - swh
+ - --log-level
+ - {{ $log_level | default "INFO" }}
+ - counters
+ - --config-file
+ - /etc/swh/config.yml
+ - journal-client
+ env:
+ - name: STATSD_HOST
+ value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ - name: STATSD_PORT
+ value: {{ .Values.statsdPort | default "9125" | quote }}
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: LOG_LEVEL
+ value: {{ $log_level | default "INFO" }}
+ {{- if .Values.counters.sentry.enabled }}
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: {{ .Values.sentry.environment }}
+ - name: SWH_MAIN_PACKAGE
+ value: swh.counters
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.counters.sentry.secretKeyRef }}
+ key: {{ .Values.counters.sentry.secretKeyName }}
+ # if the setting doesn't exist, sentry issue pushes will be disabled
+ optional: false
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ {{- end }}
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: counters-journal-client-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+{{- end -}}
diff --git a/swh/templates/counters/rpc-autoscale.yaml b/swh/templates/counters/rpc-autoscale.yaml
new file mode 100644
index 000000000..5489b7b86
--- /dev/null
+++ b/swh/templates/counters/rpc-autoscale.yaml
@@ -0,0 +1,5 @@
+{{- if and .Values.counters.enabled .Values.counters.rpc.enabled .Values.counters.rpc.autoScaling -}}
+{{- include "swh.autoscale" (dict "Values" .Values
+ "serviceType" "counters-rpc"
+ "configuration" .Values.counters.rpc) -}}
+{{- end -}}
diff --git a/swh/templates/counters/rpc-configmap.yaml b/swh/templates/counters/rpc-configmap.yaml
new file mode 100644
index 000000000..def6dae31
--- /dev/null
+++ b/swh/templates/counters/rpc-configmap.yaml
@@ -0,0 +1,22 @@
+{{ if and .Values.counters.enabled .Values.counters.rpc.enabled -}}
+{{- with .Values.counters.rpc }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: {{ $.Values.namespace }}
+ name: counters-rpc-configuration-template
+data:
+ config.yml.template: |
+ {{- include "swh.service.fromYaml"
+ (dict "service" "counters"
+ "configurationRef" .countersConfigurationRef
+ "Values" $.Values) | nindent 4 }}
+
+ {{- include "swh.service.fromYaml"
+ (dict "service" "history"
+ "configurationRef" .historyConfigurationRef
+ "Values" $.Values) | nindent 4 }}
+
+{{- end -}}
+{{- end -}}
diff --git a/swh/templates/counters/rpc-deployment.yaml b/swh/templates/counters/rpc-deployment.yaml
new file mode 100644
index 000000000..2ed07d875
--- /dev/null
+++ b/swh/templates/counters/rpc-deployment.yaml
@@ -0,0 +1,133 @@
+{{ if and .Values.counters.enabled .Values.counters.rpc.enabled -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: {{ .Values.namespace }}
+ name: counters-rpc
+ labels:
+ app: counters-rpc
+spec:
+ revisionHistoryLimit: 2
+ {{ if .Values.counters.rpc.replicas -}}
+ replicas: {{ .Values.counters.rpc.replicas }}
+ {{ end -}}
+ selector:
+ matchLabels:
+ app: counters-rpc
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: counters-rpc
+ annotations:
+ checksum/config: {{ include (print $.Template.BasePath "/counters/rpc-configmap.yaml") . | sha256sum }}
+ spec:
+ {{- if .Values.counters.affinity }}
+ affinity:
+ {{- toYaml .Values.counters.affinity | nindent 8 }}
+ {{- end }}
+ {{- if and $.Values.podPriority.enabled $.Values.counters.rpc.priorityClassName }}
+ priorityClassName: {{ $.Values.namespace }}-{{ $.Values.counters.rpc.priorityClassName }}
+ {{ end }}
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ env:
+{{ include "swh.secrets.environment" (dict "Values" .Values
+ "configurationRef" .Values.counters.rpc.countersConfigurationRef) | indent 10 }}
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ containers:
+ - name: counters-rpc
+ resources:
+ requests:
+ memory: {{ .Values.counters.rpc.requestedMemory | default "512Mi" }}
+ cpu: {{ .Values.counters.rpc.requestedCpu | default "500m" }}
+ {{- if or .Values.counters.rpc.limitedMemory .Values.counters.rpc.limitedCpu }}
+ limits:
+ {{- if .Values.counters.rpc.limitedMemory }}
+ memory: {{ .Values.counters.rpc.limitedMemory }}
+ {{- end }}
+ {{- if .Values.counters.rpc.limitedCpu }}
+ cpu: {{ .Values.counters.rpc.limitedCpu }}
+ {{- end }}
+ {{- end }}
+ image: {{ .Values.swh_counters_image }}:{{ .Values.swh_counters_image_version }}
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: {{ .Values.counters.port }}
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ {{ if .Values.counters.rpc.gunicorn -}}
+ - name: THREADS
+ value: {{ .Values.counters.rpc.gunicorn.threads | default 5 | quote }}
+ - name: WORKERS
+ value: {{ .Values.counters.rpc.gunicorn.workers | default 2 | quote }}
+ - name: TIMEOUT
+ value: {{ .Values.counters.rpc.gunicorn.timeout | default 60 | quote }}
+ {{ end -}}
+ - name: STATSD_HOST
+ value: {{ .Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ - name: STATSD_PORT
+ value: {{ .Values.statsdPort | default "9125" | quote }}
+ - name: LOG_LEVEL
+ value: {{ .Values.counters.rpc.logLevel | default "INFO" }}
+ {{- if .Values.counters.sentry.enabled }}
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: {{ .Values.sentry.environment }}
+ - name: SWH_MAIN_PACKAGE
+ value: swh.counters
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.counters.sentry.secretKeyRef }}
+ key: {{ .Values.counters.sentry.secretKeyName }}
+ # if the setting doesn't exist, sentry issue pushes will be disabled
+ optional: false
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ {{- end }}
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: counters-rpc-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+{{- end -}}
diff --git a/swh/templates/counters/rpc-ingress.yaml b/swh/templates/counters/rpc-ingress.yaml
new file mode 100644
index 000000000..2aa7aec38
--- /dev/null
+++ b/swh/templates/counters/rpc-ingress.yaml
@@ -0,0 +1,5 @@
+{{- if and .Values.counters.enabled .Values.counters.rpc.enabled .Values.counters.rpc.ingress.enabled -}}
+{{- include "swh.ingress" (dict "Values" .Values
+ "serviceType" "counters-rpc"
+ "configuration" .Values.counters.rpc) -}}
+{{ end }}
diff --git a/swh/templates/counters/rpc-service.yaml b/swh/templates/counters/rpc-service.yaml
new file mode 100644
index 000000000..c8f7e3872
--- /dev/null
+++ b/swh/templates/counters/rpc-service.yaml
@@ -0,0 +1,5 @@
+{{ if and .Values.counters.enabled .Values.counters.rpc.enabled -}}
+{{- include "swh.service" (dict "Values" .Values
+ "serviceType" "counters-rpc"
+ "configuration" .Values.counters.rpc) -}}
+{{ end }}
diff --git a/swh/values.yaml b/swh/values.yaml
index b51ff77c4..a65b4d444 100644
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -1382,3 +1382,62 @@ webhooks:
# auto_offset_reset: latest
# object_types:
# - origin_visit_status
+
+counters:
+ enabled: false
+ port: 5011
+ sentry:
+ enabled: false
+ secretKeyRef: common-secrets
+ secretKeyName: counters-sentry-dsn
+ rpc:
+ enabled: false
+ port: 5008
+ priorityClassName: frontend-rpc
+ logLevel: INFO
+ # The scheduler instance to use for rpc must be a postgresql instance
+ # schedulerConfigurationRef: postgresqlSchedulerConfiguration
+ # replicas: 2
+ # gunicorn:
+ # threads: 5
+ # workers: 2
+ # timeout: 60
+ # RPC services may have different profiles than the rest so they need their specific
+ # setup
+ # requestedMemory: 512Mi
+ # requestedCpu: 500m
+ # limitedMemory: 512Mi
+ # limitedCpu: 500m
+ # autoScaling:
+ # minReplicaCount: 2
+ # maxReplicaCount: 10
+ # cpuPercentageUsage: 100
+ ingress:
+ enabled: false
+ # Optional: the ingress classname to use
+ # className: nginx
+ # mandatory if ingress is enabled
+ # the hostname on which the storage must be reachable
+ # host: myscheduler.localdomain
+ extraAnnotations:
+ nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
+ nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
+ nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+ nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
+ nginx.ingress.kubernetes.io/proxy-body-size: "4G"
+ # Default allowed ip ranges that can be extended per ingress definitions paths
+ # whitelistSourceRangeRef: internalNetworkRanges
+ # endpoints:
+ # default:
+ # paths:
+ # - path: /
+ # read-only:
+ # paths:
+ # - path: /scheduler_metrics/get
+ # - path: /visit_stats/get
+ # # Extra allowed ip range for the paths above
+ # extraWhitelistSourceRange:
+ # - yyy.yyy.yyy.yyy/24
+ journalClient:
+ enabled: false
+ # journalConfigurationRef: journalClientConfiguration
diff --git a/swh/values/minikube.yaml b/swh/values/minikube.yaml
index 0a42af5e9..a6153a7a4 100644
--- a/swh/values/minikube.yaml
+++ b/swh/values/minikube.yaml
@@ -923,3 +923,55 @@ objstorage:
default:
paths:
- path: /
+
+fakeHistoryConfiguration:
+ cls: prometheus
+ prometheus_host: thanos.i.a.s.n
+ prometheus_port: 19191
+ live_data_start: 0
+ cache_base_directory: "/srv/softwareheritage/counters"
+ interval: 12h
+ labels:
+ environment: staging
+
+fakeLocalRedis:
+ cls: redis
+ host: localhost:6379
+
+counters:
+ enabled: false
+ rpc:
+ enabled: false
+ countersConfigurationRef: fakeLocalRedis
+ historyConfigurationRef: fakeHistoryConfiguration
+ requestedMemory: 20Mi
+ limitedMemory: 40Mi
+ requestedCpu: 10m
+ limitedCpu: 20m
+ replicas: 1
+ gunicorn:
+ threads: 1
+ workers: 1
+ timeout: 10
+ # autoScaling:
+ # minReplicaCount: 4
+ # maxReplicaCount: 20
+ # cpuPercentageUsage: 150
+ hosts:
+ - mycounters.minikube.domain
+ ingress:
+ enabled: false
+ # Default allowed ip ranges that can be extended per ingress definitions paths
+ whitelistSourceRangeRef: internalNetworkRanges
+ endpoints:
+ default:
+ paths:
+ - /
+ journalClient:
+ enabled: false
+ countersConfigurationRef: fakeRemoteCountersConfiguration
+ journalConfigurationRef: journalClientConfiguration
+ requestedMemory: 15Mi
+ limitedMemory: 30Mi
+ requestedCpu: 50m
+ limitedCpu: 100m
--
GitLab