cluster-components/templates/svix/svix-server.yaml

@@ -83,7 +83,7 @@ spec:
             {{- end }}
             {{- if .whitelistSubnets }}
             - name: SVIX_WHITELIST_SUBNETS
-              value: {{ .whitelistSubnets }}
+              value: {{ .whitelistSubnets | quote }}
             {{- end }}
 {{- end }}
 {{- end }}

swh/fake-secrets/swh-storage-postgresql-common-secret.yaml

@@ -5,5 +5,5 @@ metadata:
   name: swh-storage-postgresql-common-secret
 type: Opaque
 stringData:
-  postgres-swh-storage-password: fake-swh-storage-password
+  postgres-swh-storage-password: guest
 

swh/templates/graphql/deployment.yaml

@@ -25,6 +25,10 @@ spec:
         # Force a rollout upgrade if the configuration changes
         checksum/config: {{ include (print $.Template.BasePath "/graphql/configmap.yaml") . | sha256sum }}
     spec:
+      {{- if $.Values.graphql.affinity }}
+      affinity:
+        {{- toYaml $.Values.graphql.affinity | nindent 8 }}
+      {{- end }}
       {{- if and $.Values.podPriority.enabled $.Values.graphql.priorityClassName }}
       priorityClassName: {{ $.Values.namespace }}-{{ $.Values.graphql.priorityClassName }}
       {{ end }}

swh/templates/toolbox/configmap.yaml

@@ -47,5 +47,10 @@ data:
                             "configurationRef" (get $deployment_config "journalClientConfigurationRef")
                             "Values" $.Values) | nindent 4 -}}
 {{ end }}
+{{- if hasKey $deployment_config "webhooksConfigurationRef" }}
+  {{- include "swh.service.fromYaml" (dict "service" "webhooks"
+                            "configurationRef" $.Values.toolbox.configs.webhooks.webhooksConfigurationRef
+                            "Values" $.Values) | nindent 4 }}
+{{ end }}
 {{ end }}
 {{ end }}

swh/templates/web/deployment.yaml

@@ -63,6 +63,10 @@ spec:
             {{- include "swh.secrets.environment" (dict "Values" $.Values
                                                         "configurationRef" .Values.web.addForgeNowConfigurationRef) | nindent 12 }}
             {{- end -}}
+            {{- if .Values.web.webhooksConfigurationRef -}}
+            {{- include "swh.secrets.environment" (dict "Values" $.Values
+                                                        "configurationRef" .Values.web.webhooksConfigurationRef) | nindent 12 }}
+            {{- end -}}
             {{- if .Values.web.sentry.enabled }}
             - name: SWH_SENTRY_DSN
               valueFrom:
@@ -195,8 +199,8 @@ spec:
         - name: nginx
           resources:
             requests:
-              memory: {{ .Values.web.requestedMemory | default "512Mi" }}
-              cpu: {{ .Values.web.requestedCpu | default "500m" }}
+              memory: {{ .Values.web.nginxRequestedMemory | default "50Mi" }}
+              cpu: {{ .Values.web.nginxRequestedCpu | default "10m" }}
           image: nginx:bullseye
           imagePullPolicy: IfNotPresent
           ports:

swh/templates/webhooks/_helper.yaml

+{{ define "swh.webhooks.configmap" }}
+{{- $webhooksConfigurationRef := .Values.webhooks.svixConfigurationRef -}}
+{{- $journalClientConfigurationRef := .Values.webhooks.journalClientConfigurationRef -}}
+{{- $journalClientConfiguration := required (print "journalClientConfigurationRef " .journalClientConfigurationRef " not found in webhooks configuration") (get .Values $journalClientConfigurationRef) -}}
+{{- $journalClientOverrides := deepCopy (get .deployment_config "journalClientOverrides" | default (dict)) -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: {{ $.Values.namespace }}
+  name: webhooks-{{ .deployment }}-template
+data:
+  config.yml.template: |
+  {{- include "swh.service.fromYaml" (dict "service" "webhooks"
+                            "configurationRef" $.Values.webhooks.svixConfigurationRef
+                            "Values" $.Values) | nindent 4 }}
+  {{- include "swh.journalClientConfiguration" (dict "configurationRef" $journalClientConfigurationRef
+                            "overrides" $journalClientOverrides
+                            "Values" .Values) | nindent 4 }}
+{{ end }}

swh/templates/webhooks/configmap.yaml

+{{ if .Values.webhooks.enabled -}}
+{{- range $deployment, $deployment_config := .Values.webhooks.deployments -}}
+{{- if or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled") -}}
+{{ include "swh.webhooks.configmap" (dict "deployment" $deployment "deployment_config" $deployment_config "Values" $.Values) }}
+{{- end -}}
+{{ end -}}
+{{- end -}}

swh/templates/webhooks/deployment.yaml

+{{ if .Values.webhooks.enabled -}}
+{{- range $deployment, $deployment_config := .Values.webhooks.deployments -}}
+{{- if or (not (hasKey $deployment_config "enabled")) (get $deployment_config "enabled") -}}
+{{- $configurationChecksum := include "swh.webhooks.configmap" (dict "deployment" $deployment "deployment_config" $deployment_config "Values" $.Values) -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: webhooks-{{ $deployment }}
+  namespace: {{ $.Values.namespace }}
+  labels:
+    app: webhooks-{{ $deployment }}
+spec:
+  revisionHistoryLimit: 2
+  replicas: {{ $deployment_config.replicas | default 1 }}
+  selector:
+    matchLabels:
+      app: webhooks-{{ $deployment }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: webhooks-{{ $deployment }}
+      annotations:
+        # Force a rollout upgrade if the configuration changes
+        checksum/config: {{ $configurationChecksum | sha256sum }}
+    spec:
+      {{- if $.Values.webhooks.affinity }}
+      affinity:
+        {{ toYaml $.Values.webhooks.affinity | nindent 8 }}
+      {{- end }}
+      {{- if and $.Values.podPriority.enabled $.Values.webhooks.priorityClassName }}
+      priorityClassName: {{ $.Values.namespace }}-{{ $.Values.webhooks.priorityClassName }}
+      {{ end }}
+      initContainers:
+        - name: prepare-configuration
+          image: debian:bullseye
+          imagePullPolicy: IfNotPresent
+          env:
+{{ include "swh.secrets.environment" (dict "Values" $.Values
+                                           "configurationRef" $.Values.webhooks.svixConfigurationRef) | indent 10 }}
+{{ include "swh.secrets.environment" (dict "Values" $.Values
+                                           "configurationRef" $.Values.webhooks.journalClientConfigurationRef) | indent 10 }}
+          command:
+          - /bin/bash
+          args:
+          - -c
+          - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+          volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+          - name: configuration-template
+            mountPath: /etc/swh/configuration-template
+      containers:
+      - name: webhooks
+        resources:
+          requests:
+            memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }}
+            cpu: {{ get $deployment_config "requestedCpu" | default "500m" }}
+          {{- if or ( get $deployment_config "limitedCpu" ) ( get $deployment_config "limitedMemory" ) }}
+          limits:
+            {{- if ( get $deployment_config "limitedMemory" ) }}
+            memory: {{ get $deployment_config "limitedMemory" }}
+            {{- end }}
+            {{- if ( get $deployment_config "limitedCpu" ) }}
+            cpu: {{ get $deployment_config "limitedCpu" }}
+            {{- end }}
+          {{ end }}
+        image: {{ $.Values.swh_webhooks_image }}:{{ $.Values.swh_webhooks_image_version }}
+        imagePullPolicy: IfNotPresent
+        command:
+          - /opt/swh/entrypoint.sh
+        env:
+        - name: STATSD_HOST
+          value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+        - name: STATSD_PORT
+          value: {{ $.Values.statsdPort | default "9125" | quote }}
+        - name: MAX_TASKS_PER_CHILD
+          value: {{ get $deployment_config "maxTasksPerChild" | default 1 | quote }}
+        - name: LOGLEVEL
+          value: {{ get $deployment_config "logLevel" | default "INFO" | quote }}
+        - name: SWH_CONFIG_FILENAME
+          value: /etc/swh/config.yml
+        {{- if and $.Values.sentry.enabled $.Values.webhooks.sentry.enabled }}
+        - name: SWH_SENTRY_ENVIRONMENT
+          value: {{ $.Values.sentry.environment }}
+        - name: SWH_MAIN_PACKAGE
+          value: swh.webhooks
+        - name: SWH_SENTRY_DSN
+          valueFrom:
+            secretKeyRef:
+              name: common-secrets
+              key: webhooks-sentry-dsn
+              # 'name' secret must exist & include key "host"
+              optional: false
+        {{ end }}
+        volumeMounts:
+          - name: configuration
+            mountPath: /etc/swh
+      volumes:
+      - name: configuration
+        emptyDir: {}
+      - name: configuration-template
+        configMap:
+          name: webhooks-{{ $deployment }}-template
+          defaultMode: 0777
+          items:
+          - key: "config.yml.template"
+            path: "config.yml.template"
+{{ end }}
+{{ end }}
+{{ end }}

swh/values.yaml

@@ -699,6 +699,8 @@ web:
   logLevel: INFO
   requestedCpu: 50m
   requestedMemory: 100Mi
+  # nginxRequestedCpu: 10m
+  # nginxRequestedMemory: 50m
   # replicas: 1
   # autoScaling:
   #   minReplicaCount: 2
@@ -1353,3 +1355,30 @@ podPriority:
       range: -10000--11000
       value: -10500
       description: Background workload (replayers cassandra & postgres, ...)
+
+webhooks:
+  enabled: false
+  sentry:
+    enabled: false
+  #affinity:
+  #  nodeAffinity:
+  #    requiredDuringSchedulingIgnoredDuringExecution:
+  #      nodeSelectorTerms:
+  #      - matchExpressions:
+  #        - key: swh/webhooks
+  #          operator: In
+  #          values:
+  #          - "true"
+  #svixConfigurationRef: svixConfiguration
+  #journalClientConfigurationRef: journalClientConfiguration
+  #priorityClassName: frontend-rpc-workload
+  #deployments:
+  #  origin-visit-status:
+  #    replicas: 1
+  #    requestedCpu: 100m
+  #    requestedMemory: 100Mi
+  #    journalClientOverrides:
+  #      group_id: my-webhook-group-id
+  #      auto_offset_reset: latest
+  #      object_types:
+  #        - origin_visit_status

swh/values/default.yaml

@@ -173,8 +173,6 @@ vault:
             - "true"
 
 web:
-  requestedCpu: 50m
-  requestedMemory: 300Mi
   migrations:
     enabled: true
   sentry:
@@ -276,5 +274,18 @@ objstorage:
             values:
             - "true"
 
+webhooks:
+  sentry:
+    enabled: true
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: swh/webhooks
+            operator: In
+            values:
+            - "true"
+
 statsd_exporter:
   enabled: true

swh/values/production/default.yaml

@@ -12,6 +12,13 @@ internalNetworkRanges:
   - 10.42.0.0/16
   - 10.43.0.0/16
 
+externalAuthorizedNetworkRanges:
+  - 128.93.166.14
+  - 131.107.174.0/24
+  - 213.135.60.145
+  - 213.135.60.146
+  - 37.187.96.121
+
 memcached:
   enabled: true
 
@@ -146,23 +153,6 @@ postgresqlScrubberConfiguration:
       secretKeyRef: swh-scrubber-postgresql-common-secret
       secretKeyName: postgres-swh-scrubber-password
 
-internalNetworkRanges:
-  - 127.0.0.0/8
-  # rocq production range ip
-  - 192.168.100.0/24
-  # azure production range ip
-  - 192.168.200.0/22
-  # Cluster internal ip
-  - 10.42.0.0/16
-  - 10.43.0.0/16
-
-externalAuthorizedNetworkRanges:
-  - 128.93.166.14
-  - 131.107.174.0/24
-  - 213.135.60.145
-  - 213.135.60.146
-  - 37.187.96.121
-
 webThrottling:
   internalExemptedNetworkRangesRef: internalNetworkRanges
   externalExemptedNetworkRangesRef: externalAuthorizedNetworkRanges
@@ -202,3 +192,8 @@ journalWriteConfiguration:
 
 addForgeNowConfiguration:
   email_address: add-forge-now@archive.softwareheritage.org
+
+web:
+  # Due to the number of cpus on the production nodes
+  # nginx starts a lot of workers by default which consumes more memory than the default
+  nginxRequestedMemory: 90Mi

swh/values/staging/default.yaml

@@ -347,3 +347,18 @@ objstorage:
             operator: In
             values:
             - "storage1"
+
+svixConfiguration:
+  svix:
+    server_url: https://svix.internal.staging.swh.network
+    auth_token: ${SVIX_AUTH_TOKEN}
+  secrets:
+    SVIX_AUTH_TOKEN:
+      secretKeyRef: common-secrets
+      secretKeyName: svix-token
+
+webhooksConfiguration:
+  secrets:
+    WEBHOOKS_SECRET:
+      secretKeyRef: common-secrets
+      secretKeyName: webhooks-secret

swh/values/staging/overrides/swh-cassandra-next-version.yaml

@@ -306,3 +306,6 @@ toolbox:
 
 objstorage:
   enabled: false
+
+webhooks:
+  enabled: false

swh/values/staging/swh-cassandra.yaml

@@ -548,16 +548,17 @@ storage:
 web:
   enabled: true
   logLevel: INFO
+  # Usual consumption with visits webhook is 200m cpu (/2 replicas)
+  # Scale up if a real burst happen (>1 cpu per pod)
+  requestedCpu: 100m
+  requestedMemory: 300Mi
   autoScaling:
-    minReplicaCount: 1
-    maxReplicaCount: 3
-    cpuPercentageUsage: 50
+    minReplicaCount: 2
+    maxReplicaCount: 4 # 4 to test the autoscaling in extreme scenarios
+    cpuPercentageUsage: 1000
   hosts:
     - webapp.staging.swh.network
     - webapp-cassandra.internal.staging.swh.network
-  refreshSavecodenowStatus:
-    enabled: true
-    concurrencyPolicy: Forbid
   syncMailmaps:
     enabled: true
     concurrencyPolicy: Forbid
@@ -596,10 +597,12 @@ web:
   giveConfigurationRef: giveConfiguration
   throttlingConfigurationRef: webThrottling
   addForgeNowConfigurationRef: addForgeNowConfiguration
+  webhooksConfigurationRef: webhooksConfiguration
   extraConfig:
     keycloak:
       server_url: https://auth.softwareheritage.org/auth/
       realm_name: SoftwareHeritageStaging
+    save_code_now_webhook_secret: ${WEBHOOKS_SECRET}
     search_config:
       metadata_backend: swh-search
     content_display_max_size: 5242880
@@ -722,6 +725,8 @@ toolbox:
     scrubber-storage:
       scrubberDbConfigurationRef: postgresqlScrubberConfiguration
       storageConfigurationRef: scrubberROStorageConfiguration
+    webhooks:
+      webhooksConfigurationRef: svixConfiguration
 
 scrubber:
   enabled: true
@@ -923,3 +928,19 @@ objstorage:
             extraWhitelistSourceRange:
               # vpn network
               - 192.168.101.0/24
+
+webhooks:
+  enabled: true
+  svixConfigurationRef: svixConfiguration
+  journalClientConfigurationRef: journalClientConfiguration
+  priorityClassName: frontend-rpc-workload
+  deployments:
+    origin-visit-status:
+      replicas: 1
+      requestedCpu: 100m
+      requestedMemory: 100Mi
+      journalClientOverrides:
+        auto_offset_reset: latest
+        group_id: swh-archive-stg-webhooks
+        object_types:
+          - origin_visit_status

swh/values/staging/swh.yaml

@@ -350,7 +350,7 @@ graphql:
   searchConfigurationRef: remoteSearchConfiguration
   debug: yes
   introspection: yes
-  replicas: 2
+  replicas: 1
   gunicorn:
     threads: 4
     workers: 2
@@ -743,10 +743,10 @@ search:
 web:
   enabled: true
   logLevel: INFO
-  autoScaling:
-    minReplicaCount: 2
-    maxReplicaCount: 3
-    cpuPercentageUsage: 100
+  # small footprint because this webapp is almost not used, only for tests
+  replicas: 1
+  requestedCpu: 50m
+  requestedMemory: 250Mi
   hosts:
     - webapp-postgresql.internal.staging.swh.network
   refreshSavecodenowStatus:

values-swh-application-versions.yaml

@@ -48,3 +48,5 @@ swh_vault_image: container-registry.softwareheritage.org/swh/infra/swh-apps/vaul
 swh_vault_image_version: '20231127.1'
 swh_web_image: container-registry.softwareheritage.org/swh/infra/swh-apps/web
 swh_web_image_version: '20231127.1'
+swh_webhooks_image: container-registry.softwareheritage.org/swh/infra/swh-apps/webhooks
+swh_webhooks_image_version: '20231129.1'