production: Activate refresh counters cache cronjob
helm diff
[swh] Comparing changes between branches production and migrate-counters (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-counters branch for environment staging...
[swh] Generate config in migrate-counters branch for environment staging...
[swh] Generate config in migrate-counters branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-counters branch for environment production...
[swh] Generate config in migrate-counters branch for environment production...
[swh] Generate config in migrate-counters branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.ymvSYTqE/production-swh.before 2024-02-07 14:52:46.224539898 +0100
+++ /tmp/swh-chart.swh.ymvSYTqE/production-swh.after 2024-02-07 14:52:46.712539007 +0100
@@ -646,20 +646,73 @@
- directory
- origin
- origin_visit
- origin_visit_status
- release
- revision
- skipped_content
- snapshot
prefix: swh.journal.objects
---
+# Source: swh/templates/counters/script-utils.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: script-utils
+ namespace: swh
+data:
+ refresh-counters-cache.sh: |
+ #!/bin/bash
+
+ set -ex
+
+ cache_file=$1
+ static_file=$2
+ host=${3:-"http://counters1.internal.softwareheritage.org:5011/"}
+
+ static_file_stanza=""
+ if [ -n "${static_file}" ]; then
+ static_file_stanza=", \"static_file\": \"${static_file}\""
+ fi
+
+ tmp_file=$(mktemp)
+
+ trap "rm -f ${tmp_file}" EXIT
+
+ cat >"${tmp_file}" <<EOF
+ {
+ "cache_file": "${cache_file}",
+ "objects": ["content", "origin", "revision"]
+ ${static_file_stanza}
+ }
+ EOF
+
+ # This waits for the rpc to be up
+ while true; do
+ sleep 1
+ set +e
+ curl -f -v -XPOST -H 'Content-Type: application/json' $host/refresh_history -d @"${tmp_file}"
+ [[ $? -ne 0 ]] || break
+ set -e
+ done
+
+ fetch-static-history.sh: |
+ #!/bin/bash
+
+ set -ex
+
+ static_history_path=$1
+
+ url="https://gitlab.softwareheritage.org/swh/devel/swh-counters/-/snippets/1617/raw/main/snippetfile1.txt?inline=false"
+
+ curl -s $url > $static_history_path
+---
# Source: swh/templates/deposit/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: deposit-configuration-template
data:
config.yml.template: |
instance_name: deposit-rpc-ingress
allowed_hosts:
@@ -33809,20 +33862,87 @@
minReplicas: 4
maxReplicas: 10
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 50
---
+# Source: swh/templates/counters/refresh-counters-cache-cronjob.yaml
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: counters-refresh-counters-cache-cronjob
+spec:
+ # By default, every 4h
+ schedule: "0 */1 * * *"
+ concurrencyPolicy: Forbid
+ jobTemplate:
+ spec:
+ template:
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/counters
+ operator: In
+ values:
+ - "true"
+ containers:
+ - name: counters-refresh-counters-cache
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/counters:20240202.1
+ command:
+ - /cron/bin/refresh-counters-cache.sh
+ args:
+ - history.json
+ - static_history.json
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: LOG_LEVEL
+ value: INFO
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: script-utils
+ mountPath: /cron/bin/
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: counters-refresh-counters-cache-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: script-utils
+ configMap:
+ name: script-utils
+ defaultMode: 0555
+ items:
+ - key: "refresh-counters-cache.sh"
+ path: "refresh-counters-cache.sh"
+ restartPolicy: OnFailure
+---
# Source: swh/templates/scheduler/update-metrics-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
name: scheduler-update-metrics-cronjob
spec:
# By default, every 4h at midnight
schedule: "27 3-23/4 * * *"
concurrencyPolicy: Forbid
jobTemplate:
------------- diff for environment production namespace swh-cassandra -------------
No differencesEdited by Antoine R. Dumont