production: Migrate counters to dynamic infra
Migrate counters:
- journal client [1]. The configuration matches the production configuration [2]
[1] helm diff counters journal client
[swh] Comparing changes between branches production and migrate-rpc-counter (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-rpc-counter branch for environment staging...
[swh] Generate config in migrate-rpc-counter branch for environment staging...
[swh] Generate config in migrate-rpc-counter branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-rpc-counter branch for environment production...
[swh] Generate config in migrate-rpc-counter branch for environment production...
[swh] Generate config in migrate-rpc-counter branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.k2MsfpYj/production-swh.before 2024-02-07 13:47:46.053354965 +0100
+++ /tmp/swh-chart.swh.k2MsfpYj/production-swh.after 2024-02-07 13:47:46.865352896 +0100
@@ -615,20 +615,51 @@
swh:
level: "INFO"
celery.task:
level: "INFO"
root:
level: "INFO"
handlers:
- console
---
+# Source: swh/templates/counters/journal-client-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: counters-journal-client-configuration-template
+ namespace: swh
+data:
+ config.yml.template: |
+ counters:
+ cls: remote
+ url: http://counters1.internal.softwareheritage.org:5011/
+ journal:
+ brokers:
+ - kafka1.internal.softwareheritage.org
+ - kafka2.internal.softwareheritage.org
+ - kafka3.internal.softwareheritage.org
+ - kafka4.internal.softwareheritage.org
+ group_id: swh.counters.journal_client
+ message.max.bytes: 524288000
+ object_types:
+ - content
+ - directory
+ - origin
+ - origin_visit
+ - origin_visit_status
+ - release
+ - revision
+ - skipped_content
+ - snapshot
+ prefix: swh.journal.objects
+---
# Source: swh/templates/deposit/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: deposit-configuration-template
data:
config.yml.template: |
instance_name: deposit-rpc-ingress
allowed_hosts:
@@ -18938,20 +18969,114 @@
defaultMode: 0777
items:
- key: "pre-stop-idempotent.sh"
path: "pre-stop.sh"
- name: sentry-settings-for-celery-tasks
secret:
secretName: sentry-settings-for-celery-tasks
optional: true
---
+# Source: swh/templates/counters/journal-client-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh
+ name: counters-journal-client
+ labels:
+ app: counters-journal-client
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: counters-journal-client
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: counters-journal-client
+ annotations:
+ checksum/config: 8fa2c69f41ac8b5dd66860c2733e1c82c9b1601fa3d396169dea6fa9907191f9
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/journal_client
+ operator: In
+ values:
+ - "true"
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+ - name: counters-journal-client
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/counters:20240202.1
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ # - shell
+ # - sleep
+ # - infinity
+ - swh
+ - --log-level
+ - INFO
+ - counters
+ - --config-file
+ - /etc/swh/config.yml
+ - journal-client
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: LOG_LEVEL
+ value: INFO
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: counters-journal-client-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+---
# Source: swh/templates/deposit/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh
name: deposit
labels:
app: deposit
spec:
revisionHistoryLimit: 2
------------- diff for environment production namespace swh-cassandra -------------
No differences[2] production configuration
root@counters1:~# cat /etc/softwareheritage/counters/journal_client.yml
---
counters:
cls: remote
url: http://localhost:5011
journal:
brokers:
- kafka1.internal.softwareheritage.org
- kafka2.internal.softwareheritage.org
- kafka3.internal.softwareheritage.org
- kafka4.internal.softwareheritage.org
group_id: swh.counters.journal_client
prefix: swh.journal.objects
object_types:
- content
- directory
- origin
- origin_visit
- origin_visit_status
- release
- revision
- skipped_content
- snapshot
message.max.bytes: 524288000