swh-counters: Add template & deploy counter journal client in dynamic infra
The first commit adds the template necessary to deploy a rpc service and the counter journal client. The second commit deploys in staging the counters journal client. They are still using the remote counters rpc service in the static infra.
minikube tested with rpc and journal client ok.
make swh-helm-diff
[swh] Comparing changes between branches production and add-counters-template (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in add-counters-template branch for environment staging...
[swh] Generate config in add-counters-template branch for environment staging...
[swh] Generate config in add-counters-template branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in add-counters-template branch for environment production...
[swh] Generate config in add-counters-template branch for environment production...
[swh] Generate config in add-counters-template branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.JN4ptzmj/staging-swh-cassandra.before 2023-12-05 17:52:41.892319072 +0100
+++ /tmp/swh-chart.swh.JN4ptzmj/staging-swh-cassandra.after 2023-12-05 17:52:42.648319641 +0100
@@ -334,20 +334,49 @@
swh:
level: "INFO"
celery.task:
level: "INFO"
root:
level: "INFO"
handlers:
- console
---
+# Source: swh/templates/counters/journal-client-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: counters-journal-client-configuration-template
+ namespace: swh-cassandra
+data:
+ config.yml.template: |
+ counters:
+ cls: remote
+ url: http://counters0.internal.staging.swh.network:5011/
+ journal:
+ brokers:
+ - journal1.internal.staging.swh.network
+ - journal2.internal.staging.swh.network
+ group_id: swh.counters.journal_client
+ message.max.bytes: 524288000
+ object_types:
+ - content
+ - directory
+ - origin
+ - origin_visit
+ - origin_visit_status
+ - release
+ - revision
+ - skipped_content
+ - snapshot
+ prefix: swh.journal.objects
+---
# Source: swh/templates/deposit/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: deposit-configuration-template
data:
config.yml.template: |
instance_name: deposit-rpc-ingress
allowed_hosts:
@@ -14957,20 +14986,114 @@
defaultMode: 0777
items:
- key: "pre-stop-idempotent.sh"
path: "pre-stop.sh"
- name: sentry-settings-for-celery-tasks
secret:
secretName: sentry-settings-for-celery-tasks
optional: true
---
+# Source: swh/templates/counters/journal-client-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra
+ name: counters-journal-client
+ labels:
+ app: counters-journal-client
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: counters-journal-client
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: counters-journal-client
+ annotations:
+ checksum/config: 6597d6214907697304099de2ec36ccb1799ae3bdd8a16945276817815822d13e
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/journal_client
+ operator: In
+ values:
+ - "true"
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+ - name: counters-journal-client
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/counters:20231205.1
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ # - shell
+ # - sleep
+ # - infinity
+ - swh
+ - --log-level
+ - INFO
+ - counters
+ - --config-file
+ - /etc/swh/config.yml
+ - journal-client
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: LOG_LEVEL
+ value: INFO
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: counters-journal-client-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+---
# Source: swh/templates/deposit/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh-cassandra
name: deposit
labels:
app: deposit
spec:
revisionHistoryLimit: 2
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.JN4ptzmj/staging-swh-cassandra-next-version.before 2023-12-05 17:52:42.156319270 +0100
+++ /tmp/swh-chart.swh.JN4ptzmj/staging-swh-cassandra-next-version.after 2023-12-05 17:52:42.864319803 +0100
@@ -334,20 +334,49 @@
swh:
level: "INFO"
celery.task:
level: "INFO"
root:
level: "INFO"
handlers:
- console
---
+# Source: swh/templates/counters/journal-client-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: counters-journal-client-configuration-template
+ namespace: swh-cassandra-next-version
+data:
+ config.yml.template: |
+ counters:
+ cls: remote
+ url: http://counters0.internal.staging.swh.network:5011/
+ journal:
+ brokers:
+ - journal1.internal.staging.swh.network
+ - journal2.internal.staging.swh.network
+ group_id: swh.counters.journal_client
+ message.max.bytes: 524288000
+ object_types:
+ - content
+ - directory
+ - origin
+ - origin_visit
+ - origin_visit_status
+ - release
+ - revision
+ - skipped_content
+ - snapshot
+ prefix: swh.journal.objects
+---
# Source: swh/templates/graphql/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: graphql
namespace: swh-cassandra-next-version
data:
# TODO: rename to not have a dot in the name to allow testing
config.yml: |
storage:
@@ -14400,20 +14429,114 @@
defaultMode: 0777
items:
- key: "pre-stop-idempotent.sh"
path: "pre-stop.sh"
- name: sentry-settings-for-celery-tasks
secret:
secretName: sentry-settings-for-celery-tasks
optional: true
---
+# Source: swh/templates/counters/journal-client-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra-next-version
+ name: counters-journal-client
+ labels:
+ app: counters-journal-client
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: counters-journal-client
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: counters-journal-client
+ annotations:
+ checksum/config: e3a1e7bbe7ee89f5b9e19210728429e94c74e18c0f069fd3b3e1f04f2e56dcbf
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/journal_client
+ operator: In
+ values:
+ - "true"
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+ - name: counters-journal-client
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/counters:20231205.1
+ command:
+ - /opt/swh/entrypoint.sh
+ args:
+ # - shell
+ # - sleep
+ # - infinity
+ - swh
+ - --log-level
+ - INFO
+ - counters
+ - --config-file
+ - /etc/swh/config.yml
+ - journal-client
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: LOG_LEVEL
+ value: INFO
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: counters-journal-client-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+---
# Source: swh/templates/graphql/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: graphql
namespace: swh-cassandra-next-version
labels:
app: graphql
spec:
revisionHistoryLimit: 2
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differences