indexer_storage: Make template multi-instance & deploy read-only instance to production
This makes the indexer storage template multi-instance. This also adapts slightly the template to use the prepare configuration utils image. This simplifies and unifies the init-container with the recent change on this.
Expectedly, the diff [1] only shows:
- the renaming of the rpc configmap (due to the multi-instance change).
- The image adaptation in the init-container
The last commit actually deploys a new indexer-storage-read-only instance in production environment. [2] This instance will be used by the soon to be released webapp on the dynamic infra.
[1]
make swh-helm-diff
[swh] Comparing changes between branches production and migrate-indexers-to-prod (per environment)...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
Switched to branch 'migrate-indexers-to-prod'
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
Switched to branch 'production'
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
Switched to branch 'migrate-indexers-to-prod'
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.eUVAEZAd/staging-swh.before 2023-12-13 14:33:41.102880007 +0100
+++ /tmp/swh-chart.swh.eUVAEZAd/staging-swh.after 2023-12-13 14:33:41.838879725 +0100
@@ -287,21 +287,21 @@
realm: SoftwareHeritageStaging
client: swh-web
cache:
url: memcached://memcached:11211
---
# Source: swh/templates/indexer-storage/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
- name: indexer-storage-configuration-template
+ name: indexer-storage-rpc-configuration-template
data:
config.yml.template: |
indexer_storage:
cls: postgresql
db: host=db1.internal.staging.swh.network port=5432 user=swh-indexer dbname=swh-indexer password=${POSTGRESQL_PASSWORD}
journal_writer:
client_id: swh.idx_storage.journal_writer.storage1
cls: kafka
prefix: swh.journal.indexed
@@ -15096,57 +15096,57 @@
app: indexer-storage-rpc
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: indexer-storage-rpc
annotations:
- checksum/config: b71cc0136e069c1a6ad2041e5e764a0823db7d2b29692156e3358374ca9ba604
- checksum/config-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
+ checksum/config: e688ca7853c5421a857f84a4da8a5267fee3117c32587fd49510eaee2b23fb62
+ checksum/database-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
- "true"
priorityClassName: swh-frontend-rpc
initContainers:
- name: prepare-configuration
- image: debian:bullseye
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
imagePullPolicy: IfNotPresent
command:
- - /bin/bash
- args:
- - -c
- - eval echo "\"$(</etc/swh/configuration-template/config.yml.template)\"" > /etc/swh/config.yml
+ - /entrypoints/prepare-configuration.sh
env:
- name: POSTGRESQL_PASSWORD
valueFrom:
secretKeyRef:
name: swh-indexer-storage-postgresql-secret
key: postgres-swh-indexer-password
# 'name' secret must exist & include that ^ key
optional: false
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
- name: check-migration
image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage:20231205.1
command:
- /entrypoints/check-indexer-storage-db-version.sh
env:
- name: MODULE
value: indexer-storage
volumeMounts:
- name: configuration
@@ -15210,28 +15210,32 @@
- name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
value: "true"
volumeMounts:
- name: configuration
mountPath: /etc/swh
volumes:
- name: configuration
emptyDir: {}
- name: configuration-template
configMap:
- name: indexer-storage-configuration-template
+ name: indexer-storage-rpc-configuration-template
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: database-utils
configMap:
name: database-utils
defaultMode: 0555
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
---
# Source: swh/templates/indexers/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: indexer-origin-intrinsic
namespace: swh
labels:
app: indexer-origin-intrinsic
spec:
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differences[2] (diff on top of the previous 2 commits to only concentrate on the deployment)
make swh-helm-diff
[swh] Comparing changes between branches production and migrate-indexers-to-prod (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
[swh] Generate config in migrate-indexers-to-prod branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
[swh] Generate config in migrate-indexers-to-prod branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.ugkTtDiH/production-swh.before 2023-12-13 16:05:23.529260972 +0100
+++ /tmp/swh-chart.swh.ugkTtDiH/production-swh.after 2023-12-13 16:05:23.985260878 +0100
@@ -270,20 +270,32 @@
anonymous: 50
user: 500
auth:
server: https://auth.softwareheritage.org/auth/
realm: SoftwareHeritage
client: swh-web
cache:
url: memcached://memcached:11211
---
+# Source: swh/templates/indexer-storage/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh
+ name: indexer-storage-read-only-configuration-template
+data:
+ config.yml.template: |
+ indexer_storage:
+ cls: postgresql
+ db: host=db.internal.softwareheritage.org port=5432 user=guest dbname=softwareheritage-indexer password=${POSTGRESQL_PASSWORD}
+---
# Source: swh/templates/listers/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: lister-utils
namespace: swh
data:
pre-stop-idempotent.sh: |
#!/bin/bash
@@ -15575,20 +15587,30 @@
port: 443
protocol: TCP
targetPort: 9443
selector:
app: keda-admission-webhooks
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: indexer-storage-read-only-rpc-ingress
+ namespace: swh
+spec:
+ type: ExternalName
+ externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: search-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -15606,20 +15628,35 @@
namespace: swh
spec:
type: ClusterIP
selector:
app: graphql
ports:
- port: 5013
targetPort: 5013
name: rpc
---
+# Source: swh/templates/indexer-storage/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: indexer-storage-read-only
+ namespace: swh
+spec:
+ type: ClusterIP
+ selector:
+ app: indexer-storage-read-only
+ ports:
+ - port: 5007
+ targetPort: 5007
+ name: rpc
+---
# Source: swh/templates/memcached/service.yaml
apiVersion: v1
kind: Service
metadata:
name: memcached
namespace: swh
labels:
app: memcached
chart: "swh-0.1.0"
spec:
@@ -16287,20 +16324,169 @@
- name: config
mountPath: /etc/swh/config.yml
subPath: config.yml
readOnly: true
volumes:
- name: config
configMap:
name: graphql
defaultMode: 0444
---
+# Source: swh/templates/indexer-storage/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh
+ name: indexer-storage-read-only
+ labels:
+ app: indexer-storage-read-only
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: indexer-storage-read-only
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: indexer-storage-read-only
+ annotations:
+ checksum/config: 0413c049af8223046084d7c4f880f076d11d09b13523036d5cdeb6a2d4f537bd
+ checksum/database-utils: 95e3ec2d5d9a9195227631abff0736a8d94bcbcc795abf56981a586844da0c79
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/rpc
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+
+ - name: POSTGRESQL_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-indexer-storage-postgresql-secret
+ key: postgres-guest-password
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+
+ - name: check-migration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage:20231205.1
+ command:
+ - /entrypoints/check-indexer-storage-db-version.sh
+ env:
+ - name: MODULE
+ value: indexer-storage
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: database-utils
+ mountPath: /entrypoints
+ containers:
+ - name: indexer-storage-read-only
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer_storage:20231205.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5007
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: THREADS
+ value: "2"
+ - name: WORKERS
+ value: "4"
+ - name: TIMEOUT
+ value: "60"
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: LOG_LEVEL
+ value: "INFO"
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.indexer
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: indexer-storage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: indexer-storage-read-only-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: database-utils
+ configMap:
+ name: database-utils
+ defaultMode: 0555
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+---
# Source: swh/templates/listers/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: lister-bitbucket
namespace: swh
labels:
app: lister-bitbucket
spec:
revisionHistoryLimit: 2
@@ -28105,20 +28291,43 @@
defaultMode: 0777
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: toolbox-script-utils
configMap:
name: toolbox-script-utils
defaultMode: 0555
---
+# Source: swh/templates/indexer-storage/autoscaling.yaml
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ namespace: swh
+ name: indexer-storage-read-only
+ labels:
+ app: indexer-storage-read-only
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: indexer-storage-read-only
+ minReplicas: 2
+ maxReplicas: 4
+ metrics:
+ - type: Resource
+ resource:
+ name: cpu
+ target:
+ type: Utilization
+ averageUtilization: 90
+---
# Source: swh/templates/search/rpc-autoscale.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
namespace: swh
name: search-rpc
labels:
app: search-rpc
spec:
scaleTargetRef:
@@ -28279,20 +28488,47 @@
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: graphql
port:
number: 5013
---
+# Source: swh/templates/indexer-storage/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh
+ name: indexer-storage-read-only-ingress-default
+ annotations:
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
+ nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+ nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
+ nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
+
+spec:
+ rules:
+ - host: indexer-storage-read-only-rpc-ingress
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: indexer-storage-read-only
+ port:
+ number: 5007
+---
# Source: swh/templates/scheduler/rpc-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh
name: scheduler-rpc-ingress-default
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,127.0.0.0/8,192.168.100.0/24,192.168.130.221,192.168.200.0/22
nginx.ingress.kubernetes.io/proxy-body-size: 4G
nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
@@ -28340,20 +28576,21 @@
port:
number: 5008
---
# Source: swh/templates/search/rpc-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh
name: search-rpc-ingress-default
annotations:
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16
nginx.ingress.kubernetes.io/proxy-body-size: 4G
nginx.ingress.kubernetes.io/proxy-connect-timeout: "90"
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
nginx.ingress.kubernetes.io/proxy-send-timeout: "90"
spec:
rules:
- host: search-rpc-ingress
http:
------------- diff for environment production namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.ugkTtDiH/production-swh-cassandra.before 2023-12-13 16:05:23.701260937 +0100
+++ /tmp/swh-chart.swh.ugkTtDiH/production-swh-cassandra.after 2023-12-13 16:05:24.157260843 +0100
@@ -11095,20 +11095,30 @@
port: 443
protocol: TCP
targetPort: 9443
selector:
app: keda-admission-webhooks
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: indexer-storage-read-only-rpc-ingress
+ namespace: swh-cassandra
+spec:
+ type: ExternalName
+ externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: search-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata: