production: Deploy one instance of origin intrinsic metadata indexer
This deploys one instance of indexer origin intrinsic metadata.
helm diff
[swh] Comparing changes between branches production and deploy-origin-intrinsic-metadata (per environment)...
Your branch is ahead of 'origin/production' by 2 commits.
(use "git push" to publish your local commits)
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment staging...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment staging...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment staging...
Your branch is ahead of 'origin/production' by 2 commits.
(use "git push" to publish your local commits)
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment production...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment production...
[swh] Generate config in deploy-origin-intrinsic-metadata branch for environment production...
------------- diff for environment staging namespace swh -------------
No differences
------------- diff for environment staging namespace swh-cassandra -------------
No differences
------------- diff for environment staging namespace swh-cassandra-next-version -------------
No differences
------------- diff for environment production namespace swh -------------
--- /tmp/swh-chart.swh.dMNQLih4/production-swh.before 2024-02-09 14:57:41.259760120 +0100
+++ /tmp/swh-chart.swh.dMNQLih4/production-swh.after 2024-02-09 14:57:41.763759426 +0100
@@ -139,20 +139,31 @@
apiVersion: v1
kind: Secret
metadata:
name: keda-indexer-extrinsic-secrets
namespace: swh
type: Opaque
stringData:
sasl: "scram_sha512"
tls: "enable"
---
+# Source: swh/templates/indexers/keda-autoscaling.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+ name: keda-indexer-origin-intrinsic-secrets
+ namespace: swh
+type: Opaque
+stringData:
+ sasl: "scram_sha512"
+ tls: "enable"
+---
# Source: swh/templates/loader-metadata/keda-secrets.yaml
apiVersion: v1
kind: Secret
metadata:
name: keda-loader-metadata-kafka-secrets
namespace: swh
type: Opaque
stringData:
sasl: "scram_sha512"
username: swh-archive-prod
@@ -1172,20 +1183,146 @@
prefix: swh.journal.objects
sasl.mechanism: SCRAM-SHA-512
sasl.password: ${BROKER_USER_PASSWORD}
sasl.username: ${BROKER_USER}
security.protocol: SASL_SSL
tools:
configuration: {}
name: swh-metadata-detector
version: 0.0.2
---
+# Source: swh/templates/indexers/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: indexer-origin-intrinsic-configuration-template
+ namespace: swh
+data:
+ config.yml.template: |
+ storage:
+ cls: pipeline
+ steps:
+ - cls: retry
+ - cls: remote
+ url: http://storage-azure-read-only-rpc-ingress
+ scheduler:
+ cls: remote
+ url: http://scheduler.internal.softwareheritage.org
+ indexer_storage:
+ cls: remote
+ url: http://indexer-storage-read-write-rpc-ingress
+ objstorage:
+ cls: multiplexer
+ objstorages:
+ - cls: filtered
+ filters_conf:
+ - type: readonly
+ storage_conf:
+ cls: http
+ compression: gzip
+ timeout: 120
+ url: https://softwareheritage.s3.amazonaws.com/content/
+ - cls: filtered
+ filters_conf:
+ - type: readonly
+ storage_conf:
+ accounts:
+ "0":
+ account_name: ${ACCOUNT_NAME_0}
+ api_secret_key: ${API_SECRET_KEY_0}
+ container_name: contents
+ "1":
+ account_name: ${ACCOUNT_NAME_1}
+ api_secret_key: ${API_SECRET_KEY_1}
+ container_name: contents
+ "2":
+ account_name: ${ACCOUNT_NAME_2}
+ api_secret_key: ${API_SECRET_KEY_2}
+ container_name: contents
+ "3":
+ account_name: ${ACCOUNT_NAME_3}
+ api_secret_key: ${API_SECRET_KEY_3}
+ container_name: contents
+ "4":
+ account_name: ${ACCOUNT_NAME_4}
+ api_secret_key: ${API_SECRET_KEY_4}
+ container_name: contents
+ "5":
+ account_name: ${ACCOUNT_NAME_5}
+ api_secret_key: ${API_SECRET_KEY_5}
+ container_name: contents
+ "6":
+ account_name: ${ACCOUNT_NAME_6}
+ api_secret_key: ${API_SECRET_KEY_6}
+ container_name: contents
+ "7":
+ account_name: ${ACCOUNT_NAME_7}
+ api_secret_key: ${API_SECRET_KEY_7}
+ container_name: contents
+ "8":
+ account_name: ${ACCOUNT_NAME_8}
+ api_secret_key: ${API_SECRET_KEY_8}
+ container_name: contents
+ "9":
+ account_name: ${ACCOUNT_NAME_9}
+ api_secret_key: ${API_SECRET_KEY_9}
+ container_name: contents
+ a:
+ account_name: ${ACCOUNT_NAME_10}
+ api_secret_key: ${API_SECRET_KEY_10}
+ container_name: contents
+ b:
+ account_name: ${ACCOUNT_NAME_11}
+ api_secret_key: ${API_SECRET_KEY_11}
+ container_name: contents
+ c:
+ account_name: ${ACCOUNT_NAME_12}
+ api_secret_key: ${API_SECRET_KEY_12}
+ container_name: contents
+ d:
+ account_name: ${ACCOUNT_NAME_13}
+ api_secret_key: ${API_SECRET_KEY_13}
+ container_name: contents
+ e:
+ account_name: ${ACCOUNT_NAME_14}
+ api_secret_key: ${API_SECRET_KEY_14}
+ container_name: contents
+ f:
+ account_name: ${ACCOUNT_NAME_15}
+ api_secret_key: ${API_SECRET_KEY_15}
+ container_name: contents
+ cls: azure-prefixed
+ - cls: filtered
+ filters_conf:
+ - type: readonly
+ storage_conf:
+ cls: remote
+ url: http://objstorage-read-only-rpc-ingress
+ journal_client:
+ brokers:
+ - kafka1.internal.softwareheritage.org:9094
+ - kafka2.internal.softwareheritage.org:9094
+ - kafka3.internal.softwareheritage.org:9094
+ - kafka4.internal.softwareheritage.org:9094
+ batch_size: 200
+ cls: kafka
+ group_id: swh-indexer-prod-01-swh.indexer.journal_client.origin_intrinsic_metadata
+ prefix: swh.journal.objects
+ sasl.mechanism: SCRAM-SHA-512
+ sasl.password: ${BROKER_USER_PASSWORD}
+ sasl.username: ${BROKER_USER}
+ security.protocol: SASL_SSL
+ tools:
+ configuration: {}
+ name: swh-metadata-detector
+ version: 0.0.2
+---
# Source: swh/templates/listers/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: lister-utils
namespace: swh
data:
pre-stop-idempotent.sh: |
#!/bin/bash
@@ -21004,20 +21141,396 @@
name: config-utils
defaultMode: 0555
- name: indexer-utils
configMap:
name: indexer-utils
defaultMode: 0777
items:
- key: "pre-stop-idempotent.sh"
path: "pre-stop.sh"
---
+# Source: swh/templates/indexers/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: indexer-origin-intrinsic
+ namespace: swh
+ labels:
+ app: indexer-origin-intrinsic
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: indexer-origin-intrinsic
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: indexer-origin-intrinsic
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: 8952686da7734512383cadbd120899b7cf772bc40fad1597d7fe06659e7b8561
+ checksum/config-utils: d75ca13b805bce6a8ab59c8e24c938f2283108f6a79134f6e71db86308651dc6
+ spec:
+ affinity:
+
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/indexer
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-low-workload
+
+ terminationGracePeriodSeconds: 3600
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+
+ - name: BROKER_USER
+ valueFrom:
+ secretKeyRef:
+ name: swh-indexer-prod-01-broker-secret
+ key: BROKER_USER
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: BROKER_USER_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: swh-indexer-prod-01-broker-secret
+ key: BROKER_USER_PASSWORD
+ # 'name' secret must exist & include that ^ key
+ optional: false
+
+
+ - name: ACCOUNT_NAME_0
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 0_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_1
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 1_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_10
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 10_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_11
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 11_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_12
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 12_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_13
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 13_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_14
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 14_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_15
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 15_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_2
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 2_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_3
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 3_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_4
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 4_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_5
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 5_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_6
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 6_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_7
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 7_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_8
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 8_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: ACCOUNT_NAME_9
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 9_account_name
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_0
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 0_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_1
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 1_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_10
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 10_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_11
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 11_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_12
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 12_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_13
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 13_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_14
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 14_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_15
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 15_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_2
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 2_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_3
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 3_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_4
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 4_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_5
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 5_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_6
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 6_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_7
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 7_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_8
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 8_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ - name: API_SECRET_KEY_9
+ valueFrom:
+ secretKeyRef:
+ name: swh-objstorage-config
+ key: 9_api_secret_key
+ # 'name' secret must exist & include that ^ key
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ containers:
+ - name: indexers
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/indexer:20240202.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /opt/swh/entrypoint.sh
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ lifecycle:
+ preStop:
+ exec:
+ command: ["/pre-stop.sh"]
+ env:
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: MAX_TASKS_PER_CHILD
+ value: "10"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_INDEXER_TYPE
+ value: origin_intrinsic_metadata
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: production
+ - name: SWH_MAIN_PACKAGE
+ value: swh.indexer
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: indexer-sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+ volumeMounts:
+ - name: indexer-utils
+ mountPath: /pre-stop.sh
+ subPath: "pre-stop.sh"
+ - name: configuration
+ mountPath: /etc/swh
+ - name: localstorage
+ mountPath: /tmp
+ volumes:
+ - name: localstorage
+
+ ephemeral:
+ volumeClaimTemplate:
+ metadata:
+ labels:
+ type: ephemeral-volume
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 100Gi
+ storageClassName: local-path
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: indexer-origin-intrinsic-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: indexer-utils
+ configMap:
+ name: indexer-utils
+ defaultMode: 0777
+ items:
+ - key: "pre-stop-idempotent.sh"
+ path: "pre-stop.sh"
+---
# Source: swh/templates/listers/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: lister-bitbucket
namespace: swh
labels:
app: lister-bitbucket
spec:
revisionHistoryLimit: 2
@@ -37149,20 +37662,43 @@
triggers:
- type: kafka
metadata:
bootstrapServers: kafka1.internal.softwareheritage.org:9094,kafka2.internal.softwareheritage.org:9094,kafka3.internal.softwareheritage.org:9094,kafka4.internal.softwareheritage.org:9094
consumerGroup: swh-archive-prod-swh.indexer.journal_client.extrinsic_metadata
lagThreshold: "1000"
offsetResetPolicy: earliest
authenticationRef:
name: keda-indexer-extrinsic-authentication
---
+# Source: swh/templates/indexers/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: indexer-origin-intrinsic-scaledobject
+ namespace: swh
+spec:
+ scaleTargetRef:
+ name: indexer-origin-intrinsic
+ pollingInterval: 120
+ minReplicaCount: 1
+ maxReplicaCount: 1
+ idleReplicaCount: 0
+ triggers:
+ - type: kafka
+ metadata:
+ bootstrapServers: kafka1.internal.softwareheritage.org:9094,kafka2.internal.softwareheritage.org:9094,kafka3.internal.softwareheritage.org:9094,kafka4.internal.softwareheritage.org:9094
+ consumerGroup: swh-indexer-prod-01-swh.indexer.journal_client.origin_intrinsic_metadata
+ lagThreshold: "1000"
+ offsetResetPolicy: earliest
+ authenticationRef:
+ name: keda-indexer-origin-intrinsic-authentication
+---
# Source: swh/templates/listers/keda-autoscaling.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: lister-bitbucket-operators
namespace: swh
spec:
scaleTargetRef:
apiVersion: apps/v1 # Optional. Default: apps/v1
kind: Deployment # Optional. Default: Deployment
@@ -39976,20 +40512,41 @@
- parameter: password
name: swh-archive-broker-secret
key: BROKER_USER_PASSWORD
- parameter: sasl
name: keda-indexer-extrinsic-secrets
key: sasl
- parameter: tls
name: keda-indexer-extrinsic-secrets
key: tls
---
+# Source: swh/templates/indexers/keda-autoscaling.yaml
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+ name: keda-indexer-origin-intrinsic-authentication
+ namespace: swh
+spec:
+ secretTargetRef:
+ - parameter: username
+ name: swh-indexer-prod-01-broker-secret
+ key: BROKER_USER
+ - parameter: password
+ name: swh-indexer-prod-01-broker-secret
+ key: BROKER_USER_PASSWORD
+ - parameter: sasl
+ name: keda-indexer-origin-intrinsic-secrets
+ key: sasl
+ - parameter: tls
+ name: keda-indexer-origin-intrinsic-secrets
+ key: tls
+---
# Source: swh/templates/listers/keda-autoscaling.yaml
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: amqp-authentication-lister-bitbucket
namespace: swh
spec:
secretTargetRef:
- parameter: host # "host" is required by the scalerObject trigger metadata
name: common-secrets
------------- diff for environment production namespace swh-cassandra -------------
No differences