graph: Deploy rpc and grpc instances in staging cluster
Expose graph rpc and grpc instances in the staging clusters. Then adapt the provenance, vault and web instances to reuse those graph services (instead of the production ones).
For now, this only uses a small python3k dataset. So it's effectively diverging from the current dataset within the staging cluster.
The first commit readapts slightly the next-version services with the same idea.
helm diff
[swh] Comparing changes between branches production and mr/deploy-graph-in-staging (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in mr/deploy-graph-in-staging branch for environment staging...
[swh] Generate config in mr/deploy-graph-in-staging branch for environment staging...
[swh] Generate config in mr/deploy-graph-in-staging branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in mr/deploy-graph-in-staging branch for environment production...
[swh] Generate config in mr/deploy-graph-in-staging branch for environment production...
[swh] Generate config in mr/deploy-graph-in-staging branch for environment production...
------------- diff for environment staging namespace swh -------------
--- /tmp/swh-chart.swh.nHJQcflv/staging-swh.before 2024-10-07 17:53:13.326581370 +0200
+++ /tmp/swh-chart.swh.nHJQcflv/staging-swh.after 2024-10-07 17:53:18.686357962 +0200
@@ -2090,20 +2090,26 @@
url: http://storage-postgresql-read-only-rpc-ingress
search:
cls: remote
url: http://search-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
vault:
cls: remote
url: http://vault-rpc-ingress
+ graph:
+ max_edges:
+ anonymous: 1000
+ staff: 0
+ user: 100000
+ url: http://graph-rpc-ingress/graph
indexer_storage:
cls: remote
url: http://indexer-storage-rpc-ingress
counters_backend: swh-counters
counters:
cls: remote
url: http://counters-rpc-ingress
deposit:
private_api_url: https://deposit.staging.swh.network/1/private/
private_api_user: ${DEPOSIT_USERNAME}
@@ -2268,20 +2274,30 @@
name: deposit-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: graph-grpc-ingress
+ namespace: swh
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: indexer-storage-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -2358,20 +2374,30 @@
name: webapp-provenance-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: graph-rpc-ingress
+ namespace: swh
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: search-rpc-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -5707,21 +5733,21 @@
app: web-postgresql
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: web-postgresql
annotations:
- checksum/config: 5d893d8982004eb8518b42d6c3da06cac917191d669eaebab56ea9e136b94d1c
+ checksum/config: 5fdca1b40bce2455a62daa357af33b150bf82082e427e0cc81e2df07c750b31c
checksum/config-logging: 81fb24577eb1777be8690f58c1e92d701777fe4ff045bb8445feb924947b9f84
checksum/config-utils: d75ca13b805bce6a8ab59c8e24c938f2283108f6a79134f6e71db86308651dc6
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/web
operator: In
------------- diff for environment staging namespace swh-cassandra -------------
--- /tmp/swh-chart.swh.nHJQcflv/staging-swh-cassandra.before 2024-10-07 17:53:15.606486338 +0200
+++ /tmp/swh-chart.swh.nHJQcflv/staging-swh-cassandra.after 2024-10-07 17:53:20.974262596 +0200
@@ -277,21 +277,21 @@
# Source: swh/templates/alter/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: alter-template
namespace: swh-cassandra
data:
config.yml.template: |
graph:
timeout: null
- url: http://graph.internal.softwareheritage.org:5009/graph
+ url: http://graph-rpc-ingress/graph
recovery_bundles:
secret_sharing:
groups:
legal:
minimum_required_shares: 1
recipient_keys:
Ali: age123hpq9m25xsmx7caqvyv8k3fxaqastc3evyq9q7myur7l9ukj4dsnp7a5v
Bob: age1mrhte5tlpzpz57gg85nzcefqc5pm5usmakqpuurxux7ry2rmhdgs7r9u68
sysadmins:
minimum_required_shares: 1
@@ -942,20 +942,45 @@
},
"root": {
"handlers": [
"console"
],
"level": "INFO"
},
"version": 1
}
---
+# Source: swh/templates/graph/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: graph-grpc-python3k-configuration-template
+data:
+ config.yml.template: |
+ graph:
+ max_ram: 1g
+---
+# Source: swh/templates/graph/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra
+ name: graph-rpc-python3k-configuration-template
+data:
+ config.yml.template: |
+ graph:
+ cls: remote
+ grpc_server:
+ port: 80
+ url: graph-grpc-ingress:80
+---
# Source: swh/templates/graphql/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: graphql-cassandra-configuration-template
data:
# TODO: rename to not have a dot in the name to allow testing
config.yml: |
storage:
@@ -5489,21 +5514,21 @@
namespace: swh-cassandra
name: provenance-graph-granet-configuration-template
data:
config.yml.template: |
provenance:
cls: known_swhid_filter
filter_licenses: false
provenance:
cls: graph
max_edges: 100000
- url: graph.internal.softwareheritage.org:50091
+ url: graph-grpc-ingress:80
---
# Source: swh/templates/provenance/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra
name: provenance-graph-granet-configuration-logging
data:
logging-gunicorn.json: |
{
@@ -7882,20 +7907,26 @@
url: http://search-rpc-ingress
provenance:
cls: remote
url: http://webapp-provenance-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
vault:
cls: remote
url: http://vault-rpc-ingress
+ graph:
+ max_edges:
+ anonymous: 1000
+ staff: 0
+ user: 100000
+ url: http://graph-rpc-ingress/graph
indexer_storage:
cls: remote
url: http://indexer-storage-rpc-ingress
counters_backend: swh-counters
counters:
cls: remote
url: http://counters-rpc-ingress
deposit:
private_api_url: https://deposit.staging.swh.network/1/private/
private_api_user: ${DEPOSIT_USERNAME}
@@ -8066,20 +8097,26 @@
url: http://storage-cassandra-read-only-ingress
search:
cls: remote
url: http://search-rpc-ingress
scheduler:
cls: remote
url: http://scheduler.internal.staging.swh.network
vault:
cls: remote
url: http://vault-rpc-ingress
+ graph:
+ max_edges:
+ anonymous: 1000
+ staff: 0
+ user: 100000
+ url: http://graph-rpc-ingress/graph
indexer_storage:
cls: remote
url: http://indexer-storage-rpc-ingress
counters_backend: swh-counters
counters:
cls: remote
url: http://counters-rpc-ingress
deposit:
private_api_url: https://deposit.staging.swh.network/1/private/
private_api_user: ${DEPOSIT_USERNAME}
@@ -8271,20 +8308,54 @@
namespace: swh-cassandra
spec:
storageClassName: cephfs
accessModes:
- ReadWriteMany
volumeMode: Filesystem
resources:
requests:
storage: 30Gi
---
+# Source: swh/templates/graph/persistent-volume-claims.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: graph-python3k-inmemory-pvc
+ namespace: swh-cassandra
+ labels:
+ app: graph-grpc-python3k
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ storageClassName: local-path
+ volumeMode: Filesystem
+---
+# Source: swh/templates/graph/persistent-volume-claims.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: graph-python3k-persistent-pvc
+ namespace: swh-cassandra
+ labels:
+ app: graph-grpc-python3k
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ storageClassName: local-persistent
+ volumeMode: Filesystem
+---
# Source: swh/templates/counters/rpc-service.yaml
apiVersion: v1
kind: Service
metadata:
name: counters-rpc
namespace: swh-cassandra
labels:
app: counters-rpc
spec:
type: ClusterIP
@@ -8333,20 +8404,30 @@
name: deposit-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: graph-grpc-ingress
+ namespace: swh-cassandra
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: indexer-storage-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -8423,20 +8504,30 @@
name: webapp-provenance-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: graph-rpc-ingress
+ namespace: swh-cassandra
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: search-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -8479,20 +8570,54 @@
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
name: vault-rpc-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
+# Source: swh/templates/graph/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: graph-grpc-python3k
+ namespace: swh-cassandra
+ labels:
+ app: graph-grpc-python3k
+spec:
+ type: ClusterIP
+ selector:
+ app: graph-grpc-python3k
+ ports:
+ - port: 50091
+ targetPort: 50091
+ name: grpc
+---
+# Source: swh/templates/graph/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: graph-rpc-python3k
+ namespace: swh-cassandra
+ labels:
+ app: graph-rpc-python3k
+spec:
+ type: ClusterIP
+ selector:
+ app: graph-rpc-python3k
+ ports:
+ - port: 5009
+ targetPort: 5009
+ name: rpc
+---
# Source: swh/templates/graphql/service.yaml
apiVersion: v1
kind: Service
metadata:
name: graphql-cassandra
namespace: swh-cassandra
labels:
app: graphql-cassandra
spec:
type: ClusterIP
@@ -8807,21 +8932,21 @@
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: alter
annotations:
# Force a rollout upgrade if the configuration changes
- checksum/config: dec88cc586471f237e4e1f21eb2e70705bbe03045372f17f2333896af7c083da
+ checksum/config: 21be4ae435f4ea6d590f84ed47714a453495b501803a02e285816d2738c5ccfa
spec:
securityContext:
fsGroup: 1000
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
@@ -9818,20 +9943,361 @@
items:
- key: "config.yml.template"
path: "config.yml.template"
- name: configuration-logging
configMap:
name: deposit-configuration-logging
items:
- key: "logging-gunicorn.json"
path: "logging-gunicorn.json"
---
+# Source: swh/templates/graph/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra
+ name: graph-grpc-python3k
+ labels:
+ app: graph-grpc-python3k
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: graph-grpc-python3k
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: graph-grpc-python3k
+ annotations:
+ checksum/config: b73b013412ed4679009823fcd1967f46c6c74ce0cac466277d96a7303b1ee5e9
+ checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
+ checksum/backend-utils: 40879a26fb269aa7fb852180b3de6f7a3ab7a6910d5a00be3a6efe09cee885c1
+ spec:
+ nodeSelector:
+ kubernetes.io/hostname: rancher-node-staging-rke2-metal01
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/graph
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+
+ - name: graph-prepare-memory-volume
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/graph-prepare-memory-volume.sh
+ env:
+ - name: WITNESS_FILE
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: WITNESS_SOURCE_FILE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: PERIOD
+ value: "3"
+ - name: GRAPH_NAME
+ value: graph
+ - name: DATASET_SOURCE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed
+ - name: DATASET_LOCATION
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed
+ volumeMounts:
+ - name: backend-utils
+ mountPath: /entrypoints
+ readOnly: true
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+
+ - name: wait-for-dataset
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/graph-wait-for-dataset.sh
+ env:
+ - name: WITNESS_FILE
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: DATASET_LOCATION
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed
+ - name: PERIOD
+ value: "3"
+ volumeMounts:
+ - name: backend-utils
+ mountPath: /entrypoints
+ readOnly: true
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+
+ containers:
+ - name: graph-grpc-python3k
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/graph:20240925.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 50091
+ name: grpc
+ readinessProbe:
+ tcpSocket:
+ port: grpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ tcpSocket:
+ port: grpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: GRAPH_TYPE
+ value: grpc
+ - name: PORT
+ value: "50091"
+ - name: GRAPH_PATH
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed/graph
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: STATSD_TAGS
+ value: deployment:graph-grpc-python3k
+ - name: STATSD_SERVICE_TYPE
+ value: graph-grpc-python3k
+ - name: SWH_LOG_LEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.graph
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: storage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: graph-grpc-python3k-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: backend-utils
+ configMap:
+ name: backend-utils
+ defaultMode: 0555
+ - name: graph-python3k-inmemory
+ persistentVolumeClaim:
+ claimName: graph-python3k-inmemory-pvc
+
+ - name: graph-python3k-persistent
+ persistentVolumeClaim:
+ claimName: graph-python3k-persistent-pvc
+---
+# Source: swh/templates/graph/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra
+ name: graph-rpc-python3k
+ labels:
+ app: graph-rpc-python3k
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: graph-rpc-python3k
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: graph-rpc-python3k
+ annotations:
+ checksum/config: cd2257ef14a7e6adb8b613ab147d995cfc4b4f250daba77c5b5cdbd63dcb1a35
+ checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
+ checksum/backend-utils: 40879a26fb269aa7fb852180b3de6f7a3ab7a6910d5a00be3a6efe09cee885c1
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/graph
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+ - name: graph-rpc-python3k
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/graph:20240925.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5009
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ tcpSocket:
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: GRAPH_TYPE
+ value: rpc
+ - name: PORT
+ value: "5009"
+ - name: GRAPH_PATH
+ value: /srv/graph/graph/compressed/graph
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: STATSD_TAGS
+ value: deployment:graph-rpc-python3k
+ - name: STATSD_SERVICE_TYPE
+ value: graph-rpc-python3k
+ - name: SWH_LOG_LEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.graph
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: storage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: graph-rpc-python3k-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: backend-utils
+ configMap:
+ name: backend-utils
+ defaultMode: 0555
+---
# Source: swh/templates/graphql/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: graphql-cassandra
namespace: swh-cassandra
labels:
app: graphql-cassandra
spec:
revisionHistoryLimit: 2
@@ -17642,21 +18108,21 @@
app: provenance-graph-granet
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: provenance-graph-granet
annotations:
- checksum/config: efc6933069d30442c937aeedba25e7a0c2555e1ca65aea52be04b618e9d33d40
+ checksum/config: e466b9f13c5124eedcaa557de44e64caf7c7ff4aa9ab5dab35b7ceede6a09568
checksum/config-logging: ddcd27d991938c46f4fc0ad7ee028cb3005f186b3db022596c9ae94363881e4f
checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
@@ -21755,21 +22221,21 @@
app: web-cassandra
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: web-cassandra
annotations:
- checksum/config: ceba4f14dd0992d7dced821c914cec08666bcb41351608c49a6ed4634e67615a
+ checksum/config: 82d76d04b25c888276a045edc3029e7dd33853254200a97984ccbca2884994a8
checksum/config-logging: 21c90a039f27f4476045b8973a841bb2b3c0e4435be7fb9ab1d748372f8a96c8
checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/web
operator: In
@@ -22023,21 +22489,21 @@
app: web-webhooks
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: web-webhooks
annotations:
- checksum/config: 4cb7410d290085015b000af1700ebc5dc80015ec659dfcc0ca7e1c291083a360
+ checksum/config: 3ff3c16be901f8d966e6eec56a09532c607e53ec0ccb8afcbf5e9df6e87584d8
checksum/config-logging: 8204fa505554e2a92718b6446f5335481339d9b88337df1e300a3cdc6868c0a8
checksum/config-utils: 13a26f6add17e96ce01550153c77dcd48de60241a3f4db3c93d5467234be2a7f
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/web
operator: In
@@ -22519,20 +22985,155 @@
minReplicas: 2
maxReplicas: 4
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 75
---
+# Source: swh/templates/graph/job-fetch-dataset.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ namespace: swh-cassandra
+ name: graph-grpc-python3k-job-fetch-dataset
+ labels:
+ app: graph-grpc-python3k
+spec:
+ backoffLimit: 0
+ template:
+ metadata:
+ labels:
+ app: graph-grpc-python3k
+ spec:
+ restartPolicy: Never
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/graph
+ operator: In
+ values:
+ - "true"
+ nodeSelector:
+ kubernetes.io/hostname: rancher-node-staging-rke2-metal01
+ priorityClassName: swh-cassandra-frontend-rpc
+
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+
+ - name: fetch-graph-dataset
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/graph:20240925.1
+ command:
+ - /entrypoints/graph-fetch-dataset.sh
+ env:
+ - name: WITNESS_FILE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: GRAPH_NAME
+ value: graph
+ - name: DATASET_NAME
+ value: 2021-03-23-popular-3k-python
+ - name: DATASET_LOCATION
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed
+ - name: DATASET_URL
+ value:
+ - name: DOWNLOAD_METHOD
+ value: s3
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: backend-utils
+ mountPath: /entrypoints
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+
+ - name: graph-prepare-memory-volume
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/graph-prepare-memory-volume.sh
+ env:
+ - name: WITNESS_FILE
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: WITNESS_SOURCE_FILE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: PERIOD
+ value: "3"
+ - name: GRAPH_NAME
+ value: graph
+ - name: DATASET_SOURCE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed
+ - name: DATASET_LOCATION
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed
+ volumeMounts:
+ - name: backend-utils
+ mountPath: /entrypoints
+ readOnly: true
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: graph-grpc-python3k-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: backend-utils
+ configMap:
+ name: backend-utils
+ defaultMode: 0555
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: graph-python3k-inmemory
+ persistentVolumeClaim:
+ claimName: graph-python3k-inmemory-pvc
+
+ - name: graph-python3k-persistent
+ persistentVolumeClaim:
+ claimName: graph-python3k-persistent-pvc
+---
# Source: swh/templates/counters/refresh-counters-cache-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
namespace: swh-cassandra
name: counters-refresh-counters-cache-cronjob
spec:
# By default, every 4h
schedule: "0 */1 * * *"
concurrencyPolicy: Forbid
@@ -23128,20 +23729,101 @@
number: 5006
- path: /static
pathType: Prefix
backend:
service:
name: deposit
port:
number: 80
---
+# Source: swh/templates/graph/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh-cassandra
+ name: graph-grpc-python3k-ingress-default
+ labels:
+ app: graph-grpc-python3k
+ endpoint-definition: default
+ annotations:
+ nginx.ingress.kubernetes.io/backend-protocol: GRPC
+ nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-buffering: "on"
+ nginx.ingress.kubernetes.io/service-upstream: "true"
+ nginx.ingress.kubernetes.io/ssl-redirect: "true"
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.100.29/32,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
+spec:
+ ingressClassName: nginx
+ rules:
+ - host: graph-grpc.internal.staging.swh.network
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: graph-grpc-python3k
+ port:
+ number: 50091
+
+ - host: graph-grpc-ingress
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: graph-grpc-python3k
+ port:
+ number: 50091
+---
+# Source: swh/templates/graph/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh-cassandra
+ name: graph-rpc-python3k-ingress-default
+ labels:
+ app: graph-rpc-python3k
+ endpoint-definition: default
+ annotations:
+ nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-buffering: "on"
+ nginx.ingress.kubernetes.io/service-upstream: "true"
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.100.29/32,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
+spec:
+ rules:
+ - host: graph.internal.staging.swh.network
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: graph-rpc-python3k
+ port:
+ number: 5009
+
+ - host: graph-rpc-ingress
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: graph-rpc-python3k
+ port:
+ number: 5009
+---
# Source: swh/templates/graphql/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh-cassandra
name: graphql-cassandra-ingress-default
labels:
app: graphql-cassandra
endpoint-definition: default
annotations:
------------- diff for environment staging namespace swh-cassandra-next-version -------------
--- /tmp/swh-chart.swh.nHJQcflv/staging-swh-cassandra-next-version.before 2024-10-07 17:53:17.690399475 +0200
+++ /tmp/swh-chart.swh.nHJQcflv/staging-swh-cassandra-next-version.after 2024-10-07 17:53:23.066175401 +0200
@@ -709,29 +709,54 @@
data:
config.yml.template: |
graph:
max_ram: 1g
---
# Source: swh/templates/graph/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra-next-version
+ name: graph-grpc-python3k-configuration-template
+data:
+ config.yml.template: |
+ graph:
+ max_ram: 1g
+---
+# Source: swh/templates/graph/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra-next-version
name: graph-rpc-example-configuration-template
data:
config.yml.template: |
graph:
cls: remote
grpc_server:
port: 80
url: graph-grpc-next-version-ingress:80
---
+# Source: swh/templates/graph/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ namespace: swh-cassandra-next-version
+ name: graph-rpc-python3k-configuration-template
+data:
+ config.yml.template: |
+ graph:
+ cls: remote
+ grpc_server:
+ port: 80
+ url: graph-grpc-next-version-ingress:80
+---
# Source: swh/templates/graphql/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra-next-version
name: graphql-cassandra-configuration-template
data:
# TODO: rename to not have a dot in the name to allow testing
config.yml: |
storage:
@@ -4790,21 +4815,21 @@
namespace: swh-cassandra-next-version
name: provenance-graph-granet-configuration-template
data:
config.yml.template: |
provenance:
cls: known_swhid_filter
filter_licenses: false
provenance:
cls: graph
max_edges: 100000
- url: grpc-next-version.internal.staging.swh.network:50091
+ url: graph-grpc-next-version-ingress:80
---
# Source: swh/templates/provenance/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh-cassandra-next-version
name: provenance-graph-granet-configuration-logging
data:
logging-gunicorn.json: |
{
@@ -7148,21 +7173,25 @@
provenance:
cls: remote
url: http://webapp-provenance-ingress-next-version
scheduler:
cls: remote
url: http://scheduler-rpc:5008
vault:
cls: remote
url: http://vault-rpc:5005
graph:
- url: http://graph-next-version.internal.staging.swh.network/graph
+ max_edges:
+ anonymous: 1000
+ staff: 0
+ user: 100000
+ url: http://graph-rpc-next-version-ingress/graph
indexer_storage:
cls: remote
url: http://indexer-storage-rw:5007
counters_backend: swh-counters
counters:
cls: remote
url: http://counters-rpc:5011
deposit:
private_api_url: https://deposit-next-version.internal.staging.swh.network/1/private/
private_api_user: ${DEPOSIT_USERNAME}
@@ -7338,20 +7367,54 @@
app: graph-grpc-example
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
storageClassName: local-path
volumeMode: Filesystem
---
+# Source: swh/templates/graph/persistent-volume-claims.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: graph-python3k-inmemory-pvc
+ namespace: swh-cassandra-next-version
+ labels:
+ app: graph-grpc-python3k
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ storageClassName: local-path
+ volumeMode: Filesystem
+---
+# Source: swh/templates/graph/persistent-volume-claims.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: graph-python3k-persistent-pvc
+ namespace: swh-cassandra-next-version
+ labels:
+ app: graph-grpc-python3k
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ storageClassName: local-persistent
+ volumeMode: Filesystem
+---
# Source: swh/templates/counters/rpc-service.yaml
apiVersion: v1
kind: Service
metadata:
name: counters-rpc
namespace: swh-cassandra-next-version
labels:
app: counters-rpc
spec:
type: ClusterIP
@@ -7420,20 +7483,30 @@
name: graph-rpc-next-version-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: graph-grpc-ingress
+ namespace: swh-cassandra-next-version
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: indexer-storage-rpc-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -7510,20 +7583,30 @@
name: webapp-provenance-ingress-next-version
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
+ name: graph-rpc-ingress
+ namespace: swh-cassandra-next-version
+spec:
+ type: ExternalName
+ externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+# Source: swh/templates/external-services/cname.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: search-rpc-ingress
namespace: swh-cassandra-next-version
spec:
type: ExternalName
externalName: archive-staging-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
@@ -7587,33 +7670,67 @@
app: graph-grpc-example
ports:
- port: 50091
targetPort: 50091
name: grpc
---
# Source: swh/templates/graph/service.yaml
apiVersion: v1
kind: Service
metadata:
+ name: graph-grpc-python3k
+ namespace: swh-cassandra-next-version
+ labels:
+ app: graph-grpc-python3k
+spec:
+ type: ClusterIP
+ selector:
+ app: graph-grpc-python3k
+ ports:
+ - port: 50091
+ targetPort: 50091
+ name: grpc
+---
+# Source: swh/templates/graph/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
name: graph-rpc-example
namespace: swh-cassandra-next-version
labels:
app: graph-rpc-example
spec:
type: ClusterIP
selector:
app: graph-rpc-example
ports:
- port: 5009
targetPort: 5009
name: rpc
---
+# Source: swh/templates/graph/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: graph-rpc-python3k
+ namespace: swh-cassandra-next-version
+ labels:
+ app: graph-rpc-python3k
+spec:
+ type: ClusterIP
+ selector:
+ app: graph-rpc-python3k
+ ports:
+ - port: 5009
+ targetPort: 5009
+ name: rpc
+---
# Source: swh/templates/graphql/service.yaml
apiVersion: v1
kind: Service
metadata:
name: graphql-cassandra
namespace: swh-cassandra-next-version
labels:
app: graphql-cassandra
spec:
type: ClusterIP
@@ -8972,20 +9089,227 @@
- name: swh-graph-grpc-inmemory
persistentVolumeClaim:
claimName: swh-graph-grpc-inmemory-pvc
---
# Source: swh/templates/graph/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh-cassandra-next-version
+ name: graph-grpc-python3k
+ labels:
+ app: graph-grpc-python3k
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: graph-grpc-python3k
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: graph-grpc-python3k
+ annotations:
+ checksum/config: 776cb520c1cb3c498dab9cef0737a3cf6103775dd3d2167f6fbac5f9284b54ea
+ checksum/config-utils: 94d255131467f84bef964a4c72b2b792c5ebaf711bb1c77829d7cd1007a8ac22
+ checksum/backend-utils: eca303b18ba9654db85a0f8d4838403916b673d7f06c09aedd8771b5028c7f62
+ spec:
+ nodeSelector:
+ kubernetes.io/hostname: rancher-node-staging-rke2-metal01
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/graph
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-next-version-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+
+ - name: graph-prepare-memory-volume
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/graph-prepare-memory-volume.sh
+ env:
+ - name: WITNESS_FILE
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: WITNESS_SOURCE_FILE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: PERIOD
+ value: "3"
+ - name: GRAPH_NAME
+ value: graph
+ - name: DATASET_SOURCE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed
+ - name: DATASET_LOCATION
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed
+ volumeMounts:
+ - name: backend-utils
+ mountPath: /entrypoints
+ readOnly: true
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+
+ - name: wait-for-dataset
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/graph-wait-for-dataset.sh
+ env:
+ - name: WITNESS_FILE
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: DATASET_LOCATION
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed
+ - name: PERIOD
+ value: "3"
+ volumeMounts:
+ - name: backend-utils
+ mountPath: /entrypoints
+ readOnly: true
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+
+ containers:
+ - name: graph-grpc-python3k
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/graph:20240925.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 50091
+ name: grpc
+ readinessProbe:
+ tcpSocket:
+ port: grpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ tcpSocket:
+ port: grpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: GRAPH_TYPE
+ value: grpc
+ - name: PORT
+ value: "50091"
+ - name: GRAPH_PATH
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed/graph
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: STATSD_TAGS
+ value: deployment:graph-grpc-python3k
+ - name: STATSD_SERVICE_TYPE
+ value: graph-grpc-python3k
+ - name: SWH_LOG_LEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.graph
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: storage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: graph-grpc-python3k-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: backend-utils
+ configMap:
+ name: backend-utils
+ defaultMode: 0555
+ - name: graph-python3k-inmemory
+ persistentVolumeClaim:
+ claimName: graph-python3k-inmemory-pvc
+
+ - name: graph-python3k-persistent
+ persistentVolumeClaim:
+ claimName: graph-python3k-persistent-pvc
+---
+# Source: swh/templates/graph/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra-next-version
name: graph-rpc-example
labels:
app: graph-rpc-example
spec:
revisionHistoryLimit: 2
selector:
matchLabels:
app: graph-rpc-example
strategy:
type: RollingUpdate
@@ -9103,20 +9427,154 @@
path: "config.yml.template"
- name: config-utils
configMap:
name: config-utils
defaultMode: 0555
- name: backend-utils
configMap:
name: backend-utils
defaultMode: 0555
---
+# Source: swh/templates/graph/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ namespace: swh-cassandra-next-version
+ name: graph-rpc-python3k
+ labels:
+ app: graph-rpc-python3k
+spec:
+ revisionHistoryLimit: 2
+ selector:
+ matchLabels:
+ app: graph-rpc-python3k
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: graph-rpc-python3k
+ annotations:
+ checksum/config: 544f402e62b812494b8c6b592a7c6d0fdde975637c47c69b66dc6c955c89f081
+ checksum/config-utils: 94d255131467f84bef964a4c72b2b792c5ebaf711bb1c77829d7cd1007a8ac22
+ checksum/backend-utils: eca303b18ba9654db85a0f8d4838403916b673d7f06c09aedd8771b5028c7f62
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/graph
+ operator: In
+ values:
+ - "true"
+ priorityClassName: swh-cassandra-next-version-frontend-rpc
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+ - name: graph-rpc-python3k
+ resources:
+ requests:
+ memory: 512Mi
+ cpu: 500m
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/graph:20240925.1
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 5009
+ name: rpc
+ readinessProbe:
+ httpGet:
+ path: /
+ port: rpc
+ initialDelaySeconds: 15
+ failureThreshold: 30
+ periodSeconds: 5
+ livenessProbe:
+ tcpSocket:
+ port: rpc
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ env:
+ - name: GRAPH_TYPE
+ value: rpc
+ - name: PORT
+ value: "5009"
+ - name: GRAPH_PATH
+ value: /srv/graph/graph/compressed/graph
+ - name: STATSD_HOST
+ value: prometheus-statsd-exporter
+ - name: STATSD_PORT
+ value: "9125"
+ - name: STATSD_TAGS
+ value: deployment:graph-rpc-python3k
+ - name: STATSD_SERVICE_TYPE
+ value: graph-rpc-python3k
+ - name: SWH_LOG_LEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: staging
+ - name: SWH_MAIN_PACKAGE
+ value: swh.graph
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: common-secrets
+ key: storage-sentry-dsn
+ # 'name' secret should exist & include key
+ # if the setting doesn't exist, sentry pushes will be disabled
+ optional: true
+ - name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
+ value: "true"
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: graph-rpc-python3k-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: backend-utils
+ configMap:
+ name: backend-utils
+ defaultMode: 0555
+---
# Source: swh/templates/graphql/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: graphql-cassandra
namespace: swh-cassandra-next-version
labels:
app: graphql-cassandra
spec:
revisionHistoryLimit: 2
@@ -16443,21 +16901,21 @@
app: provenance-graph-granet
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: provenance-graph-granet
annotations:
- checksum/config: c32d2ea55573517d9fb19e347a8710eb38312879154cd1ec77d3d37a436ed2c3
+ checksum/config: 3a920cab49ad7bb0f2c6e36ef83ad7764740050a70f204675c9b36eb544a59b1
checksum/config-logging: 3ec68ca129865387885cf527bf08f90bda9e6d3ae5e50d948534cbe73306d6fb
checksum/config-utils: 94d255131467f84bef964a4c72b2b792c5ebaf711bb1c77829d7cd1007a8ac22
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
@@ -20249,21 +20707,21 @@
app: web-cassandra
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: web-cassandra
annotations:
- checksum/config: fcafed4055d435e856f502b51cabe7096609d0052f4851787c3e5d60f1f76f90
+ checksum/config: 8ae9a1fdae5979eb03624d7953e151fedaf2ba62f7694195406a4cb30c86e343
checksum/config-logging: f266f784128ac9c57c6d0f154a646e15f06d0ad7557f191487df0d1b385acb48
checksum/config-utils: 94d255131467f84bef964a4c72b2b792c5ebaf711bb1c77829d7cd1007a8ac22
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/web
operator: In
@@ -20698,20 +21156,155 @@
name: config-utils
defaultMode: 0555
- name: swh-graph-grpc-dataset-example
persistentVolumeClaim:
claimName: swh-graph-grpc-dataset-example-pvc
- name: swh-graph-grpc-inmemory
persistentVolumeClaim:
claimName: swh-graph-grpc-inmemory-pvc
---
+# Source: swh/templates/graph/job-fetch-dataset.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ namespace: swh-cassandra-next-version
+ name: graph-grpc-python3k-job-fetch-dataset
+ labels:
+ app: graph-grpc-python3k
+spec:
+ backoffLimit: 0
+ template:
+ metadata:
+ labels:
+ app: graph-grpc-python3k
+ spec:
+ restartPolicy: Never
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/graph
+ operator: In
+ values:
+ - "true"
+ nodeSelector:
+ kubernetes.io/hostname: rancher-node-staging-rke2-metal01
+ priorityClassName: swh-cassandra-next-version-frontend-rpc
+
+ initContainers:
+ - name: prepare-configuration
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/prepare-configuration.sh
+ env:
+
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: config-utils
+ mountPath: /entrypoints
+ readOnly: true
+ containers:
+
+ - name: fetch-graph-dataset
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/graph:20240925.1
+ command:
+ - /entrypoints/graph-fetch-dataset.sh
+ env:
+ - name: WITNESS_FILE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: GRAPH_NAME
+ value: graph
+ - name: DATASET_NAME
+ value: 2021-03-23-popular-3k-python
+ - name: DATASET_LOCATION
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed
+ - name: DATASET_URL
+ value:
+ - name: DOWNLOAD_METHOD
+ value: s3
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ - name: backend-utils
+ mountPath: /entrypoints
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+
+ - name: graph-prepare-memory-volume
+ image: container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1
+ imagePullPolicy: IfNotPresent
+ command:
+ - /entrypoints/graph-prepare-memory-volume.sh
+ env:
+ - name: WITNESS_FILE
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: WITNESS_SOURCE_FILE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed/.graph-is-initialized
+ - name: PERIOD
+ value: "3"
+ - name: GRAPH_NAME
+ value: graph
+ - name: DATASET_SOURCE
+ value: /srv/dataset/2021-03-23-popular-3k-python/compressed
+ - name: DATASET_LOCATION
+ value: /srv/graph/2021-03-23-popular-3k-python/compressed
+ volumeMounts:
+ - name: backend-utils
+ mountPath: /entrypoints
+ readOnly: true
+ - name: graph-python3k-inmemory
+ mountPath: /srv/graph
+ readOnly: false
+
+ - name: graph-python3k-persistent
+ mountPath: /srv/dataset
+ readOnly: false
+
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: graph-grpc-python3k-configuration-template
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - name: backend-utils
+ configMap:
+ name: backend-utils
+ defaultMode: 0555
+ - name: config-utils
+ configMap:
+ name: config-utils
+ defaultMode: 0555
+ - name: graph-python3k-inmemory
+ persistentVolumeClaim:
+ claimName: graph-python3k-inmemory-pvc
+
+ - name: graph-python3k-persistent
+ persistentVolumeClaim:
+ claimName: graph-python3k-persistent-pvc
+---
# Source: swh/templates/deposit/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh-cassandra-next-version
name: deposit-ingress-authenticated
labels:
app: deposit
endpoint-definition: authenticated
annotations:
@@ -20815,20 +21408,62 @@
service:
name: graph-grpc-example
port:
number: 50091
---
# Source: swh/templates/graph/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh-cassandra-next-version
+ name: graph-grpc-python3k-ingress-default
+ labels:
+ app: graph-grpc-python3k
+ endpoint-definition: default
+ annotations:
+ nginx.ingress.kubernetes.io/backend-protocol: GRPC
+ nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-buffering: "on"
+ nginx.ingress.kubernetes.io/service-upstream: "true"
+ nginx.ingress.kubernetes.io/ssl-redirect: "true"
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.100.29/32,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
+spec:
+ ingressClassName: nginx
+ rules:
+ - host: graph-grpc.internal.staging.swh.network
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: graph-grpc-python3k
+ port:
+ number: 50091
+
+ - host: graph-grpc-ingress
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: graph-grpc-python3k
+ port:
+ number: 50091
+---
+# Source: swh/templates/graph/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh-cassandra-next-version
name: graph-rpc-example-ingress-default
labels:
app: graph-rpc-example
endpoint-definition: default
annotations:
nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
nginx.ingress.kubernetes.io/proxy-body-size: 4G
nginx.ingress.kubernetes.io/proxy-buffering: "on"
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.100.29/32,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
@@ -20849,20 +21484,59 @@
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: graph-rpc-example
port:
number: 5009
---
+# Source: swh/templates/graph/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ namespace: swh-cassandra-next-version
+ name: graph-rpc-python3k-ingress-default
+ labels:
+ app: graph-rpc-python3k
+ endpoint-definition: default
+ annotations:
+ nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
+ nginx.ingress.kubernetes.io/proxy-body-size: 4G
+ nginx.ingress.kubernetes.io/proxy-buffering: "on"
+ nginx.ingress.kubernetes.io/service-upstream: "true"
+ nginx.ingress.kubernetes.io/whitelist-source-range: 10.42.0.0/16,10.43.0.0/16,192.168.100.29/32,192.168.101.0/24,192.168.130.0/24,192.168.50.0/24
+spec:
+ rules:
+ - host: graph.internal.staging.swh.network
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: graph-rpc-python3k
+ port:
+ number: 5009
+
+ - host: graph-rpc-ingress
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: graph-rpc-python3k
+ port:
+ number: 5009
+---
# Source: swh/templates/graphql/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh-cassandra-next-version
name: graphql-cassandra-ingress-default
labels:
app: graphql-cassandra
endpoint-definition: default
annotations:
------------- diff for environment production namespace swh -------------
No differences
------------- diff for environment production namespace swh-cassandra -------------
No differences
Edited by Antoine R. Dumont