production: Deploy provenance rpc service
This deploys the provenance rpc in production. Then adapts the (webapp) archive.s.o to open the provenance api endpoints.
http://provenance.internal.softwareheritage.org (ingress) will be usable from the vpn.
helm diff
[swh] Comparing changes between branches production and mr/deploy-provenance-in-production (per environment)...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment staging, namespace swh...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra...
[swh] Generate config in production branch for environment staging, namespace swh-cassandra-next-version...
[swh] Generate config in mr/deploy-provenance-in-production branch for environment staging...
[swh] Generate config in mr/deploy-provenance-in-production branch for environment staging...
[swh] Generate config in mr/deploy-provenance-in-production branch for environment staging...
Your branch is up to date with 'origin/production'.
[swh] Generate config in production branch for environment production, namespace swh...
[swh] Generate config in production branch for environment production, namespace swh-cassandra...
[swh] Generate config in production branch for environment production, namespace swh-cassandra-next-version...
[swh] Generate config in mr/deploy-provenance-in-production branch for environment production...
[swh] Generate config in mr/deploy-provenance-in-production branch for environment production...
[swh] Generate config in mr/deploy-provenance-in-production branch for environment production...
------------- diff for environment staging namespace swh -------------
_ __ __
_| |_ _ / _|/ _| between /tmp/swh-chart.swh.w5HHVmsi/staging-swh.before, 139 documents
/ _' | | | | |_| |_ and /tmp/swh-chart.swh.w5HHVmsi/staging-swh.after, 139 documents
| (_| | |_| | _| _|
\__,_|\__, |_| |_| returned no differences
|___/
------------- diff for environment staging namespace swh-cassandra -------------
_ __ __
_| |_ _ / _|/ _| between /tmp/swh-chart.swh.w5HHVmsi/staging-swh-cassandra.before, 438 documents
/ _' | | | | |_| |_ and /tmp/swh-chart.swh.w5HHVmsi/staging-swh-cassandra.after, 438 documents
| (_| | |_| | _| _|
\__,_|\__, |_| |_| returned no differences
|___/
------------- diff for environment staging namespace swh-cassandra-next-version -------------
_ __ __
_| |_ _ / _|/ _| between /tmp/swh-chart.swh.w5HHVmsi/staging-swh-cassandra-next-version.before, 345 documents
/ _' | | | | |_| |_ and /tmp/swh-chart.swh.w5HHVmsi/staging-swh-cassandra-next-version.after, 345 documents
| (_| | |_| | _| _|
\__,_|\__, |_| |_| returned no differences
|___/
------------- diff for environment production namespace swh -------------
_ __ __
_| |_ _ / _|/ _| between /tmp/swh-chart.swh.w5HHVmsi/production-swh.before, 442 documents
/ _' | | | | |_| |_ and /tmp/swh-chart.swh.w5HHVmsi/production-swh.after, 449 documents
| (_| | |_| | _| _|
\__,_|\__, |_| |_| returned seven differences
|___/
(file level)
---
# Source: swh/templates/provenance/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: provenance-graph-granet-configuration-template
namespace: swh
data:
config.yml.template: |
provenance:
cls: graph
url: graph.internal.softwareheritage.org:50091
# Source: swh/templates/provenance/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
namespace: swh
name: provenance-graph-granet-configuration-logging
data:
logging-gunicorn.json: |
{
"disable_existing_loggers": true,
"filters": {
"throttle_accesslog": {
"interval": 60,
"status_codes": [
200,
400
]
}
},
"formatters": {
"default": {
"class": "pythonjsonlogger.jsonlogger.JsonFormatter",
"format": "[%(asctime)s: %(levelname)s/%(processName)s-%(process)d %(threadName)s(%(pathname)s:%(lineno)s:%(funcName)s)]%(name)s %(message)s"
}
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"formatter": "default",
"level": "INFO",
"stream": "ext://sys.stdout"
}
},
"loggers": {
"azure": {
"level": "WARN"
},
"gunicorn": {
"level": "INFO"
},
"gunicorn.access": {
"filters": [
"throttle_accesslog"
],
"handlers": [
"console"
],
"level": "WARN"
},
"gunicorn.error": {
"level": "INFO"
},
"swh": {
"level": "INFO"
},
"swh.core.statsd": {
"level": "INFO"
}
},
"root": {
"handlers": [
"console"
],
"level": "INFO"
},
"version": 1
}
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
name: webapp-provenance-ingress
namespace: swh
spec:
type: ExternalName
externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.local
# Source: swh/templates/provenance/service.yaml
apiVersion: v1
kind: Service
metadata:
name: provenance-graph-granet
namespace: swh
labels:
app: provenance-graph-granet
spec:
type: ClusterIP
selector:
app: provenance-graph-granet
ports:
- port: 5014
targetPort: 5014
name: rpc
# Source: swh/templates/provenance/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: swh
name: provenance-graph-granet
labels:
app: provenance-graph-granet
spec:
revisionHistoryLimit: 2
replicas: 2
selector:
matchLabels:
app: provenance-graph-granet
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: provenance-graph-granet
annotations:
checksum/config: d88314e75975d1e772541700770d914cb4e347c5fa5fb98e31feb4500629bdbf
checksum/config-logging: 7416a1f8e95281590d6d73606ed60a7cee7917af47c56dccb30356e3da306a7b
checksum/config-utils: d75ca13b805bce6a8ab59c8e24c938f2283108f6a79134f6e71db86308651dc6
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: swh/rpc
operator: In
values:
- "true"
priorityClassName: swh-frontend-rpc
initContainers:
- name: prepare-configuration
image: "container-registry.softwareheritage.org/swh/infra/swh-apps/utils:20231211.1"
imagePullPolicy: IfNotPresent
command:
- /entrypoints/prepare-configuration.sh
env:
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-template
mountPath: /etc/swh/configuration-template
- name: config-utils
mountPath: /entrypoints
readOnly: true
containers:
- name: provenance-graph-granet
resources:
requests:
memory: 512Mi
cpu: 500m
image: "container-registry.softwareheritage.org/swh/infra/swh-apps/provenance:20240830.1"
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5014
name: rpc
readinessProbe:
httpGet:
path: /
port: rpc
initialDelaySeconds: 15
failureThreshold: 30
periodSeconds: 5
livenessProbe:
tcpSocket:
port: rpc
initialDelaySeconds: 10
periodSeconds: 5
command:
- /bin/bash
args:
- "-c"
- /opt/swh/entrypoint.sh
env:
- name: WORKERS
value: 4
- name: THREADS
value: 1
- name: TIMEOUT
value: 60
- name: STATSD_HOST
value: prometheus-statsd-exporter
- name: STATSD_PORT
value: 9125
- name: STATSD_TAGS
value: "deployment:provenance-graph-granet"
- name: STATSD_SERVICE_TYPE
value: provenance-graph-granet
- name: SWH_LOG_LEVEL
value: INFO
- name: SWH_LOG_CONFIG_JSON
value: /etc/swh/logging/logging-gunicorn.json
- name: SWH_SENTRY_ENVIRONMENT
value: production
- name: SWH_MAIN_PACKAGE
value: swh.provenance
- name: SWH_SENTRY_DSN
valueFrom:
secretKeyRef:
name: common-secrets
key: provenance-sentry-dsn
# 'name' secret should exist & include key
# if the setting doesn't exist, sentry pushes will be disabled
optional: true
- name: SWH_SENTRY_DISABLE_LOGGING_EVENTS
value: "true"
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: configuration-logging
mountPath: /etc/swh/logging
volumes:
- name: configuration
emptyDir: {}
- name: configuration-template
configMap:
name: provenance-graph-granet-configuration-template
items:
- key: config.yml.template
path: config.yml.template
- name: configuration-logging
configMap:
name: provenance-graph-granet-configuration-logging
items:
- key: logging-gunicorn.json
path: logging-gunicorn.json
- name: config-utils
configMap:
name: config-utils
defaultMode: 0555
# Source: swh/templates/provenance/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh
name: provenance-graph-granet-ingress-default
labels:
app: provenance-graph-granet
endpoint-definition: default
annotations:
nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
nginx.ingress.kubernetes.io/proxy-body-size: 4G
nginx.ingress.kubernetes.io/proxy-buffering: on
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/whitelist-source-range: "10.42.0.0/16,10.43.0.0/16"
spec:
rules:
- host: webapp-provenance-ingress
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: provenance-graph-granet
port:
number: 5014
# Source: swh/templates/provenance/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: swh
name: provenance-graph-granet-ingress-extra-1-default
labels:
app: provenance-graph-granet
endpoint-definition: default
annotations:
nginx.ingress.kubernetes.io/client-body-buffer-size: 128K
nginx.ingress.kubernetes.io/proxy-body-size: 4G
nginx.ingress.kubernetes.io/proxy-buffering: on
nginx.ingress.kubernetes.io/service-upstream: "true"
nginx.ingress.kubernetes.io/whitelist-source-range: "10.42.0.0/16,10.43.0.0/16,127.0.0.0/8,192.168.100.0/24,192.168.101.0/24,192.168.200.0/22,192.168.50.0/24"
spec:
rules:
- host: provenance.internal.softwareheritage.org
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: provenance-graph-granet
port:
number: 5014
data.config.yml.template (v1/ConfigMap/swh/web-archive-configuration-template)
± value change in multiline text (one insert, no deletions)
+ provenance:
+ cls: remote
+ url: http://webapp-provenance-ingress
spec.template.metadata.annotations.checksum/config (apps/v1/Deployment/swh/web-archive)
± value change
- 9dd8ed77be16c5a4f8ac85d5175cf7e2b6c72dacfda1d3baf55f986a76a26205
+ 063dad00b39ebd1276f4ac81b195c7346156b55e068aca8cce6939e5dad5ade5
spec.rules.archive.softwareheritage.org.http.paths (networking.k8s.io/v1/Ingress/swh/web-archive-ingress-authenticated)
- one list entry removed:
- path: /api/1/provenance/
pathType: Prefix
backend:
service:
name: web-archive
port:
number: 5004
spec.rules.base.softwareheritage.org.http.paths (networking.k8s.io/v1/Ingress/swh/web-archive-ingress-authenticated)
- one list entry removed:
- path: /api/1/provenance/
pathType: Prefix
backend:
service:
name: web-archive
port:
number: 5004
spec.rules.archive.internal.softwareheritage.org.http.paths (networking.k8s.io/v1/Ingress/swh/web-archive-ingress-authenticated)
- one list entry removed:
- path: /api/1/provenance/
pathType: Prefix
backend:
service:
name: web-archive
port:
number: 5004
spec.rules.archive-dynamic.internal.softwareheritage.org.http.paths (networking.k8s.io/v1/Ingress/swh/web-archive-ingress-authenticated)
- one list entry removed:
- path: /api/1/provenance/
pathType: Prefix
backend:
service:
name: web-archive
port:
number: 5004
------------- diff for environment production namespace swh-cassandra -------------
_ __ __
_| |_ _ / _|/ _| between /tmp/swh-chart.swh.w5HHVmsi/production-swh-cassandra.before, 125 documents
/ _' | | | | |_| |_ and /tmp/swh-chart.swh.w5HHVmsi/production-swh-cassandra.after, 126 documents
| (_| | |_| | _| _|
\__,_|\__, |_| |_| returned one difference
|___/
(file level)
---
# Source: swh/templates/external-services/cname.yaml
apiVersion: v1
kind: Service
metadata:
name: webapp-provenance-ingress
namespace: swh-cassandra
spec:
type: ExternalName
externalName: archive-production-rke2-ingress-nginx-controller.ingress-nginx.svc.cluster.localEdited by Antoine R. Dumont