Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • ardumont/swh-charts
  • vlorentz/swh-charts
  • swh/infra/ci-cd/swh-charts
  • vsellier/swh-charts
  • anlambert/swh-charts
5 results
Show changes
Commits on Source (30)
Hide whitespace changes
Inline Side-by-side
Showing
with 444 additions and 29 deletions
swh/templates/graph/_helper-volume.yaml
View file @ 5d13a8b0
......@@ -88,7 +88,7 @@
image: {{ .Values.swh_utils_image }}:{{ .Values.swh_utils_image_version }}
imagePullPolicy: IfNotPresent
command:
- /entrypoints/graph-wait-for-dataset.sh
- /entrypoints/wait-for-dataset.sh
env:
{{- include "swh.graph.volume.witnessfile" (dict "graphPath" .graphPath) | nindent 4 }}
- name: DATASET_LOCATION
......
swh/templates/graph/ingress.yaml
View file @ 5d13a8b0
......@@ -4,6 +4,10 @@
{{- if and (hasKey $graphConfig "ingress") $graphConfig.ingress.enabled -}}
{{- $type := $graphConfig.type -}}
{{- $port := $graphConfig.port | default (ternary $.Values.graph.rpcPort $.Values.graph.grpcPort (eq $type "rpc")) -}}
{{- if not (hasKey $graphConfig "port") }}
{{- $graphConfig := set $graphConfig "port" $port -}}
{{- end -}}
{{- $annotations := $graphConfig.ingress.extraAnnotations | default dict }}
{{- if (eq $type "grpc") }}
{{- $annotations = mustMergeOverwrite $annotations
......
swh/templates/provenance/_helper-config.yaml
View file @ 5d13a8b0
......@@ -2,7 +2,8 @@
* Create a provenance configmap for service .serviceType
*/}}
{{ define "swh.provenance.configmap" }}
{{- $graphConfigurationRef := required (print "<.configuration.graphConfigurationRef> key is mandatory" ) .configuration.graphConfigurationRef -}}
{{- $graphConfigurationRef := .configuration.graphConfigurationRef -}}
{{- if $graphConfigurationRef -}}
{{- $graphConfiguration := get .Values $graphConfigurationRef -}}
---
apiVersion: v1
......@@ -14,6 +15,7 @@ data:
config.yml.template: |
provenance:
{{- toYaml $graphConfiguration | nindent 6 -}}
{{ end }}
{{- end -}}
{{/*
......@@ -30,3 +32,51 @@ data:
"Values" .Values) -}}
{{- end }}
{{/*
* Configure provenance configuration with default values.
* This also raises when mandatory configuration is missing.
*/}}
{{ define "swh.provenance.config" }}
{{- $configuration := .configuration -}}
{{- $provenanceImageVersion := get $configuration "imageVersion" | default $.Values.provenance.imageVersion | default $.Values.swh_provenance_image_version -}}
{{- $configuration := set $configuration "imageVersion" $provenanceImageVersion -}}
{{- if not (hasKey $configuration "type") }}
{{- $configuration := set $configuration "type" "rpc" -}}
{{- end -}}
{{- $type := $configuration.type -}}
{{- if not (hasKey $configuration "port") }}
{{- $port := eq $type "rpc" | ternary $.Values.provenance.rpcPort $.Values.provenance.grpcPort -}}
{{- $configuration := set $configuration "port" $port -}}
{{- end -}}
{{- $logLevel := $configuration.logLevel | default $.Values.provenance.logLevel | default "INFO" | upper -}}
{{- $configuration := set $configuration "logLevel" $logLevel -}}
{{- if eq $type "rpc" -}}
{{- $graphConfigurationRef := required (print "<.configuration.graphConfigurationRef> key is mandatory" ) .configuration.graphConfigurationRef -}}
{{- $gunicornConfig := fromYaml
(include "swh.gunicorn.config"
(dict "defaultConfiguration" $.Values.provenance.gunicorn
"configuration" $configuration.gunicorn)) -}}
{{- $configuration := set $configuration "gunicornConfig" $gunicornConfig -}}
{{- else -}}
{{- $provenanceDatasetFull := dig "dataset" "provenance" "full" true $configuration -}}
{{- $configuration := set $configuration "provenanceDatasetFull" $provenanceDatasetFull -}}
{{- $provenanceIndexDataset := dig "dataset" "provenance" "index" false $configuration -}}
{{- $configuration := set $configuration "provenanceIndexDataset" $provenanceIndexDataset -}}
{{- $provenanceFetchDataset := dig "dataset" "provenance" "fetch" false $configuration -}}
{{- $configuration := set $configuration "provenanceFetchDataset" $provenanceFetchDataset -}}
{{- if or $provenanceIndexDataset $provenanceFetchDataset -}}
{{- $datasetName := required (print "<.configuration.dataset.name> key is mandatory") $configuration.dataset.name -}}
{{- $configuration := set $configuration "datasetName" $datasetName -}}
{{- end -}}
{{- $graphPath := required (print "<.configuration.dataset.graph.path> key is mandatory") $configuration.dataset.graph.path -}}
{{- $configuration := set $configuration "graphPath" $graphPath -}}
{{- $provenancePath := required (print "<.configuration.dataset.provenance.path> key is mandatory") $configuration.dataset.provenance.path -}}
{{- $configuration := set $configuration "provenancePath" $provenancePath -}}
{{- end -}}
{{- toYaml $configuration -}}
{{- end }}
swh/templates/provenance/_helper-volume.yaml 0 → 100644
View file @ 5d13a8b0
{{/* Generate the initialize provenance backend container configuration if needed */}}
{{- define "swh.provenance.volume.witnessfile" -}}
- name: {{ .witness_file_env_variable_name | default "WITNESS_FILE" }}
value: {{ .provenancePath }}/{{ .filename | default ".provenance-is-initialized" }}
{{- end -}}
{{/* Generate the initialize provenance backend container configuration if needed */}}
{{- define "swh.provenance.fetchDataset" -}}
{{- $imagePrefixName := .imagePrefixName | default "swh_provenance_image" -}}
{{- $imageVersion := .imageVersion | default ( get .Values (print $imagePrefixName "_version") ) -}}
- name: {{ .containerName | default "fetch-provenance-dataset" }}
image: {{ get .Values $imagePrefixName }}:{{ $imageVersion }}
command:
- /entrypoints/provenance-fetch-datasets.sh
env:
{{- include "swh.provenance.volume.witnessfile" (dict "witness_file_env_variable_name" "WITNESS_FETCH_FILE"
"provenancePath" .provenancePath) | nindent 2 }}
- name: SWH_CONFIG_FILENAME
value: /etc/swh/config.yml
- name: PROVENANCE_PATH
value: {{ .provenancePath }}
- name: PROVENANCE_DATASET_FULL
value: {{ .provenanceDatasetFull }}
- name: GRAPH_PATH
value: {{ .graphPath }}
- name: DATASET_VERSION
value: {{ .datasetName | default "" }}
volumeMounts:
- name: configuration
mountPath: /etc/swh
- name: backend-utils
mountPath: /entrypoints
{{- range $volumeName, $volumeConfig := .extraVolumes }}
- name: {{ $volumeName }}
mountPath: {{ $volumeConfig.mountPath }}
readOnly: {{ $volumeConfig.readOnly | default "false" }}
{{ end }}
{{- end -}}
{{/* init-container to wait for dataset presence. */}}
{{- define "swh.provenance.waitForDataset" -}}
- name: wait-for-dataset
image: {{ .Values.swh_utils_image }}:{{ .Values.swh_utils_image_version }}
imagePullPolicy: IfNotPresent
command:
- /entrypoints/wait-for-dataset.sh
env:
{{- include "swh.provenance.volume.witnessfile" (dict "provenancePath" .provenancePath) | nindent 4 }}
- name: PERIOD
value: {{ .period | default "3" | quote }}
volumeMounts:
- name: backend-utils
mountPath: /entrypoints
readOnly: true
{{- range $volumeName, $volumeConfig := .extraVolumes }}
- name: {{ $volumeName }}
mountPath: {{ $volumeConfig.mountPath }}
readOnly: {{ $volumeConfig.readOnly | default "false" }}
{{ end }}
{{- end -}}
{{/* init-container to index provenance dataset. */}}
{{- define "swh.provenance.indexDataset" -}}
{{- $imagePrefixName := .imagePrefixName | default "swh_provenance_image" -}}
{{- $imageVersion := .imageVersion | default ( get .Values (print $imagePrefixName "_version") ) -}}
- name: {{ .containerName | default "index-provenance-dataset" }}
image: {{ get .Values $imagePrefixName }}:{{ $imageVersion }}
imagePullPolicy: IfNotPresent
command:
- /entrypoints/provenance-index-dataset.sh
env:
{{- include "swh.provenance.volume.witnessfile" (dict "witness_file_env_variable_name" "WITNESS_DATASETS_FILE"
"provenancePath" .provenancePath) | nindent 4 }}
{{- include "swh.provenance.volume.witnessfile" (dict "witness_file_env_variable_name" "WITNESS_INDEX_FILE"
"provenancePath" .provenancePath
"filename" ".provenance-is-indexed") | nindent 4 }}
- name: PROVENANCE_PATH
value: {{ .provenancePath }}
- name: PERIOD
value: {{ .period | default "3" | quote }}
volumeMounts:
- name: backend-utils
mountPath: /entrypoints
readOnly: true
{{- range $volumeName, $volumeConfig := .extraVolumes }}
- name: {{ $volumeName }}
mountPath: {{ $volumeConfig.mountPath }}
readOnly: {{ $volumeConfig.readOnly | default "false" }}
{{ end }}
{{- end -}}
swh/templates/provenance/configmap.yaml
View file @ 5d13a8b0
{{ if .Values.provenance.enabled -}}
{{ range $provenanceType, $deploymentConfig := .Values.provenance.deployments }}
{{- if or (not (hasKey $deploymentConfig "enabled")) (get $deploymentConfig "enabled") -}}
{{- $provenanceConfig := include "swh.provenance.config" (dict "configuration" $deploymentConfig
"Values" $.Values) | fromYaml -}}
{{- $type := $provenanceConfig.type -}}
{{- if eq $type "rpc" -}}
{{- $argsDict := dict "serviceType" (print "provenance-" $provenanceType)
"configuration" $deploymentConfig
"configuration" $provenanceConfig
"Values" $.Values -}}
{{ include "swh.provenance.configmap" $argsDict }}
{{ include "swh.provenance.gunicorn.logging" $argsDict }}
{{- end -}}
{{- end -}}
{{ end -}}
{{- end -}}
swh/templates/provenance/deployment.yaml
View file @ 5d13a8b0
{{ if .Values.provenance.enabled -}}
{{- $configUtilsChecksum := include (print $.Template.BasePath "/utils/config-utils.yaml") . | sha256sum -}}
{{- $backendUtilsChecksum := include (print $.Template.BasePath "/utils/backend-utils.yaml") . | sha256sum -}}
{{ range $provenanceType, $provenanceConfig := .Values.provenance.deployments }}
{{- if or (not (hasKey $provenanceConfig "enabled")) (get $provenanceConfig "enabled") -}}
{{- if and (or (not (hasKey $provenanceConfig "enabled")) (get $provenanceConfig "enabled"))
(or (not (hasKey $provenanceConfig "startService")) (get $provenanceConfig "startService")) -}}
{{- $serviceType := ( print "provenance-" $provenanceType ) -}}
{{- $configArgsDict := dict "serviceType" $serviceType
"configuration" $provenanceConfig
......@@ -9,9 +12,19 @@
{{- $configChecksum := include "swh.provenance.configmap" $configArgsDict | sha256sum -}}
{{- $configLoggingChecksum := include "swh.provenance.gunicorn.logging" $configArgsDict | sha256sum -}}
{{- $securityContext := $provenanceConfig.securityContext | default $.Values.provenance.securityContext -}}
{{ $gunicornConfig := fromYaml (include "swh.gunicorn.config"
(dict "configuration" $provenanceConfig.gunicorn
"defaultConfiguration" $.Values.provenance.gunicorn)) -}}
{{- $provenanceConfig := include "swh.provenance.config" (dict "configuration" $provenanceConfig
"Values" $.Values) | fromYaml -}}
{{- $type := $provenanceConfig.type -}}
{{- $port := $provenanceConfig.port -}}
{{- $logLevel := $provenanceConfig.logLevel -}}
{{- $datasetName := $provenanceConfig.datasetName -}}
{{- $graphPath := $provenanceConfig.graphPath -}}
{{- $provenancePath := $provenanceConfig.provenancePath -}}
{{- $gunicornConfig := $provenanceConfig.gunicornConfig -}}
{{- $provenanceFetchDataset := $provenanceConfig.provenanceFetchDataset -}}
{{- $provenanceIndexDataset := $provenanceConfig.provenanceIndexDataset -}}
{{- $provenanceImageVersion := $provenanceConfig.imageVersion -}}
{{- $provenanceDatasetFull := $provenanceConfig.provenanceDatasetFull -}}
---
apiVersion: apps/v1
kind: Deployment
......@@ -40,6 +53,7 @@ spec:
checksum/config: {{ $configChecksum }}
checksum/config-logging: {{ $configLoggingChecksum }}
checksum/config-utils: {{ $configUtilsChecksum }}
checksum/backend-utils: {{ $backendUtilsChecksum }}
spec:
{{- if $securityContext }}
securityContext:
......@@ -60,6 +74,7 @@ spec:
priorityClassName: {{ $.Values.namespace }}-{{ $.Values.provenance.priorityClassName }}
{{- end }}
initContainers:
{{- if eq $type "rpc" }}
- name: prepare-configuration
image: {{ $.Values.swh_utils_image }}:{{ $.Values.swh_utils_image_version }}
imagePullPolicy: IfNotPresent
......@@ -75,6 +90,26 @@ spec:
- name: config-utils
mountPath: /entrypoints
readOnly: true
{{- end }}
{{- if $provenanceFetchDataset }}
{{- include "swh.provenance.fetchDataset" (dict "Values" $.Values
"datasetName" $datasetName
"datasetFull" $provenanceDatasetFull
"provenancePath" $provenancePath
"graphPath" $graphPath
"extraVolumes" $provenanceConfig.extraVolumes) | nindent 8 -}}
{{- end }}
{{- if $provenanceIndexDataset }}
{{- include "swh.provenance.indexDataset" (dict "Values" $.Values
"imageVersion" $provenanceImageVersion
"provenancePath" $provenancePath
"extraVolumes" $provenanceConfig.extraVolumes) | nindent 8 -}}
{{- end }}
{{- if $provenanceFetchDataset }}
{{- include "swh.provenance.waitForDataset" (dict "Values" $.Values
"provenancePath" $provenancePath
"extraVolumes" $provenanceConfig.extraVolumes) | nindent 8 -}}
{{- end }}
containers:
- name: {{ $serviceType }}
resources:
......@@ -93,18 +128,23 @@ spec:
image: {{ $.Values.swh_provenance_image }}:{{ $.Values.swh_provenance_image_version }}
imagePullPolicy: IfNotPresent
ports:
- containerPort: {{ $provenanceConfig.port | default $.Values.provenance.port }}
name: rpc
- containerPort: {{ $port }}
name: {{ $type }}
readinessProbe:
{{- if (eq $type "rpc") }}
httpGet:
path: /
port: rpc
port: {{ $type }}
{{- else }}
tcpSocket:
port: {{ $type }}
{{- end }}
initialDelaySeconds: 15
failureThreshold: 30
periodSeconds: 5
livenessProbe:
tcpSocket:
port: rpc
port: {{ $type }}
initialDelaySeconds: 10
periodSeconds: 5
command:
......@@ -113,13 +153,26 @@ spec:
- -c
- /opt/swh/entrypoint.sh
env:
{{ if $gunicornConfig -}}
- name: PROVENANCE_TYPE
value: {{ $type }}
- name: PORT
value: {{ $port | quote }}
{{ if eq $type "rpc" -}}
- name: WORKERS
value: {{ $gunicornConfig.workers | quote }}
- name: THREADS
value: {{ $gunicornConfig.threads | quote }}
- name: TIMEOUT
value: {{ $gunicornConfig.timeout | quote }}
- name: SWH_LOG_CONFIG_JSON
value: /etc/swh/logging/logging-gunicorn.json
- name: STATSD_SERVICE_TYPE
value: {{ $serviceType }}
{{ else -}}
- name: PROVENANCE_PATH
value: {{ $provenancePath }}
- name: GRAPH_PATH
value: {{ $graphPath }}
{{ end -}}
- name: STATSD_HOST
value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
......@@ -127,12 +180,8 @@ spec:
value: {{ $.Values.statsdPort | default "9125" | quote }}
- name: STATSD_TAGS
value: deployment:{{ $serviceType }}
- name: STATSD_SERVICE_TYPE
value: {{ $serviceType }}
- name: SWH_LOG_LEVEL
value: {{ $provenanceConfig.logLevel | default $.Values.provenance.logLevel | default "INFO" | quote }}
- name: SWH_LOG_CONFIG_JSON
value: /etc/swh/logging/logging-gunicorn.json
value: {{ $logLevel }}
{{- if $.Values.provenance.sentry.enabled }}
- name: SWH_SENTRY_ENVIRONMENT
value: {{ $.Values.sentry.environment }}
......@@ -150,10 +199,12 @@ spec:
value: "true"
{{- end }}
volumeMounts:
{{- if eq $type "rpc" }}
- name: configuration
mountPath: /etc/swh
- name: configuration-logging
mountPath: /etc/swh/logging
{{ end }}
{{- range $volumeName, $volumeConfig := $provenanceConfig.extraVolumes }}
- name: {{ $volumeName }}
mountPath: {{ $volumeConfig.mountPath }}
......@@ -162,6 +213,7 @@ spec:
volumes:
- name: configuration
emptyDir: {}
{{- if eq $type "rpc" }}
- name: configuration-template
configMap:
name: {{ $serviceType }}-configuration-template
......@@ -174,10 +226,15 @@ spec:
items:
- key: "logging-gunicorn.json"
path: "logging-gunicorn.json"
{{ end }}
- name: config-utils
configMap:
name: config-utils
defaultMode: 0555
- name: backend-utils
configMap:
name: backend-utils
defaultMode: 0555
{{- range $volumeName, $volumeConfig := $provenanceConfig.extraVolumes }}
- name: {{ $volumeName }}
{{- toYaml $volumeConfig.volumeDefinition | nindent 8 }}
......
swh/templates/provenance/ingress.yaml
View file @ 5d13a8b0
......@@ -3,8 +3,11 @@
{{- $serviceType := ( print "provenance-" $provenanceType ) -}}
{{- if or (not (hasKey $deploymentConfig "enabled")) (get $deploymentConfig "enabled") }}
{{- if and (hasKey $deploymentConfig "ingress") $deploymentConfig.ingress.enabled -}}
{{- $configuration := include "swh.provenance.config" (dict "configuration" $deploymentConfig
"Values" $.Values) | fromYaml -}}
{{- include "swh.ingress" (dict "serviceType" $serviceType
"configuration" $deploymentConfig
"configuration" $configuration
"Values" $.Values) -}}
{{- end -}}
{{- if $deploymentConfig.extraIngresses -}}
......@@ -14,8 +17,10 @@
{{- $mergedIngressConfig :=
mustMergeOverwrite (omit $deploymentConfig "extraIngresses" "ingress")
(dict "ingress" (mustMergeOverwrite $deploymentConfig.ingress $extraIngressConfig)) -}}
{{- $config := include "swh.provenance.config" (dict "configuration" $mergedIngressConfig
"Values" $.Values) | fromYaml -}}
{{- include "swh.ingress" (dict "serviceType" $serviceType
"configuration" $mergedIngressConfig
"configuration" $config
"extraNameLabel" ( print "extra-" $extraNameLabelCtr )
"Values" $.Values) -}}
{{- end -}}
......
swh/templates/provenance/service.yaml
View file @ 5d13a8b0
{{ if .Values.provenance.enabled -}}
{{ range $provenanceType, $deploymentConfig := .Values.provenance.deployments }}
{{- if or (not (hasKey $deploymentConfig "enabled")) (get $deploymentConfig "enabled") -}}
{{- if not (hasKey $deploymentConfig "port") }}
{{- $deploymentConfig := set $deploymentConfig "port" $.Values.provenance.port -}}
{{- end -}}
{{- $configuration := include "swh.provenance.config" (dict "configuration" $deploymentConfig
"Values" $.Values) | fromYaml -}}
{{- include "swh.service" (dict "serviceType" ( print "provenance-" $provenanceType )
"configuration" $deploymentConfig
"configuration" $configuration
"Values" $.Values) -}}
{{- end -}}
{{ end -}}
......
swh/templates/utils/backend-utils.yaml
View file @ 5d13a8b0
......@@ -62,7 +62,7 @@ data:
# Finally, we make explicit the graph is ready
touch ${WITNESS_FILE}
graph-wait-for-dataset.sh: |
wait-for-dataset.sh: |
#!/usr/bin/env bash
# Uses env variables WITNESS_FILE
[ -z "${WITNESS_FILE}" ] && \
......@@ -158,6 +158,110 @@ data:
swh graph reindex --ef ${DATASET_LOCATION}/${GRAPH_NAME} && \
touch $WITNESS_REINDEX_FILE
provenance-fetch-datasets.sh: |
#!/usr/bin/env bash
[ -z "${WITNESS_FETCH_FILE}" ] && \
echo "<WITNESS_FETCH_FILE> env variable must be set" && exit 1
[ -z "${DATASET_VERSION}" ] && \
echo "<DATASET_VERSION> env variable must be set" && exit 1
[ -z "${PROVENANCE_PATH}" ] && \
echo "<PROVENANCE_PATH> env variable must be set" && exit 1
[ -z "${GRAPH_PATH}" ] && \
echo "<GRAPH_PATH> env variable must be set" && exit 1
[ -f ${WITNESS_FETCH_FILE} ] && \
echo "Datasets graph & provenance <${DATASET_VERSION}> already present. Skip." && \
exit 0
set -e
# Create destination paths
mkdir -p ${PROVENANCE_PATH} ${GRAPH_PATH}
echo "Fetching datasets..."
if [ ${PROVENANCE_DATASET_FULL} = true ]; then
# Retrieve all the provenance dataset
REFS=all
else
# This excludes revisions not targetted by a snapshot
# Ok to use for test purposes
REFS=heads
fi
URL_PROVENANCE="s3://softwareheritage/derived_datasets/${DATASET_VERSION}/provenance/${REFS}/"
CMD_GET="aws s3 cp --no-progress --no-sign-request"
echo "1. Fetching provenance dataset (parquet files)..."
${CMD_GET} --recursive "${URL_PROVENANCE}" "${PROVENANCE_PATH}"
echo "1. Provenance datasets installed!"
echo "2. Fetching extra graph files..."
URL_GRAPH="s3://softwareheritage/graph/${DATASET_VERSION}/compressed"
mkdir -p "${GRAPH_PATH}"
for filename in graph.pthash graph.pthash.order graph.node2swhid.bin.zst graph.node2type.bin.zst; do
${CMD_GET} "${URL_GRAPH}/${filename}" "${GRAPH_PATH}"
done
echo "2. Extra graph files installed!"
echo "3. Uncompressing graph files..."
set -x
# Uncompress the compressed graph *.zst files
for filename in graph.node2type.bin.zst graph.node2swhid.bin.zst; do
filepath="${GRAPH_PATH}/${filename}"
# Uncompress and delete the .zst file
[ -f "${filepath}" ] && unzstd --rm "${filepath}"
done
set +x
echo "3. Graph files uncompressed!"
# Make explicit the provenance datasets are fetched
touch ${WITNESS_FETCH_FILE}
echo "Provenance datasets installed!"
provenance-index-dataset.sh: |
#!/usr/bin/env bash
[ -z "${WITNESS_DATASETS_FILE}" ] && \
echo "<WITNESS_DATASETS_FILE> env variable must be set" && exit 1
[ -z "${WITNESS_INDEX_FILE}" ] && \
echo "<WITNESS_INDEX_FILE> env variable must be set" && exit 1
[ -z "${PERIOD}" ] && \
echo "<PERIOD> env variable must be set" && exit 1
[ -z "${PROVENANCE_PATH}" ] && \
echo "<PROVENANCE_PATH> env variable must be set" && exit 1
# [ -f ${WITNESS_INDEX_FILE} ] && echo "Provenance already indexed, do nothing." && \
# exit 0
set -eu
# Let's wait for the dataset installation
while [ ! -f "${WITNESS_DATASETS_FILE}" ]; do
echo "${WITNESS_DATASETS_FILE} missing, waiting provenance dataset installation..."
sleep $PERIOD
done
echo "Datasets file installed, build provenance dataset indexes..."
echo "provenance path: $PROVENANCE_PATH"
set -x
ls -lah /srv/dataset
ls -lah $PROVENANCE_PATH
ls -lah /srv/dataset/graph/
# To make the query faster, the provenance needs to build index out of the
# current dataset files. We store the output indexes in the same path as
# the dataset.
swh-provenance-index \
--database file://${PROVENANCE_PATH} && \
touch "${WITNESS_INDEX_FILE}" && \
echo "Provenance indexes built!" || \
echo "Provenance indexes failed!"
initialize-search-backend.sh: |
#!/usr/bin/env bash
......
swh/values.yaml
View file @ 5d13a8b0
......@@ -1895,22 +1895,55 @@ storage_backfiller:
provenance:
enabled: false
# Default service port (overridable per instance)
port: 5014
# Default rpc port
rpcPort: 5014
# Default grpc port
grpcPort: 50141
priorityClassName: frontend-rpc
sentry:
enabled: false
secretKeyRef: common-secrets
secretKeyName: provenance-sentry-dsn
deployments: {}
# instance:
# instance-rpc:
# enabled: false
# # The server type to deploy, either a rpc or a grpc
# type: rpc
# # Port to use, either inferred from the type or explicitly set (can be
# # overridden per instance)
# # port: 5014
# # For type rpc, the graph configuration to use
# graphConfigurationRef: graphConfiguration
# hosts: []
# ingress:
# enabled: true
# endpoints: []
# instance-grpc:
# enabled: false
# # The server type to deploy, either a rpc or a grpc
# type: grpc
# # For type grpc, datasets need to be available for the service to use
# dataset:
# # Name of the dataset, graph and provenance must be in sync so same
# # version must be used
# name: 2024-08-23-popular-500-python
# # Graph setup
# graph:
# # Where to store the files
# path: /srv/dataset/graph
# provenance:
# # Whether or not to fetch provenance dataset
# fetch: true
# # Whether or not to build the provenance dataset indices
# index: true
# # Where to store the parquet files
# path: /srv/dataset/provenance
# # Whether to start/stop the service
# startService: true
# hosts: []
# ingress:
# enabled: true
# endpoints: []
rpcLocalRustWithGrpcGraphConfiguration:
cls: local_rust
......
swh/values/local-cluster.yaml
View file @ 5d13a8b0
......@@ -1455,7 +1455,7 @@ objstorageReplayer:
provenance:
enabled: false
deployments:
test:
test-rpc:
enabled: true
graphConfigurationRef: fakeGraphConfiguration
replicas: 1
......@@ -1463,6 +1463,32 @@ provenance:
# workers: 4
# threads: 1
# timeout: 60
test-grpc:
enabled: true
type: grpc
dataset:
name: 2024-08-23-popular-500-python
provenance:
fetch: true
index: true
# For test purpose, use a dataset subset
full: false
path: /srv/dataset/provenance
graph:
path: /srv/dataset/graph
startService: true
extraVolumes:
dataset-persistent:
mountPath: /srv/dataset
volumeDefinition:
persistentVolumeClaim:
claimName: provenance-popular-persistent-pvc
replicas: 1
# gunicorn:
# workers: 4
# threads: 1
# timeout: 60
fakeGraphConfiguration:
cls: graph
......@@ -1516,6 +1542,17 @@ externalServices:
volumes:
enabled: true
persistentVolumeClaims:
provenance-popular-persistent-pvc:
enabled: true
appName: provenance-test-grpc
spec:
storageClassName: local-persistent
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
alter-recovery-bundles-pvc:
enabled: true
appName: alter
......
swh/values/staging/next-version.yaml
View file @ 5d13a8b0
......@@ -1250,6 +1250,25 @@ provenance:
extraWhitelistSourceRange:
- 192.168.50.0/24 # for blackbox monitoring
- 192.168.101.0/24 # vpn network
grpc:
enabled: true
type: grpc
dataset:
name: 2024-08-23-popular-500-python
provenance:
fetch: true
index: true
path: /srv/dataset/provenance
graph:
path: /srv/dataset/graph
startService: true
extraVolumes:
dataset-persistent:
mountPath: /srv/dataset
volumeDefinition:
persistentVolumeClaim:
claimName: provenance-popular-persistent-pvc
# Rpc Rust graph configuration example (for a service rpc spawning an internal grpc
# instance)
......@@ -1294,6 +1313,17 @@ volumes:
requests:
storage: 1Gi
volumeMode: Filesystem
provenance-popular-persistent-pvc:
enabled: true
appName: provenance-grpc
spec:
storageClassName: local-persistent
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
graph:
enabled: true
......
values-swh-application-versions.yaml
View file @ 5d13a8b0
......@@ -41,7 +41,7 @@ swh_objstorage_image_version: '20250319.1'
swh_objstorage_replayer_image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage_replayer
swh_objstorage_replayer_image_version: '20250319.1'
swh_provenance_image: container-registry.softwareheritage.org/swh/infra/swh-apps/provenance
swh_provenance_image_version: '20250319.1'
swh_provenance_image_version: '20250319.2'
swh_scheduler_image: container-registry.softwareheritage.org/swh/infra/swh-apps/scheduler
swh_scheduler_image_version: '20250319.1'
swh_scrubber_image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber
......