Compare revisions

Antoine R. Dumont · Antoine R. Dumont · Antoine R. Dumont · Antoine R. Dumont · Antoine R. Dumont · Antoine R. Dumont
--- a/swh/templates/graph/_helper-volume.yaml
+++ b/swh/templates/graph/_helper-volume.yaml
@@ -88,7 +88,7 @@
  image: {{ .Values.swh_utils_image }}:{{ .Values.swh_utils_image_version }}
  imagePullPolicy: IfNotPresent
  command:
-  - /entrypoints/graph-wait-for-dataset.sh
+  - /entrypoints/wait-for-dataset.sh
  env:
    {{- include "swh.graph.volume.witnessfile" (dict "graphPath" .graphPath) | nindent 4 }}
    - name: DATASET_LOCATION

--- a/swh/templates/graph/ingress.yaml
+++ b/swh/templates/graph/ingress.yaml
@@ -4,6 +4,10 @@
 {{- if and (hasKey $graphConfig "ingress") $graphConfig.ingress.enabled -}}
 {{- $type := $graphConfig.type -}}
 {{- $port := $graphConfig.port | default (ternary $.Values.graph.rpcPort $.Values.graph.grpcPort (eq $type "rpc")) -}}
+{{- if not (hasKey $graphConfig "port") }}
+{{- $graphConfig := set $graphConfig "port" $port -}}
+{{- end -}}
+
 {{- $annotations := $graphConfig.ingress.extraAnnotations | default dict }}
 {{- if (eq $type "grpc") }}
 {{- $annotations = mustMergeOverwrite $annotations

--- a/swh/templates/provenance/_helper-config.yaml
+++ b/swh/templates/provenance/_helper-config.yaml
@@ -2,7 +2,8 @@
   * Create a provenance configmap for service .serviceType
   */}}
 {{ define "swh.provenance.configmap" }}
-{{- $graphConfigurationRef := required (print "<.configuration.graphConfigurationRef> key is mandatory" ) .configuration.graphConfigurationRef -}}
+{{- $graphConfigurationRef := .configuration.graphConfigurationRef -}}
+{{- if $graphConfigurationRef -}}
 {{- $graphConfiguration := get .Values $graphConfigurationRef -}}
 ---
 apiVersion: v1
@@ -14,6 +15,7 @@ data:
  config.yml.template: |
    provenance:
      {{- toYaml $graphConfiguration | nindent 6 -}}
+{{ end }}
 {{- end -}}

 {{/*
@@ -30,3 +32,49 @@ data:
        "Values" .Values) -}}
 {{- end }}

+
+{{/*
+   * Configure provenance configuration with default values.
+   * This also raises when mandatory configuration is missing.
+   */}}
+{{ define "swh.provenance.config" }}
+{{- $configuration := .configuration -}}
+{{- $provenanceImageVersion := get $configuration "imageVersion" | default $.Values.provenance.imageVersion | default $.Values.swh_provenance_image_version -}}
+{{- $configuration := set $configuration "imageVersion" $provenanceImageVersion -}}
+{{- if not (hasKey $configuration "type") }}
+{{- $configuration := set $configuration "type" "rpc" -}}
+{{- end -}}
+{{- $type := $configuration.type -}}
+{{- if not (hasKey $configuration "port") }}
+{{- $port := eq $type "rpc" | ternary $.Values.provenance.rpcPort $.Values.provenance.grpcPort -}}
+{{- $configuration := set $configuration "port" $port -}}
+{{- end -}}
+
+{{- $logLevel := $configuration.logLevel | default $.Values.provenance.logLevel | default "INFO" | upper -}}
+{{- $configuration := set $configuration "logLevel" $logLevel -}}
+
+{{- if eq $type "rpc" -}}
+{{- $graphConfigurationRef := required (print "<.configuration.graphConfigurationRef> key is mandatory" ) .configuration.graphConfigurationRef -}}
+{{- $gunicornConfig := fromYaml
+                       (include "swh.gunicorn.config"
+                         (dict "defaultConfiguration" $.Values.provenance.gunicorn
+                               "configuration" $configuration.gunicorn)) -}}
+{{- $configuration := set $configuration "gunicornConfig" $gunicornConfig -}}
+{{- else -}}
+{{- $provenanceIndexDataset := dig "dataset" "provenance" "index" false $configuration -}}
+{{- $configuration := set $configuration "provenanceIndexDataset" $provenanceIndexDataset -}}
+{{- $provenanceFetchDataset := dig "dataset" "provenance" "fetch" false $configuration -}}
+{{- $configuration := set $configuration "provenanceFetchDataset" $provenanceFetchDataset -}}
+{{- if or $provenanceIndexDataset $provenanceFetchDataset -}}
+{{- $datasetName := required (print "<.configuration.dataset.name> key is mandatory") $configuration.dataset.name -}}
+{{- $configuration := set $configuration "datasetName" $datasetName -}}
+{{- end -}}
+{{- $graphPath := required (print "<.configuration.dataset.graph.path> key is mandatory") $configuration.dataset.graph.path -}}
+{{- $configuration := set $configuration "graphPath" $graphPath -}}
+{{- $provenancePath := required (print "<.configuration.dataset.provenance.path> key is mandatory") $configuration.dataset.provenance.path -}}
+{{- $configuration := set $configuration "provenancePath" $provenancePath -}}
+{{- end -}}
+{{- toYaml $configuration -}}
+{{- end }}
+
+
--- a/swh/templates/provenance/_helper-volume.yaml
+++ b/swh/templates/provenance/_helper-volume.yaml
+{{/* Generate the initialize provenance backend container configuration if needed */}}
+{{- define "swh.provenance.volume.witnessfile" -}}
+- name: {{ .witness_file_env_variable_name | default "WITNESS_FILE" }}
+  value: {{ .provenancePath }}/{{ .filename | default ".provenance-is-initialized" }}
+{{- end -}}
+
+{{/* Generate the initialize provenance backend container configuration if needed */}}
+{{- define "swh.provenance.fetchDataset" -}}
+{{- $imagePrefixName := .imagePrefixName | default "swh_provenance_image" -}}
+{{- $imageVersion := .imageVersion | default ( get .Values (print $imagePrefixName "_version") ) -}}
+- name: {{ .containerName | default "fetch-provenance-dataset" }}
+  image: {{ get .Values $imagePrefixName }}:{{ $imageVersion }}
+  command:
+  - /entrypoints/provenance-fetch-datasets.sh
+  env:
+  {{- include "swh.provenance.volume.witnessfile" (dict "witness_file_env_variable_name" "WITNESS_FETCH_FILE"
+                                                        "provenancePath" .provenancePath) | nindent 2 }}
+  - name: SWH_CONFIG_FILENAME
+    value: /etc/swh/config.yml
+  - name: PROVENANCE_PATH
+    value: {{ .provenancePath }}
+  - name: GRAPH_PATH
+    value: {{ .graphPath }}
+  - name: DATASET_VERSION
+    value: {{ .datasetName | default "" }}
+  volumeMounts:
+  - name: configuration
+    mountPath: /etc/swh
+  - name: backend-utils
+    mountPath: /entrypoints
+  {{- range $volumeName, $volumeConfig := .extraVolumes }}
+  - name: {{ $volumeName }}
+    mountPath: {{ $volumeConfig.mountPath }}
+    readOnly: {{ $volumeConfig.readOnly | default "false" }}
+  {{ end }}
+{{- end -}}
+
+{{/* init-container to wait for dataset presence. */}}
+{{- define "swh.provenance.waitForDataset" -}}
+- name: wait-for-dataset
+  image: {{ .Values.swh_utils_image }}:{{ .Values.swh_utils_image_version }}
+  imagePullPolicy: IfNotPresent
+  command:
+  - /entrypoints/wait-for-dataset.sh
+  env:
+    {{- include "swh.provenance.volume.witnessfile" (dict "provenancePath" .provenancePath) | nindent 4 }}
+    - name: PERIOD
+      value: {{ .period | default "3" | quote }}
+  volumeMounts:
+  - name: backend-utils
+    mountPath: /entrypoints
+    readOnly: true
+  {{- range $volumeName, $volumeConfig := .extraVolumes }}
+  - name: {{ $volumeName }}
+    mountPath: {{ $volumeConfig.mountPath }}
+    readOnly: {{ $volumeConfig.readOnly | default "false" }}
+  {{ end }}
+{{- end -}}
+
+{{/* init-container to index provenance dataset. */}}
+{{- define "swh.provenance.indexDataset" -}}
+{{- $imagePrefixName := .imagePrefixName | default "swh_provenance_image" -}}
+{{- $imageVersion := .imageVersion | default ( get .Values (print $imagePrefixName "_version") ) -}}
+- name: {{ .containerName | default "index-provenance-dataset" }}
+  image: {{ get .Values $imagePrefixName }}:{{ $imageVersion }}
+  imagePullPolicy: IfNotPresent
+  command:
+  - /entrypoints/provenance-index-dataset.sh
+  env:
+    {{- include "swh.provenance.volume.witnessfile" (dict "witness_file_env_variable_name" "WITNESS_DATASETS_FILE"
+                                                          "provenancePath" .provenancePath) | nindent 4 }}
+    {{- include "swh.provenance.volume.witnessfile" (dict "witness_file_env_variable_name" "WITNESS_INDEX_FILE"
+                                                          "provenancePath" .provenancePath
+                                                          "filename" ".provenance-is-indexed") | nindent 4 }}
+    - name: PROVENANCE_PATH
+      value: {{ .provenancePath }}
+    - name: PERIOD
+      value: {{ .period | default "3" | quote }}
+  volumeMounts:
+  - name: backend-utils
+    mountPath: /entrypoints
+    readOnly: true
+  {{- range $volumeName, $volumeConfig := .extraVolumes }}
+  - name: {{ $volumeName }}
+    mountPath: {{ $volumeConfig.mountPath }}
+    readOnly: {{ $volumeConfig.readOnly | default "false" }}
+  {{ end }}
+{{- end -}}
+
--- a/swh/templates/provenance/configmap.yaml
+++ b/swh/templates/provenance/configmap.yaml
 {{ if .Values.provenance.enabled -}}
 {{ range $provenanceType, $deploymentConfig := .Values.provenance.deployments }}
 {{- if or (not (hasKey $deploymentConfig "enabled")) (get $deploymentConfig "enabled") -}}
+{{- $provenanceConfig := include "swh.provenance.config" (dict "configuration" $deploymentConfig
+                                                               "Values" $.Values) | fromYaml -}}
+{{- $type := $provenanceConfig.type -}}
+{{- if eq $type "rpc" -}}
 {{- $argsDict := dict "serviceType" (print "provenance-" $provenanceType)
-                      "configuration" $deploymentConfig
+                      "configuration" $provenanceConfig
                      "Values" $.Values -}}
 {{ include "swh.provenance.configmap" $argsDict }}
 {{ include "swh.provenance.gunicorn.logging" $argsDict }}
 {{- end -}}
+{{- end -}}
 {{ end -}}
 {{- end -}}
--- a/swh/templates/provenance/deployment.yaml
+++ b/swh/templates/provenance/deployment.yaml
 {{ if .Values.provenance.enabled -}}
 {{- $configUtilsChecksum := include (print $.Template.BasePath "/utils/config-utils.yaml") . | sha256sum -}}
+{{- $backendUtilsChecksum := include (print $.Template.BasePath "/utils/backend-utils.yaml") . | sha256sum -}}
 {{ range $provenanceType, $provenanceConfig := .Values.provenance.deployments }}
-{{- if or (not (hasKey $provenanceConfig "enabled")) (get $provenanceConfig "enabled") -}}
+{{- if and (or (not (hasKey $provenanceConfig "enabled")) (get $provenanceConfig "enabled"))
+           (or (not (hasKey $provenanceConfig "startService")) (get $provenanceConfig "startService")) -}}
+
 {{- $serviceType := ( print "provenance-" $provenanceType ) -}}
 {{- $configArgsDict := dict "serviceType" $serviceType
                            "configuration" $provenanceConfig
@@ -9,9 +12,18 @@
 {{- $configChecksum := include "swh.provenance.configmap" $configArgsDict | sha256sum -}}
 {{- $configLoggingChecksum := include "swh.provenance.gunicorn.logging" $configArgsDict | sha256sum -}}
 {{- $securityContext := $provenanceConfig.securityContext | default $.Values.provenance.securityContext -}}
-{{ $gunicornConfig := fromYaml (include "swh.gunicorn.config"
-                                 (dict "configuration" $provenanceConfig.gunicorn
-                                       "defaultConfiguration" $.Values.provenance.gunicorn)) -}}
+{{- $provenanceConfig := include "swh.provenance.config" (dict "configuration" $provenanceConfig
+                                                               "Values" $.Values) | fromYaml -}}
+{{- $type := $provenanceConfig.type -}}
+{{- $port := $provenanceConfig.port -}}
+{{- $logLevel := $provenanceConfig.logLevel -}}
+{{- $datasetName := $provenanceConfig.datasetName -}}
+{{- $graphPath := $provenanceConfig.graphPath -}}
+{{- $provenancePath := $provenanceConfig.provenancePath -}}
+{{- $gunicornConfig := $provenanceConfig.gunicornConfig -}}
+{{- $provenanceFetchDataset := $provenanceConfig.provenanceFetchDataset -}}
+{{- $provenanceIndexDataset := $provenanceConfig.provenanceIndexDataset -}}
+{{- $provenanceImageVersion := $provenanceConfig.imageVersion -}}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -40,6 +52,7 @@ spec:
        checksum/config: {{ $configChecksum }}
        checksum/config-logging: {{ $configLoggingChecksum }}
        checksum/config-utils: {{ $configUtilsChecksum }}
+        checksum/backend-utils: {{ $backendUtilsChecksum }}
    spec:
      {{- if $securityContext }}
      securityContext:
@@ -60,6 +73,7 @@ spec:
      priorityClassName: {{ $.Values.namespace }}-{{ $.Values.provenance.priorityClassName }}
      {{- end }}
      initContainers:
+        {{- if eq $type "rpc" }}
        - name: prepare-configuration
          image: {{ $.Values.swh_utils_image }}:{{ $.Values.swh_utils_image_version }}
          imagePullPolicy: IfNotPresent
@@ -75,6 +89,25 @@ spec:
          - name: config-utils
            mountPath: /entrypoints
            readOnly: true
+        {{- end }}
+        {{- if $provenanceFetchDataset }}
+        {{- include "swh.provenance.fetchDataset" (dict "Values" $.Values
+                                                        "datasetName" $datasetName
+                                                        "provenancePath" $provenancePath
+                                                        "graphPath" $graphPath
+                                                        "extraVolumes" $provenanceConfig.extraVolumes) | nindent 8 -}}
+        {{- end }}
+        {{- if $provenanceIndexDataset }}
+        {{- include "swh.provenance.indexDataset" (dict "Values" $.Values
+                                                        "imageVersion" $provenanceImageVersion
+                                                        "provenancePath" $provenancePath
+                                                        "extraVolumes" $provenanceConfig.extraVolumes) | nindent 8 -}}
+        {{- end }}
+        {{- if $provenanceFetchDataset }}
+        {{- include "swh.provenance.waitForDataset" (dict "Values" $.Values
+                                                          "provenancePath" $provenancePath
+                                                          "extraVolumes" $provenanceConfig.extraVolumes) | nindent 8 -}}
+        {{- end }}
      containers:
        - name: {{ $serviceType }}
          resources:
@@ -93,18 +126,23 @@ spec:
          image: {{ $.Values.swh_provenance_image }}:{{ $.Values.swh_provenance_image_version }}
          imagePullPolicy: IfNotPresent
          ports:
-            - containerPort: {{ $provenanceConfig.port | default $.Values.provenance.port }}
-              name: rpc
+            - containerPort: {{ $port }}
+              name: {{ $type }}
          readinessProbe:
+          {{- if (eq $type "rpc") }}
            httpGet:
              path: /
-              port: rpc
+              port: {{ $type }}
+          {{- else }}
+            tcpSocket:
+              port: {{ $type }}
+          {{- end }}
            initialDelaySeconds: 15
            failureThreshold: 30
            periodSeconds: 5
          livenessProbe:
            tcpSocket:
-              port: rpc
+              port: {{ $type }}
            initialDelaySeconds: 10
            periodSeconds: 5
          command:
@@ -113,13 +151,26 @@ spec:
          - -c
          - /opt/swh/entrypoint.sh
          env:
-            {{ if $gunicornConfig -}}
+            - name: PROVENANCE_TYPE
+              value: {{ $type }}
+            - name: PORT
+              value: {{ $port | quote }}
+            {{ if eq $type "rpc" -}}
            - name: WORKERS
              value: {{ $gunicornConfig.workers | quote }}
            - name: THREADS
              value: {{ $gunicornConfig.threads | quote }}
            - name: TIMEOUT
              value: {{ $gunicornConfig.timeout | quote }}
+            - name: SWH_LOG_CONFIG_JSON
+              value: /etc/swh/logging/logging-gunicorn.json
+            - name: STATSD_SERVICE_TYPE
+              value: {{ $serviceType }}
+            {{ else -}}
+            - name: PROVENANCE_PATH
+              value: {{ $provenancePath }}
+            - name: GRAPH_PATH
+              value: {{ $graphPath }}
            {{ end -}}
            - name: STATSD_HOST
              value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
@@ -127,12 +178,8 @@ spec:
              value: {{ $.Values.statsdPort | default "9125" | quote }}
            - name: STATSD_TAGS
              value: deployment:{{ $serviceType }}
-            - name: STATSD_SERVICE_TYPE
-              value: {{ $serviceType }}
            - name: SWH_LOG_LEVEL
-              value: {{ $provenanceConfig.logLevel | default $.Values.provenance.logLevel | default "INFO" | quote }}
-            - name: SWH_LOG_CONFIG_JSON
-              value: /etc/swh/logging/logging-gunicorn.json
+              value: {{ $logLevel }}
          {{- if $.Values.provenance.sentry.enabled }}
            - name: SWH_SENTRY_ENVIRONMENT
              value: {{ $.Values.sentry.environment }}
@@ -150,10 +197,12 @@ spec:
              value: "true"
          {{- end }}
          volumeMounts:
+          {{- if eq $type "rpc" }}
          - name: configuration
            mountPath: /etc/swh
          - name: configuration-logging
            mountPath: /etc/swh/logging
+          {{ end }}
          {{- range $volumeName, $volumeConfig := $provenanceConfig.extraVolumes }}
          - name: {{ $volumeName }}
            mountPath: {{ $volumeConfig.mountPath }}
@@ -162,6 +211,7 @@ spec:
      volumes:
      - name: configuration
        emptyDir: {}
+      {{- if eq $type "rpc" }}
      - name: configuration-template
        configMap:
          name: {{ $serviceType }}-configuration-template
@@ -174,10 +224,15 @@ spec:
          items:
          - key: "logging-gunicorn.json"
            path: "logging-gunicorn.json"
+      {{ end }}
      - name: config-utils
        configMap:
          name: config-utils
          defaultMode: 0555
+      - name: backend-utils
+        configMap:
+          name: backend-utils
+          defaultMode: 0555
      {{- range $volumeName, $volumeConfig := $provenanceConfig.extraVolumes }}
      - name: {{ $volumeName }}
        {{- toYaml $volumeConfig.volumeDefinition | nindent 8 }}

--- a/swh/templates/provenance/ingress.yaml
+++ b/swh/templates/provenance/ingress.yaml
@@ -3,8 +3,11 @@
 {{- $serviceType := ( print "provenance-" $provenanceType ) -}}
 {{- if or (not (hasKey $deploymentConfig "enabled")) (get $deploymentConfig "enabled") }}
 {{- if and (hasKey $deploymentConfig "ingress") $deploymentConfig.ingress.enabled -}}
+
+{{- $configuration := include "swh.provenance.config" (dict "configuration" $deploymentConfig
+                                                            "Values" $.Values) | fromYaml -}}
 {{- include "swh.ingress" (dict "serviceType"   $serviceType
-                                "configuration" $deploymentConfig
+                                "configuration" $configuration
                                "Values"        $.Values) -}}
 {{- end -}}
 {{- if $deploymentConfig.extraIngresses -}}
@@ -14,8 +17,10 @@
 {{- $mergedIngressConfig :=
  mustMergeOverwrite (omit $deploymentConfig "extraIngresses" "ingress")
                     (dict "ingress" (mustMergeOverwrite $deploymentConfig.ingress $extraIngressConfig)) -}}
+{{- $config := include "swh.provenance.config" (dict "configuration" $mergedIngressConfig
+                                                     "Values" $.Values) | fromYaml -}}
 {{- include "swh.ingress" (dict "serviceType"    $serviceType
-                                "configuration"  $mergedIngressConfig
+                                "configuration"  $config
                                "extraNameLabel" ( print "extra-" $extraNameLabelCtr )
                                "Values"         $.Values) -}}
 {{- end -}}

--- a/swh/templates/provenance/service.yaml
+++ b/swh/templates/provenance/service.yaml
 {{ if .Values.provenance.enabled -}}
 {{ range $provenanceType, $deploymentConfig := .Values.provenance.deployments }}
 {{- if or (not (hasKey $deploymentConfig "enabled")) (get $deploymentConfig "enabled") -}}
-{{- if not (hasKey $deploymentConfig "port") }}
-{{- $deploymentConfig := set $deploymentConfig "port" $.Values.provenance.port -}}
-{{- end -}}
+{{- $configuration := include "swh.provenance.config" (dict "configuration" $deploymentConfig
+                                                            "Values" $.Values) | fromYaml -}}
 {{- include "swh.service" (dict "serviceType"   ( print "provenance-" $provenanceType )
-                                "configuration" $deploymentConfig
+                                "configuration" $configuration
                                "Values"        $.Values) -}}
 {{- end -}}
 {{ end -}}

--- a/swh/templates/utils/backend-utils.yaml
+++ b/swh/templates/utils/backend-utils.yaml
@@ -62,7 +62,7 @@ data:
    # Finally, we make explicit the graph is ready
    touch ${WITNESS_FILE}

-  graph-wait-for-dataset.sh: |
+  wait-for-dataset.sh: |
    #!/usr/bin/env bash
    # Uses env variables WITNESS_FILE
    [ -z "${WITNESS_FILE}" ] && \
@@ -158,6 +158,100 @@ data:
    swh graph reindex --ef ${DATASET_LOCATION}/${GRAPH_NAME} && \
      touch $WITNESS_REINDEX_FILE

+  provenance-fetch-datasets.sh: |
+    #!/usr/bin/env bash
+    [ -z "${WITNESS_FETCH_FILE}" ] && \
+      echo "<WITNESS_FETCH_FILE> env variable must be set" && exit 1
+    [ -z "${DATASET_VERSION}" ] && \
+      echo "<DATASET_VERSION> env variable must be set" && exit 1
+    [ -z "${PROVENANCE_PATH}" ] && \
+      echo "<PROVENANCE_PATH> env variable must be set" && exit 1
+    [ -z "${GRAPH_PATH}" ] && \
+      echo "<GRAPH_PATH> env variable must be set" && exit 1
+
+    [ -f ${WITNESS_FETCH_FILE} ] && \
+        echo "Datasets graph & provenance <${DATASET_VERSION}> already present. Skip." && \
+        exit 0
+
+    set -e
+
+    # Create destination paths
+    mkdir -p ${PROVENANCE_PATH} ${GRAPH_PATH}
+
+    echo "Fetching datasets..."
+
+    URL_PROVENANCE="s3://softwareheritage/derived_datasets/${DATASET_VERSION}/provenance/all/"
+
+    CMD_GET="aws s3 cp --no-progress --no-sign-request"
+
+    echo "1. Fetching provenance dataset (parquet files)..."
+    ${CMD_GET} --recursive "${URL_PROVENANCE}" "${PROVENANCE_PATH}"
+    echo "1. Provenance datasets installed!"
+
+    echo "2. Fetching extra graph files..."
+    URL_GRAPH="s3://softwareheritage/graph/${DATASET_VERSION}/compressed"
+
+    mkdir -p "${GRAPH_PATH}"
+    for filename in graph.pthash graph.pthash.order graph.node2swhid.bin.zst graph.node2type.bin.zst; do
+        ${CMD_GET} "${URL_GRAPH}/${filename}" "${GRAPH_PATH}"
+    done
+    echo "2. Extra graph files installed!"
+
+    echo "3. Uncompressing graph files..."
+    # Uncompress the compressed graph *.zst files
+    for filename in graph.node2type.bin.zst graph.node2swhid.bin.zst; do
+        filepath="${GRAPH_PATH}/${filename}"
+        CMD="unzstd --rm \"${filepath}\""
+        # Uncompress and delete the .zst file
+        [ -f "${filepath}" ] && echo "${CMD}" && "${CMD}"
+    done
+    echo "3. Graph files uncompressed!"
+
+    # Make explicit the provenance datasets are fetched
+    touch ${WITNESS_FETCH_FILE}
+
+    echo "Provenance datasets installed!"
+
+  provenance-index-dataset.sh: |
+    #!/usr/bin/env bash
+    [ -z "${WITNESS_DATASETS_FILE}" ] && \
+      echo "<WITNESS_DATASETS_FILE> env variable must be set" && exit 1
+    [ -z "${WITNESS_INDEX_FILE}" ] && \
+      echo "<WITNESS_INDEX_FILE> env variable must be set" && exit 1
+    [ -z "${PERIOD}" ] && \
+      echo "<PERIOD> env variable must be set" && exit 1
+    [ -z "${PROVENANCE_PATH}" ] && \
+      echo "<PROVENANCE_PATH> env variable must be set" && exit 1
+
+    # [ -f ${WITNESS_INDEX_FILE} ] && echo "Provenance already indexed, do nothing." && \
+    #   exit 0
+
+    set -eu
+
+    # Let's wait for the dataset installation
+    while [ ! -f "${WITNESS_DATASETS_FILE}" ]; do
+        echo "${WITNESS_DATASETS_FILE} missing, waiting provenance dataset installation..."
+        sleep $PERIOD
+    done
+
+    echo "Datasets file installed, build provenance dataset indexes..."
+
+    echo "provenance path: $PROVENANCE_PATH"
+    set -x
+
+    ls -lah /srv/dataset
+    ls -lah $PROVENANCE_PATH
+    ls -lah /srv/dataset/graph/
+    # To make the query faster, the provenance needs to build index out of the
+    # current dataset files. We store the output indexes in the same path as
+    # the dataset.
+    swh-provenance-index \
+      --database file://${PROVENANCE_PATH} && \
+      touch "${WITNESS_INDEX_FILE}" && \
+      echo "Provenance indexes built!" || \
+
+    echo "Provenance indexes failed!"
+
  initialize-search-backend.sh: |
    #!/usr/bin/env bash


--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -1895,22 +1895,55 @@ storage_backfiller:

 provenance:
  enabled: false
-  # Default service port (overridable per instance)
-  port: 5014
+  # Default rpc port
+  rpcPort: 5014
+  # Default grpc port
+  grpcPort: 50141
  priorityClassName: frontend-rpc
  sentry:
    enabled: false
    secretKeyRef: common-secrets
    secretKeyName: provenance-sentry-dsn
  deployments: {}
-    # instance:
+    # instance-rpc:
    #   enabled: false
+    #   # The server type to deploy, either a rpc or a grpc
+    #   type: rpc
+    #   # Port to use, either inferred from the type or explicitly set (can be
+    #   # overridden per instance)
    #   # port: 5014
+    #   # For type rpc, the graph configuration to use
    #   graphConfigurationRef: graphConfiguration
    #   hosts: []
    #   ingress:
    #     enabled: true
    #     endpoints: []
+    # instance-grpc:
+    #   enabled: false
+    #   # The server type to deploy, either a rpc or a grpc
+    #   type: grpc
+    #   # For type grpc, datasets need to be available for the service to use
+    #   dataset:
+    #     # Name of the dataset, graph and provenance must be in sync so same
+    #     # version must be used
+    #     name: 2024-08-23-popular-500-python
+    #     # Graph setup
+    #     graph:
+    #       # Where to store the files
+    #       path: /srv/dataset/graph
+    #     provenance:
+    #       # Whether or not to fetch provenance dataset
+    #       fetch: true
+    #       # Whether or not to build the provenance dataset indices
+    #       index: true
+    #       # Where to store the parquet files
+    #       path: /srv/dataset/provenance
+    #   # Whether to start/stop the service
+    #   startService: true
+    #   hosts: []
+    #   ingress:
+    #     enabled: true
+    #     endpoints: []

 rpcLocalRustWithGrpcGraphConfiguration:
  cls: local_rust

--- a/swh/values/local-cluster.yaml
+++ b/swh/values/local-cluster.yaml
@@ -1455,7 +1455,7 @@ objstorageReplayer:
 provenance:
  enabled: false
  deployments:
-    test:
+    test-rpc:
      enabled: true
      graphConfigurationRef: fakeGraphConfiguration
      replicas: 1
@@ -1463,6 +1463,30 @@ provenance:
      #   workers: 4
      #   threads: 1
      #   timeout: 60
+    test-grpc:
+      enabled: true
+      type: grpc
+      dataset:
+        name: 2024-08-23-popular-500-python
+        provenance:
+          fetch: true
+          index: true
+          path: /srv/dataset/provenance
+        graph:
+          path: /srv/dataset/graph
+      startService: true
+      extraVolumes:
+        dataset-persistent:
+          mountPath: /srv/dataset
+          volumeDefinition:
+            persistentVolumeClaim:
+              claimName: provenance-popular-persistent-pvc
+
+      replicas: 1
+      # gunicorn:
+      #   workers: 4
+      #   threads: 1
+      #   timeout: 60

 fakeGraphConfiguration:
  cls: graph
@@ -1516,6 +1540,17 @@ externalServices:
 volumes:
  enabled: true
  persistentVolumeClaims:
+    provenance-popular-persistent-pvc:
+      enabled: true
+      appName: provenance-test-grpc
+      spec:
+        storageClassName: local-persistent
+        volumeMode: Filesystem
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
    alter-recovery-bundles-pvc:
      enabled: true
      appName: alter

--- a/swh/values/staging/next-version.yaml
+++ b/swh/values/staging/next-version.yaml
@@ -1250,6 +1250,25 @@ provenance:
              extraWhitelistSourceRange:
                - 192.168.50.0/24 # for blackbox monitoring
                - 192.168.101.0/24 # vpn network
+    grpc:
+      enabled: true
+      type: grpc
+      dataset:
+        name: 2024-08-23-popular-500-python
+        provenance:
+          fetch: true
+          index: true
+          path: /srv/dataset/provenance
+        graph:
+          path: /srv/dataset/graph
+      startService: true
+      extraVolumes:
+        dataset-persistent:
+          mountPath: /srv/dataset
+          volumeDefinition:
+            persistentVolumeClaim:
+              claimName: provenance-popular-persistent-pvc
+

 # Rpc Rust graph configuration example (for a service rpc spawning an internal grpc
 # instance)
@@ -1294,6 +1313,17 @@ volumes:
          requests:
            storage: 1Gi
        volumeMode: Filesystem
+    provenance-popular-persistent-pvc:
+      enabled: true
+      appName: provenance-grpc
+      spec:
+        storageClassName: local-persistent
+        volumeMode: Filesystem
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi

 graph:
  enabled: true

--- a/values-swh-application-versions.yaml
+++ b/values-swh-application-versions.yaml
@@ -41,7 +41,7 @@ swh_objstorage_image_version: '20250319.1'
 swh_objstorage_replayer_image: container-registry.softwareheritage.org/swh/infra/swh-apps/objstorage_replayer
 swh_objstorage_replayer_image_version: '20250319.1'
 swh_provenance_image: container-registry.softwareheritage.org/swh/infra/swh-apps/provenance
-swh_provenance_image_version: '20250319.1'
+swh_provenance_image_version: '20250319.2'
 swh_scheduler_image: container-registry.softwareheritage.org/swh/infra/swh-apps/scheduler
 swh_scheduler_image_version: '20250319.1'
 swh_scrubber_image: container-registry.softwareheritage.org/swh/infra/swh-apps/scrubber
No results found