.pre-commit-config.yaml

 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: check-json
       - id: check-yaml
 
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
+    rev: 7.1.0
     hooks:
       - id: flake8
         additional_dependencies: [flake8-bugbear==22.9.23]
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.3.0
     hooks:
       - id: codespell
         name: Check source code spelling
@@ -30,11 +30,11 @@ repos:
         types: [python]
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.13.2
     hooks:
       - id: isort
 
   - repo: https://github.com/python/black
-    rev: 22.10.0
+    rev: 24.4.2
     hooks:
       - id: black

README.rst

@@ -38,3 +38,184 @@ file. Out of this, a virtualenv is generated and frozen into a
 
 The frozen requirements file can also be used to deploy virtual environments
 directly, e.g. on an existing VM or bare metal server.
+
+Local development
+-----------------
+
+App-manager
+~~~~~~~~~~~
+
+It's the cli tools which allows to execute various actions:
+- list dependency between swh modules
+- manipulate the generation of frozen requirements for our python applications
+- helm chart dependency version update
+- etc...
+
+.. code::
+
+   # Build the app-manager image
+   docker build -t app-manager scripts
+
+List dependent applications
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is the listing command to list whatever other applications is depending
+on a specification application version.
+
+.. code::
+
+   docker run --rm -v $workspace/swh-apps:/src \
+     app-manager list-dependent-apps \
+       --application swh.graph --version v6.3.0
+
+Example:
+
+.. code::
+
+   docker run --rm -v $workspace/swh-apps:/src \
+     app-manager list-dependent-apps \
+       --application swh.graph --version v6.3.0
+   swh-alter
+   swh-graph
+   swh-provenance
+   swh-vault-cookers
+   swh-web
+
+
+Freeze application dependencies
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is in charge of using the existing requirements.txt declared besides the
+application's Dockerfile and generate an updated frozen-requirements.txt. This
+file is then used within the docker image to build an identical python
+environments within the image.
+
+Note that it does update any dependencies within the environments so any new
+upstream libraries will get updated if new releases happened.
+
+.. code::
+
+   docker run --rm -v $workspace/swh-apps:/src \
+     app-manager generate-frozen-requirements swh-provenance
+
+.. code::
+
+   $ docker run --rm -v $workspace/swh-apps:/src \
+       app-manager generate-frozen-requirements swh-provenance
+   Collecting uv
+     Downloading uv-0.5.26-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.2 MB)
+        ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 16.2/16.2 MB 36.4 MB/s eta 0:00:00
+   Installing collected packages: uv
+   Successfully installed uv-0.5.26
+
+   [notice] A new release of pip is available: 23.0.1 -> 25.0
+   [notice] To update, run: python3 -m pip install --upgrade pip
+   Using Python 3.10.15 environment at: /tmp/swh-provenancep16_3kh7
+   Resolved 3 packages in 46ms
+   Prepared 3 packages in 86ms
+   Uninstalled 2 packages in 51ms
+   Installed 3 packages in 32ms
+    - pip==23.0.1
+    + pip==25.0
+    - setuptools==65.5.0
+    + setuptools==75.8.0
+    + wheel==0.45.1
+   # This file was autogenerated by uv via the following command:
+   #    uv pip compile /src/apps/swh-provenance/requirements.txt -o /src/apps/swh-provenance/tmpk6i4m4rz
+   Resolved 76 packages in 850ms
+   aiohappyeyeballs==2.4.4
+       # via aiohttp
+       # via -r /src/apps/swh-provenance/requirements.txt
+   python-magic==0.4.27
+       # via swh-core
+   python-mimeparse==2.0.0
+       # via aiohttp-utils
+   pyyaml==6.0.2
+       # via swh-core
+   redis==5.2.1
+       # via
+       # via -r /src/apps/swh-provenance/requirements.txt
+   ...
+   swh-storage==2.9.0
+       # via
+       #   swh-dataset
+       #   swh-provenance
+   tenacity==9.0.0
+       # via
+       #   swh-core
+       #   swh-journal
+       #   swh-storage
+   tqdm==4.67.1
+       # via swh-dataset
+   types-protobuf==5.29.1.20241207
+       # via mypy-protobuf
+   types-requests==2.32.0.20241016
+       # via swh-dataset
+   typing-extensions==4.12.2
+       # via
+       #   multidict
+       #   swh-core
+       #   swh-model
+       #   swh-objstorage
+       #   swh-storage
+   urllib3==2.3.0
+       # via
+       #   botocore
+       #   requests
+       #   sentry-sdk
+       #   types-requests
+   werkzeug==3.1.3
+       # via flask
+   wrapt==1.17.2
+       # via deprecated
+   yarl==1.18.3
+       # via aiohttp
+
+Update swh-charts' version
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Providing access to both the swh-apps and the swh-charts repositories, this
+allows to update the values of values-swh-application-versions.yaml with the
+most recent docker image versions. This also bumps the Charts.yaml's version
+incrementally.
+
+.. code::
+
+   docker run --rm \
+     --volume $workspace/swh-apps:/src \
+     --volume $workspace/swh-charts:/tmp/swh-charts \
+     app-manager update-versions \
+       --applications-filepath /tmp/s-charts/values-swh-application-versions.yaml \
+       --chart-filepath /tmp/swh-charts/swh/Chart.yaml
+
+Build application image
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Each application is stored in swh-apps:/apps/$application_dir/ folder.
+It usually holds the same set of files:
+- Dockerfile: The set of instructions to build the application's docker image
+- entrypoint.sh: The last Dockerfile instruction refers to this executable to run in the
+    image.
+- requirements.txt: The set of python dependencies the application requires to run.
+- requirements-frozen.txt: The derivative frozen set of dependencies to reproduce the
+    environment in the docker image. It's built by the app-manager out of the
+    requirements.txt file.
+
+Cli call to build the application locally:
+
+.. code::
+
+   $ DOCKER_BUILDKIT=1 docker build -t swh-toolbox:latest apps/swh-toolbox \
+     --build-arg REGISTRY=
+
+Note: The REGISTRY is empty so we only rely to local docker image. You can avoid
+changing it, in which case, it will use the swh's gitlab registry.
+
+Cli call to run the application and be dropped in a shell:
+
+.. code::
+
+   $ docker run -it swh-toolbox:latest shell
+
+Note: This is specific command depending on the entrypoint.sh. In that case, the
+entrypoint.sh allows a shell option to be dropped in an interactive bash session.

apps/cassandra-checks/Dockerfile

+ARG REGISTRY=container-registry.softwareheritage.org/swh/infra/swh-apps/
+ARG base_image=${REGISTRY}base
+ARG base_image_version=latest
+
+FROM ${base_image}:${base_image_version}
+
+ARG user=swh
+ARG workdir=/opt/${user}
+ARG configdir=/etc/${user}
+
+USER root
+RUN mkdir /volume && chown root: /volume
+
+COPY --chmod=0644 requirements-frozen.txt ${workdir}
+RUN --mount=type=cache,target=.cache,uid=1000,gid=1000,uid=1000,gid=1000 \
+    uv pip sync requirements-frozen.txt
+
+COPY --chmod=0755 entrypoint.sh ${workdir}
+
+USER ${user}
+
+ENV SWH_CONFIG_FILENAME=${configdir}/config.yml
+
+ENTRYPOINT ["/opt/swh/entrypoint.sh"]

apps/cassandra-checks/entrypoint.sh

+#!/bin/bash
+
+set -e
+
+CLONE_DIR=/opt/swh/snippets
+
+if [ ! -d $CLONE_DIR ]; then
+    git clone --depth=1 https://gitlab.softwareheritage.org/swh/devel/snippets.git $CLONE_DIR
+else
+    pushd $CLONE_DIR
+    git fetch
+    git merge origin/master
+    popd
+fi
+
+case "$1" in
+    "shell")
+        shift
+        if (( $# == 0)); then
+            exec bash -i
+        else
+            "$@"
+        fi
+        ;;
+    *)
+        echo Starting Cassandra objects checker.
+        exec $CLONE_DIR/sysadmin/cassandra_checks/get_journal_check_and_replay.py
+        ;;
+esac

apps/cassandra-checks/requirements-frozen.txt

+# This file was autogenerated by uv via the following command:
+#    uv pip compile /src/apps/cassandra-checks/requirements.txt -o /src/apps/cassandra-checks/tmpkbxxtj2z
+aiohappyeyeballs==2.4.6
+    # via aiohttp
+aiohttp==3.11.12
+    # via
+    #   aiohttp-utils
+    #   swh-core
+aiohttp-utils==3.2.1
+    # via swh-core
+aiosignal==1.3.2
+    # via aiohttp
+attrs==25.1.0
+    # via
+    #   aiohttp
+    #   attrs-strict
+    #   hypothesis
+    #   swh-model
+attrs-strict==1.0.1
+    # via swh-model
+backports-entry-points-selectable==1.3.0
+    # via
+    #   swh-core
+    #   swh-storage
+blinker==1.9.0
+    # via
+    #   flask
+    #   swh-core
+cassandra-driver==3.29.2
+    # via swh-storage
+certifi==2025.1.31
+    # via
+    #   requests
+    #   sentry-sdk
+cffi==1.17.1
+    # via swh-perfecthash
+charset-normalizer==3.4.1
+    # via requests
+click==8.1.8
+    # via
+    #   flask
+    #   geomet
+    #   swh-core
+    #   swh-objstorage
+    #   swh-storage
+confluent-kafka==2.8.0
+    # via swh-journal
+deprecated==1.2.18
+    # via
+    #   swh-core
+    #   swh-model
+    #   swh-objstorage
+    #   swh-storage
+flask==3.1.0
+    # via
+    #   swh-core
+    #   swh-counters
+    #   swh-storage
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+geomet==0.2.1.post1
+    # via cassandra-driver
+gunicorn==23.0.0
+    # via aiohttp-utils
+hypothesis==6.125.2
+    # via swh-model
+idna==3.10
+    # via
+    #   requests
+    #   yarl
+iso8601==2.1.0
+    # via
+    #   swh-core
+    #   swh-model
+    #   swh-storage
+itsdangerous==2.2.0
+    # via flask
+jinja2==3.1.5
+    # via flask
+markupsafe==3.0.2
+    # via
+    #   jinja2
+    #   werkzeug
+msgpack==1.1.0
+    # via
+    #   swh-core
+    #   swh-journal
+    #   swh-objstorage
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.0.0
+    # via swh-storage
+packaging==24.2
+    # via gunicorn
+propcache==0.2.1
+    # via
+    #   aiohttp
+    #   yarl
+psycopg2==2.9.10
+    # via
+    #   swh-core
+    #   swh-storage
+pycparser==2.22
+    # via cffi
+python-dateutil==2.9.0.post0
+    # via swh-model
+python-magic==0.4.27
+    # via swh-core
+python-mimeparse==2.0.0
+    # via aiohttp-utils
+pyyaml==6.0.2
+    # via swh-core
+redis==5.2.1
+    # via
+    #   swh-counters
+    #   swh-storage
+requests==2.32.3
+    # via
+    #   swh-core
+    #   swh-objstorage
+sentry-sdk==2.20.0
+    # via swh-core
+six==1.17.0
+    # via
+    #   geomet
+    #   python-dateutil
+sortedcontainers==2.4.0
+    # via hypothesis
+swh-core==3.8.0
+    # via
+    #   swh-counters
+    #   swh-journal
+    #   swh-objstorage
+    #   swh-storage
+swh-counters==0.11.0
+    # via swh-storage
+swh-journal==1.5.3
+    # via swh-counters
+swh-model==7.0.0
+    # via
+    #   -r /src/apps/cassandra-checks/requirements.txt
+    #   swh-journal
+    #   swh-objstorage
+    #   swh-storage
+swh-objstorage==3.4.0
+    # via swh-storage
+swh-perfecthash==1.3.2
+    # via swh-objstorage
+swh-storage==2.9.1
+    # via -r /src/apps/cassandra-checks/requirements.txt
+tenacity==9.0.0
+    # via
+    #   swh-core
+    #   swh-journal
+    #   swh-storage
+typing-extensions==4.12.2
+    # via
+    #   swh-core
+    #   swh-model
+    #   swh-objstorage
+    #   swh-storage
+urllib3==2.3.0
+    # via
+    #   requests
+    #   sentry-sdk
+werkzeug==3.1.3
+    # via flask
+wrapt==1.17.2
+    # via deprecated
+yarl==1.18.3
+    # via aiohttp

apps/cassandra-checks/requirements.txt

+swh-storage
+swh-model

apps/software-stories/Dockerfile

@@ -10,13 +10,11 @@ USER node
 ENV NODE_OPTIONS=--openssl-legacy-provider
 WORKDIR /opt/swh/swh-stories
 
-COPY --chown=node:node entrypoint.sh /opt/swh
-
-RUN chmod u+x /opt/swh/entrypoint.sh && \
-    git clone https://github.com/ScienceStories/swh-stories.git /opt/swh/swh-stories && \
-    git checkout 04168585b48edb77b3410cfe0e5e06af545bb18c && \
-    rm package-lock.json && \
+RUN git clone https://github.com/ScienceStories/swh-stories.git /opt/swh/swh-stories && \
     npm install --no-audit && \
     npm run build
 
-ENTRYPOINT "/opt/swh/entrypoint.sh"
+COPY --chown=node:node entrypoint.sh /opt/swh
+RUN chmod u+x /opt/swh/entrypoint.sh
+
+ENTRYPOINT ["/opt/swh/entrypoint.sh"]

apps/swh-add-forge-now/Dockerfile

+ARG REGISTRY=container-registry.softwareheritage.org/swh/infra/swh-apps/
+ARG base_image=${REGISTRY}base
+ARG base_image_version=latest
+
+FROM ${base_image}:${base_image_version}
+
+ARG user=swh
+ARG workdir=/opt/${user}
+ARG configdir=/etc/${user}
+
+USER root
+RUN apt-get -y update && \
+    apt-get install -y netcat-openbsd jq && \
+    apt-get clean
+
+COPY --chmod=0644 requirements-frozen.txt ${workdir}
+RUN --mount=type=cache,target=.cache,uid=1000,gid=1000 \
+    uv pip sync requirements-frozen.txt
+
+USER ${user}
+
+CMD [ "/bin/bash", "-i" ]

apps/swh-add-forge-now/requirements-frozen.txt

+# This file was autogenerated by uv via the following command:
+#    uv pip compile /src/apps/swh-add-forge-now/requirements.txt -o /src/apps/swh-add-forge-now/tmp2856xk2o
+aiofiles==24.1.0
+    # via python-keycloak
+aiohappyeyeballs==2.4.6
+    # via aiohttp
+aiohttp==3.11.13
+    # via
+    #   aiohttp-utils
+    #   swh-core
+aiohttp-utils==3.2.1
+    # via swh-core
+aiosignal==1.3.2
+    # via aiohttp
+amqp==5.3.1
+    # via kombu
+anyio==4.8.0
+    # via httpx
+async-property==0.2.2
+    # via python-keycloak
+attrs==25.1.0
+    # via
+    #   aiohttp
+    #   attrs-strict
+    #   hypothesis
+    #   swh-model
+    #   swh-scheduler
+attrs-strict==1.0.1
+    # via
+    #   swh-model
+    #   swh-scheduler
+backports-entry-points-selectable==1.3.0
+    # via
+    #   swh-core
+    #   swh-storage
+billiard==4.2.1
+    # via celery
+blinker==1.9.0
+    # via
+    #   flask
+    #   swh-core
+cassandra-driver==3.29.2
+    # via swh-storage
+celery==5.4.0
+    # via swh-scheduler
+certifi==2025.1.31
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+    #   sentry-sdk
+cffi==1.17.1
+    # via
+    #   cryptography
+    #   swh-perfecthash
+charset-normalizer==3.4.1
+    # via requests
+click==8.1.8
+    # via
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+    #   flask
+    #   geomet
+    #   swh-auth
+    #   swh-core
+    #   swh-objstorage
+    #   swh-scheduler
+    #   swh-storage
+click-didyoumean==0.3.1
+    # via celery
+click-plugins==1.1.1
+    # via celery
+click-repl==0.3.0
+    # via celery
+confluent-kafka==2.8.0
+    # via swh-journal
+cryptography==44.0.1
+    # via jwcrypto
+deprecated==1.2.18
+    # via
+    #   swh-core
+    #   swh-model
+    #   swh-objstorage
+    #   swh-storage
+deprecation==2.1.0
+    # via python-keycloak
+flask==3.1.0
+    # via
+    #   swh-core
+    #   swh-counters
+    #   swh-scheduler
+    #   swh-storage
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+geomet==0.2.1.post1
+    # via cassandra-driver
+gunicorn==23.0.0
+    # via aiohttp-utils
+h11==0.14.0
+    # via httpcore
+httpcore==1.0.7
+    # via httpx
+httpx==0.28.1
+    # via python-keycloak
+humanize==4.12.1
+    # via swh-scheduler
+hypothesis==6.127.2
+    # via swh-model
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+importlib-metadata==8.6.1
+    # via swh-scheduler
+iso8601==2.1.0
+    # via
+    #   swh-core
+    #   swh-model
+    #   swh-storage
+itsdangerous==2.2.0
+    # via flask
+jinja2==3.1.5
+    # via flask
+jwcrypto==1.5.6
+    # via python-keycloak
+kombu==5.4.2
+    # via celery
+markupsafe==3.0.2
+    # via
+    #   jinja2
+    #   werkzeug
+msgpack==1.1.0
+    # via
+    #   swh-core
+    #   swh-journal
+    #   swh-objstorage
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.0.0
+    # via swh-storage
+packaging==24.2
+    # via
+    #   deprecation
+    #   gunicorn
+pika==1.3.2
+    # via swh-scheduler
+prompt-toolkit==3.0.50
+    # via click-repl
+propcache==0.3.0
+    # via
+    #   aiohttp
+    #   yarl
+psycopg2==2.9.10
+    # via
+    #   swh-core
+    #   swh-scheduler
+    #   swh-storage
+pycparser==2.22
+    # via cffi
+python-dateutil==2.9.0.post0
+    # via
+    #   celery
+    #   swh-model
+python-keycloak==5.3.1
+    # via swh-auth
+python-magic==0.4.27
+    # via swh-core
+python-mimeparse==2.0.0
+    # via aiohttp-utils
+pyyaml==6.0.2
+    # via
+    #   swh-auth
+    #   swh-core
+    #   swh-scheduler
+redis==5.2.1
+    # via
+    #   swh-counters
+    #   swh-storage
+requests==2.32.3
+    # via
+    #   python-keycloak
+    #   requests-toolbelt
+    #   swh-core
+    #   swh-objstorage
+    #   swh-scheduler
+requests-toolbelt==1.0.0
+    # via python-keycloak
+sentry-sdk==2.22.0
+    # via
+    #   swh-core
+    #   swh-scheduler
+setuptools==75.8.2
+    # via swh-scheduler
+six==1.17.0
+    # via
+    #   geomet
+    #   python-dateutil
+sniffio==1.3.1
+    # via anyio
+sortedcontainers==2.4.0
+    # via hypothesis
+swh-auth==0.10.0
+    # via -r /src/apps/swh-add-forge-now/requirements.txt
+swh-core==3.8.0
+    # via
+    #   swh-auth
+    #   swh-counters
+    #   swh-journal
+    #   swh-objstorage
+    #   swh-scheduler
+    #   swh-storage
+swh-counters==0.11.0
+    # via swh-storage
+swh-journal==1.5.3
+    # via swh-counters
+swh-model==7.0.0
+    # via
+    #   swh-journal
+    #   swh-objstorage
+    #   swh-storage
+swh-objstorage==3.4.0
+    # via swh-storage
+swh-perfecthash==1.3.2
+    # via swh-objstorage
+swh-scheduler==2.7.3
+    # via -r /src/apps/swh-add-forge-now/requirements.txt
+swh-storage==2.9.1
+    # via swh-scheduler
+tabulate==0.9.0
+    # via swh-scheduler
+tenacity==9.0.0
+    # via
+    #   swh-core
+    #   swh-journal
+    #   swh-storage
+typing-extensions==4.12.2
+    # via
+    #   anyio
+    #   jwcrypto
+    #   swh-core
+    #   swh-model
+    #   swh-objstorage
+    #   swh-scheduler
+    #   swh-storage
+tzdata==2025.1
+    # via
+    #   celery
+    #   kombu
+urllib3==2.3.0
+    # via
+    #   requests
+    #   sentry-sdk
+vine==5.1.0
+    # via
+    #   amqp
+    #   celery
+    #   kombu
+wcwidth==0.2.13
+    # via prompt-toolkit
+werkzeug==3.1.3
+    # via flask
+wrapt==1.17.2
+    # via deprecated
+yarl==1.18.3
+    # via aiohttp
+zipp==3.21.0
+    # via importlib-metadata

apps/swh-add-forge-now/requirements.txt

+swh-auth
+swh-scheduler

apps/swh-alter/Dockerfile

+ARG REGISTRY=container-registry.softwareheritage.org/swh/infra/swh-apps/
+ARG base_image=${REGISTRY}base
+ARG base_image_version=latest
+
+FROM ${base_image}:${base_image_version}
+
+ARG user=swh
+ARG workdir=/opt/${user}
+
+USER root
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get install -y \
+      curl \
+      dnsutils \
+      emacs-nox \
+      host \
+      jq \
+      libcmph-dev \
+      librdkafka-dev \
+      libsystemd-dev \
+      netcat-openbsd \
+      postgresql-client \
+      sed \
+      strace \
+      telnet \
+      tmux \
+      vim && \
+    apt-get clean
+
+COPY --chmod=0644 requirements-frozen.txt ${workdir}
+RUN --mount=type=cache,target=.cache,uid=1000,gid=1000 \
+    uv pip sync requirements-frozen.txt
+
+USER ${user}
+ENV CARGO_HOME="${workdir}/.cargo"
+ENV RUSTUP_HOME="${workdir}/.rustup"
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o /tmp/rustup-init.sh && \
+  chmod +x /tmp/rustup-init.sh && \
+  /tmp/rustup-init.sh -y && \
+  rm /tmp/rustup-init.sh
+
+ENV PATH="${CARGO_HOME}/bin:${PATH}"
+# install rage (for swh-alter)
+
+# Use the latest available version so we can detect if rage behavior has changed when
+# testing swh-alter. Otherwise, this might only be detected when processing a
+# takedown notice or restoring a bundle… which would not be a nice experience.
+RUN --mount=type=cache,target=.cache,uid=1000,gid=1000 \
+  cargo install rage
+
+COPY --chmod=0755 entrypoint.sh ${workdir}
+
+ENTRYPOINT ["/opt/swh/entrypoint.sh"]

apps/swh-alter/entrypoint.sh

+#!/bin/bash
+
+set -e
+
+case "$1" in
+    "shell")
+        shift
+        if (( $# == 0)); then
+            exec bash -i
+        else
+            "$@"
+        fi
+        exit 0
+        ;;
+    *)
+        echo "swh-alter pod: running a tmux server"
+        tmux set -g exit-empty off \; start
+
+        trap "tmux kill-server" INT TERM EXIT
+
+        while tmux list-sessions 2>/dev/null; do sleep 5; done
+        echo "tmux server has exited, bailing"
+        ;;
+esac

apps/swh-alter/requirements-frozen.txt

+# This file was autogenerated by uv via the following command:
+#    uv pip compile /src/apps/swh-alter/requirements.txt -o /src/apps/swh-alter/tmpyzbd0djh
+aiohappyeyeballs==2.6.1
+    # via aiohttp
+aiohttp==3.11.13
+    # via
+    #   aiohttp-utils
+    #   swh-core
+    #   swh-graph
+aiohttp-utils==3.2.1
+    # via swh-core
+aiosignal==1.3.2
+    # via aiohttp
+apache-libcloud==3.8.0
+    # via swh-objstorage
+attrs==25.3.0
+    # via
+    #   aiohttp
+    #   attrs-strict
+    #   hypothesis
+    #   swh-alter
+    #   swh-model
+attrs-strict==1.0.1
+    # via swh-model
+azure-core==1.32.0
+    # via azure-storage-blob
+azure-storage-blob==12.25.0
+    # via swh-objstorage
+backports-entry-points-selectable==1.3.0
+    # via
+    #   swh-core
+    #   swh-storage
+blinker==1.9.0
+    # via
+    #   flask
+    #   swh-core
+boto3==1.37.12
+    # via swh-graph
+botocore==1.37.12
+    # via
+    #   boto3
+    #   s3transfer
+cachetools==5.5.2
+    # via pyld
+cassandra-driver==3.29.2
+    # via swh-storage
+certifi==2025.1.31
+    # via
+    #   elasticsearch
+    #   requests
+    #   sentry-sdk
+cffi==1.17.1
+    # via
+    #   cryptography
+    #   swh-perfecthash
+charset-normalizer==3.4.1
+    # via requests
+click==8.1.8
+    # via
+    #   flask
+    #   geomet
+    #   swh-alter
+    #   swh-core
+    #   swh-graph
+    #   swh-indexer
+    #   swh-objstorage
+    #   swh-search
+    #   swh-storage
+confluent-kafka==2.8.2
+    # via swh-journal
+cryptography==44.0.2
+    # via azure-storage-blob
+deprecated==1.2.18
+    # via
+    #   swh-core
+    #   swh-model
+    #   swh-objstorage
+    #   swh-storage
+elasticsearch==7.17.12
+    # via swh-search
+flask==3.1.0
+    # via
+    #   swh-core
+    #   swh-counters
+    #   swh-storage
+frozendict==2.4.6
+    # via
+    #   pyld
+    #   swh-indexer
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+geomet==0.2.1.post1
+    # via cassandra-driver
+grpcio==1.71.0
+    # via grpcio-tools
+grpcio-tools==1.71.0
+    # via swh-graph
+gunicorn==23.0.0
+    # via aiohttp-utils
+humanize==4.12.1
+    # via swh-alter
+hypothesis==6.129.1
+    # via swh-model
+idna==3.10
+    # via
+    #   requests
+    #   yarl
+igraph==0.11.8
+    # via swh-alter
+iso8601==2.1.0
+    # via
+    #   swh-core
+    #   swh-indexer
+    #   swh-model
+    #   swh-search
+    #   swh-storage
+isodate==0.7.2
+    # via azure-storage-blob
+itsdangerous==2.2.0
+    # via flask
+jinja2==3.1.6
+    # via flask
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+latexcodec==3.0.0
+    # via pybtex
+lxml==5.3.1
+    # via pyld
+markupsafe==3.0.2
+    # via
+    #   jinja2
+    #   werkzeug
+msgpack==1.1.0
+    # via
+    #   swh-core
+    #   swh-journal
+    #   swh-objstorage
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+mypy-extensions==1.0.0
+    # via swh-storage
+mypy-protobuf==3.6.0
+    # via swh-graph
+packaging==24.2
+    # via gunicorn
+propcache==0.3.0
+    # via
+    #   aiohttp
+    #   yarl
+protobuf==5.29.3
+    # via
+    #   grpcio-tools
+    #   mypy-protobuf
+    #   swh-graph
+psutil==7.0.0
+    # via swh-graph
+psycopg2==2.9.10
+    # via
+    #   swh-core
+    #   swh-storage
+py4j==0.10.9.9
+    # via swh-graph
+pybtex==0.24.0
+    # via swh-indexer
+pycparser==2.22
+    # via cffi
+pyld==2.0.4
+    # via swh-indexer
+pyparsing==3.2.1
+    # via rdflib
+python-dateutil==2.9.0.post0
+    # via
+    #   botocore
+    #   swh-model
+python-magic==0.4.27
+    # via
+    #   swh-core
+    #   swh-indexer
+python-mimeparse==2.0.0
+    # via aiohttp-utils
+pyyaml==6.0.2
+    # via
+    #   pybtex
+    #   swh-alter
+    #   swh-core
+rdflib==7.1.3
+    # via swh-indexer
+redis==5.2.1
+    # via
+    #   swh-counters
+    #   swh-storage
+requests==2.32.3
+    # via
+    #   apache-libcloud
+    #   azure-core
+    #   swh-core
+    #   swh-objstorage
+s3transfer==0.11.4
+    # via boto3
+sentry-sdk==2.22.0
+    # via
+    #   swh-core
+    #   swh-indexer
+setuptools==76.0.0
+    # via grpcio-tools
+shamir-mnemonic==0.3.0
+    # via swh-alter
+six==1.17.0
+    # via
+    #   azure-core
+    #   geomet
+    #   pybtex
+    #   python-dateutil
+sortedcontainers==2.4.0
+    # via hypothesis
+swh-alter==1.3.0
+    # via -r /src/apps/swh-alter/requirements.txt
+swh-core==3.8.0
+    # via
+    #   swh-alter
+    #   swh-counters
+    #   swh-graph
+    #   swh-indexer
+    #   swh-journal
+    #   swh-objstorage
+    #   swh-search
+    #   swh-storage
+swh-counters==0.11.0
+    # via swh-storage
+swh-graph==6.7.1
+    # via swh-alter
+swh-indexer==3.9.1
+    # via swh-search
+swh-journal==1.5.3
+    # via
+    #   swh-counters
+    #   swh-indexer
+    #   swh-search
+swh-model==7.0.0
+    # via
+    #   swh-alter
+    #   swh-graph
+    #   swh-indexer
+    #   swh-journal
+    #   swh-objstorage
+    #   swh-search
+    #   swh-storage
+swh-objstorage==3.4.0
+    # via
+    #   -r /src/apps/swh-alter/requirements.txt
+    #   swh-alter
+    #   swh-indexer
+    #   swh-storage
+swh-perfecthash==1.3.2
+    # via swh-objstorage
+swh-search==0.22.1
+    # via swh-alter
+swh-storage==2.9.1
+    # via
+    #   swh-alter
+    #   swh-indexer
+tabulate==0.9.0
+    # via swh-alter
+tenacity==9.0.0
+    # via
+    #   swh-core
+    #   swh-journal
+    #   swh-storage
+texttable==1.7.0
+    # via igraph
+tree-sitter==0.21.3
+    # via swh-search
+types-protobuf==5.29.1.20250208
+    # via mypy-protobuf
+typing-extensions==4.12.2
+    # via
+    #   azure-core
+    #   azure-storage-blob
+    #   swh-alter
+    #   swh-core
+    #   swh-indexer
+    #   swh-model
+    #   swh-objstorage
+    #   swh-search
+    #   swh-storage
+urllib3==1.26.20
+    # via
+    #   botocore
+    #   elasticsearch
+    #   requests
+    #   sentry-sdk
+werkzeug==3.1.3
+    # via flask
+wrapt==1.17.2
+    # via deprecated
+xmltodict==0.14.2
+    # via swh-indexer
+yarl==1.18.3
+    # via aiohttp

apps/swh-alter/requirements.txt

+swh.alter
+swh.objstorage[azure,libcloud]

apps/swh-base/Dockerfile

+FROM python:3.11-bookworm
+
+ARG userid=1000
+ARG groupid=1000
+ARG user=swh
+ARG workdir=/opt/${user}
+ARG venv=${workdir}/venv
+ARG configdir=/etc/${user}
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y git sed \
+      gzip lzip zstd cpio \
+      less && \
+    apt-get clean && \
+    addgroup --gid ${userid} ${user} && \
+    useradd --gid ${groupid} --uid ${userid} -m -d ${workdir} ${user} && \
+    mkdir ${configdir}
+
+USER ${user}
+RUN python -m venv ${venv}
+WORKDIR ${workdir}
+
+ENV PYTHONPATH=${venv}
+ENV PATH=${venv}/bin:$PATH
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/

apps/swh-counters/Dockerfile

+ARG REGISTRY=container-registry.softwareheritage.org/swh/infra/swh-apps/
+ARG base_image=${REGISTRY}base
+ARG base_image_version=latest
+
+FROM ${base_image}:${base_image_version}
+
+ARG user=swh
+ARG workdir=/opt/${user}
+ARG configdir=/etc/${user}
+
+USER root
+# retry command is used by the refresh-history cron job
+# to ensure the rpc service is well up
+RUN apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get install retry && \
+    apt-get clean
+
+COPY --chmod=0644 requirements-frozen.txt ${workdir}
+RUN --mount=type=cache,target=.cache,uid=1000,gid=1000 \
+    uv pip sync requirements-frozen.txt
+
+COPY --chmod=0755 entrypoint.sh ${workdir}
+
+USER ${user}
+ENV SWH_CONFIG_FILENAME=${configdir}/config.yml
+ENV PORT 5011
+EXPOSE $PORT
+ENV THREADS 2
+ENV WORKERS 2
+ENV TIMEOUT 3600
+
+ENTRYPOINT ["/opt/swh/entrypoint.sh"]

apps/swh-counters/entrypoint.sh

+#!/bin/bash
+
+set -e
+
+case "$1" in
+    "shell")
+        shift
+        if (( $# == 0)); then
+            exec bash -i
+        else
+            "$@"
+        fi
+        ;;
+    "swh")
+        shift
+        echo "Running swh command $@"
+        exec swh $@
+        ;;
+    *)
+        EXTRA_CLI_FLAGS=()
+        if [ -n "${SWH_LOG_CONFIG_JSON}" ]; then
+            EXTRA_CLI_FLAGS+=('--log-config-json' "${SWH_LOG_CONFIG_JSON}")
+        fi
+        if [ -n "${STATSD_HOST}" -a -n "${STATSD_PORT}" ]; then
+            EXTRA_CLI_FLAGS+=('--statsd-host' "${STATSD_HOST}:${STATSD_PORT}")
+        fi
+        if [ -n "${STATSD_SERVICE_TYPE}" ]; then
+            EXTRA_CLI_FLAGS+=('--statsd-prefix' "${STATSD_SERVICE_TYPE}")
+        fi
+
+        echo 'Starting the swh-counters API server'
+        exec gunicorn --bind "0.0.0.0:${PORT}" \
+             --log-level "${SWH_LOG_LEVEL:-INFO}" \
+             "${EXTRA_CLI_FLAGS[@]}" \
+             --threads "${THREADS}" \
+             --workers "${WORKERS}" \
+             --timeout "${TIMEOUT}" \
+             --reload \
+             --config 'python:swh.core.api.gunicorn_config' \
+             'swh.counters.api.server:make_app_from_configfile()'
+esac

apps/swh-counters/requirements-frozen.txt

+# This file was autogenerated by uv via the following command:
+#    uv pip compile /src/apps/swh-counters/requirements.txt -o /src/apps/swh-counters/tmpapltmhyk
+aiohappyeyeballs==2.4.6
+    # via aiohttp
+aiohttp==3.11.12
+    # via
+    #   aiohttp-utils
+    #   swh-core
+aiohttp-utils==3.2.1
+    # via swh-core
+aiosignal==1.3.2
+    # via aiohttp
+attrs==25.1.0
+    # via
+    #   aiohttp
+    #   attrs-strict
+    #   hypothesis
+    #   swh-model
+attrs-strict==1.0.1
+    # via swh-model
+backports-entry-points-selectable==1.3.0
+    # via swh-core
+blinker==1.9.0
+    # via
+    #   flask
+    #   swh-core
+certifi==2025.1.31
+    # via
+    #   requests
+    #   sentry-sdk
+charset-normalizer==3.4.1
+    # via requests
+click==8.1.8
+    # via
+    #   flask
+    #   swh-core
+confluent-kafka==2.8.0
+    # via swh-journal
+deprecated==1.2.18
+    # via
+    #   swh-core
+    #   swh-model
+flask==3.1.0
+    # via
+    #   swh-core
+    #   swh-counters
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
+gunicorn==23.0.0
+    # via
+    #   -r /src/apps/swh-counters/requirements.txt
+    #   aiohttp-utils
+hypothesis==6.125.2
+    # via swh-model
+idna==3.10
+    # via
+    #   requests
+    #   yarl
+iso8601==2.1.0
+    # via
+    #   swh-core
+    #   swh-model
+itsdangerous==2.2.0
+    # via flask
+jinja2==3.1.5
+    # via flask
+markupsafe==3.0.2
+    # via
+    #   jinja2
+    #   werkzeug
+msgpack==1.1.0
+    # via
+    #   swh-core
+    #   swh-journal
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+packaging==24.2
+    # via gunicorn
+propcache==0.2.1
+    # via
+    #   aiohttp
+    #   yarl
+python-dateutil==2.9.0.post0
+    # via swh-model
+python-json-logger==3.2.1
+    # via -r /src/apps/swh-counters/requirements.txt
+python-magic==0.4.27
+    # via swh-core
+python-mimeparse==2.0.0
+    # via aiohttp-utils
+pyyaml==6.0.2
+    # via swh-core
+redis==5.2.1
+    # via swh-counters
+requests==2.32.3
+    # via swh-core
+sentry-sdk==2.20.0
+    # via swh-core
+six==1.17.0
+    # via python-dateutil
+sortedcontainers==2.4.0
+    # via hypothesis
+swh-core==3.8.0
+    # via
+    #   swh-counters
+    #   swh-journal
+swh-counters==0.11.0
+    # via -r /src/apps/swh-counters/requirements.txt
+swh-journal==1.5.3
+    # via swh-counters
+swh-model==7.0.0
+    # via swh-journal
+tenacity==9.0.0
+    # via
+    #   swh-core
+    #   swh-journal
+typing-extensions==4.12.2
+    # via swh-model
+urllib3==2.3.0
+    # via
+    #   requests
+    #   sentry-sdk
+werkzeug==3.1.3
+    # via flask
+wrapt==1.17.2
+    # via deprecated
+yarl==1.18.3
+    # via aiohttp

apps/swh-counters/requirements.txt

+swh.counters
+python-json-logger
+gunicorn

apps/swh-deposit-checkers/Dockerfile

-# Deeply inspired from the Dockerfile of the swh-graph project
-FROM python:3.10-bullseye
+ARG REGISTRY=container-registry.softwareheritage.org/swh/infra/swh-apps/
+ARG base_image=${REGISTRY}base
+ARG base_image_version=latest
 
-RUN apt-get -y update && \
-    apt-get -y upgrade && \
-    apt-get install -y libcmph-dev && \
-    apt clean && \
-    addgroup --gid 1000 swh && \
-    useradd --gid 1000 --uid 1000 -m -d /opt/swh swh && \
-    mkdir /etc/swh
+FROM ${base_image}:${base_image_version}
 
-USER swh
-WORKDIR /opt/swh
+ARG user=swh
+ARG workdir=/opt/${user}
+ARG configdir=/etc/${user}
 
-COPY --chown=swh:swh requirements-frozen.txt /opt/swh
-COPY --chown=swh:swh entrypoint.sh /opt/swh
+COPY --chmod=0644 requirements-frozen.txt ${workdir}
+RUN --mount=type=cache,target=.cache,uid=1000,gid=1000 \
+    uv pip sync requirements-frozen.txt
 
-ENV PYTHONPATH=/opt/swh
-ENV PATH=/opt/swh/.local/bin:$PATH
+COPY --chmod=0755 entrypoint.sh ${workdir}
 
-RUN chmod u+x /opt/swh/entrypoint.sh && \
-    /usr/local/bin/python -m pip install --upgrade pip && \
-    pip install --no-cache-dir -r requirements-frozen.txt && \
-    pip install gunicorn
-
-ENV SWH_CONFIG_FILENAME=/etc/swh/config.yml
+USER ${user}
+ENV SWH_CONFIG_FILENAME=${configdir}/config.yml
 ENV SWH_WORKER_INSTANCE=checker
 ENV CONCURRENCY=1
 ENV MAX_TASKS_PER_CHILD=5
-ENV LOGLEVEL=INFO
 
-ENTRYPOINT "/opt/swh/entrypoint.sh"
+ENTRYPOINT ["/opt/swh/entrypoint.sh"]