From 2045b62caca89fb679f93e3c293871c37d1f3641 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Fri, 31 Jan 2025 16:42:40 +0100
Subject: [PATCH 1/8] graphql: Install dependencies faster
This gains around 6 to 7 seconds for this application only. It's probably a
lot more for some packages with a high number of dependencies.
In any case, if we generalize this, we can gain some minutes when we have to
build lots of images (e.g. from any tag set on swh-storage, swh-objstorage,
swh-core, swh-scheduler, swh-model, etc...).
[1] master
```
/usr/bin/time docker build --no-cache -t swh-graphql apps/swh-graphql
Successfully tagged swh-graphql:latest
0.02user 0.06system 0:33.23elapsed 0%CPU (0avgtext+0avgdata 41112maxresident)k
0inputs+0outputs (0major+2522minor)pagefaults 0swaps
Successfully tagged swh-graphql:latest
0.05user 0.02system 0:32.30elapsed 0%CPU (0avgtext+0avgdata 40768maxresident)k
0inputs+0outputs (0major+2583minor)pagefaults 0swaps
Successfully tagged swh-graphql:latest
0.03user 0.04system 0:32.61elapsed 0%CPU (0avgtext+0avgdata 38588maxresident)k
0inputs+0outputs (0major+2562minor)pagefaults 0swaps
```
[2] dev branch
```
/usr/bin/time docker build --no-cache -t swh-graphql-with-uv apps/swh-graphql
Successfully tagged swh-graphql-with-uv:latest
0.01user 0.03system 0:26.56elapsed 0%CPU (0avgtext+0avgdata 38856maxresident)k
0inputs+0outputs (0major+2570minor)pagefaults 0swaps
Successfully tagged swh-graphql-with-uv:latest
0.03user 0.01system 0:26.87elapsed 0%CPU (0avgtext+0avgdata 39852maxresident)k
0inputs+0outputs (0major+2471minor)pagefaults 0swaps
Successfully tagged swh-graphql-with-uv:latest
0.02user 0.02system 0:25.86elapsed 0%CPU (0avgtext+0avgdata 39524maxresident)k
0inputs+0outputs (0major+2268minor)pagefaults 0swaps
```
---
apps/swh-graphql/Dockerfile | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/apps/swh-graphql/Dockerfile b/apps/swh-graphql/Dockerfile
index c24d8cad8..ac62dd6c1 100644
--- a/apps/swh-graphql/Dockerfile
+++ b/apps/swh-graphql/Dockerfile
@@ -6,22 +6,22 @@ RUN apt-get -y update && \
apt-get install -y libcmph-dev librdkafka-dev && \
apt clean && \
addgroup --gid 1000 swh && \
- useradd --gid 1000 --uid 1000 -m -d /opt/graphql swh && \
+ useradd --gid 1000 --uid 1000 -m -d /opt/swh swh && \
mkdir /etc/swh
USER swh
-WORKDIR /opt/graphql
+RUN /usr/local/bin/python -m venv /opt/swh/venv
+WORKDIR /opt/swh
-COPY --chown=swh:swh requirements-frozen.txt /opt/graphql
+ENV PYTHONPATH=/opt/swh/venv
+ENV PATH=/opt/swh/venv/bin:$PATH
-ENV PYTHONPATH=/opt/graphql
-ENV PATH=/opt/graphql/.local/bin:$PATH
+RUN python -m pip install uv
+COPY --chown=swh:swh requirements-frozen.txt /opt/swh/
+RUN uv pip sync --no-cache requirements-frozen.txt
-RUN /usr/local/bin/python -m pip install --upgrade pip && \
- pip install --no-cache-dir -r requirements-frozen.txt
-
-COPY --chown=swh:swh entrypoint.sh /opt/graphql
-RUN chmod u+x /opt/graphql/entrypoint.sh
+COPY --chown=swh:swh entrypoint.sh /opt/swh/
+RUN chmod u+x /opt/swh/entrypoint.sh
ENV SWH_CONFIG_FILENAME=/etc/swh/config.yml
ENV PORT 5013
@@ -30,4 +30,4 @@ ENV THREADS 4
ENV WORKERS 2
ENV TIMEOUT 3600
-ENTRYPOINT ["/opt/graphql/entrypoint.sh"]
+ENTRYPOINT ["/opt/swh/entrypoint.sh"]
--
GitLab
From 732ba8f8a98ebff2250268f69c498404fbefbfad Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Fri, 31 Jan 2025 17:14:44 +0100
Subject: [PATCH 2/8] graph: Install dependencies faster
This gains only 4 seconds for the graph. Unsurprisingly the gain is less
significant for the graph as most of the time is spent in installing debian
package or crates dependencies (with lots of compilation).
[1] master
```
$ /usr/bin/time docker build --no-cache -t swh-graph apps/swh-graph
...
Successfully tagged swh-graph:latest
0.23user 0.21system 6:16.24elapsed 0%CPU (0avgtext+0avgdata 41756maxresident)k
0inputs+0outputs (0major+2669minor)pagefaults 0swaps
```
[2]
```
$ /usr/bin/time docker build --no-cache -t swh-graph-with-uv apps/swh-graph
...
Successfully tagged swh-graph-with-uv:latest
0.20user 0.21system 6:12.39elapsed 0%CPU (0avgtext+0avgdata 38716maxresident)k
0inputs+0outputs (0major+2782minor)pagefaults 0swaps
```
---
apps/swh-graph/Dockerfile | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/apps/swh-graph/Dockerfile b/apps/swh-graph/Dockerfile
index a15d9fb3d..24a23831c 100644
--- a/apps/swh-graph/Dockerfile
+++ b/apps/swh-graph/Dockerfile
@@ -36,15 +36,15 @@ RUN chown $UID:$GID /srv/graph /srv/dataset
VOLUME /srv/graph /srv/dataset
USER swh
+RUN /usr/local/bin/python -m venv /opt/swh/venv
WORKDIR /opt/swh
-COPY --chown=swh:swh requirements-frozen.txt /opt/swh
-
-ENV PYTHONPATH=/opt/swh
-ENV PATH=/opt/swh/.local/bin:$PATH
+ENV PYTHONPATH=/opt/swh/venv
+ENV PATH=/opt/swh/venv/bin:$PATH
-RUN /usr/bin/python3 -m pip install --upgrade pip --break-system-packages && \
- pip install --no-cache-dir -r requirements-frozen.txt --break-system-packages
+RUN python -m pip install uv
+COPY --chown=swh:swh requirements-frozen.txt /opt/swh
+RUN uv pip sync --no-cache requirements-frozen.txt
COPY --from=rust_build /usr/local/cargo/bin/swh-graph* /usr/local/bin/
ENV SWH_CONFIG_FILENAME=/etc/swh/config.yml
--
GitLab
From 04e60fd92e6779dc7a6d7a4b31a169b8e52249b1 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Fri, 31 Jan 2025 17:38:48 +0100
Subject: [PATCH 3/8] deposit: Install dependencies faster
This gains around 20 seconds. Unsurprisingly the gain is a bit more
significant since there is a lot of dependencies for that kind of frontend
applications.
[1] master
```
$ /usr/bin/time docker build --no-cache -t swh-deposit apps/swh-deposit
...
Successfully tagged swh-deposit:latest
0.11user 0.07system 0:49.05elapsed 0%CPU (0avgtext+0avgdata 38876maxresident)k
8inputs+0outputs (0major+2568minor)pagefaults 0swaps
```
*** uv
```
Successfully tagged swh-deposit-with-uv:latest
0.02user 0.02system 0:28.45elapsed 0%CPU (0avgtext+0avgdata 40412maxresident)k
0inputs+0outputs (0major+2471minor)pagefaults 0swaps
```
---
apps/swh-deposit/Dockerfile | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/apps/swh-deposit/Dockerfile b/apps/swh-deposit/Dockerfile
index d674ff86c..7f1ae6847 100644
--- a/apps/swh-deposit/Dockerfile
+++ b/apps/swh-deposit/Dockerfile
@@ -10,15 +10,15 @@ RUN apt-get -y update && \
mkdir /etc/swh
USER swh
+RUN /usr/local/bin/python -m venv /opt/swh/venv
WORKDIR /opt/swh
-COPY --chown=swh:swh requirements-frozen.txt /opt/swh
-
-ENV PYTHONPATH=/opt/swh
-ENV PATH=/opt/swh/.local/bin:$PATH
+ENV PYTHONPATH=/opt/swh/venv
+ENV PATH=/opt/swh/venv/bin:$PATH
-RUN /usr/local/bin/python -m pip install --upgrade pip && \
- pip install --no-cache-dir -r requirements-frozen.txt
+RUN python -m pip install uv
+COPY --chown=swh:swh requirements-frozen.txt /opt/swh
+RUN uv pip sync --no-cache requirements-frozen.txt
COPY --chown=swh:swh entrypoint.sh /opt/swh
RUN chmod u+x /opt/swh/entrypoint.sh
--
GitLab
From 72af3192bdb21c5168ed3614969aec41a798c98e Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Fri, 31 Jan 2025 18:02:03 +0100
Subject: [PATCH 4/8] loader-savecodenow: Install dependencies faster
This gains around 45 seconds. Unsurprisingly the gain is a bit more
significant since there is a lot of python dependencies in this application.
[1] master
```
$ /usr/bin/time docker build --no-cache -t swh-loader-savecodenow apps/swh-loader-savecodenow
Successfully tagged swh-loader-savecodenow:latest
0.10user 0.05system 1:37.69elapsed 0%CPU (0avgtext+0avgdata 40896maxresident)k
8inputs+0outputs (0major+2645minor)pagefaults 0swaps
```
[2] with improvments
```
$ /usr/bin/time docker build --no-cache -t swh-loader-savecodenow-with-uv apps/swh-loader-savecodenow
...
Successfully tagged swh-loader-savecodenow-with-uv:latest
0.06user 0.05system 0:55.73elapsed 0%CPU (0avgtext+0avgdata 40932maxresident)k
0inputs+0outputs (0major+2587minor)pagefaults 0swaps
```
---
apps/swh-loader-savecodenow/Dockerfile | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/apps/swh-loader-savecodenow/Dockerfile b/apps/swh-loader-savecodenow/Dockerfile
index 2c5091542..71bca8ebe 100644
--- a/apps/swh-loader-savecodenow/Dockerfile
+++ b/apps/swh-loader-savecodenow/Dockerfile
@@ -17,15 +17,15 @@ RUN apt-get -y update && \
mkdir /etc/swh
USER swh
+RUN /usr/local/bin/python -m venv /opt/swh/venv
WORKDIR /opt/swh
-COPY --chown=swh:swh requirements-frozen.txt /opt/swh
-
-ENV PYTHONPATH=/opt/swh
-ENV PATH=/opt/swh/.local/bin:$PATH
+ENV PYTHONPATH=/opt/swh/venv
+ENV PATH=/opt/swh/venv/bin:$PATH
-RUN /usr/local/bin/python -m pip install --upgrade pip && \
- pip install --no-cache-dir -r requirements-frozen.txt
+RUN python -m pip install uv
+COPY --chown=swh:swh requirements-frozen.txt /opt/swh
+RUN uv pip sync --no-cache requirements-frozen.txt
COPY --chown=swh:swh entrypoint.sh /opt/swh
RUN chmod u+x /opt/swh/entrypoint.sh
@@ -35,4 +35,4 @@ ENV SWH_WORKER_INSTANCE=loader
ENV CONCURRENCY=1
ENV MAX_TASKS_PER_CHILD=5
-ENTRYPOINT ["/opt/swh/entrypoint.sh"]
+ENTRYPOINT [ "/opt/swh/entrypoint.sh" ]
--
GitLab
From be1f53a4b413c5088e399b36f4fbc003ef17d106 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Fri, 31 Jan 2025 18:05:26 +0100
Subject: [PATCH 5/8] loader-bzr: Install dependencies faster
This gains around 10 seconds.
[1]
```
Successfully tagged swh-loader-bzr:latest
0.05user 0.04system 0:44.04elapsed 0%CPU (0avgtext+0avgdata 39236maxresident)k
8inputs+0outputs (0major+2291minor)pagefaults 0swaps
```
[2]
```
Successfully tagged swh-loader-bzr-with-uv:latest
0.03user 0.03system 0:34.54elapsed 0%CPU (0avgtext+0avgdata 40980maxresident)k
0inputs+0outputs (0major+2240minor)pagefaults 0swaps
```
---
apps/swh-loader-bzr/Dockerfile | 12 ++++++------
apps/swh-loader-bzr/entrypoint.sh | 1 +
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/apps/swh-loader-bzr/Dockerfile b/apps/swh-loader-bzr/Dockerfile
index bc37a1722..aa52b3e1f 100644
--- a/apps/swh-loader-bzr/Dockerfile
+++ b/apps/swh-loader-bzr/Dockerfile
@@ -11,15 +11,15 @@ RUN apt-get -y update && \
mkdir /etc/swh
USER swh
+RUN /usr/local/bin/python -m venv /opt/swh/venv
WORKDIR /opt/swh
-COPY --chown=swh:swh requirements-frozen.txt /opt/swh
-
-ENV PYTHONPATH=/opt/swh
-ENV PATH=/opt/swh/.local/bin:$PATH
+ENV PYTHONPATH=/opt/swh/venv
+ENV PATH=/opt/swh/venv/bin:$PATH
-RUN /usr/local/bin/python -m pip install --upgrade pip && \
- pip install --no-cache-dir -r requirements-frozen.txt
+RUN python -m pip install uv
+COPY --chown=swh:swh requirements-frozen.txt /opt/swh
+RUN uv pip sync --no-cache requirements-frozen.txt
COPY --chown=swh:swh entrypoint.sh /opt/swh
RUN chmod u+x /opt/swh/entrypoint.sh
diff --git a/apps/swh-loader-bzr/entrypoint.sh b/apps/swh-loader-bzr/entrypoint.sh
index 564221961..f36a37357 100644
--- a/apps/swh-loader-bzr/entrypoint.sh
+++ b/apps/swh-loader-bzr/entrypoint.sh
@@ -16,6 +16,7 @@ case "$1" in
# workdir as the ephemeral volumes are not recreated
echo "Cleaning /tmp..."
rm -rf /tmp/*
+
echo Starting the swh Celery worker for ${SWH_WORKER_INSTANCE}
exec python -m celery \
--app=swh.scheduler.celery_backend.config.app \
--
GitLab
From 1db963ad6e97bee0a55d27adab0ba304dc712450 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Sat, 1 Feb 2025 11:58:04 +0100
Subject: [PATCH 6/8] loader-svn: Install python dependencies faster
Current pip install [1] vs uv pip install [2].
[1]
```
$ /usr/bin/time docker build --no-cache -t swh-loader-svn apps/swh-loader-svn
...
Successfully tagged swh-loader-svn:latest
0.05user 0.05system 0:58.04elapsed 0%CPU (0avgtext+0avgdata 38948maxresident)k
0inputs+0outputs (0major+2652minor)pagefaults 0swaps
```
[2]
```
$ /usr/bin/time docker build --no-cache -t swh-loader-svn-with-uv apps/swh-loader-svn
...
Successfully tagged swh-loader-svn-with-uv:latest
0.04user 0.03system 0:46.99elapsed 0%CPU (0avgtext+0avgdata 40972maxresident)k
0inputs+0outputs (0major+2282minor)pagefaults 0swaps
---
apps/swh-loader-svn/Dockerfile | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/apps/swh-loader-svn/Dockerfile b/apps/swh-loader-svn/Dockerfile
index 718f42af5..62bdf5858 100644
--- a/apps/swh-loader-svn/Dockerfile
+++ b/apps/swh-loader-svn/Dockerfile
@@ -16,15 +16,15 @@ RUN apt-get -y update && \
mkdir /etc/swh
USER swh
+RUN /usr/local/bin/python -m venv /opt/swh/venv
WORKDIR /opt/swh
-COPY --chown=swh:swh requirements-frozen.txt /opt/swh
-
-ENV PYTHONPATH=/opt/swh
-ENV PATH=/opt/swh/.local/bin:$PATH
+ENV PYTHONPATH=/opt/swh/venv
+ENV PATH=/opt/swh/venv/bin:$PATH
-RUN /usr/local/bin/python -m pip install --upgrade pip && \
- pip install --no-cache-dir -r requirements-frozen.txt
+RUN python -m pip install uv
+COPY --chown=swh:swh requirements-frozen.txt /opt/swh
+RUN uv pip sync --no-cache requirements-frozen.txt
COPY --chown=swh:swh entrypoint.sh /opt/swh
RUN chmod u+x /opt/swh/entrypoint.sh
--
GitLab
From 8ad3d96dd31806ae80f0e5a2817c91a6fb4ed9b0 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Sat, 1 Feb 2025 12:16:54 +0100
Subject: [PATCH 7/8] loader-package: Install python dependencies faster
Current pip install [1] vs uv pip install [2].
[1]
```
$ /usr/bin/time docker build --no-cache -t swh-loader-package apps/swh-loader-package
...
Successfully tagged swh-loader-package:latest
0.03user 0.04system 0:57.30elapsed 0%CPU (0avgtext+0avgdata 40880maxresident)k
8inputs+0outputs (0major+2588minor)pagefaults 0swaps
```
[2]
```
$ /usr/bin/time docker build --no-cache -t swh-loader-package-with-uv apps/swh-loader-package
...
Successfully tagged swh-loader-package-with-uv:latest
0.03user 0.02system 0:51.90elapsed 0%CPU (0avgtext+0avgdata 40516maxresident)k
0inputs+0outputs (0major+2554minor)pagefaults 0swaps
```
---
apps/swh-loader-package/Dockerfile | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/apps/swh-loader-package/Dockerfile b/apps/swh-loader-package/Dockerfile
index 55bea9984..72cc48834 100644
--- a/apps/swh-loader-package/Dockerfile
+++ b/apps/swh-loader-package/Dockerfile
@@ -12,15 +12,15 @@ RUN apt-get -y update && \
mkdir /etc/swh
USER swh
+RUN /usr/local/bin/python -m venv /opt/swh/venv
WORKDIR /opt/swh
-COPY --chown=swh:swh requirements-frozen.txt /opt/swh
-
-ENV PYTHONPATH=/opt/swh
-ENV PATH=/opt/swh/.local/bin:$PATH
+ENV PYTHONPATH=/opt/swh/venv
+ENV PATH=/opt/swh/venv/bin:$PATH
-RUN /usr/local/bin/python -m pip install --upgrade pip && \
- pip install --no-cache-dir -r requirements-frozen.txt
+RUN python -m pip install uv
+COPY --chown=swh:swh requirements-frozen.txt /opt/swh
+RUN uv pip sync --no-cache requirements-frozen.txt
COPY --chown=swh:swh entrypoint.sh /opt/swh
RUN chmod u+x /opt/swh/entrypoint.sh
--
GitLab
From 7889a4ba78c50cefc9850745dae3ca20034d959f Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Sat, 1 Feb 2025 12:22:19 +0100
Subject: [PATCH 8/8] toolbox: Install python dependencies faster
Current pip install [1] vs uv pip install [2].
[1]
```
$ /usr/bin/time docker build --no-cache -t swh-loader-package apps/swh-loader-package
...
Successfully tagged toolbox:latest
0.08user 0.12system 2:23.18elapsed 0%CPU (0avgtext+0avgdata 40712maxresident)k
0inputs+0outputs (0major+2589minor)pagefaults 0swaps
```
[2]
```
$ /usr/bin/time docker build --no-cache -t swh-loader-package-with-uv apps/swh-loader-package
Successfully tagged toolbox-with-uv:latest
0.09user 0.05system 1:45.10elapsed 0%CPU (0avgtext+0avgdata 38656maxresident)k
0inputs+0outputs (0major+2675minor)pagefaults 0swaps
...
```
---
apps/swh-toolbox/Dockerfile | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/apps/swh-toolbox/Dockerfile b/apps/swh-toolbox/Dockerfile
index 03c13ff09..33fe3aecb 100644
--- a/apps/swh-toolbox/Dockerfile
+++ b/apps/swh-toolbox/Dockerfile
@@ -19,15 +19,15 @@ RUN wget -q https://github.com/fullstorydev/grpcurl/releases/download/v1.9.1/grp
dpkg -i grpcurl_1.9.1_linux_amd64.deb
USER swh
+RUN /usr/local/bin/python -m venv /opt/swh/venv
WORKDIR /opt/swh
-COPY --chown=swh:swh requirements-frozen.txt /opt/swh
-
-ENV PYTHONPATH=/opt/swh
-ENV PATH=/opt/swh/.local/bin:$PATH
+ENV PYTHONPATH=/opt/swh/venv
+ENV PATH=/opt/swh/venv/bin:$PATH
-RUN /usr/local/bin/python -m pip install --upgrade pip && \
- pip install --no-cache-dir -r requirements-frozen.txt
+RUN python -m pip install uv
+COPY --chown=swh:swh requirements-frozen.txt /opt/swh
+RUN uv pip sync --no-cache requirements-frozen.txt
COPY --chown=swh:swh entrypoint.sh /opt/swh
RUN chmod u+x /opt/swh/entrypoint.sh
--
GitLab