From 41ee36a37d74f11ebc76b66886da4c920ab9c1bb Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Wed, 12 Mar 2025 15:44:41 +0100
Subject: [PATCH 1/4] swh-environment: Autodetect when cargo build must be
called
---
jobs/templates/swh-environment.groovy.j2 | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/jobs/templates/swh-environment.groovy.j2 b/jobs/templates/swh-environment.groovy.j2
index 097b68f..eeabff5 100644
--- a/jobs/templates/swh-environment.groovy.j2
+++ b/jobs/templates/swh-environment.groovy.j2
@@ -49,11 +49,7 @@ pipeline {
pip install --upgrade pip
# install dependencies
./bin/install
- (cd swh-graph &&
- prepare-cargo-cache &&
- RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 cargo build --all-features --locked)
- (cd swh-provenance &&
- RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 cargo build --all-features --locked)
+ prepare-cargo-cache
(cd swh-web && yarn install)
# execute tests in each swh module
pip install pytest-xdist
@@ -61,6 +57,10 @@ pipeline {
swh_env_dir=$PWD
for module in $(./bin/ls-py-modules); do
cd $swh_env_dir/$module
+ if [ -f Cargo.toml ]; then
+ # build rust binaries
+ RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 cargo build --all-features --locked
+ fi
echo Executing $module checks and tests
tests_dir=$(echo $module | sed 's/-/\\//g')/tests
if [ -d "$tests_dir" ]; then
--
GitLab
From be0918286f73fd8bd50f0ea2fab17aa35e561dac Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Wed, 12 Mar 2025 15:45:29 +0100
Subject: [PATCH 2/4] docker: Fix automatic jobs creation when starting jenkins
service
Due to different user permissions on the mounted volume, git now
refuses to clone a repository from the volume if not explicitly
marked as safe.
---
docker/create_swh_jobs.groovy | 23 ++++++-----------------
1 file changed, 6 insertions(+), 17 deletions(-)
diff --git a/docker/create_swh_jobs.groovy b/docker/create_swh_jobs.groovy
index 87cb1cb..e5c0c53 100644
--- a/docker/create_swh_jobs.groovy
+++ b/docker/create_swh_jobs.groovy
@@ -12,22 +12,6 @@ def jobXml = '''<?xml version='1.1' encoding='UTF-8'?>
<project>
<description></description>
<keepDependencies>false</keepDependencies>
- <scm class="hudson.plugins.git.GitSCM" plugin="git@5.2.1">
- <configVersion>2</configVersion>
- <userRemoteConfigs>
- <hudson.plugins.git.UserRemoteConfig>
- <url>file:///opt/swh-jenkins-jobs</url>
- </hudson.plugins.git.UserRemoteConfig>
- </userRemoteConfigs>
- <branches>
- <hudson.plugins.git.BranchSpec>
- <name>*/master</name>
- </hudson.plugins.git.BranchSpec>
- </branches>
- <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
- <submoduleCfg class="empty-list"/>
- <extensions/>
- </scm>
<canRoam>true</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
@@ -36,7 +20,12 @@ def jobXml = '''<?xml version='1.1' encoding='UTF-8'?>
<concurrentBuild>false</concurrentBuild>
<builders>
<hudson.tasks.Shell>
- <command>tox -- update --delete-old --jobs-only</command>
+ <command>
+ git config --global --add safe.directory /opt/swh-jenkins-jobs/.git
+ git clone file:///opt/swh-jenkins-jobs
+ cd swh-jenkins-jobs
+ tox -- update --delete-old --jobs-only
+ </command>
<configuredLocalRules/>
</hudson.tasks.Shell>
</builders>
--
GitLab
From 9f6921d33d4542599fac569711937a4a71afd1f5 Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Wed, 12 Mar 2025 15:59:56 +0100
Subject: [PATCH 3/4] Add a script to configure and start local jenkins with
docker
It ensures same docker group id as the host is used in the jenkins
docker image or jenkins jobs will end up with errors when executed.
---
start-docker-jenkins.sh | 5 +++++
1 file changed, 5 insertions(+)
create mode 100755 start-docker-jenkins.sh
diff --git a/start-docker-jenkins.sh b/start-docker-jenkins.sh
new file mode 100755
index 0000000..8bde6e7
--- /dev/null
+++ b/start-docker-jenkins.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker_gid=$(stat -c '%g' /var/run/docker.sock)
+docker compose build --build-arg DOCKER_GROUP=$docker_gid
+docker compose up -d
--
GitLab
From 10e01d4597e0b06c3c944b594b10d44c7eb9dfdf Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Wed, 12 Mar 2025 16:03:13 +0100
Subject: [PATCH 4/4] README.md: Update instructions to run Jenkins with docker
---
README.md | 50 ++++++++++++----------------------------------
docker-compose.yml | 2 --
2 files changed, 13 insertions(+), 39 deletions(-)
diff --git a/README.md b/README.md
index 4ae8dbc..4164a39 100644
--- a/README.md
+++ b/README.md
@@ -25,19 +25,24 @@ in a docker container. The local ``swh-jenkins-jobs`` repository will be mounted
volume and cloned by Jenkins so do not forget to commit the changes you want to test.
- Launch jenkins
+
+Executing the following script located in the root directory of that repository will automatically configure the docker image build and start the compose session.
```
-docker-compose build
-docker-compose up
+./start-docker-jenkins.sh
```
-Connect to localhost:8080, then within the jenkins ui:
+Jenkins jobs for Software Heritage should be automatically registered when the jenkins service is starting.
+
+If the jobs did not get automatically registered, you can trigger their creation by following these instructions:
+
+- Connect to localhost:8080, then within the jenkins ui:
- Create a jenkins folder `jenkins-tools`
-- Create a new `free-style` job named `job-builder` inside the `jenkins-tools` targeting
- this git repository `file:///opt/swh-jenkins-jobs`
- - Configure the branch your are developing on (e.g. `*/master`, `*/awesome-feature`,
- ...)
- - Add a `build` step `Execute shell` with this content
+- Create a new `free-style` job named `job-builder` inside the `jenkins-tools`
+- Add a `build` step `Execute shell` with this content
```
+git config --global --add safe.directory /opt/swh-jenkins-jobs/.git
+git clone file:///opt/swh-jenkins-jobs
+cd swh-jenkins-jobs
tox -- update --delete-old --jobs-only
```
- Save your build configuration
@@ -46,32 +51,3 @@ tox -- update --delete-old --jobs-only
This will install the jobs in your local jenkins. Jobs that can be run directly on the
built-in node can be executed. Other jobs that may need to run docker needs the docker
agent to be configured.
-
-# Configure a docker agent
-
-For making a docker agent runnable, it needs the 50000 port to be
-available.
-
-Then, within your local jenkins, click on the local jenkins interface:
-
-> Manage jenkins
-> Manage nodes and clouds
-> New nodes
-
-Then fill in the form, keeping the default values and adapting the rest:
-
-- name: docker agent
-- remote root dir: /var/tmp/jenkins
-- labels: docker
-- launch method: launch agent by connecting it to the controller
-- custom workdir path: /var/tmp/jenkins
-
-Save, then click on 'docker agent' and follow the proposed instructions:
-
-```
-$ curl -sO http://localhost:8080/jnlpJars/agent.jar
-$ mkdir -p /var/tmp/jenkins
-$ java -jar agent.jar \
- -jnlpUrl http://localhost:8080/manage/computer/docker%20agent/jenkins-agent.jnlp \
- -workDir "/var/tmp/jenkins"
-```
diff --git a/docker-compose.yml b/docker-compose.yml
index c9161dd..f3ee604 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -23,6 +23,4 @@ services:
-Dhudson.security.csrf.GlobalCrumbIssuerConfiguration.DISABLE_CSRF_PROTECTION=true
ports:
- 8080:8080
- # For docker agent
- - 50000:50000
entrypoint: /docker/entrypoint.sh
--
GitLab