jobs/dependency-packages: Add GitLab support for building debian deps

Debian dependencies packages will be built when pushing gbp buildpackage tags
on the associated gitlab repositories.

jobs/dependency-packages.yaml

 - job-group:
-    name: 'dependency-jobs-{name}'
+    name: "dependency-jobs-{name}"
+    gitlab_project_name: "swh/infra/ci-cd/debs/{repo_name}"
     jobs:
       - "debian/deps/{name}"
       - "debian/deps/{name}/incoming-tag"
       - "debian/deps/{name}/gbp-buildpackage"
       - "debian/deps/{name}/automatic-backport"
+      - "debian/deps/{name}/gitlab-incoming-tag"
+      - "debian/deps/{name}/gitlab-gbp-buildpackage"
+      - "debian/deps/{name}/gitlab-automatic-backport"
 
 - project:
     name: PPDW
@@ -12,7 +16,7 @@
     repo_name: python3-dulwich
     pkg: dulwich
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PTDTM
@@ -20,7 +24,7 @@
     repo_name: python3-django-test-migrations
     pkg: django-test-migrations
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PMK
@@ -28,7 +32,7 @@
     repo_name: python3-mirakuru
     pkg: mirakuru
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PPL
@@ -36,7 +40,7 @@
     repo_name: python3-plotille
     pkg: plotille
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PTPP
@@ -44,7 +48,7 @@
     repo_name: python3-pytest-postgresql
     pkg: pytest-postgresql
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PPF
@@ -52,7 +56,7 @@
     repo_name: python3-port-for
     pkg: port-for
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PSD
@@ -60,7 +64,7 @@
     repo_name: python3-sqlitedict
     pkg: sqlitedict
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PDJR
@@ -68,7 +72,7 @@
     repo_name: python3-django-js-reverse
     pkg: django-js-reverse
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PJOS
@@ -76,7 +80,7 @@
     repo_name: python3-jose
     pkg: python3-jose
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PRMD
@@ -84,7 +88,7 @@
     repo_name: python3-repomd
     pkg: repomd
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PKCL
@@ -92,7 +96,7 @@
     repo_name: python3-keycloak
     pkg: python3-keycloak
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PATS
@@ -100,7 +104,7 @@
     repo_name: python3-attr-strict
     pkg: python3-attr-strict
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PPTRE
@@ -108,7 +112,7 @@
     repo_name: python3-pytest-redis
     pkg: pytest_redis
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PCASS
@@ -116,7 +120,7 @@
     repo_name: python-cassandra-driver
     pkg: python-cassandra-driver
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PPSASL
@@ -124,7 +128,7 @@
     repo_name: python-pure-sasl
     pkg: python-pure-sasl
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PDWL
@@ -140,7 +144,7 @@
     repo_name: python3-prometheus-pve-exporter
     pkg: prometheus-pve-exporter
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"
 
 - project:
     name: PSTATIO
@@ -156,4 +160,4 @@
     repo_name: python3-tree-sitter
     pkg: tree-sitter
     jobs:
-      - 'dependency-jobs-{name}'
+      - "dependency-jobs-{name}"

jobs/templates/debian/automatic-backport.yaml

@@ -31,13 +31,18 @@
           default: false
     dsl: !include-jinja2: automatic-backport.groovy.j2
 
+- job-template:
+    name: "debian/packages/{name}/gitlab-automatic-backport"
+    description: |
+      Backport Debian package to the given release (GitLab)
+    gitlab_project: true
+    <<: *automatic_backport
+
 - job-template:
     name: "debian/deps/{name}/automatic-backport"
     <<: *automatic_backport
 
 - job-template:
-    name: "debian/packages/{name}/gitlab-automatic-backport"
-    description: |
-      Backport Debian package to the given release (GitLab)
+    name: "debian/deps/{name}/gitlab-automatic-backport"
     gitlab_project: true
     <<: *automatic_backport

jobs/templates/debian/gbp-buildpackage.yaml

@@ -65,12 +65,17 @@
           default-value: "{deb-backport-on-success}"
     dsl: !include-jinja2: gbp-buildpackage.groovy.j2
 
+- job-template:
+    name: "debian/packages/{name}/gitlab-gbp-buildpackage"
+    description: Build Debian package (GitLab)
+    gitlab_project: true
+    <<: *gbp_buildpackage
+
 - job-template:
     name: "debian/deps/{name}/gbp-buildpackage"
     <<: *gbp_buildpackage
 
 - job-template:
-    name: "debian/packages/{name}/gitlab-gbp-buildpackage"
-    description: Build Debian package (GitLab)
+    name: "debian/deps/{name}/gitlab-gbp-buildpackage"
     gitlab_project: true
     <<: *gbp_buildpackage

jobs/templates/dependency-incoming-tag.groovy.j2

@@ -4,8 +4,12 @@ pipeline {
     stage('Refresh tag list') {
       agent any
       steps {
+        {%- if gitlab_project %}
+        updateGitlabCommitStatus name: 'jenkins', state: 'running'
+        {%- endif %}
         checkout([
           $class: 'GitSCM',
+          {% if not gitlab_project %}
           userRemoteConfigs: [[
             url: 'https://forge.softwareheritage.org/source/{{repo_name}}.git',
           ]],
@@ -17,25 +21,67 @@ pipeline {
             repo: '{{repo_name}}',
             repoUrl: 'https://forge.softwareheritage.org/',
           ],
+          {%- else %}
+          userRemoteConfigs: [[
+            name:'origin', url: '{{gitlab_url}}/{{gitlab_project_name}}.git',
+            refspec: '+refs/tags/*:refs/remotes/origin/tags*'
+          ]],
+          branches: [[
+            name: "${env.gitlabSourceBranch}"
+          ]],
+          browser: [
+            $class: 'GitLab',
+            repoUrl: '{{gitlab_url}}/{{gitlab_project_name}}'
+          ],
+          extensions: [[$class: 'CloneOption', honorRefspec: true]],
+          {%- endif %}
         ])
       }
     }
     stage('Build Debian package') {
       when {
+        {%- if not gitlab_project %}
         expression { params.GIT_TAG ==~ /debian\/.*/ }
         expression { !(params.GIT_TAG ==~ /debian\/upstream\/.*/) }
         expression { jobExists('/debian/deps/{{name}}/gbp-buildpackage') }
+        {%- else %}
+        expression { "${env.gitlabSourceBranch}" ==~ /refs\/tags\/debian\/.*/ }
+        expression { !("${env.gitlabSourceBranch}" ==~ /refs\/tags\/debian\/upstream\/.*/) }
+        expression { jobExists('/debian/deps/{{name}}/gitlab-gbp-buildpackage') }
+        {%- endif %}
       }
       steps {
         build(
+          {%- if not gitlab_project %}
           job: '/debian/deps/{{name}}/gbp-buildpackage',
           parameters: [
             string(name: 'GIT_REVISION', value: params.GIT_TAG),
             booleanParam(name: 'DO_UPLOAD', value: true),
           ],
-          wait: false,
+          {%- else %}
+          job: '/debian/deps/{{name}}/gitlab-gbp-buildpackage',
+          parameters: [
+            // remove refs/tags/ prefix from source branch
+            string(name: 'GIT_REVISION', value: env.gitlabSourceBranch.substring(10)),
+            booleanParam(name: 'DO_UPLOAD', value: false),
+          ],
+          {%- endif %}
+          wait: true,
         )
       }
     }
   }
+  {%- if gitlab_project %}
+  post {
+    failure {
+      updateGitlabCommitStatus name: 'jenkins', state: 'failed'
+    }
+    success {
+      updateGitlabCommitStatus name: 'jenkins', state: 'success'
+    }
+    aborted {
+      updateGitlabCommitStatus name: 'jenkins', state: 'canceled'
+    }
+  }
+  {%- endif %}
 }

jobs/templates/incoming-tag.yaml

@@ -18,12 +18,29 @@
     dsl: !include-jinja2: incoming-tag.groovy.j2
 
 - job-template:
+    name: "{name}/gitlab-incoming-tag"
+    display-name: Incoming tag (GitLab)
+    gitlab_project: true
+    parameters:
+    properties:
+      - gitlab:
+          connection: "{gitlab_connection_name}"
+    triggers:
+      - gitlab:
+          trigger-push: true
+          trigger-merge-request: false
+          add-ci-message: true
+          # secret jenkins token is generated when executing tox
+          secret-token: !include-raw: jenkins-token
+    <<: *incoming_tag
+
+- job-template: &dep_incoming_tag
     name: "debian/deps/{name}/incoming-tag"
     dsl: !include-jinja2: dependency-incoming-tag.groovy.j2
     <<: *incoming_tag
 
 - job-template:
-    name: "{name}/gitlab-incoming-tag"
+    name: "debian/deps/{name}/gitlab-incoming-tag"
     display-name: Incoming tag (GitLab)
     gitlab_project: true
     parameters:
@@ -37,4 +54,4 @@
           add-ci-message: true
           # secret jenkins token is generated when executing tox
           secret-token: !include-raw: jenkins-token
-    <<: *incoming_tag
+    <<: *dep_incoming_tag

jobs/tools/setup-gitlab-webhooks.groovy.j2

@@ -44,6 +44,17 @@ pipeline {
                                  false, false, true)
             }
           }
+
+          projects = readYaml(file: 'jobs/dependency-packages.yaml')
+          for (project in projects) {
+            if (project.containsKey("project")) {
+              def jenkinsFolder = project.get('project').get('name')
+              jenkinsFolder = "debian/deps/${jenkinsFolder}/gitlab-incoming-tag"
+              def repoName= project.get('project').get('repo_name')
+              def gitlabProjectName = "swh/infra/ci-cd/debs/${repoName}"
+              setupGitlabWebhook(gitlabProjectName, jenkinsFolder, false, false, true)
+            }
+          }
         }
       }
     }