Overhaul debian packaging bootstrap scripts

 - bin/debpkg-get-dependencies: update for recent versions of dh-python
 - bin/debpkg-update-automatic-dependencies: new script to update
 debian/control from the output of debpkg-get-dependencies
 - bin/debpkg-bootstrap-branches: generate a packaging branch out of a
 single tag, using files from the `debian-template` directory

bin/debpkg-bootstrap-branches

@@ -7,48 +7,70 @@ set -o pipefail
 package=$(basename $(pwd))
 module=${package//-/.}
 
-if [ $# -ne 0 ]; then
-  last_debian_rev=$1
-else
-  last_debian_rev=master
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 <initial_upstream_tag> <extra build depends>"
+  exit 1
 fi
 
-git branch -D pristine-tar debian/upstream debian/unstable-swh debian/stretch-swh || true
+initial_upstream_tag=$1
 
-for tag in `git tag -l debian/*`; do
-  git tag -d $tag
-done
+shift
 
-for tag in `git tag -l --sort=v:refname v\*`; do
-  ver=${tag/v/}
-  firstver=${firstver:-${ver}}
-  if [ -f ../packages/${package}_${ver}.orig.tar.gz ]; then
-     continue
-  fi
+read -p "This script will remove local changes and delete debian branches without confirmation. Use it on a clean checkout. Proceed? " -n 1 -r
+echo    # (optional) move to a new line
+if [[ ! $REPLY =~ ^[Yy]$ ]]
+then
+    exit 1
+fi
 
-  git checkout $tag
-  if [ -d swh ]; then
-    git clean -dfx swh
-  fi
-  if [ -f yarn.lock ]; then
-    (yarn install --frozen-lockfile && yarn build) || true
-  fi
-  find . -maxdepth 1 -type d -name '*.egg-info' -exec rm -r '{}' \+
-  python3 setup.py egg_info
-  pname=$(awk '/^Name:/{print $2}' *.egg-info/PKG-INFO)
-  pver=$(awk '/^Version:/{print $2}' *.egg-info/PKG-INFO)
-  python3 setup.py sdist -d ../packages
-  mv ../packages/${pname}-${pver}.tar.gz ../packages/${package}_${ver}.orig.tar.gz
+build_depends=()
+extra_build_depends=("$@")
+extra_binary_depends=()
+pkg_architecture=all
+
+# Clean up existing debian branches and tags
+
+git branch -D pristine-tar debian/upstream debian/unstable-swh || true
+
+for tag in $(git tag -l debian/*); do
+  git tag -d "$tag"
 done
 
-upstream_tag=$(git describe --abbrev=0)
-ver=${upstream_tag/v/}
+# create sdist for the initial tag
+
+firstver=${initial_upstream_tag/v/}
+git checkout $initial_upstream_tag
 
-author_name=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggername)%(else)%(authorname)%(end)" "${upstream_tag}")
-author_email=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggeremail)%(else)%(authoremail)%(end)" "${upstream_tag}")
+if [ -d swh ]; then
+  git clean -dfx swh
+fi
+if [ -f yarn.lock ]; then
+  (yarn install --frozen-lockfile && yarn build) || true
+fi
+
+destdir=$(mktemp -d)
+
+find . -maxdepth 1 -type d -name '*.egg-info' -exec rm -r '{}' \+
+python3 setup.py egg_info
+pname=${module}
+pver=$(grep-dctrl -n -s Version -FName $module 2>/dev/null < *.egg-info/PKG-INFO || true)
+python3 setup.py sdist -d $destdir
+python3 setup.py bdist_wheel -d $destdir
+mv ${destdir}/${pname}-${pver}.tar.gz ${destdir}/${package}_${firstver}.orig.tar.gz
+
+# Check if package is going to be architecture-dependent
+wheels=(${destdir}/${pname}-${pver}*.whl)
+if [[ "${wheels[0]}" = *$(uname -m)* ]]; then
+  pkg_architecture=any
+  extra_build_depends+=('python3-all-dev')
+  extra_binary_depends+=('${shlibs:Depends}')
+fi
+
+author_name=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggername)%(else)%(authorname)%(end)" "${initial_upstream_tag}")
+author_email=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggeremail)%(else)%(authoremail)%(end)" "${initial_upstream_tag}")
 # Strip <>
 author_email=${author_email:1:-1}
-author_date=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggerdate:iso)%(else)%(authordate:iso)%(end)" "${upstream_tag}")
+author_date=$(git tag -l --format="%(if)%(*objecttype)%(then)%(taggerdate:iso)%(else)%(authordate:iso)%(end)" "${initial_upstream_tag}")
 
 export DEBEMAIL="${author_email}"
 export DEBFULLNAME="${author_name}"
@@ -59,52 +81,57 @@ export GIT_COMMITTER_NAME="${author_name}"
 export GIT_COMMITTER_EMAIL="${author_email}"
 export GIT_COMMITTER_DATE="${author_date}"
 
-git checkout $upstream_tag
+git checkout $initial_upstream_tag
 git clean -dfx
 git checkout -b debian/upstream
 
 git ls-tree --name-only HEAD | xargs rm -r
-tar -x --strip-components 1 -f ../packages/${package}_${firstver}.orig.tar.gz
+tar -x --strip-components 1 -f ${destdir}/${package}_${firstver}.orig.tar.gz
 git add .
 git commit --no-verify -m "Import upstream version ${firstver}"
 git tag debian/upstream/${firstver}
 
 git checkout --orphan pristine-tar
-pristine-tar commit ../packages/${package}_${firstver}.orig.tar.gz
+pristine-tar commit ${destdir}/${package}_${firstver}.orig.tar.gz
 
 git checkout debian/upstream
 git checkout -b debian/unstable-swh
-git checkout ${last_debian_rev} -- debian
-cat > debian/gbp.conf << EOF
-[DEFAULT]
-upstream-branch=debian/upstream
-upstream-tag=debian/upstream/%(version)s
-upstream-vcs-tag=v%(version)s
-debian-branch=debian/unstable-swh
-pristine-tar=True
-EOF
-
-rm debian/changelog
+cp -r $(dirname $0)/../debian-template debian/
+
+summary=$(grep-dctrl -n -s Summary -FName ${module} 2>/dev/null < *.egg-info/PKG-INFO || true)
+summary_escaped="${summary//\//\\\/}"
+
+extra_build_depends_sub="$(printf "\\\\n %s," "${extra_build_depends[@]}")"
+extra_binary_depends_sub="$(printf "\\\\n %s," "${extra_binary_depends[@]}")"
+
+for file in debian/*; do
+  if [ -f "$file" ]; then
+    sed -i \
+      -e "s/@DOTTED_PKG_NAME@/$module/g" \
+      -e "s/@DASHED_PKG_NAME@/$package/g" \
+      -e "s/@YEAR@/$(date +%Y)/g" \
+      -e "s/@PKG_DESCRIPTION@/${summary_escaped}/g" \
+      -e "s/@EXTRA_BUILD_DEPENDS@/${extra_build_depends_sub:2}/" \
+      -e "s/@EXTRA_BINARY_DEPENDS@/${extra_binary_depends_sub:2}/" \
+      -e "s/@PKG_ARCHITECTURE@/${pkg_architecture}/g" \
+      "$file"
+  fi
+done
+
+#wrap-and-sort -an
+
 faketime "${author_date}" dch --create --package ${package} -v ${firstver}-1~swh1 'Initial release'
 
+
 git tag -l --format="%(contents:subject)%(if)%(contents:body)%(then)%0a%(contents:body)%(end)" "${upstream_tag}" | sed -E -e '/^$/d' -e 's/^ *(- *)?//' | while read line; do
   faketime "${author_date}" dch "${line}"
 done
 
 faketime "${author_date}" dch -D unstable-swh --force-distribution ''
 
+$(dirname $0)/debpkg-update-automatic-dependencies | sponge debian/control
+
 git add debian
 git commit --no-verify -m "Updated debian directory for version ${firstver}"
 
-git checkout -b debian/buster-swh
-faketime "${author_date}" dch --bpo -D buster-swh --force-distribution ''
-git add debian/changelog
-sed -i s/unstable/buster/ debian/gbp.conf
-git add debian/gbp.conf
-git commit --no-verify -m "Updated debian buster backport directory for version ${firstver}"
-
-for tag in `git tag -l --sort=v:refname v\* | tail -n +2`; do
-  version=${tag/v/}
-  echo $tag: $version
-  ../bin/debpkg-bump-version $version ../packages/${package}_${version}.orig.tar.gz
-done
+rm -r ${destdir}

bin/debpkg-get-dependencies

+#!/usr/bin/env python3
+
+import glob
+import logging
+import os
+import sys
+
+sys.path.append("/usr/share/dh-python")
+
+from dhpython.pydist import REQUIRES_RE, guess_dependency, sensible_pname
+from dhpython.version import Version
+
+log = logging.getLogger(__name__)
+
+
+def parse_requirements(filename):
+    requirements = []
+    if not os.path.exists(filename):
+        return requirements
+
+    with open(filename) as f:
+        for line in f.readlines():
+            if "#" in line:
+                line = line[: line.index("#")]
+            line = line.strip()
+            if not line:
+                continue
+            requirements.append(line)
+    return requirements
+
+
+def dependencies_from_requirements(directory):
+    requirements = []
+    for filename in glob.glob(os.path.join(directory, "requirements*.txt")):
+        requirements.extend(parse_requirements(filename))
+
+    for req in set(requirements):
+        dep = guess_dependency("cpython3", req, accept_upstream_versions=True)
+        if dep:
+            yield dep
+        else:
+            # fallback to sensible_pname
+            req_d = REQUIRES_RE.match(req)
+            if not req_d:
+                log.info(
+                    "please ask dh_python3 author to fix REQUIRES_RE "
+                    "or your upstream author to fix requires.txt"
+                )
+                raise Exception("requirement is not valid: %s" % req)
+            req_d = req_d.groupdict()
+            name = req_d["name"]
+            pname = sensible_pname("cpython3", name)
+            if req_d.get("operator") == ">=":
+                yield f"{pname} (>= {req_d['version']})"
+            else:
+                yield pname
+
+            if name == "swh.core" and "db" in req_d["enabled_extras"]:
+                yield "python3-swh.core.db.pytestplugin"
+
+
+def get_all_dependencies(directory):
+    dependencies = (
+        [
+            "debhelper-compat (= 13)",
+            "python3-all",
+            "python3-setuptools",
+            "python3-setuptools-scm",
+            "dh-python (>= 3)",
+        ]
+        + sys.argv[1:]
+        + list(dependencies_from_requirements(directory))
+    )
+
+    deduped_dependencies = set()
+    versions_dict = {}
+    for dep in dependencies:
+        if " " not in dep:
+            deduped_dependencies.add(dep)
+            continue
+        pkg, constraint = dep.split(" ", 1)
+        if not constraint.startswith("(>= "):
+            deduped_dependencies.add(dep)
+            continue
+
+        version = Version(constraint[4:-1])
+        if pkg in versions_dict:
+            versions_dict[pkg] = max(versions_dict[pkg], version)
+        else:
+            versions_dict[pkg] = version
+
+    for pkg, minver in versions_dict.items():
+        if minver.micro is not None:
+            minver_str = f"{minver.major}.{minver.minor}.{minver.micro}"
+        else:
+            minver_str = str(minver)
+        deduped_dependencies.add(f"{pkg} (>= {minver_str})")
+
+    return sorted(deduped_dependencies)
+
+
+if __name__ == "__main__":
+    for dep in get_all_dependencies("."):
+        print(dep)

bin/debpkg-update-automatic-dependencies

+#!/usr/bin/perl
+
+use strict;
+use v5.26;
+
+use File::Basename;
+
+open( my $control, "<", "debian/control" ) or die $!;
+
+my $dir = dirname $0;
+open( my $deps, "-|", "$dir/debpkg-get-dependencies" ) or die $!;
+
+my $in_build_depends = 0;
+
+while (<$control>) {
+    if ( $in_build_depends && /^[^ ]/ ) {
+        $in_build_depends = 0;
+    }
+
+    print unless $in_build_depends;
+
+    if (/^Build-Depends:/) {
+        $in_build_depends = 1;
+        while (<$deps>) {
+            chomp;
+            print " $_,\n";
+        }
+    }
+}
+
+0;

bin/debpkg-update-metadata

-#!/usr/bin/env python3
-
-import glob
-import os
-import sys
-
-sys.path.append("/usr/share/dh-python")
-
-from dhpython.pydist import guess_dependency  # noqa
-
-
-def parse_requirements(filename):
-    requirements = []
-    if not os.path.exists(filename):
-        return requirements
-
-    with open(filename) as f:
-        for line in f.readlines():
-            line = line.strip()
-            if not line or line.startswith("#"):
-                continue
-            requirements.append(line)
-    return requirements
-
-
-def dependencies_from_requirements(directory):
-    requirements = []
-    for filename in glob.glob(os.path.join(directory, "requirements*.txt")):
-        requirements.extend(parse_requirements(filename))
-
-    for req in set(requirements):
-        yield guess_dependency("cpython3", req, accept_upstream_versions=True)
-
-
-def get_all_dependencies(directory):
-    dependencies = ["debhelper (>= 11)", "python3-all", "dh-python (>= 3)"]
-    yield from dependencies
-    yield from dependencies_from_requirements(directory)
-
-
-if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        dir = "."
-    else:
-        dir = sys.argv[1]
-    for dep in get_all_dependencies(dir):
-        print(dep)

bin/ls-all-repos

@@ -30,6 +30,7 @@ find . -mindepth 1 -maxdepth 1 -type d -not -path '*/\.*' |
 	dir=${dir#./}  # strip "./" prefix
 	if [ "$dir" == ".git" \
 	     -o "$dir" == "bin" \
+         -o "$dir" == "debian-template" \
 	     -o "$dir" == "doc" \
 	     -o "$dir" == "docker" \
 	     -o "$dir" == "packages" \

debian-template/control

+Source: @DASHED_PKG_NAME@
+Maintainer: Software Heritage developers <swh-devel@inria.fr>
+Section: python
+Priority: optional
+Build-Depends:
+# The above dependencies are automatically generated. Add extra dependencies below:
+@EXTRA_BUILD_DEPENDS@
+Rules-Requires-Root: no
+Standards-Version: 4.6.0
+Homepage: https://forge.softwareheritage.org/source/@DASHED_PKG_NAME@
+
+Package: python3-@DOTTED_PKG_NAME@
+Architecture: @PKG_ARCHITECTURE@
+Depends: ${misc:Depends}, ${python3:Depends},
+@EXTRA_BINARY_DEPENDS@
+Description: @PKG_DESCRIPTION@

debian-template/copyright

+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+
+Files: *
+Copyright: 2015-@YEAR@ The Software Heritage developers
+License: GPL-3+
+
+License: GPL-3+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ .
+ On Debian systems, the complete text of the GNU General Public
+ License version 3 can be found in `/usr/share/common-licenses/GPL-3'.

debian-template/gbp.conf

+[DEFAULT]
+upstream-branch=debian/upstream
+upstream-tag=debian/upstream/%(version)s
+upstream-vcs-tag=v%(version)s
+debian-branch=debian/unstable-swh
+pristine-tar=True

debian-template/rules

+#!/usr/bin/make -f
+
+export PYBUILD_NAME=@DOTTED_PKG_NAME@
+export PYBUILD_TEST_ARGS=-vv
+
+%:
+	dh $@ --with python3 --buildsystem=pybuild
+
+override_dh_install:
+	dh_install
+	rm -v $(CURDIR)/debian/python3-*/usr/lib/python*/dist-packages/swh/__init__.py

debian-template/source/format

+3.0 (quilt)