scripts: Manage applications through one cli with subcommands

At the moment 2:
- generate-frozen-requirements
- list (with/without parameters)

This will ease reusing the same docker image for multiple calls to subcommands.

Ref. swh/infra/sysadm-environment#4724

scripts/Dockerfile

@@ -12,18 +12,18 @@ RUN apt-get -y update && \
       librdkafka-dev \
       # svn loader (subvertpy)
       libsvn-dev libapr1-dev libaprutil1-dev \
+      libpython3-dev \
       libpq-dev \
+      python3-click \
       python3-venv \
     && \
     apt clean
 
 RUN mkdir -p /opt/scripts/
-COPY common.py /opt/scripts/
-COPY generate-frozen-requirements /opt/scripts/
-RUN chmod +x /opt/scripts/generate-frozen-requirements
+COPY app_manager.py /opt/scripts/
+RUN chmod +x /opt/scripts/app_manager.py
 
 # This expects the /src/ to be mounted on swh-apps repository folder at runtime
 ENV SWH_APPS_DIR=/src/apps
 
-WORKDIR /opt/scripts
-ENTRYPOINT [ "./generate-frozen-requirements" ]
+ENTRYPOINT [ "/opt/scripts/app_manager.py" ]

scripts/generate-frozen-requirements → scripts/app_manager.py

-#!/usr/bin/env python3
+#!/usr/bin/python3
 
-# Copyright (C) 2022  The Software Heritage developers
+# Copyright (C) 2022-2023  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License v3 or later
 # See top-level LICENSE file for more information
 
-"""Generate the requirements-frozen.txt file for given app(s)"""
+"""Cli to list applications (with/without filters) or generate the
+requirements-frozen.txt file for app(s) provided as parameters.
 
+"""
+
+import click
 import os
 import pathlib
 import subprocess
 import sys
 import tempfile
 from venv import EnvBuilder
+from typing import Iterator
+
+
+APPS_DIR = pathlib.Path(__file__).absolute().parent.parent / "apps"
+
+# The file to read to determine impacted image to rebuild.
+# Note: We cannot use the requirements.txt as it does not explicit the python module it
+# depends upon
+
+requirements_frozen = "requirements-frozen.txt"
+
+
+requirements = "requirements.txt"
+
+
+@click.group("app")
+@click.pass_context
+def app(ctx):
+    apps_dir = os.environ.get("SWH_APPS_DIR")
+    if not apps_dir:
+        absolute_apps_dirpath = APPS_DIR
+    else:
+        absolute_apps_dirpath = pathlib.Path(apps_dir)
+
+    ctx.ensure_object(dict)
+    ctx.obj["apps_dir"] = absolute_apps_dirpath
 
 
 class AppEnvBuilder(EnvBuilder):
@@ -42,28 +72,23 @@ class AppEnvBuilder(EnvBuilder):
         return subprocess.run(cmd, capture_output=capture_output, check=True)
 
 
-def usage():
-    print(__doc__, file=sys.stderr)
-    print("", file=sys.stderr)
-    print(f"Usage: {sys.argv[0]} app1 ... appN", file=sys.stderr)
-
-
 def generate_requirements_frozen(app: str, absolute_apps_dirpath: pathlib.Path) -> None:
-    """Generate the ``requirements-frozen.txt`` file out of the
-    ``requirements.txt`` file present in the ``app`` directory
+    """Generate the ``requirements-frozen.txt`` file out of the ``requirements.txt``
+    file present in the ``app`` directory
 
     Args:
         app: Name of the application to generate the frozen dependencies set
-        apps_dir: Absolute folder holding all application declarations
+        absolute_apps_dirpath: Absolute path to folder holding application declarations
+
     """
 
     app_dir = absolute_apps_dirpath / app
-    src_req_file = app_dir / "requirements.txt"
-    dst_req_file = app_dir / "requirements-frozen.txt"
+    src_req_file = app_dir / requirements
+    dst_req_file = app_dir / requirements_frozen
 
     if not src_req_file.is_file():
         raise FileNotFoundError(
-            f"requirements.txt file for app {app} not found (checked {src_req_file})"
+            f"{requirements} file for app {app} not found (checked {src_req_file})"
         )
 
     with tempfile.TemporaryDirectory(prefix=app) as envdir:
@@ -79,18 +104,80 @@ def generate_requirements_frozen(app: str, absolute_apps_dirpath: pathlib.Path)
             p.rename(dst_req_file)
 
 
-if __name__ == "__main__":
-    if not sys.argv[1:]:
-        usage()
-        sys.exit(2)
+@app.command("generate-frozen-requirements")
+@click.argument("applications", required=True)
+@click.pass_context
+def generate_frozen_requirements(ctx, applications):
+    """Generate frozen requirements for applications passed as parameters"""
 
-    apps_dir = os.environ.get("SWH_APPS_DIR")
-    if not apps_dir:
-        from common import APPS_DIR
+    absolute_apps_dirpath = ctx.obj["apps_dir"]
 
-        absolute_apps_dirpath = APPS_DIR
-    else:
-        absolute_apps_dirpath = pathlib.Path(apps_dir)
+    apps = [applications] if isinstance(applications, str) else applications
 
-    for app in sys.argv[1:]:
+    for app in apps:
         generate_requirements_frozen(app, absolute_apps_dirpath)
+
+
+def from_application_to_module(app_name: str) -> str:
+    """Compute python module name from the application name."""
+    return app_name.replace("-", ".")
+
+
+def from_tag_to_version(version: str) -> str:
+    """Compute python module version from a version tag (prefixed with 'v')."""
+    return version.lstrip("v")
+
+
+def list_impacted_apps(apps_dir: pathlib.Path, application: str, version: str) -> Iterator[str]:
+    """List all apps whose constraint does not match `application==version.`.
+
+    Expectedly, only applications who have `application` in their
+    requirements-frozen.txt have a chance to be listed.
+
+    """
+    app_module = from_application_to_module(application)
+    version = from_tag_to_version(version)
+    for req_file in sorted(apps_dir.glob(f"*/{requirements_frozen}")):
+        with open(req_file, 'r') as f:
+            for line in f:
+                line = line.rstrip()
+                if '==' in line:  # we ignore the `@` case for now
+                    module, version_ = line.rstrip().split('==')
+                    if module == app_module:
+                        if version == version_:
+                            # Application is already built for the right version, stop
+                            break
+                        # Application has the dependency for another version, rebuild
+                        yield req_file.parent.stem
+
+
+def list_apps(apps_dir: pathlib.Path) -> Iterator[str]:
+    """List all known apps with a requirements.txt file"""
+    for req_file in sorted(apps_dir.glob(f"*/{requirements}")):
+        yield req_file.parent.stem
+
+
+@app.command("list")
+@click.option("-a", "--application", "application", default=None, help="Application name")
+@click.option("-v", "--version", "version", default=None, help="Version of the application")
+@click.pass_context
+def list(ctx, application: str, version: str) -> None:
+    """With no parameters, list all known applications with a requirements.txt. With
+    application and version, list known applications (with requirements.txt) whose
+    constraint does not match exactly application & version. Applications without
+    constraint  of `application` are ignored.
+
+    """
+    absolute_apps_dirpath = ctx.obj["apps_dir"]
+
+    if application is not None and version is not None:
+        apps = list_impacted_apps(absolute_apps_dirpath, application, version)
+    else:
+        apps = list_apps(absolute_apps_dirpath)
+
+    for app in apps:
+        print(app)
+
+
+if __name__ == "__main__":
+    app()

scripts/common.py

-#!/usr/bin/env python3
-
-# Copyright (C) 2022  The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License v3 or later
-# See top-level LICENSE file for more information
-
-import pathlib
-
-APPS_DIR = pathlib.Path(__file__).absolute().parent.parent / "apps"

scripts/list-apps

-#!/usr/bin/env python3
-
-# Copyright (C) 2022  The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License v3 or later
-# See top-level LICENSE file for more information
-
-from common import APPS_DIR
-
-
-def list_apps():
-    """List all known apps with a requirements.txt file"""
-    for req_file in sorted(APPS_DIR.glob("*/requirements.txt")):
-        yield req_file.parent.stem
-
-
-if __name__ == "__main__":
-    for app in list_apps():
-        print(app)

scripts/list-apps-to-rebuild

-#!/usr/bin/env python3
-
-# Copyright (C) 2023  The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License v3 or later
-# See top-level LICENSE file for more information
-
-"""A script to generate the list of applications impacted by an updated dependency
-application.
-
-"""
-
-from typing import Iterator
-from common import APPS_DIR
-import click
-
-# The file to read to determine impacted image to rebuild.
-# Note: We cannot use the requirements.txt as it does not explicit the python module it
-# depends upon
-requirements_file="requirements-frozen.txt"
-
-
-def from_application_to_module(app_name: str) -> str:
-    """Compute python module name from the application name."""
-    return app_name.replace("-", ".")
-
-
-def from_tag_to_version(version: str) -> str:
-    """Compute python module version from a version tag (prefixed with 'v')."""
-    return version.lstrip("v")
-
-
-def list_impacted_apps(application: str, version: str) -> Iterator[str]:
-    """List all apps whose constraint does not match `application==version.`.
-
-    Expectedly, only applications who have `application` in their
-    requirements-frozen.txt have a chance to be listed.
-
-    """
-    app_module = from_application_to_module(application)
-    version = from_tag_to_version(version)
-    for req_file in sorted(APPS_DIR.glob(f"*/{requirements_file}")):
-        with open(req_file, 'r') as f:
-            for line in f:
-                line = line.rstrip()
-                if '==' in line:  # we ignore the `@` case for now
-                    module, version_ = line.rstrip().split('==')
-                    if module == app_module:
-                        if version == version_:
-                            # Application is already built for the right version, stop
-                            break
-                        # Application has the dependency for another version, rebuild
-                        yield req_file.parent.stem
-
-
-@click.command()
-@click.argument("application")
-@click.argument("version")
-def main(application: str, version: str) -> None:
-    """Determine list of applications impacted by change of application @ version."""
-    for app in list_impacted_apps(application, version):
-        print(app)
-
-if __name__ == "__main__":
-    main()