Populate every software origin's page with logo and basic links

docs/user/software-origins/tuleap.rst

@@ -5,3 +5,5 @@ Tuleap
 
 .. todo::
    This page is a work in progress.
+
+.. include:: dynamic/tuleap_status.inc

swh/docs/generate_software_origin_status.py

+# Copyright (C) 2023 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+"""Reads :file:`docs/software-origins-support.yml` and generates ReST documents
+which are included each in a forge's page to summarize its status and link
+to other documentation page"""
+
+from pathlib import Path
+import sys
+
+from .software_origins import parse
+
+
+def write_status(data, forge_id, file) -> None:
+    forge = data.forges[forge_id]
+
+    has_loader = forge["loader"]["status"] != "N/A"
+    has_lister = forge["lister"]["status"] != "N/A"
+
+    links_cell = ""
+
+    if forge.status == "dev":
+        status_cell = f"Archival for {forge.name} is currently in development:\n\n"
+    elif forge.status in ("staging", "prod"):
+        if forge.status == "staging":
+            status_cell = (
+                f"{forge.name} is currently archived only on "
+                f"the staging infrastructure:\n\n"
+            )
+        else:
+            status_cell = (
+                f"{forge.name} is currently archived by Software Heritage:\n\n"
+            )
+
+        if has_loader:
+            links_cell += f"* `Browse\N{NBSP}origins <{forge.origins}>`__\n"
+        if has_lister:
+            links_cell += f"* `See\N{NBSP}coverage <{forge.coverage}>`__\n"
+    else:
+        assert False, f"Unexpected status {forge.status!r} for {forge_id}"
+
+    if has_loader:
+        links_cell += (
+            f"* `Loader Source Code <{forge.loader.source}>`__\n"
+            f"* :mod:`Loader Developer documentation <{forge.loader.package_name}>`"
+            f"\n"
+        )
+    if has_lister:
+        links_cell += (
+            f"* `Lister Source Code <{forge.lister.source}>`__\n"
+            f"* :mod:`Lister Developer documentation <{forge.lister.package_name}>`"
+            f"\n"
+        )
+
+    loader_issue = forge["loader"].get("issue")
+    lister_issue = forge["lister"].get("issue")
+    if loader_issue and lister_issue:
+        if loader_issue == lister_issue:
+            links_cell += f"* `Tracking issue <{loader_issue}>`__\n\n"
+        else:
+            links_cell += (
+                f"* `Tracking loader issue <{loader_issue}>`__\n\n"
+                f"* `Tracking lister issue <{lister_issue}>`__\n\n"
+            )
+    elif not loader_issue and not lister_issue:
+        pass
+    elif lister_issue:
+        if has_loader:
+            status_cell += f"* `Lister Tracking issue <{lister_issue}>`__\n\n"
+        else:
+            status_cell += f"* `Tracking issue <{lister_issue}>`__\n\n"
+    elif loader_issue:
+        if has_loader:
+            status_cell += f"* `Loader Tracking issue <{loader_issue}>`__\n\n"
+        else:
+            status_cell += f"* `Tracking issue <{loader_issue}>`__\n\n"
+    else:
+        assert False, f"The impossible happened for {forge_id}"
+
+    notes = forge.get("notes")
+    if notes:
+        status_cell = f"{status_cell}\n\n{notes}"
+
+    grant_id = forge.get("grant")
+    grant = data["grants"][grant_id] if grant_id else None
+    developer_id = forge.get("developer")
+    if grant:
+        assert developer_id, f"{forge_id} has grant but no developer"
+        developer = data["developers"][developer_id]
+
+        # TODO: grammar is hard; we may want to write 'grant from the XXX Foundation'
+        # but also 'grant from XXX' sometimes.
+        # For now, only foundations provide grants.
+        assert grant.funder.endswith(" Foundation")
+
+        grant_cell = (
+            f"Developed by `{developer['name']} <{developer['url']}>`__ "
+            f"thanks to a `grant from the {grant.funder} <{grant.announcement}>`__ "
+        )
+    else:
+        assert not developer_id, f"{forge_id} has developer but no grant"
+        grant_cell = ""
+
+    file.write(status_cell)
+    file.write(links_cell)
+    file.write("\n")
+    file.write(grant_cell)
+
+    file.write("\n")
+
+
+def write_logo(data, forge_id, file) -> None:
+    forge = data.forges[forge_id]
+    file.write(
+        f".. image:: ../logos/{forge_id}.png\n"
+        f"  :alt: {forge['name']} logo\n"
+        f"  :align: right"
+        f"\n\n"
+    )
+
+
+def main(input_path: Path, output_dir: Path) -> None:
+    data = parse(input_path)
+
+    for forge_id in data.forges:
+        with (output_dir / f"{forge_id}_status.inc").open("wt") as output_file:
+            write_logo(data, forge_id, output_file)
+            write_status(data, forge_id, output_file)
+
+
+if __name__ == "__main__":
+    try:
+        (_, input_path, output_path) = sys.argv
+    except ValueError:
+        print(
+            f"Syntax: {sys.argv[0]} docs/devel/forge-support.yml "
+            f"docs/user/software-origins/",
+            sys.stderr,
+        )
+        exit(1)
+    main(Path(input_path), Path(output_path))

swh/docs/generate_software_origins_list.py

@@ -11,7 +11,8 @@ import sys
 import textwrap
 
 import tabulate
-import yaml
+
+from .software_origins import parse
 
 PRELUDE = """
 .. This file was generated by swh/docs/generate_software_origins_list.py
@@ -40,108 +41,41 @@ def write_table(data, file) -> None:
             f"<user-software-origins-{forge_id}>`"
         )
 
+        links.append((f"{forge_id}-loader-source", forge.loader.source))
+        links.append((f"{forge_id}-lister-source", forge.lister.source))
+
         has_loader = forge["loader"]["status"] != "N/A"
         has_lister = forge["lister"]["status"] != "N/A"
 
-        assert has_loader or has_lister, f"{forge_id} has neither loader nor lister"
-
-        if has_loader and has_lister:
-            # We may want to do this eventually, but it never happened so far.
-            assert (
-                forge["loader"]["status"] == forge["lister"]["status"]
-            ), f"Loader and lister statuses for {forge_id} are not the same."
-
-        status = forge["loader"]["status"] if has_loader else forge["lister"]["status"]
-
-        default_loader_source = (
-            f"https://gitlab.softwareheritage.org/swh/devel/swh-loader-core/-/"
-            f"tree/master/swh/loader/package/{forge_id}"
-        )
-        default_loader_package_name = f"swh.loader.package.{forge_id}"
-        default_lister_source = (
-            f"https://gitlab.softwareheritage.org/swh/devel/swh-lister/-/"
-            f"tree/master/swh/lister/{forge_id}"
-        )
-        default_lister_package_name = f"swh.lister.{forge_id}"
-
-        loader_package_name = forge["loader"].get(
-            "package_name", default_loader_package_name
-        )
-        lister_package_name = forge["lister"].get(
-            "package_name", default_lister_package_name
-        )
-        links.append(
-            (
-                f"{forge_id}-loader-source",
-                forge["loader"].get("source", default_loader_source),
-            )
-        )
-        links.append(
-            (
-                f"{forge_id}-lister-source",
-                forge["lister"].get("source", default_lister_source),
-            )
-        )
-
-        loader_id_in_swh_web = forge["loader"].get("id_in_swh_web", forge_id)
-        lister_id_in_swh_web = forge["lister"].get("id_in_swh_web", forge_id)
-
-        if status == "dev":
+        if forge.status == "dev":
             status_cell = "In\N{NBSP}development\n\n"
-        elif status in ("staging", "prod"):
-            if status == "staging":
+        elif forge.status in ("staging", "prod"):
+            links.append((f"{forge_id}-origins", forge.origins))
+            links.append((f"{forge_id}-coverage", forge.coverage))
+            if forge.status == "staging":
                 status_cell = "In\N{NBSP}staging\n\n"
-                links.append(
-                    (
-                        f"{forge_id}-origins",
-                        "https://webapp.staging.swh.network/browse/search/"
-                        f"?with_visit=true&with_content=true"
-                        f"&visit_type={loader_id_in_swh_web}",
-                    )
-                )
-                links.append(
-                    (
-                        f"{forge_id}-coverage",
-                        "https://webapp.staging.swh.network/coverage/"
-                        f"?focus={lister_id_in_swh_web}#{lister_id_in_swh_web}",
-                    )
-                )
             else:
-                # status == "prod"
                 status_cell = "In\N{NBSP}production\n\n"
-                links.append(
-                    (
-                        f"{forge_id}-origins",
-                        f"https://archive.softwareheritage.org/browse/search/"
-                        f"?with_visit=true&with_content=true"
-                        f"&visit_type={loader_id_in_swh_web}",
-                    )
-                )
-                links.append(
-                    (
-                        f"{forge_id}-coverage",
-                        f"https://archive.softwareheritage.org/coverage/"
-                        f"?focus={lister_id_in_swh_web}#{lister_id_in_swh_web}",
-                    )
-                )
 
             if has_loader:
                 status_cell += f"`Browse\N{NBSP}origins <{forge_id}-origins_>`__\n\n"
             if has_lister:
                 status_cell += f"`See\N{NBSP}coverage <{forge_id}-coverage_>`__\n\n"
         else:
-            assert False, f"Unexpected status {status!r} for {forge_id}"
+            assert False, f"Unexpected status {forge.status!r} for {forge_id}"
 
         links_cell = ""
         if has_loader:
             links_cell += (
                 f"* `Loader Source Code <{forge_id}-loader-source_>`__\n"
-                f"* :mod:`Loader Developer documentation <{loader_package_name}>`\n"
+                f"* :mod:`Loader Developer documentation <{forge.loader.package_name}>`"
+                f"\n"
             )
         if has_lister:
             links_cell += (
                 f"* `Lister Source Code <{forge_id}-lister-source_>`__\n"
-                f"* :mod:`Lister Developer documentation <{lister_package_name}>`\n"
+                f"* :mod:`Lister Developer documentation <{forge.lister.package_name}>`"
+                f"\n"
             )
 
         loader_issue = forge["loader"].get("issue")
@@ -231,7 +165,7 @@ def write_grants(data, file) -> None:
 
 
 def main(input_path: Path, output_path: Path) -> None:
-    data = yaml.safe_load(input_path.read_text())
+    data = parse(input_path)
 
     with output_path.open("wt") as output_file:
         output_file.write(PRELUDE.format(source_yml=input_path))

swh/docs/software_origins.py

+# Copyright (C) 2023 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+"""Parser for :file:`docs/devel/software-origins-support.yml`
+"""
+
+import collections
+from pathlib import Path
+
+import yaml
+
+
+class DotDict(collections.UserDict):
+    """Like a dictionary, but can be accessed with ``.key`` in addition to
+    ``[key]``, to simplify f-string."""
+
+    def __init__(self, d):
+        d = {
+            key: DotDict(value) if isinstance(value, dict) else value
+            for (key, value) in d.items()
+        }
+        super().__setattr__("data", d)
+
+    def __getattr__(self, key):
+        try:
+            return self.data[key]
+        except KeyError as e:
+            raise AttributeError(*e.args) from None
+
+    def __setattr__(self, key, value):
+        self.data[key] = value
+
+
+def parse(input_path: Path):
+    data = DotDict(yaml.safe_load(input_path.read_text()))
+
+    for (forge_id, forge) in sorted(data["forges"].items()):
+        forge.loader.setdefault(
+            "source",
+            f"https://gitlab.softwareheritage.org/swh/devel/swh-loader-core/-/"
+            f"tree/master/swh/loader/package/{forge_id}",
+        )
+        forge.lister.setdefault(
+            "source",
+            f"https://gitlab.softwareheritage.org/swh/devel/swh-lister/-/"
+            f"tree/master/swh/lister/{forge_id}",
+        )
+
+        forge.loader.setdefault("package_name", f"swh.loader.package.{forge_id}")
+        forge.lister.setdefault("package_name", f"swh.lister.{forge_id}")
+
+        forge.loader.setdefault("id_in_swh_web", forge_id)
+        forge.lister.setdefault("id_in_swh_web", forge_id)
+
+        has_loader = forge["loader"]["status"] != "N/A"
+        has_lister = forge["lister"]["status"] != "N/A"
+
+        assert has_loader or has_lister, f"{forge_id} has neither loader nor lister"
+
+        if has_loader and has_lister:
+            # We may want to do this eventually, but it never happened so far.
+            assert (
+                forge["loader"]["status"] == forge["lister"]["status"]
+            ), f"Loader and lister statuses for {forge_id} are not the same."
+
+        forge.status = (
+            forge["loader"]["status"] if has_loader else forge["lister"]["status"]
+        )
+
+        if forge.status == "dev":
+            forge.origins = None
+            forge.coverage = None
+        elif forge.status in ("staging", "prod"):
+            if forge.status == "staging":
+                archive_base_url = "https://webapp.staging.swh.network"
+            else:
+                archive_base_url = "https://archive.softwareheritage.org"
+            forge.origins = (
+                f"{archive_base_url}/browse/search/?with_visit=true&with_content=true"
+                f"&visit_type={forge.loader.id_in_swh_web}"
+            )
+            forge.coverage = (
+                f"{archive_base_url}/coverage"
+                f"focus={forge.lister.id_in_swh_web}#{forge.lister.id_in_swh_web}"
+            )
+
+    return data