sphinx/conf.py: Speedup documentation development build significantly

The pydata sphinx theme we are using generates a custom sidebar for each documentation page but it is known that this step takes a long time if the whole documentation TOC tree has a large size (see pydata-sphinx-theme documentation and the flamegraph below I generated to profile the sphinx-build execution ).

So apply workaround suggested by the pydata sphinx theme documentation: remove from the TOC tree the API documentation pages generated by apidoc, except the root package indices, in order to significantly reduce the TOC tree size and thus the build time of the documentation.

Also rework the sphinx hook to include API reference of a package in its documentation index: prefer to modify source of documentation index instead of hacking the TOC tree and include links to submodules API.

Before:

(swh) ✔ ~/swh/swh-environment/swh-docs/docs [master|⚑ 9] 
$ time make
...
real    13m27,279s
user    14m23,286s
sys     0m9,899s

After:

(swh) ✔ ~/swh/swh-environment/swh-docs/docs [speedup-doc-build L|⚑ 9] 
$ time make
...
real    4m32,027s
user    5m29,672s
sys     0m9,270s

This performance optimization is only applied to the development builds of the full SWH documentation, either when calling make in the root directory of swh-docs or on Jenkins when pushing a merge request. For the latter, we gain more than 10 minutes for building the full SWH documentation, see screenshot below.

swh/docs/sphinx/conf.py

@@ -8,11 +8,10 @@ import os
 from pathlib import Path
 import re
 import sys
+import textwrap
 from typing import Dict
 
-from sphinx import addnodes
 from sphinx.ext import autodoc
-from sphinx.transforms import SphinxTransform
 
 from swh.docs.django_settings import force_django_settings
 
@@ -47,6 +46,7 @@ extensions = [
     "sphinx_celery.setting_crossref",
     "sphinx_carousel.carousel",
     "sphinx_design",
+    "sphinx_remove_toctrees",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -233,6 +233,18 @@ redirects = {
     "devel/swh-web/uri-scheme-identifiers": "uri-scheme-swhids.html",
 }
 
+if os.environ.get("SWH_DOCS_DEV_BUILD", ""):
+    # only keep root package indices in the toctree displayed in the sidebar
+    # to greatly speedup documentation build
+    # https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/performance.html
+    remove_from_toctrees = [
+        os.path.join(root, file)
+        for root, _, files in os.walk("devel/apidoc")
+        for file in files
+        if len(file.split(".")) > 3
+        and not re.match(r"^swh.loader.[^.]+.rst$", file)
+        and file not in ("swh.objstorage.replayer.rst", "swh.web.client.rst")
+    ]
 
 # -- autodoc configuration ----------------------------------------------
 autodoc_default_flags = [
@@ -372,6 +384,38 @@ def ensure_readme(app, config):
             break
 
 
+def add_swh_package_index(app, docname, source):
+    # Act only on indexes of swh packages
+    match = re.match(r"^devel/(swh-[^/]+)/index$", docname)
+    if not match:
+        return
+
+    # Compute path to the module index generated by apidoc
+    swh_package = match.group(1).replace("-", ".")
+    swh_package_apidoc = f"devel/apidoc/{swh_package}"
+    swh_package_apidoc_file = os.path.join(app.env.srcdir, swh_package_apidoc) + ".rst"
+
+    if not os.path.exists(swh_package_apidoc_file):
+        return
+
+    # Add API reference at the end of the package documentation index
+    source[0] += textwrap.dedent(
+        """
+        Package API reference
+        ---------------------
+
+        .. toctree::
+           :maxdepth: 1
+           :glob:
+
+           /swh_package_apidoc
+           /swh_package_apidoc.*
+        """.replace(
+            "swh_package_apidoc", swh_package_apidoc
+        )
+    )
+
+
 def setup(app):
     app.connect("config-inited", ensure_readme)
     # env-purge-doc event is fired before source-read
@@ -415,31 +459,4 @@ def setup(app):
             get_sphinx_warning_handler().filters.insert(0, HttpDomainRefWarningFilter())
 
     else:
-
-        class SwhPackageTocTreeAddApidoc(SphinxTransform):
-            # Post processing. Deadline to modify text and referencing.
-            default_priority = 700
-
-            def apply(self, **kwargs):
-                # Act only on indexes of swh packages
-                match = re.match(r"^devel/(swh-[^/]+)/index$", self.env.docname)
-                if not match:
-                    return
-
-                # Compute path to the module index generated by apidoc
-                swh_package = match.group(1).replace("-", ".")
-                swh_package_apidoc = f"devel/apidoc/{swh_package}"
-                swh_package_apidoc_file = (
-                    os.path.join(self.env.srcdir, swh_package_apidoc) + ".rst"
-                )
-
-                # If the path exists, add it to the toc
-                if not os.path.exists(swh_package_apidoc_file):
-                    return
-                nodes = list(self.document.findall(addnodes.toctree))
-                if nodes:
-                    main_toc = nodes[0]
-                    main_toc["entries"].append((None, swh_package_apidoc))
-                    main_toc["includefiles"].append(swh_package_apidoc)
-
-        app.add_transform(SwhPackageTocTreeAddApidoc)
+        app.connect("source-read", add_swh_package_index)