make-doc: refactor doc build system to build the doc as a whole

- back to assets only building in each package
- run sphinx-autodoc on the whole swh package at once

we do not really need a local standalone buildable sphinx project in each swh
subpackage, so running a full 'make html' in each of them is not required any
more.

Since we want to build the doc at once instead of building each swh subpackage
independantly, we need to run apidoc on the whole swh package.

This is done by actually running the apidoc on the pip installed swh
packages instead of the source repositories, since sphinx-apidoc does
not seem to support multiple MODULE_PATH as cmdline argument.

So it's best to use tox to build the doc.

Related to T1330.

.gitignore

@@ -5,4 +5,6 @@
 .eggs/
 __pycache__
 *.egg-info/
-version.txt
\ No newline at end of file
+version.txt
+.tox
+docs/apidoc

docs/Makefile

@@ -3,17 +3,23 @@ SPHINXBUILD = python3 -c 'import sphinx, sys; sys.exit(sphinx.main(sys.argv))'
 SOURCEDIR = .
 BUILDDIR = _build
 HTMLDIR = $(BUILDDIR)/html
+SWHPKGDIR = `python3 -c 'import swh; print(swh.__path__[0])'`
 
 INSTALL_HOST = pergamon.internal.softwareheritage.org
 INSTALL_DIR = /srv/softwareheritage/docs/webroot/devel
 INSTALL_GROUP = swhdev
 INSTALL_PERMS = g+rwX
 
-html: fix-indices-stamp sphinx/html
+SPHINXAPIDOC = sphinx-apidoc
+APIDOC_DIR = apidoc
+APIDOC_OPTS = --ext-viewcode
+APIDOC_EXCLUDES = */tests */tests/* */*/tests/* */*/*/tests/*
+APIDOC_EXCLUDES += */migrations */migrations/* */*/migrations/* */*/*/migrations/*
+APIDOC_SWH_EXCLUDES = $(patsubst %,$(SWHPKGDIR)/%,$(APIDOC_EXCLUDES))
 
-fix-indices-stamp: sphinx/html
-	bin/copy-and-fix-subprojects-indices
-	touch $@
+apidoc_dep = apidoc-stamp
+
+html: sphinx/html
 
 sphinx/html: links-stamp apidoc-stamp images-stamp rec-build-stamp
 
@@ -21,8 +27,9 @@ links-stamp:
 	bin/ln-sphinx-subprojects
 	touch $@
 
+apidoc: $(apidoc_dep)
 apidoc-stamp:
-	$(MAKE) -C ../../ docs-apidoc
+	$(SPHINXAPIDOC) $(APIDOC_OPTS) -o $(APIDOC_DIR) $(SWHPKGDIR) $(APIDOC_SWH_EXCLUDES)
 	touch $@
 
 images-stamp:
@@ -33,13 +40,14 @@ images-stamp:
 # non-sphinx managed documentation artifacts (e.g., schema diagrams) are also
 # built.
 rec-build-stamp: $(wildcard ../../swh-*/docs/*.rst)
-	$(MAKE) -C ../../ docs
+	$(MAKE) -C ../../ docs-assets
 	touch $@
 
 clean: sphinx/clean
 	bin/ln-sphinx-subprojects --remove
 	$(MAKE) -C images clean
 	rm -f *-stamp
+	rm -f $(APIDOC_DIR)/*
 
 distclean: clean
 	make -C ../../ docs-clean

requirements-swh.txt

 # Add here internal Software Heritage dependencies, one per line.
+swh-core
+swh-model
+swh-objstorage[testing]
+swh-scheduler
+swh-storage[schemadata]
+swh-loader-core
+swh-lister
+swh-journal
+swh-vault
+swh-loader-dir
+swh-loader-tar
+swh-loader-pypi
+swh-loader-debian
+swh-loader-mercurial
+swh-loader-svn
+swh-loader-git
+swh-archiver
+swh-web
+swh-deposit
+swh-indexer
+swh-mirror-forge

swh/docs/django_settings.py

+#from swh.web.settings.development import *  # noqa
+from swh.deposit.settings.development import *  # noqa
+
+SECRET_KEY = 'change me'

swh/docs/sphinx/conf.py

@@ -20,6 +20,7 @@ extensions = ['sphinx.ext.autodoc',
               'sphinxcontrib.httpdomain',
               'sphinx.ext.extlinks',
               'sphinxcontrib.images',
+              'sphinx.ext.viewcode',
               ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -40,8 +41,9 @@ master_doc = 'index'
 
 # A string of reStructuredText that will be included at the beginning of every
 # source file that is read.
+# A bit hackish but should work both for each swh package and the whole swh-doc
 rst_prolog = '''
-.. include:: /swh_substitutions
+.. include:: /../../swh-docs/docs/swh_substitutions
 '''
 
 # The version info for the project you're documenting, acts as replacement for
@@ -130,14 +132,9 @@ extlinks = {}
 # hack to set the adequate django settings when building global swh doc
 # to avoid build errors
 def source_read_handler(app, docname, source):
-    if 'swh-deposit' in docname:
-        os.environ.setdefault('DJANGO_SETTINGS_MODULE',
-                              'swh.deposit.settings.development')
-        django.setup()
-    elif 'swh-web' in docname:
-        os.environ.setdefault('DJANGO_SETTINGS_MODULE',
-                              'swh.web.settings.development')
-        django.setup()
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE',
+                          'swh.docs.django_settings')
+    django.setup()
 
 
 def setup(app):

tox.ini

+[tox]
+envlist=flake8
+
+[testenv:sphinx]
+deps =
+  django < 2
+  .[testing]
+  pifpaf
+commands =
+  {envpython} -m pifpaf run postgresql -- make -C docs html
+
+[testenv:flake8]
+skip_install = true
+deps =
+  flake8
+commands =
+  {envpython} -m flake8