.copier-answers.yml

+# Changes here will be overwritten by Copier
+_commit: v0.3.3
+_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
+description: Software Heritage Documentation
+distribution_name: swh-docs
+have_cli: false
+have_workers: false
+package_root: swh/docs
+project_name: swh.docs
+python_minimal_version: '3.7'
+readme_format: rst

.git-blame-ignore-revs

+# black
+d71d13ea72434aa9337e2331ba0fbb034ad39acb
+a2b344ef0b824894d2d7bbbed249f3ac68f89eb1

.gitignore

+*.egg-info/
 *.pyc
-*.sw?
-*~
 .coverage
 .eggs/
-__pycache__
-*.egg-info/
-version.txt
+.hypothesis
+.mypy_cache
 .tox
-docs/apidoc
+__pycache__
+build/
+dist/
+# these are symlinks created by a hook in swh-docs' main sphinx conf.py
+docs/README.rst
+docs/README.md
+# this should be a symlink for people who want to build the sphinx doc
+# without using tox, generally created by the swh-env/bin/update script
+docs/Makefile.sphinx

.pre-commit-config.yaml

+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-json
+      - id: check-yaml
+
+  - repo: https://github.com/python/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.0
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        name: Check source code spelling
+        additional_dependencies:
+          - tomli
+        stages: [pre-commit]
+      - id: codespell
+        name: Check commit message spelling
+        additional_dependencies:
+          - tomli
+        stages: [commit-msg]
+
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        args: [swh]
+        pass_filenames: false
+        language: system
+        types: [python]
+      - id: twine-check
+        name: twine check
+        description: call twine check when pushing an annotated release tag
+        entry: bash -c "ref=$(git describe) &&
+          [[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+          (python3 -m build --sdist && twine check $(ls -t dist/* | head -1)) || true"
+        pass_filenames: false
+        stages: [pre-push]
+        language: python
+        additional_dependencies: [twine, build]

AUTHORS

-Copyright (C) 2015-2018 The Software Heritage developers
+Copyright (C) 2015-2023 The Software Heritage developers
 
 See http://www.softwareheritage.org/ for more information.

CODE_OF_CONDUCT.md

+# Software Heritage Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as Software
+Heritage contributors and maintainers pledge to making participation in our
+project and our community a harassment-free experience for everyone, regardless
+of age, body size, disability, ethnicity, sex characteristics, gender identity
+and expression, level of experience, education, socioeconomic status,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at `conduct@softwareheritage.org`. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an
+incident.  Further details of specific enforcement policies may be posted
+separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTORS

+Archit Agrawal
+Ishan Bhanuka
+Loïc Dachary
+Matthew Vernon
+Thibault Allançon

MANIFEST.in

-include Makefile
-include requirements.txt
-include requirements-swh.txt
-include version.txt

Makefile.local

+check-mypy:
+	@echo "no op - swh-docs does not need typechecking, skipping"
+
+all:
+	@echo Building the whole documentation from source code repositories
+	@echo available in ../ via tox. This can take a while.
+	@echo Please be patient.
+	tox -vv -r -e sphinx-dev
+
+clean:
+	make -C docs clean
+
+# inline *build* dependencies to allow installing requirements-swh-dev.txt without build
+# isolation
+BUILD_DEPS :=
+
+# swh.perfecthash
+BUILD_DEPS += cffi
+
+# swh.search
+BUILD_DEPS += tree-sitter\<0.22.0
+
+# swh.loader.bzr>breezy
+BUILD_DEPS += configobj
+
+# swh.docs
+BUILD_DEPS += pifpaf
+
+# psycopg-c
+BUILD_DEPS += tomli
+
+# docutils >= 0.21
+BUILD_DEPS += flit-core
+
+pip-install-swh-dev:
+	python3 -m pip install --upgrade pip wheel setuptools setuptools-scm
+	python3 -m pip install --upgrade $(BUILD_DEPS)
+
+	python3 -m pip install -r requirements-swh-dev.txt --no-build-isolation
+	python3 -m pip install .

Makefile.sphinx

@@ -2,22 +2,19 @@
 # automatically build sphinx documentation.
 
 APIDOC_DIR = apidoc
-APIDOC_OPTS = --separate
-SPHINXBUILD = python3 -c 'import sphinx, sys; sys.exit(sphinx.main(sys.argv))'
-SPHINXAPIDOC = python3 -c 'import sphinx.apidoc, sys; sys.exit(sphinx.apidoc.main(sys.argv))'
-SPHINX_OPTS =
+APIDOC_OPTS = --ext-viewcode --separate --implicit-namespaces
+SPHINXBUILD = sphinx-build
+SPHINXAPIDOC = sphinx-apidoc
+SPHINX_OPTS = -t standalone_package_doc
 SPHINX_BUILDDIR = _build
 SPHINX_SRCDIR = .
 
 # relative to docs/ dir. Hence "." exclude the docs/ dir itself
-APIDOC_EXCLUDES =
-APIDOC_EXCLUDES += . ../setup.py
-APIDOC_EXCLUDES += ../swh/*/tests ../swh/*/tests/* ../swh/*/*/tests/* ../swh/*/*/*/tests/*
-APIDOC_EXCLUDES += ../swh/*/migrations ../swh/*/migrations/* ../swh/*/*/migrations/* ../swh/*/*/*/migrations/*
-APIDOC_EXCLUDES += ../bin/*
-APIDOC_EXCLUDES += ../utils/*
-APIDOC_EXCLUDES += ../node_modules/*
-
+APIDOC_EXCLUDES = conftest.py setup.py
+APIDOC_EXCLUDES += */tests/* */migrations/* */wsgi.py */conftest.py
+APIDOC_EXCLUDES += */deposit/settings/* */web/settings/*
+APIDOC_EXCLUDES += bin build dist utils node_modules
+APIDOC_SWH_EXCLUDES = $(patsubst %,"../%",$(APIDOC_EXCLUDES))
 
 html: sphinx/html
 help: sphinx/help
@@ -31,13 +28,13 @@ apidoc_dep = apidoc-stamp
 endif
 
 sphinx/clean:
-	$(SPHINXBUILD) -M clean $(SPHINX_SRCDIR) $(SPHINX_BUILDDIR) $(SPHINXOPTS)
+	$(SPHINXBUILD) -M clean $(SPHINX_SRCDIR) $(SPHINX_BUILDDIR) $(SPHINX_OPTS) $(SPHINXOPTS)
 sphinx/%: $(apidoc_dep)
-	$(SPHINXBUILD) -b $* $(SPHINX_SRCDIR) $(SPHINX_BUILDDIR)/$* $(SPHINXOPTS)
+	$(SPHINXBUILD) -b $* $(SPHINX_SRCDIR) $(SPHINX_BUILDDIR)/$* $(SPHINX_OPTS) $(SPHINXOPTS)
 
 apidoc: $(apidoc_dep)
 apidoc-stamp:
-	$(SPHINXAPIDOC) $(APIDOC_OPTS) -o $(APIDOC_DIR) .. $(APIDOC_EXCLUDES)
+	$(SPHINXAPIDOC) $(APIDOC_OPTS) -o $(APIDOC_DIR) `if [ -d ../src/swh ]; then echo ../src/swh; else echo ../swh; fi` $(APIDOC_SWH_EXCLUDES)
 	# to silent Sphinx warnings about apidoc documents not included in any toctree
 	find $(shell pwd)/apidoc -name "*.rst" | xargs sed -i '1i:orphan:\n'
 	touch $@

README.md

-swh-docs
-========
-
-This module contains (the logics for generating) the Software Heritage
-development documentation.
-
-Specifically, it contains some general information about Software Heritage
-internals (stuff that would not fit in any other specific software component of
-the Software Heritage stack) and bundle them together component-specific
-documentation coming from other modules of the stack.
-
-All documentation is written and typeset using [Sphinx][1]. General
-documentation is shipped as part of this module. Module-specific documentation
-is centralized here via symlinks to the `docs/` dirs of individual modules.
-Therefore to build the full documentation you need a working and
-complete [Software Heritage development environment][2].
-
-
-
-How to build the doc
---------------------
-
-Install the [Software Heritage development environment][2]
-
-    $ git clone https://forge.softwareheritage.org/source/swh-environment
-    $ cd swh-environment
-    $ ./bin/update  # this will clone needed git repos, inc. swh-docs
-    $ cd swh-docs
-
-Ensure you have the required tools to generate images ([graphviz][3]'s `dot`
-and [plantuml][4]). On a Debian system:
-
-    $ sudo apt install plantuml graphviz
-
-
-It is also recomanded to build the doc using [tox][5], so ensure you have it
-installed, eg. on a Debian system:
-
-    $ sudo apt install tox
-
-
-Then (from the `swh-environment/swh-docs/` directory):
-
-    $ tox -e sphinx-dev
-
-This tox environment will build the documentation from the sources available in
-the parent directory (`swh-environment`).
-
-Behind the scene, this tox environment will run the sphinx documentation
-building process via [pifpaf][6] to encapsulate the need os Postgresql to
-generate database schemas. The documentation building process itself consists
-mainly in 3 steps:
-
-### 1. Generate documentation assets for all modules
-
-    $ cd swh-environment
-    $ make docs-assets
-
-This will *not* build the documentation in each module (there is `make docs`
-for that).
-
-
-### 2. Build the api docs for all swh python packages
-
-    $ cd swh-docs/docs
-    $ make apidoc
-
-### 3. Build the documentation
-
-    $ cd swh-docs/docs
-    $ make
-
-The HTML documentation is now available starting from `_build/html/index.html`.
-
-
-Cleaning up
------------
-
-    $ cd docs
-    $ make distclean
-
-The former (`make clean`) will only clean the local Sphinx build, without
-touching other modules. The latter (`make distclean`) will also clean Sphinx
-builds in all other modules.
-
-
-Publishing the doc
-------------------
-
-The publication of the documentation is now managed by the [CI][7].
-
-
-
-[1]: http://www.sphinx-doc.org/
-[2]: https://forge.softwareheritage.org/source/swh-environment/
-[3]: https://graphviz.org
-[4]: http://plantuml.com
-[5]: https://tox.readthedocs.io/
-[6]: https://github.com/jd/pifpaf

README.rst

+Software Heritage Technical Documentation
+=========================================
+
+This module contains (the logics for generating) the Software Heritage
+technical documentation.
+
+Specifically, it contains some general information about Software Heritage
+internals (stuff that would not fit in any other specific software component of
+the Software Heritage stack) and bundle them together component-specific
+documentation coming from other modules of the stack.
+
+All documentation is written and typeset using Sphinx_. General
+documentation is shipped as part of this module. Module-specific documentation
+is centralized here via symlinks to the ``docs/`` dirs of individual modules.
+Therefore to build the full documentation you need a working and
+complete `Software Heritage development environment`_.
+
+
+How to build the doc
+--------------------
+
+Install the `Software Heritage development environment`_:
+
+.. code-block:: shell
+
+   $ git clone https://gitlab.softwareheritage.org/swh/devel/swh-environment.git
+   $ cd swh-environment
+   $ ./bin/update  # this will clone needed git repos, inc. swh-docs
+   $ cd swh-docs
+
+Ensure you have the required tools to generate images (graphviz_'s ``dot``,
+plantuml_ and inkscape_). On a Debian system:
+
+.. code-block:: shell
+
+   $ sudo apt install plantuml graphviz
+
+These additional packages are required on Debian 10.x (and newer) systems:
+
+- libapr1-dev
+- libaprutil1-dev
+- libsvn-dev
+- postgresql-11
+- dia
+- postgresql-autodoc
+
+It is also recommended to build the doc using tox_, so ensure you have it
+installed, eg. on a Debian system:
+
+.. code-block:: shell
+
+   $ sudo apt install tox
+
+
+Then (from the ``swh-environment/swh-docs/`` directory):
+
+.. code-block:: shell
+
+    $ tox run -e sphinx-dev
+
+This tox environment will build the documentation from the sources available in
+the parent directory (``swh-environment``).
+
+Behind the scene, this tox environment will run the sphinx documentation
+building process via pifpaf_ to encapsulate the need os Postgresql to
+generate database schemas. The documentation building process itself consists
+mainly in 3 steps:
+
+### 1. Generate documentation assets for all modules
+
+.. code-block:: shell
+
+   $ cd swh-environment
+   $ pifpaf run postgresql -- make docs-assets
+
+This will *not* build the documentation in each module (there is ``make docs``
+for that).
+
+
+### 2. Build the api docs for all swh python packages
+
+.. code-block:: shell
+
+   $ cd swh-docs/docs
+   $ pifpaf run postgresql -- make apidoc
+
+### 3. Build the documentation
+
+.. code-block:: shell
+
+   $ cd swh-docs/docs
+   $ make
+
+The HTML documentation is now available starting from
+``_build/html/index.html``.
+
+
+Cleaning up
+-----------
+
+.. code-block:: shell
+
+   $ cd docs
+   $ make distclean
+
+The former (``make clean``) will only clean the local Sphinx build, without
+touching other modules. The latter (``make distclean``) will also clean Sphinx
+builds in all other modules.
+
+
+Publishing the doc
+------------------
+
+The publication of the documentation is now managed by the CI_.
+
+
+Building standalone package documentation
+-----------------------------------------
+
+Each documentation local to a swh package can also be built with [tox][6].
+
+For instance to build the standalone documentation of ``swh-web``, proceed as
+follows:
+
+.. code-block:: shell
+
+   $ cd swh-environment/swh-web
+   $ tox run -e sphinx-dev
+
+Sphinx warnings related to unresolved references located in other swh packages
+are suppressed because expected.
+
+Please also note that Sphinx warnings are turned into errors in that case.
+
+The HTML documentation is now available starting from
+``docs/_build/html/index.html``.
+
+
+.. _Sphinx: http://www.sphinx-doc.org/
+.. _`Software Heritage development environment`: https://gitlab.softwareheritage.org/swh/devel/swh-environment.git
+.. _graphviz: https://graphviz.org
+.. _plantuml: http://plantuml.com
+.. _inkscape: https://inkscape.org/
+.. _tox: https://tox.readthedocs.io/
+.. _CI: https://jenkins.softwareheritage.org/job/DDOC/
+.. _pifpaf: https://github.com/jd/pifpaf

debian/changelog

-<module-name> (0.0.1-1) unstable; urgency=low
-
-  *
-
- -- AUTHOR-NAME <AUTHOR-EMAIL>  <UTC-DATE>

debian/compat

-9

debian/control

-Source: <module-name>
-Maintainer: Software Heritage developers <swh-devel@inria.fr>
-Section: python
-Priority: optional
-Build-Depends: debhelper (>= 9),
-               dh-python (>= 2),
-               python3-all,
-               python3-nose,
-               python3-setuptools,
-               python3-swh.core,
-               python3-swh.storage,
-               python3-vcversioner
-Standards-Version: 3.9.6
-Homepage: https://forge.softwareheritage.org/diffusion/<PHABRICATOR_CALLSIGN>/
-
-Package: python3-<dot-separated-name-module>
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}
-Description: Software Heritage <module-description>

debian/copyright

-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-
-Files: *
-Copyright: 2015 The Software Heritage developers
-License: GPL-3+
-
-License: GPL-3+
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License, or
- (at your option) any later version.
- .
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- GNU General Public License for more details.
- .
- You should have received a copy of the GNU General Public License
- along with this program.  If not, see <http://www.gnu.org/licenses/>.
- .
- On Debian systems, the complete text of the GNU General Public
- License version 3 can be found in `/usr/share/common-licenses/GPL-3'.

debian/rules

-#!/usr/bin/make -f
-
-export PYBUILD_NAME=swh.docs
-export PYBUILD_TEST_ARGS=--with-doctest -sv -a !db,!fs
-
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-
-override_dh_install:
-	dh_install
-	rm -v $(CURDIR)/debian/python3-*/usr/lib/python*/dist-packages/swh/__init__.py

debian/source/format

-3.0 (quilt)

docs/.gitignore

 *-stamp
 _build/
-/swh-*
+devel/swh-*
+errors.log
+!/swh-loader.rst
+sources/
+user/software-origins/dynamic/*.inc
+apidoc/

docs/Makefile

-SPHINXOPTS ?=
-SPHINXBUILD = python3 -c 'import sphinx, sys; sys.exit(sphinx.main(sys.argv))'
-SOURCEDIR = .
-BUILDDIR = _build
-HTMLDIR = $(BUILDDIR)/html
-SWHPKGDIR = `python3 -c 'import swh; print(swh.__path__[0])'`
-
-INSTALL_HOST = pergamon.internal.softwareheritage.org
-INSTALL_DIR = /srv/softwareheritage/docs/webroot/devel
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS  ?= --jobs auto
+SPHINXBUILD ?= sphinx-build
+SOURCEDIR    = .
+BUILDDIR     = _build
+HTMLDIR      = $(BUILDDIR)/html
+
+INSTALL_HOST  = pergamon.internal.softwareheritage.org
+INSTALL_DIR   = /srv/softwareheritage/docs/webroot
 INSTALL_GROUP = swhdev
 INSTALL_PERMS = g+rwX
 
-SPHINXAPIDOC = sphinx-apidoc
-APIDOC_DIR = apidoc
-APIDOC_OPTS = --ext-viewcode
-APIDOC_EXCLUDES = */tests */tests/* */*/tests/* */*/*/tests/*
-APIDOC_EXCLUDES += */migrations */migrations/* */*/migrations/* */*/*/migrations/*
-APIDOC_SWH_EXCLUDES = $(patsubst %,$(SWHPKGDIR)/%,$(APIDOC_EXCLUDES))
-
-html: sphinx/html
-
-sphinx/html: links-stamp apidoc-stamp images-stamp rec-build-stamp
-
-links-stamp:
-	bin/ln-sphinx-subprojects
-	touch $@
+.PHONY: help images apidoc dynamic-rst html clean install
 
-apidoc: apidoc-stamp
-apidoc-stamp: images-stamp
-	$(SPHINXAPIDOC) $(APIDOC_OPTS) -o $(APIDOC_DIR) $(SWHPKGDIR) $(APIDOC_SWH_EXCLUDES)
-	touch $@
+all: html
 
-images-stamp:
-	$(MAKE) -C images
-	touch $@
+dynamic-rst:
+	make -C user dynamic-rst
 
-# Build documentation recursively in all packages, to make sure that auxiliary,
-# non-sphinx managed documentation artifacts (e.g., schema diagrams) are also
-# built.
-rec-build-stamp: $(wildcard ../../swh-*/docs/*.rst)
-	$(MAKE) -C ../../ docs-assets
-	touch $@
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+sphinx/%: Makefile images apidoc dynamic-rst
+	@$(SPHINXBUILD) -M $* "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-clean: sphinx/clean
-	bin/ln-sphinx-subprojects --remove
-	$(MAKE) -C images clean
-	rm -f *-stamp
-	rm -f $(APIDOC_DIR)/*
+sphinx/clean:
+	@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-distclean: clean
-	make -C ../../ docs-clean
+images:
+	$(MAKE) -C devel images-stamp
+	$(MAKE) -C sysadm images-stamp
 
-help: sphinx/help
+apidoc:
+	$(MAKE) -C devel apidoc
 
-sphinx/clean:
-	$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)
+html: sphinx/html
 
-sphinx/%:
-	$(SPHINXBUILD) -b $* "$(SOURCEDIR)" "$(BUILDDIR)/$*" $(SPHINXOPTS)
+clean: sphinx/clean
+	$(MAKE) -C devel clean
+	$(MAKE) -C sysadm clean
+	$(MAKE) -C user clean
 
 install: html
 	test -d $(HTMLDIR)
@@ -65,5 +50,3 @@ install: html
 		"find $(INSTALL_DIR) -not -group $(INSTALL_GROUP) -exec chgrp -v $(INSTALL_GROUP) {} + ; \
 		 find $(INSTALL_DIR) -not -perm -ug=rw,o=r -exec chmod -v ug+rw,o+r {} + ; \
 		 find $(INSTALL_DIR) -type d -not -perm -g=xs,ou=x -exec chmod -v g+xs,ou+x {} +"
-
-.PHONY: help html clean distclean install