From b5d48b043119612805c8ba07616c4efaf768c90c Mon Sep 17 00:00:00 2001
From: David Douard <david.douard@sdfa3.org>
Date: Mon, 27 Nov 2023 18:08:21 +0100
Subject: [PATCH] Migrate to copier-based swh-py-template
---
.copier-answers.yml | 11 +++++++
.gitignore | 19 ++++--------
.pre-commit-config.yaml | 25 ++++++++-------
MANIFEST.in | 16 ----------
Makefile | 2 +-
docs/Makefile | 3 +-
mypy.ini | 11 ++++---
pyproject.toml | 42 ++++++++++++++++++++++++++
pytest.ini | 9 ++++--
setup.py | 62 ++------------------------------------
swh/__init__.py | 3 --
swh/web/settings/common.py | 20 ++++++++++--
swh/web/utils/exc.py | 1 -
tox.ini | 62 +++++++++++++++++---------------------
14 files changed, 134 insertions(+), 152 deletions(-)
create mode 100644 .copier-answers.yml
delete mode 100644 MANIFEST.in
delete mode 100644 swh/__init__.py
diff --git a/.copier-answers.yml b/.copier-answers.yml
new file mode 100644
index 000000000..517130ca0
--- /dev/null
+++ b/.copier-answers.yml
@@ -0,0 +1,11 @@
+# Changes here will be overwritten by Copier
+_commit: v0.1.5
+_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
+description: Software Heritage web UI
+distribution_name: swh-web
+have_cli: false
+have_workers: false
+package_root: swh/web
+project_name: swh.web
+python_minimal_version: '3.7'
+readme_format: rst
diff --git a/.gitignore b/.gitignore
index d82de44c1..5b16d985d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,20 +1,15 @@
+*.egg-info/
*.pyc
-*.sw?
-*~
-\#*
-.\#*
-/.coverage
-/.coverage.*
+.coverage
.eggs/
-resources/test/
+.hypothesis
+.mypy_cache
+.tox
__pycache__
-version.txt
-swh.web.egg-info
docs/build/
docs/uri-scheme.md
docs/dev-info.md
*.sqlite3*
-.vscode/
.directory
node_modules/
static/*.*
@@ -25,9 +20,6 @@ static/jssources/
static/img/thirdParty/
build/
dist/
-.hypothesis
-.cache
-.pytest_cache
.tox/
.mypy_cache/
package-lock.json
@@ -41,3 +33,4 @@ cypress/junit/
cypress/downloads/
.eslintcache
docs/README.rst
+docs/README.md
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4b295e380..ee848cd03 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,15 +2,24 @@ exclude: "^swh/web/tests/resources/"
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.3.0
+ rev: v4.4.0
hooks:
- id: trailing-whitespace
- exclude: ".eml$"
- id: check-json
- id: check-yaml
+ - repo: https://github.com/python/black
+ rev: 23.1.0
+ hooks:
+ - id: black
+
+ - repo: https://github.com/PyCQA/isort
+ rev: 5.12.0
+ hooks:
+ - id: isort
+
- repo: https://github.com/pycqa/flake8
- rev: 5.0.4
+ rev: 6.0.0
hooks:
- id: flake8
additional_dependencies: [flake8-bugbear==22.9.23]
@@ -45,16 +54,6 @@ repos:
language: system
types: [javascript]
- - repo: https://github.com/PyCQA/isort
- rev: 5.11.5
- hooks:
- - id: isort
-
- - repo: https://github.com/python/black
- rev: 22.10.0
- hooks:
- - id: black
-
- repo: https://github.com/Riverside-Healthcare/djLint
rev: v1.31.1
hooks:
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index 7e69941a4..000000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1,16 +0,0 @@
-include pytest.ini
-include README.md
-include requirements*.txt
-include tox.ini
-include version.txt
-recursive-include swh py.typed
-
-recursive-include assets *
-recursive-include swh/web/*/templates *
-recursive-include swh/web/*/tests/assets *
-recursive-include swh/web/*/tests/data *
-recursive-include swh/web/*/tests/resources *
-recursive-include swh/web/tests/resources *
-
-include package.json
-include yarn.lock
diff --git a/Makefile b/Makefile
index 5db623f03..524175c2b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-# Makefile driver for swh Python modules. DO NOT CHANGE.
+# Makefile driver for SWH Python modules. DO NOT CHANGE.
# You can add custom Makefile rules to Makefile.local
include ../Makefile.python
diff --git a/docs/Makefile b/docs/Makefile
index ceee056eb..faf78f40f 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -1,3 +1,2 @@
-include ../../swh-docs/Makefile.sphinx
-
+include swh-docs/Makefile.sphinx
APIDOC_EXCLUDES += ../assets ../cypress ../static ../swh/*/settings
diff --git a/mypy.ini b/mypy.ini
index 4d0608893..bb0b3ed5f 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -1,6 +1,10 @@
[mypy]
namespace_packages = True
warn_unused_ignores = True
+explicit_package_bases = True
+# ^ Needed for mypy to detect py.typed from swh packages installed
+# in editable mode
+
# support for django magic: https://github.com/typeddjango/django-stubs
plugins = mypy_django_plugin.main, mypy_drf_plugin.main
@@ -8,7 +12,6 @@ plugins = mypy_django_plugin.main, mypy_drf_plugin.main
django_settings_module = swh.web.settings.development
# 3rd party libraries without stubs (yet)
-
[mypy-bs4.*]
ignore_missing_imports = True
@@ -42,9 +45,6 @@ ignore_missing_imports = True
[mypy-pygments.*]
ignore_missing_imports = True
-[mypy-pytest.*]
-ignore_missing_imports = True
-
[mypy-requests_mock.*]
ignore_missing_imports = True
@@ -56,3 +56,6 @@ ignore_missing_imports = True
[mypy-swh.docs.*]
ignore_missing_imports = True
+
+# [mypy-add_your_lib_here.*]
+# ignore_missing_imports = True
diff --git a/pyproject.toml b/pyproject.toml
index d4d28af19..cf0572f87 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,3 +1,45 @@
+[project]
+name = "swh.web"
+authors = [
+ {name="Software Heritage developers", email="swh-devel@inria.fr"},
+]
+
+description = "Software Heritage web UI"
+readme = {file = "README.rst", content-type = "text/x-rst"}
+requires-python = ">=3.7"
+classifiers = [
+ "Programming Language :: Python :: 3",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+ "Operating System :: OS Independent",
+ "Development Status :: 5 - Production/Stable",
+ "Framework :: Django",
+]
+dynamic = ["version", "dependencies", "optional-dependencies"]
+
+[tool.setuptools.packages.find]
+include = ["swh.*"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt", "requirements-swh.txt"]}
+
+[tool.setuptools.dynamic.optional-dependencies]
+testing = {file = ["requirements-test.txt"]}
+
+[project.urls]
+"Homepage" = "https://gitlab.softwareheritage.org/swh/devel/swh-web"
+"Bug Reports" = "https://gitlab.softwareheritage.org/swh/devel/swh-web/-/issues"
+"Funding" = "https://www.softwareheritage.org/donate"
+"Documentation" = "https://docs.softwareheritage.org/devel/swh-web/"
+"Source" = "https://gitlab.softwareheritage.org/swh/devel/swh-web.git"
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+fallback_version = "0.0.1"
+
[tool.black]
target-version = ['py37']
exclude = 'swh/web/tests/resources/contents'
diff --git a/pytest.ini b/pytest.ini
index b82ccd747..00c98462e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,6 +1,11 @@
[pytest]
-addopts = -p no:flask -p no:pytest_swh_storage -p swh.web.tests.pytest_plugin --ignore=swh/web/tests/random_fixtures_test.py
-norecursedirs = build docs node_modules .tox
+addopts =
+ -p no:flask
+ -p no:pytest_swh_storage
+ -p swh.web.tests.pytest_plugin
+ --ignore-glob=*random_fixtures_test.py
+ --ignore-glob=*create_test_*.py
+norecursedirs = build docs node_modules resources .tox
DJANGO_SETTINGS_MODULE = swh.web.settings.tests
filterwarnings =
ignore:.*Plural value must be an integer, got float
diff --git a/setup.py b/setup.py
index 2b7fdac14..54e4e10b1 100755
--- a/setup.py
+++ b/setup.py
@@ -1,40 +1,12 @@
#!/usr/bin/env python3
-# Copyright (C) 2015-2019 The Software Heritage developers
+# Copyright (C) 2015-2023 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License v3 or later
# See top-level LICENSE file for more information
-from io import open
import os
-from os import path
-
-from setuptools import find_packages, setup
-
-here = path.abspath(path.dirname(__file__))
-
-# Get the long description from the README file
-with open(path.join(here, "README.rst"), encoding="utf-8") as f:
- long_description = f.read()
-
-
-def parse_requirements(name=None):
- if name:
- reqf = "requirements-%s.txt" % name
- else:
- reqf = "requirements.txt"
-
- requirements = []
- if not path.exists(reqf):
- return requirements
-
- with open(reqf) as f:
- for line in f.readlines():
- line = line.strip()
- if not line or line.startswith("#"):
- continue
- requirements.append(line)
- return requirements
+from setuptools import setup
# package generated static assets as module data files
data_files = []
@@ -43,35 +15,7 @@ for folder in ("static/", "assets/"):
root_files = [os.path.join(root, i) for i in files]
data_files.append((os.path.join("share/swh/web", root), root_files))
+# TODO: data_files usage has been deprecated, we should move away from using it...
setup(
- name="swh.web",
- description="Software Heritage Web UI",
- long_description=long_description,
- long_description_content_type="text/x-rst",
- python_requires=">=3.7",
- author="Software Heritage developers",
- author_email="swh-devel@inria.fr",
- url="https://gitlab.softwareheritage.org/swh/devel/swh-web",
- packages=find_packages(),
- scripts=[],
- install_requires=parse_requirements() + parse_requirements("swh"),
- setup_requires=["setuptools-scm"],
- use_scm_version=True,
- extras_require={"testing": parse_requirements("test")},
- include_package_data=True,
- classifiers=[
- "Programming Language :: Python :: 3",
- "Intended Audience :: Developers",
- "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", # noqa
- "Operating System :: OS Independent",
- "Development Status :: 5 - Production/Stable",
- "Framework :: Django",
- ],
- project_urls={
- "Bug Reports": "https://gitlab.softwareheritage.org/swh/devel/swh-web/-/issues",
- "Funding": "https://www.softwareheritage.org/donate",
- "Source": "https://gitlab.softwareheritage.org/swh/devel/swh-web.git",
- "Documentation": "https://docs.softwareheritage.org/devel/swh-web/",
- },
data_files=data_files,
)
diff --git a/swh/__init__.py b/swh/__init__.py
deleted file mode 100644
index b36383a61..000000000
--- a/swh/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from pkgutil import extend_path
-
-__path__ = extend_path(__path__, __name__)
diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py
index b2aaaf2e8..fc1b08db9 100644
--- a/swh/web/settings/common.py
+++ b/swh/web/settings/common.py
@@ -10,9 +10,11 @@ Django common settings for swh-web.
from importlib.util import find_spec
import os
+from pathlib import Path
import site
import sys
from typing import Any, Dict
+import warnings
from django.utils import encoding
@@ -198,15 +200,27 @@ STATIC_DIR = os.path.join(sys.prefix, "share/swh/web/static")
if not os.path.exists(STATIC_DIR):
# static folder location when swh-web has been installed with "pip --user"
STATIC_DIR = os.path.join(site.getuserbase(), "share/swh/web/static")
+
+if not os.path.exists(STATIC_DIR):
+ ROOT_DIR = Path(PROJECT_DIR).absolute()
+ while not (ROOT_DIR / "static").is_dir():
+ ROOT_DIR = ROOT_DIR = ROOT_DIR.parent
+ if (ROOT_DIR / ".git").is_dir() or (ROOT_DIR / "pyproject.toml").is_file():
+ break
+ if ROOT_DIR == ROOT_DIR.parent:
+ break
+ STATIC_DIR = str(ROOT_DIR / "static")
+
if not os.path.exists(STATIC_DIR):
- # static folder location when developping swh-web
- STATIC_DIR = os.path.join(PROJECT_DIR, "../../../static")
+ warnings.warn(
+ "Unable to find the static assets directory. Check your installation of swh.web."
+ )
+
STATICFILES_DIRS = [STATIC_DIR]
if "static_path" in SWH_MIRROR_CONFIG:
STATICFILES_DIRS.append(SWH_MIRROR_CONFIG["static_path"])
-
INTERNAL_IPS = ["127.0.0.1"]
throttle_rates = {}
diff --git a/swh/web/utils/exc.py b/swh/web/utils/exc.py
index a3367ea89..ae4079c1f 100644
--- a/swh/web/utils/exc.py
+++ b/swh/web/utils/exc.py
@@ -81,7 +81,6 @@ http_status_code_message = {
def _generate_error_page(
request: HttpRequest, error_code: int, error_description: str
) -> HttpResponse:
-
error_data = {
"error": http_status_code_message[error_code],
"reason": error_description,
diff --git a/tox.ini b/tox.ini
index fe028a604..e48d0e475 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,10 @@
[tox]
-requires =
- tox>4
-envlist=black,flake8,mypy,py3
+minversion = 4
+envlist =
+ black
+ flake8
+ mypy
+ py3
[testenv]
extras =
@@ -10,10 +13,24 @@ deps =
pytest-cov
commands =
pytest \
- slow: --hypothesis-profile=swh-web \
- !slow: --hypothesis-profile=swh-web-fast \
- --ignore-glob=*random_fixtures_test.py \
- --cov {envsitepackagesdir}/swh/web --cov-branch {posargs} {envsitepackagesdir}/swh/web
+ --import-mode importlib \
+ --rootdir {envsitepackagesdir} \
+ --cov={envsitepackagesdir}/swh/web \
+ --cov-branch \
+ slow: --hypothesis-profile=swh-web \
+ !slow: --hypothesis-profile=swh-web-fast \
+ {envsitepackagesdir}/swh/web \
+ {posargs}
+# --rootdir and --import-mode are required to make tests that depends
+# on the test file to be a proper submodule of the swh namespace after
+# migration to PEP420 (implicit namespace).
+
+[testenv:black]
+skip_install = true
+deps =
+ black==22.10.0
+commands =
+ {envpython} -m black --check swh
[testenv:flake8]
skip_install = true
@@ -34,44 +51,19 @@ deps =
commands =
mypy swh
-[testenv:black]
-skip_install = true
-deps =
- black==22.10.0
-commands =
- {envpython} -m black --exclude swh/web/tests/resources --check swh
-
# build documentation outside swh-environment using the current
# git HEAD of swh-docs, is executed on CI for each diff to prevent
# breaking doc build
[testenv:sphinx]
allowlist_externals = make
-usedevelop = true
-extras =
- testing
-deps =
- # fetch and install swh-docs in develop mode
- -e git+https://gitlab.softwareheritage.org/swh/devel/swh-docs.git\#egg=swh.docs
-setenv =
- SWH_PACKAGE_DOC_TOX_BUILD = 1
- # turn warnings into errors
- SPHINXOPTS = -W
-commands =
- make -I ../.tox/sphinx/src/swh-docs/swh/ -C docs
-
-# build documentation only inside swh-environment using local state
-# of swh-docs package
-[testenv:sphinx-dev]
-allowlist_externals = make
-usedevelop = true
extras =
testing
deps =
- # install swh-docs in develop mode
- -e ../swh-docs
+ # fetch and install swh-docs
+ git+https://gitlab.softwareheritage.org/swh/devel/swh-docs.git\#egg=swh.docs
setenv =
SWH_PACKAGE_DOC_TOX_BUILD = 1
# turn warnings into errors
SPHINXOPTS = -W
commands =
- make -I ../.tox/sphinx-dev/src/swh-docs/swh/ -C docs
+ make -I {env_dir}/share/ -C docs
--
GitLab