From d4d718502e161527736bf8d8104f6234e79a66c1 Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Fri, 14 Feb 2025 11:12:23 +0100
Subject: [PATCH 1/3] Apply swh-py-template v0.3.3 with copier
Bump development tools: mypy, codespell, isort, ...
Move all tools configuration in pyproject.toml.
Remove no longer needed mypy overrides.
---
.copier-answers.yml | 2 +-
.pre-commit-config.yaml | 12 +++++-----
mypy.ini | 50 -----------------------------------------
pyproject.toml | 40 +++++++++++++++++++++++++++++++++
pytest.ini | 5 -----
requirements-test.txt | 4 ++++
setup.cfg | 9 --------
swh/graph/download.py | 5 ++---
tox.ini | 8 ++++---
9 files changed, 58 insertions(+), 77 deletions(-)
delete mode 100644 mypy.ini
delete mode 100644 pytest.ini
delete mode 100644 setup.cfg
diff --git a/.copier-answers.yml b/.copier-answers.yml
index 5e2536575..22baaec97 100644
--- a/.copier-answers.yml
+++ b/.copier-answers.yml
@@ -1,5 +1,5 @@
# Changes here will be overwritten by Copier
-_commit: v0.2.3
+_commit: v0.3.3
_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
description: Software Heritage graph service
distribution_name: swh-graph
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3483a4203..cf77ac527 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,18 +1,18 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.6.0
+ rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: check-json
- id: check-yaml
- repo: https://github.com/python/black
- rev: 24.8.0
+ rev: 25.1.0
hooks:
- id: black
- repo: https://github.com/PyCQA/isort
- rev: 5.13.2
+ rev: 6.0.0
hooks:
- id: isort
@@ -20,10 +20,10 @@ repos:
rev: 7.1.1
hooks:
- id: flake8
- additional_dependencies: [flake8-bugbear==24.4.26]
+ additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
- repo: https://github.com/codespell-project/codespell
- rev: v2.3.0
+ rev: v2.4.1
hooks:
- id: codespell
name: Check source code spelling
@@ -31,7 +31,7 @@ repos:
- "-L te,wth,alledges,afterall,mmapped,crate,beling,ser,\
implementor,implementors"
- "--skip=rust/tests/data/*"
- stages: [commit]
+ stages: [pre-commit]
- id: codespell
name: Check commit message spelling
stages: [commit-msg]
diff --git a/mypy.ini b/mypy.ini
deleted file mode 100644
index 8a99503f5..000000000
--- a/mypy.ini
+++ /dev/null
@@ -1,50 +0,0 @@
-[mypy]
-plugins = luigi.mypy
-namespace_packages = True
-warn_unused_ignores = True
-explicit_package_bases = True
-# ^ Needed for mypy to detect py.typed from swh packages installed
-# in editable mode
-exclude = (?x)(
- ^swh/graph/grpc
- )
-
-# 3rd party libraries without stubs (yet)
-[mypy-botocore.*]
-ignore_missing_imports = True
-
-[mypy-boto3.*]
-ignore_missing_imports = True
-
-[mypy-datafusion.*]
-ignore_missing_imports = True
-
-[mypy-grpc.aio.*]
-ignore_missing_imports = True
-
-[mypy-luigi.*]
-ignore_missing_imports = True
-
-[mypy-magic.*]
-ignore_missing_imports = True
-
-[mypy-pkg_resources.*]
-ignore_missing_imports = True
-
-[mypy-psutil.*]
-ignore_missing_imports = True
-
-[mypy-pyarrow.*]
-ignore_missing_imports = True
-
-[mypy-py4j.*]
-ignore_missing_imports = True
-
-[mypy-pyzstd.*]
-ignore_missing_imports = True
-
-[mypy-scancode.*]
-ignore_missing_imports = True
-
-# [mypy-add_your_lib_here.*]
-# ignore_missing_imports = True
diff --git a/pyproject.toml b/pyproject.toml
index 212fc471b..00718b4ef 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,3 +67,43 @@ ensure_newline_before_comments = true
line_length = 88
force_sort_within_sections = true
known_first_party = ['swh']
+
+[tool.mypy]
+namespace_packages = true
+warn_unused_ignores = true
+explicit_package_bases = true
+# ^ Needed for mypy to detect py.typed from swh packages installed
+# in editable mode
+exclude = [
+ "^swh/graph/grpc"
+]
+
+plugins = ["luigi.mypy"]
+
+# 3rd party libraries without stubs (yet)
+[[tool.mypy.overrides]]
+module = [
+ "datafusion.*",
+ "grpc.aio.*",
+ "luigi.*",
+ "magic.*",
+ "pkg_resources.*",
+ "pyzstd.*",
+ "scancode.*",
+]
+ignore_missing_imports = true
+
+[tool.flake8]
+select = ["C", "E", "F", "W", "B950"]
+ignore = [
+ "E203", # whitespaces before ':' <https://github.com/psf/black/issues/315>
+ "E231", # missing whitespace after ','
+ "E501", # line too long, use B950 warning from flake8-bugbear instead
+ "W503" # line break before binary operator <https://github.com/psf/black/issues/52>
+]
+max-line-length = 88
+
+[tool.pytest.ini_options]
+norecursedirs = "build docs .*"
+asyncio_mode = "strict"
+consider_namespace_packages = true
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index c017966a0..000000000
--- a/pytest.ini
+++ /dev/null
@@ -1,5 +0,0 @@
-[pytest]
-norecursedirs = build docs
-asyncio_mode = strict
-consider_namespace_packages = true
-addopts = --ignore=target/
diff --git a/requirements-test.txt b/requirements-test.txt
index 4b48f4e48..8a53ea997 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -4,8 +4,12 @@ pytest-postgresql
swh.core[testing] >= 3.0.0
types-click
+types-psutil
types-pyyaml
types-requests
types-protobuf
types-tqdm
+boto3-stubs
+botocore-stubs
grpc-stubs
+pyarrow-stubs
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 63fd1d0de..000000000
--- a/setup.cfg
+++ /dev/null
@@ -1,9 +0,0 @@
-[flake8]
-# E203: whitespaces before ':' <https://github.com/psf/black/issues/315>
-# E231: missing whitespace after ','
-# E501: line too long, use B950 warning from flake8-bugbear instead
-# W503: line break before binary operator <https://github.com/psf/black/issues/52>
-select = C,E,F,W,B950
-ignore = E203,E231,E501,E704,W503
-max-line-length = 88
-extend_exclude = swh/graph/grpc build
diff --git a/swh/graph/download.py b/swh/graph/download.py
index ac61e7fa6..dff867e7c 100644
--- a/swh/graph/download.py
+++ b/swh/graph/download.py
@@ -9,6 +9,7 @@ from typing import Callable
import boto3
import botocore
+from botocore.handlers import disable_signing
import tqdm
@@ -24,9 +25,7 @@ class GraphDownloader:
self.s3 = boto3.resource("s3")
# don't require credentials to list the bucket
- self.s3.meta.client.meta.events.register(
- "choose-signer.s3.*", botocore.handlers.disable_signing
- )
+ self.s3.meta.client.meta.events.register("choose-signer.s3.*", disable_signing)
self.client = boto3.client(
"s3",
config=botocore.client.Config(
diff --git a/tox.ini b/tox.ini
index 9cc3872f6..ae584e054 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,7 +21,7 @@ commands =
[testenv:black]
skip_install = true
deps =
- black==24.8.0
+ black==25.1.0
commands =
{envpython} -m black --check swh
@@ -29,8 +29,10 @@ commands =
skip_install = true
deps =
flake8==7.1.1
- flake8-bugbear==24.4.26
+ flake8-bugbear==24.12.12
+ flake8-pyproject==1.2.3
pycodestyle==2.12.1
+
commands =
{envpython} -m flake8
@@ -39,7 +41,7 @@ extras =
testing
luigi
deps =
- mypy==1.11.1
+ mypy==1.15.0
commands =
mypy swh
--
GitLab
From 5cf8107f32806348f927a97ee5a729f8ccf526ec Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Fri, 14 Feb 2025 11:13:49 +0100
Subject: [PATCH 2/3] webgraph: Fix black formatting
---
swh/graph/webgraph.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/swh/graph/webgraph.py b/swh/graph/webgraph.py
index e54119fe0..ee7280b06 100644
--- a/swh/graph/webgraph.py
+++ b/swh/graph/webgraph.py
@@ -3,9 +3,7 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-"""WebGraph driver
-
-"""
+"""WebGraph driver"""
from datetime import datetime
import difflib
--
GitLab
From 22c98c5be39aab6216f9347f354ef62e6bfb1c18 Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Fri, 14 Feb 2025 11:14:24 +0100
Subject: [PATCH 3/3] luigi/compressed_graph: Fix flake8 warnings
---
swh/graph/luigi/compressed_graph.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/swh/graph/luigi/compressed_graph.py b/swh/graph/luigi/compressed_graph.py
index 78a00346c..4228bb67d 100644
--- a/swh/graph/luigi/compressed_graph.py
+++ b/swh/graph/luigi/compressed_graph.py
@@ -486,7 +486,7 @@ class _CompressionStepTask(luigi.Task):
conf: dict[str, Any] = {
"object_types": ",".join(self.object_types),
- "max_ram": f"{(self._large_java_allocations() + _LOW_XMX)//(1024*1024)}M",
+ "max_ram": f"{(self._large_java_allocations() + _LOW_XMX) // (1024 * 1024)}M",
# TODO: make this more configurable
}
if self.batch_size:
--
GitLab