diff --git a/.copier-answers.yml b/.copier-answers.yml index 5e2536575335d59c4ac72043b55829538f6808c2..22baaec976a0f9bd9d7f04ff4ffe19a2e00c2e4e 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.2.3 +_commit: v0.3.3 _src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git description: Software Heritage graph service distribution_name: swh-graph diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3483a42035a7c3df3265a14f4e1d2a672674c18c..cf77ac52736942cbc637d088899f56df33f2e9f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,18 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: check-json - id: check-yaml - repo: https://github.com/python/black - rev: 24.8.0 + rev: 25.1.0 hooks: - id: black - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 6.0.0 hooks: - id: isort @@ -20,10 +20,10 @@ repos: rev: 7.1.1 hooks: - id: flake8 - additional_dependencies: [flake8-bugbear==24.4.26] + additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject] - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell name: Check source code spelling @@ -31,7 +31,7 @@ repos: - "-L te,wth,alledges,afterall,mmapped,crate,beling,ser,\ implementor,implementors" - "--skip=rust/tests/data/*" - stages: [commit] + stages: [pre-commit] - id: codespell name: Check commit message spelling stages: [commit-msg] diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 8a99503f5f78bf21a197ca025f3363862658d82c..0000000000000000000000000000000000000000 --- a/mypy.ini +++ /dev/null @@ -1,50 +0,0 @@ -[mypy] -plugins = luigi.mypy -namespace_packages = True -warn_unused_ignores = True -explicit_package_bases = True -# ^ Needed for mypy to detect py.typed from swh packages installed -# in editable mode -exclude = (?x)( - ^swh/graph/grpc - ) - -# 3rd party libraries without stubs (yet) -[mypy-botocore.*] -ignore_missing_imports = True - -[mypy-boto3.*] -ignore_missing_imports = True - -[mypy-datafusion.*] -ignore_missing_imports = True - -[mypy-grpc.aio.*] -ignore_missing_imports = True - -[mypy-luigi.*] -ignore_missing_imports = True - -[mypy-magic.*] -ignore_missing_imports = True - -[mypy-pkg_resources.*] -ignore_missing_imports = True - -[mypy-psutil.*] -ignore_missing_imports = True - -[mypy-pyarrow.*] -ignore_missing_imports = True - -[mypy-py4j.*] -ignore_missing_imports = True - -[mypy-pyzstd.*] -ignore_missing_imports = True - -[mypy-scancode.*] -ignore_missing_imports = True - -# [mypy-add_your_lib_here.*] -# ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 212fc471b090744e086bb62f8f6dbe3f1701fdf3..00718b4ef474a22991d36d7ceece70022f869722 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,3 +67,43 @@ ensure_newline_before_comments = true line_length = 88 force_sort_within_sections = true known_first_party = ['swh'] + +[tool.mypy] +namespace_packages = true +warn_unused_ignores = true +explicit_package_bases = true +# ^ Needed for mypy to detect py.typed from swh packages installed +# in editable mode +exclude = [ + "^swh/graph/grpc" +] + +plugins = ["luigi.mypy"] + +# 3rd party libraries without stubs (yet) +[[tool.mypy.overrides]] +module = [ + "datafusion.*", + "grpc.aio.*", + "luigi.*", + "magic.*", + "pkg_resources.*", + "pyzstd.*", + "scancode.*", +] +ignore_missing_imports = true + +[tool.flake8] +select = ["C", "E", "F", "W", "B950"] +ignore = [ + "E203", # whitespaces before ':' <https://github.com/psf/black/issues/315> + "E231", # missing whitespace after ',' + "E501", # line too long, use B950 warning from flake8-bugbear instead + "W503" # line break before binary operator <https://github.com/psf/black/issues/52> +] +max-line-length = 88 + +[tool.pytest.ini_options] +norecursedirs = "build docs .*" +asyncio_mode = "strict" +consider_namespace_packages = true diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index c017966a0e7e3e27093f6f877f487dc5cfeab769..0000000000000000000000000000000000000000 --- a/pytest.ini +++ /dev/null @@ -1,5 +0,0 @@ -[pytest] -norecursedirs = build docs -asyncio_mode = strict -consider_namespace_packages = true -addopts = --ignore=target/ diff --git a/requirements-test.txt b/requirements-test.txt index 4b48f4e484b2093717dc7270ad29546e692e9226..8a53ea997dbfd1db8328d646e2ae62fbbd0636a8 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,8 +4,12 @@ pytest-postgresql swh.core[testing] >= 3.0.0 types-click +types-psutil types-pyyaml types-requests types-protobuf types-tqdm +boto3-stubs +botocore-stubs grpc-stubs +pyarrow-stubs diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 63fd1d0de44bb7608e931c749375db72cdba8a2d..0000000000000000000000000000000000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[flake8] -# E203: whitespaces before ':' <https://github.com/psf/black/issues/315> -# E231: missing whitespace after ',' -# E501: line too long, use B950 warning from flake8-bugbear instead -# W503: line break before binary operator <https://github.com/psf/black/issues/52> -select = C,E,F,W,B950 -ignore = E203,E231,E501,E704,W503 -max-line-length = 88 -extend_exclude = swh/graph/grpc build diff --git a/swh/graph/download.py b/swh/graph/download.py index ac61e7fa6c8aeec610d15f97c089149fa96e86f1..dff867e7cfae7fa179031a5ee4ec4f4effdcf82e 100644 --- a/swh/graph/download.py +++ b/swh/graph/download.py @@ -9,6 +9,7 @@ from typing import Callable import boto3 import botocore +from botocore.handlers import disable_signing import tqdm @@ -24,9 +25,7 @@ class GraphDownloader: self.s3 = boto3.resource("s3") # don't require credentials to list the bucket - self.s3.meta.client.meta.events.register( - "choose-signer.s3.*", botocore.handlers.disable_signing - ) + self.s3.meta.client.meta.events.register("choose-signer.s3.*", disable_signing) self.client = boto3.client( "s3", config=botocore.client.Config( diff --git a/swh/graph/luigi/compressed_graph.py b/swh/graph/luigi/compressed_graph.py index 78a00346c5f2e095dbec3c2532ee43af91fdd6aa..4228bb67d5e2aa6b77d191d7119724d229fe283c 100644 --- a/swh/graph/luigi/compressed_graph.py +++ b/swh/graph/luigi/compressed_graph.py @@ -486,7 +486,7 @@ class _CompressionStepTask(luigi.Task): conf: dict[str, Any] = { "object_types": ",".join(self.object_types), - "max_ram": f"{(self._large_java_allocations() + _LOW_XMX)//(1024*1024)}M", + "max_ram": f"{(self._large_java_allocations() + _LOW_XMX) // (1024 * 1024)}M", # TODO: make this more configurable } if self.batch_size: diff --git a/swh/graph/webgraph.py b/swh/graph/webgraph.py index e54119fe0090842dd266eaeb6b6dc335372e6672..ee7280b0648a5d8fefaabc082fa7fe501987d99a 100644 --- a/swh/graph/webgraph.py +++ b/swh/graph/webgraph.py @@ -3,9 +3,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -"""WebGraph driver - -""" +"""WebGraph driver""" from datetime import datetime import difflib diff --git a/tox.ini b/tox.ini index 9cc3872f626ec2bb3cc19c744d283f41fecb5489..ae584e054284a1d5dfa58d49a9e8434ea135dacc 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ commands = [testenv:black] skip_install = true deps = - black==24.8.0 + black==25.1.0 commands = {envpython} -m black --check swh @@ -29,8 +29,10 @@ commands = skip_install = true deps = flake8==7.1.1 - flake8-bugbear==24.4.26 + flake8-bugbear==24.12.12 + flake8-pyproject==1.2.3 pycodestyle==2.12.1 + commands = {envpython} -m flake8 @@ -39,7 +41,7 @@ extras = testing luigi deps = - mypy==1.11.1 + mypy==1.15.0 commands = mypy swh