diff --git a/conftest.py b/conftest.py index 888e3c3cd10fe5fcc696550d6627cddd449fa85d..8b6908fe913ef3d9c3fecb3b1b6c2e7fdf2b9b1f 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,5 @@ -import pytest from hypothesis import settings +import pytest from swh.core.cli import swh as _swhmain diff --git a/setup.py b/setup.py index f7426e5b817ed6c3f7c2426da411f15e11e7a174..0c550c95be43226a694b7da49f2825cc022ba3a4 100755 --- a/setup.py +++ b/setup.py @@ -4,11 +4,11 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from io import open import os -from setuptools import setup, find_packages - from os import path -from io import open + +from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) diff --git a/swh/core/api/__init__.py b/swh/core/api/__init__.py index 376397d528208dcb9ba91bc2dd38ff326296a635..2094698f0d51914b8a220d56343f49fa3118f68c 100644 --- a/swh/core/api/__init__.py +++ b/swh/core/api/__init__.py @@ -8,8 +8,6 @@ import functools import inspect import logging import pickle -import requests - from typing import ( Any, Callable, @@ -23,25 +21,22 @@ from typing import ( Union, ) -from flask import Flask, Request, Response, request, abort +from flask import Flask, Request, Response, abort, request +import requests from werkzeug.exceptions import HTTPException +from .negotiation import Formatter as FormatterBase +from .negotiation import Negotiator as NegotiatorBase +from .negotiation import negotiate as _negotiate from .serializers import ( - decode_response, - encode_data_client as encode_data, - msgpack_dumps, - msgpack_loads, + exception_to_dict, json_dumps, json_loads, - exception_to_dict, -) - -from .negotiation import ( - Formatter as FormatterBase, - Negotiator as NegotiatorBase, - negotiate as _negotiate, + msgpack_dumps, + msgpack_loads, ) - +from .serializers import decode_response +from .serializers import encode_data_client as encode_data logger = logging.getLogger(__name__) diff --git a/swh/core/api/asynchronous.py b/swh/core/api/asynchronous.py index 04830439e544ebdbb1afc29f39bd174b415121f4..4522834a6371b011fad6e6ded1136d6bd039802c 100644 --- a/swh/core/api/asynchronous.py +++ b/swh/core/api/asynchronous.py @@ -8,14 +8,17 @@ import logging from typing import Tuple, Type import aiohttp.web +from aiohttp_utils import Response, negotiation from deprecated import deprecated import multidict -from .serializers import msgpack_dumps, msgpack_loads -from .serializers import json_dumps, json_loads -from .serializers import exception_to_dict - -from aiohttp_utils import negotiation, Response +from .serializers import ( + exception_to_dict, + json_dumps, + json_loads, + msgpack_dumps, + msgpack_loads, +) def encode_msgpack(data, **kwargs): diff --git a/swh/core/api/classes.py b/swh/core/api/classes.py index 0aefcae2ff9af849609b5fc3fcee636fc4ec459e..c84db28536f938ed90c7cfdee09a22d9fd0c190b 100644 --- a/swh/core/api/classes.py +++ b/swh/core/api/classes.py @@ -4,16 +4,7 @@ # See top-level LICENSE file for more information from dataclasses import dataclass, field - -from typing import ( - Callable, - Generic, - Iterable, - List, - Optional, - TypeVar, -) - +from typing import Callable, Generic, Iterable, List, Optional, TypeVar TResult = TypeVar("TResult") TToken = TypeVar("TToken") diff --git a/swh/core/api/negotiation.py b/swh/core/api/negotiation.py index 4e2abab6aa89ab27cdebfaf312755445e4fa5433..8f6398bf3f91a1bebc7a582358acddd803339efb 100644 --- a/swh/core/api/negotiation.py +++ b/swh/core/api/negotiation.py @@ -24,11 +24,10 @@ # from collections import defaultdict -from decorator import decorator from inspect import getcallargs +from typing import Any, Callable, DefaultDict, List, NoReturn, Optional, Type -from typing import Any, List, Optional, Callable, Type, NoReturn, DefaultDict - +from decorator import decorator from requests import Response diff --git a/swh/core/api/serializers.py b/swh/core/api/serializers.py index 8d8cbf120cab34a596660f83d23b6c066b17c551..3b6c139c77a29f954eb0f49e642ff4703f0727a8 100644 --- a/swh/core/api/serializers.py +++ b/swh/core/api/serializers.py @@ -9,13 +9,12 @@ from enum import Enum import json import traceback import types +from typing import Any, Dict, Tuple, Union from uuid import UUID import arrow import iso8601 import msgpack - -from typing import Any, Dict, Union, Tuple from requests import Response from swh.core.api.classes import PagedResult diff --git a/swh/core/api/tests/test_async.py b/swh/core/api/tests/test_async.py index 7b8400a2009773ce3563f0e415098dab571e2e37..4b25aa311e829547e18ed3bbee75fb5b9265d058 100644 --- a/swh/core/api/tests/test_async.py +++ b/swh/core/api/tests/test_async.py @@ -4,16 +4,18 @@ # See top-level LICENSE file for more information import datetime -import msgpack import json +import msgpack import pytest -from swh.core.api.asynchronous import RPCServerApp, Response -from swh.core.api.asynchronous import encode_msgpack, decode_request - -from swh.core.api.serializers import msgpack_dumps, SWHJSONEncoder - +from swh.core.api.asynchronous import ( + Response, + RPCServerApp, + decode_request, + encode_msgpack, +) +from swh.core.api.serializers import SWHJSONEncoder, msgpack_dumps pytest_plugins = ["aiohttp.pytest_plugin", "pytester"] diff --git a/swh/core/api/tests/test_classes.py b/swh/core/api/tests/test_classes.py index d63d4108fdf3781d0b813e83f389dc007e2858eb..7509f51c7039528b8c611aacd6b6573ad898d73d 100644 --- a/swh/core/api/tests/test_classes.py +++ b/swh/core/api/tests/test_classes.py @@ -3,10 +3,10 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.core.api.classes import PagedResult as CorePagedResult, stream_results - from typing import TypeVar +from swh.core.api.classes import PagedResult as CorePagedResult +from swh.core.api.classes import stream_results T = TypeVar("T") TestPagedResult = CorePagedResult[T, bytes] diff --git a/swh/core/api/tests/test_gunicorn.py b/swh/core/api/tests/test_gunicorn.py index c0d12ef5d763421c38b7ba62f53a55ca1dba86ae..92a3284d9fb748a92b7598295c60bdc6e156baf2 100644 --- a/swh/core/api/tests/test_gunicorn.py +++ b/swh/core/api/tests/test_gunicorn.py @@ -4,9 +4,10 @@ # See top-level LICENSE file for more information import os -import pkg_resources from unittest.mock import patch +import pkg_resources + import swh.core.api.gunicorn_config as gunicorn_config diff --git a/swh/core/api/tests/test_rpc_client.py b/swh/core/api/tests/test_rpc_client.py index 24a2fba6be1cab06a44ed6596d002409dad24874..7dffac198ba64e91dc3dac39f61ea35f4db8ffe5 100644 --- a/swh/core/api/tests/test_rpc_client.py +++ b/swh/core/api/tests/test_rpc_client.py @@ -4,11 +4,12 @@ # See top-level LICENSE file for more information import re + import pytest -from swh.core.api import remote_api_endpoint, RPCClient +from swh.core.api import RPCClient, remote_api_endpoint -from .test_serializers import ExtraType, extra_encoders, extra_decoders +from .test_serializers import ExtraType, extra_decoders, extra_encoders @pytest.fixture diff --git a/swh/core/api/tests/test_rpc_client_server.py b/swh/core/api/tests/test_rpc_client_server.py index ec651eb2882b7f7a84391177c68fcf7decee8320..81b0afa596d31fd1f6a41fa3254533d274714f36 100644 --- a/swh/core/api/tests/test_rpc_client_server.py +++ b/swh/core/api/tests/test_rpc_client_server.py @@ -5,8 +5,14 @@ import pytest -from swh.core.api import remote_api_endpoint, RPCServerApp, RPCClient -from swh.core.api import error_handler, encode_data_server, RemoteException +from swh.core.api import ( + RemoteException, + RPCClient, + RPCServerApp, + encode_data_server, + error_handler, + remote_api_endpoint, +) # this class is used on the server part diff --git a/swh/core/api/tests/test_rpc_server.py b/swh/core/api/tests/test_rpc_server.py index 31fdb27895dcd746a82195946279b8e3bc6cdcc7..84bdf2f93f42861bcb2a6eb4514e51fd7e7705d6 100644 --- a/swh/core/api/tests/test_rpc_server.py +++ b/swh/core/api/tests/test_rpc_server.py @@ -3,15 +3,21 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pytest import json -import msgpack from flask import url_for +import msgpack +import pytest + +from swh.core.api import ( + JSONFormatter, + MsgpackFormatter, + RPCServerApp, + negotiate, + remote_api_endpoint, +) -from swh.core.api import remote_api_endpoint, RPCServerApp -from swh.core.api import negotiate, JSONFormatter, MsgpackFormatter -from .test_serializers import ExtraType, extra_encoders, extra_decoders +from .test_serializers import ExtraType, extra_decoders, extra_encoders class MyRPCServerApp(RPCServerApp): diff --git a/swh/core/api/tests/test_serializers.py b/swh/core/api/tests/test_serializers.py index 0b4803d8efb8a87e8bc1df143ce1c94829af9766..9d7c261dc309f9f55e5600ce8a250a11a8450041 100644 --- a/swh/core/api/tests/test_serializers.py +++ b/swh/core/api/tests/test_serializers.py @@ -5,23 +5,21 @@ import datetime import json - from typing import Any, Callable, List, Tuple, Union - -from arrow import Arrow from uuid import UUID -import pytest import arrow +from arrow import Arrow +import pytest import requests from swh.core.api.classes import PagedResult from swh.core.api.serializers import ( SWHJSONDecoder, SWHJSONEncoder, + decode_response, msgpack_dumps, msgpack_loads, - decode_response, ) diff --git a/swh/core/cli/__init__.py b/swh/core/cli/__init__.py index a840d552d755ea25e5b6ca27c810f622554307bd..1488bd334f7c635adfb02f9008db924637eca92b 100644 --- a/swh/core/cli/__init__.py +++ b/swh/core/cli/__init__.py @@ -85,7 +85,9 @@ def swh(ctx, log_level, log_config, sentry_dsn, sentry_debug): """Command line interface for Software Heritage. """ import signal + import yaml + from ..sentry import init_sentry signal.signal(signal.SIGTERM, clean_exit_on_signal) diff --git a/swh/core/cli/db.py b/swh/core/cli/db.py index 8090df645116e0fb5ea74ce5bb409f392a97b759..86f81bee296f463401aae428490aede22c24f2fd 100755 --- a/swh/core/cli/db.py +++ b/swh/core/cli/db.py @@ -5,16 +5,14 @@ # See top-level LICENSE file for more information import logging -from os import path, environ +from os import environ, path from typing import Tuple - import warnings import click from swh.core.cli import CONTEXT_SETTINGS - warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t @@ -152,6 +150,7 @@ def db_init(module, db_name, create_db): def get_sql_for_package(modname): import glob from importlib import import_module + from swh.core.utils import numfile_sortkey as sortkey if not modname.startswith("swh."): @@ -181,6 +180,7 @@ def populate_database_for_package(modname: str, conninfo: str) -> Tuple[bool, in version of the database. """ import subprocess + from swh.core.db.tests.db_testing import swh_db_version current_version = swh_db_version(conninfo) diff --git a/swh/core/config.py b/swh/core/config.py index f05983a23e839758039dc2c8b3b4c2300045c8cc..91a8a4a8c28d421f55aaf829e0918b3287acca5a 100644 --- a/swh/core/config.py +++ b/swh/core/config.py @@ -3,14 +3,13 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from copy import deepcopy +from itertools import chain import logging import os -import yaml -from itertools import chain -from copy import deepcopy - from typing import Any, Callable, Dict, List, Optional, Tuple +import yaml logger = logging.getLogger(__name__) diff --git a/swh/core/db/__init__.py b/swh/core/db/__init__.py index a98d5e90643927213682323f52c1ed89b482e992..eb1341f6277fbc655a1231e628bf9f34e1e233e5 100644 --- a/swh/core/db/__init__.py +++ b/swh/core/db/__init__.py @@ -3,6 +3,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from contextlib import contextmanager import datetime import enum import json @@ -12,13 +13,10 @@ import sys import threading from typing import Any, Callable, Iterable, Iterator, Mapping, Optional, Type, TypeVar -from contextlib import contextmanager - import psycopg2 import psycopg2.extras import psycopg2.pool - logger = logging.getLogger(__name__) diff --git a/swh/core/db/common.py b/swh/core/db/common.py index 17c46be67b6ffadb0350a2e720bfb8b8c3fcb450..65e6d752a793c577d7e2b9851fc86b134a93d708 100644 --- a/swh/core/db/common.py +++ b/swh/core/db/common.py @@ -3,8 +3,8 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import inspect import functools +import inspect def remove_kwargs(names): diff --git a/swh/core/db/db_utils.py b/swh/core/db/db_utils.py index aa97a9376df1fd5cef1bc0c5ac20fe04227f5bae..d656a5df00d206d6a2e162758837c9396bc916bb 100644 --- a/swh/core/db/db_utils.py +++ b/swh/core/db/db_utils.py @@ -13,8 +13,8 @@ # later version. -import re import functools +import re import psycopg2.extensions diff --git a/swh/core/db/tests/db_testing.py b/swh/core/db/tests/db_testing.py index 89bd2342df839b904c31c1fc743a2ffceffbb90e..9f6c01bfb40318df412e34c2d6b7c596f16e5424 100644 --- a/swh/core/db/tests/db_testing.py +++ b/swh/core/db/tests/db_testing.py @@ -3,17 +3,15 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import os import glob +import os import subprocess +from typing import Dict, Iterable, Optional, Tuple, Union import psycopg2 -from typing import Dict, Iterable, Optional, Tuple, Union - from swh.core.utils import numfile_sortkey as sortkey - DB_DUMP_TYPES = {".sql": "psql", ".dump": "pg_dump"} # type: Dict[str, str] diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py index d8152711c5c9916af56fe64ea88532bf2393373e..87b40eb40ca010937f68600b76d2de7a0f56d7d5 100644 --- a/swh/core/db/tests/test_cli.py +++ b/swh/core/db/tests/test_cli.py @@ -4,7 +4,6 @@ from click.testing import CliRunner from swh.core.cli.db import db as swhdb - help_msg = """Usage: swh [OPTIONS] COMMAND [ARGS]... Command line interface for Software Heritage. diff --git a/swh/core/db/tests/test_db.py b/swh/core/db/tests/test_db.py index 93385d4d989caad1b8be616ae0099b99b9813ccc..bb7f80b9099f668d76154c506fc9ff6e447c59d1 100644 --- a/swh/core/db/tests/test_db.py +++ b/swh/core/db/tests/test_db.py @@ -11,24 +11,20 @@ import os.path from string import printable import tempfile from typing import Any -from typing_extensions import Protocol import unittest -from unittest.mock import Mock, MagicMock +from unittest.mock import MagicMock, Mock import uuid -from hypothesis import strategies, given +from hypothesis import given, strategies from hypothesis.extra.pytz import timezones import psycopg2 import pytest +from typing_extensions import Protocol from swh.core.db import BaseDb from swh.core.db.common import db_transaction, db_transaction_generator -from .db_testing import ( - SingleDbTestFixture, - db_create, - db_destroy, - db_close, -) + +from .db_testing import SingleDbTestFixture, db_close, db_create, db_destroy # workaround mypy bug https://github.com/python/mypy/issues/5485 diff --git a/swh/core/logger.py b/swh/core/logger.py index f0163a66b10db5976bfb817e15fd304335457e25..1c88883f3143eb108b9edd8a73bd2d4be278db79 100644 --- a/swh/core/logger.py +++ b/swh/core/logger.py @@ -7,7 +7,8 @@ import datetime import logging from typing import Any, Generator, List, Tuple -from systemd.journal import JournalHandler as _JournalHandler, send +from systemd.journal import JournalHandler as _JournalHandler +from systemd.journal import send try: from celery import current_task diff --git a/swh/core/pytest_plugin.py b/swh/core/pytest_plugin.py index 233d5544ba347b79197691c89081d07bc6fb8d37..7a450197cb0c524bb9bccd3735e5dbbd6a7b03bf 100644 --- a/swh/core/pytest_plugin.py +++ b/swh/core/pytest_plugin.py @@ -3,21 +3,19 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import logging -import re -import pytest -import requests - from functools import partial +import logging from os import path +import re from typing import Dict, List, Optional -from urllib.parse import urlparse, unquote +from urllib.parse import unquote, urlparse +import pytest +import requests from requests.adapters import BaseAdapter from requests.structures import CaseInsensitiveDict from requests.utils import get_encoding_from_headers - logger = logging.getLogger(__name__) diff --git a/swh/core/sentry.py b/swh/core/sentry.py index 2af66a62c37919fcaa9a0fb921e7c3d7c9ad5d1a..75e3e39ae675902c664e7533cd592a7047019351 100644 --- a/swh/core/sentry.py +++ b/swh/core/sentry.py @@ -3,9 +3,10 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pkg_resources import os +import pkg_resources + def get_sentry_release(): main_package = os.environ.get("SWH_MAIN_PACKAGE") diff --git a/swh/core/statsd.py b/swh/core/statsd.py index c212e715839309fe8a52dd377da349e59dc4e180..1747328bcacc9cfb0ceffd9741b5fc63e6c9f5d4 100644 --- a/swh/core/statsd.py +++ b/swh/core/statsd.py @@ -53,16 +53,15 @@ from asyncio import iscoroutinefunction from functools import wraps -from random import random -from time import monotonic import itertools import logging import os +from random import random import socket import threading +from time import monotonic import warnings - log = logging.getLogger("swh.core.statsd") diff --git a/swh/core/tarball.py b/swh/core/tarball.py index d557efd35112eac26e781e16da4cefd864a73019..3f3c29a9654f82e4c94c836a244b7dc8f282f00a 100644 --- a/swh/core/tarball.py +++ b/swh/core/tarball.py @@ -6,11 +6,10 @@ import os import shutil import stat +from subprocess import run import tarfile import zipfile -from subprocess import run - from . import utils diff --git a/swh/core/tests/__init__.py b/swh/core/tests/__init__.py index e70ce2aa0cbf13e895d9d1fb31a4e582874ce7c9..688a7de686927670f3a70c54aafe097ba582ca07 100644 --- a/swh/core/tests/__init__.py +++ b/swh/core/tests/__init__.py @@ -1,5 +1,5 @@ from os import path -import swh.core +import swh.core SQL_DIR = path.join(path.dirname(swh.core.__file__), "sql") diff --git a/swh/core/tests/fixture/conftest.py b/swh/core/tests/fixture/conftest.py index 412d102b898cd8d788d2e0815842cb9c3c6d0096..0c3cb69c9d028051936ffc05e3924fded145d2f7 100644 --- a/swh/core/tests/fixture/conftest.py +++ b/swh/core/tests/fixture/conftest.py @@ -3,10 +3,9 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pytest - from os import path +import pytest DATADIR = path.join(path.abspath(path.dirname(__file__)), "data") diff --git a/swh/core/tests/test_cli.py b/swh/core/tests/test_cli.py index 089eb93665b29e2eaccd74d4f1da9d22e27e023c..6e710609a15e6991a8960b8ff60db49ca5a9b5d9 100644 --- a/swh/core/tests/test_cli.py +++ b/swh/core/tests/test_cli.py @@ -4,15 +4,14 @@ # See top-level LICENSE file for more information import logging -import pkg_resources import textwrap from unittest.mock import patch import click from click.testing import CliRunner +import pkg_resources import pytest - help_msg = """Usage: swh [OPTIONS] COMMAND [ARGS]... Command line interface for Software Heritage. diff --git a/swh/core/tests/test_collections.py b/swh/core/tests/test_collections.py index c40a1211085eb24064ad368932d571dc2f624263..22efbc06646eadd5c013c3393495a5d4232f3cb7 100644 --- a/swh/core/tests/test_collections.py +++ b/swh/core/tests/test_collections.py @@ -5,10 +5,8 @@ import pytest - from swh.core.collections import SortedList - parametrize = pytest.mark.parametrize( "items", [ diff --git a/swh/core/tests/test_config.py b/swh/core/tests/test_config.py index 6ee50d50c2abc789938757e5cc103d4eacafe86a..3227ebce9135325afa42eeff31b6ab8fed656e07 100644 --- a/swh/core/tests/test_config.py +++ b/swh/core/tests/test_config.py @@ -4,10 +4,10 @@ # See top-level LICENSE file for more information import os -import pytest import shutil -import pkg_resources.extern.packaging.version +import pkg_resources.extern.packaging.version +import pytest import yaml from swh.core import config @@ -17,8 +17,8 @@ if pytest_v < pkg_resources.extern.packaging.version.parse("3.9"): @pytest.fixture def tmp_path(request): - import tempfile import pathlib + import tempfile with tempfile.TemporaryDirectory() as tmpdir: yield pathlib.Path(tmpdir) diff --git a/swh/core/tests/test_logger.py b/swh/core/tests/test_logger.py index 6980e6e1d0a82a6861b7bba9d6873b71ecdfc309..677082e639edd3761d7834ece2f2ec964a810d2e 100644 --- a/swh/core/tests/test_logger.py +++ b/swh/core/tests/test_logger.py @@ -4,12 +4,12 @@ # See top-level LICENSE file for more information from datetime import datetime -import logging -import pytz import inspect - +import logging from unittest.mock import patch +import pytz + from swh.core import logger diff --git a/swh/core/tests/test_pytest_plugin.py b/swh/core/tests/test_pytest_plugin.py index 7c72df319fba1342d454d71e7db5d44c8a8ddbcb..f8d23efa087c4fd05a4d8f538a7e57129534e64b 100644 --- a/swh/core/tests/test_pytest_plugin.py +++ b/swh/core/tests/test_pytest_plugin.py @@ -3,11 +3,11 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import requests - from os import path from urllib.parse import unquote +import requests + from swh.core.pytest_plugin import requests_mock_datadir_factory diff --git a/swh/core/tests/test_tarball.py b/swh/core/tests/test_tarball.py index 3c0449f175ad829334f1542023cb3ee6905a1212..add1becb336d37d8400e56fb4e1496a27c0710a2 100644 --- a/swh/core/tests/test_tarball.py +++ b/swh/core/tests/test_tarball.py @@ -4,9 +4,10 @@ # See top-level LICENSE file for more information import os -import pytest import shutil +import pytest + from swh.core import tarball diff --git a/swh/core/utils.py b/swh/core/utils.py index 0b6fcbd44d38b4e5bb2ecbeb9a95b5fe98e8ec91..a14daa5776aee38059024d1751b89787f5b2d2f2 100644 --- a/swh/core/utils.py +++ b/swh/core/utils.py @@ -3,12 +3,11 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import os -import itertools import codecs -import re - from contextlib import contextmanager +import itertools +import os +import re @contextmanager