.copier-answers.yml

+# Changes here will be overwritten by Copier
+_commit: v0.3.3
+_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
+description: Software Heritage core and package loaders
+distribution_name: swh-loader-core
+have_cli: true
+have_workers: true
+package_root: swh/loader
+project_name: swh.loader.core
+python_minimal_version: '3.7'
+readme_format: rst

.git-blame-ignore-revs

+# python: Reformat code with black
+cf496e18440a073ec3d2b65657882e1bdb69a4d2
+9c8a00123c37db204c59f6e6d17af520d29c7b65
+8148efe14edafda15ce468e2ac2d5c9fbcae5623

.gitignore

+*.egg-info/
 *.pyc
-*.sw?
-*~
-/.coverage
-/.coverage.*
+.coverage
 .eggs/
+.hypothesis
+.mypy_cache
+.tox
 __pycache__
-*.egg-info/
 build/
 dist/
-version.txt
-.tox/
-.mypy_cache/
+# these are symlinks created by a hook in swh-docs' main sphinx conf.py
+docs/README.rst
+docs/README.md
+# this should be a symlink for people who want to build the sphinx doc
+# without using tox, generally created by the swh-env/bin/update script
+docs/Makefile.sphinx

.pre-commit-config.yaml

 repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v2.4.0
-  hooks:
-  - id: trailing-whitespace
-  - id: flake8
-  - id: check-json
-  - id: check-yaml
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-json
+      - id: check-yaml
 
-- repo: https://github.com/codespell-project/codespell
-  rev: v1.16.0
-  hooks:
-  - id: codespell
-    exclude: ^(swh/loader/package/.*[/]+tests/data/.*)$
+  - repo: https://github.com/python/black
+    rev: 25.1.0
+    hooks:
+      - id: black
 
-- repo: local
-  hooks:
-  - id: mypy
-    name: mypy
-    entry: mypy
-    args: [swh]
-    pass_filenames: false
-    language: system
-    types: [python]
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.0
+    hooks:
+      - id: isort
 
-# unfortunately, we are far from being able to enable this...
-# - repo: https://github.com/PyCQA/pydocstyle.git
-#   rev: 4.0.0
-#   hooks:
-#   - id: pydocstyle
-#     name: pydocstyle
-#     description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions.
-#     entry: pydocstyle --convention=google
-#     language: python
-#     types: [python]
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
 
-# black requires py3.6+
-#- repo: https://github.com/python/black
-#  rev: 19.3b0
-#  hooks:
-#  - id: black
-#    language_version: python3
-#- repo: https://github.com/asottile/blacken-docs
-#  rev: v1.0.0-1
-#  hooks:
-#  - id: blacken-docs
-#    additional_dependencies: [black==19.3b0]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        name: Check source code spelling
+        additional_dependencies:
+          - tomli
+        exclude: ^(swh/loader/.*/tests/data/.*)$
+        stages: [pre-commit]
+      - id: codespell
+        name: Check commit message spelling
+        additional_dependencies:
+          - tomli
+        stages: [commit-msg]
+
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        args: [swh]
+        pass_filenames: false
+        language: system
+        types: [python]
+      - id: twine-check
+        name: twine check
+        description: call twine check when pushing an annotated release tag
+        entry: bash -c "ref=$(git describe) &&
+          [[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+          (python3 -m build --sdist && twine check $(ls -t dist/* | head -1)) || true"
+        pass_filenames: false
+        stages: [pre-push]
+        language: python
+        additional_dependencies: [twine, build]

CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as Software
 Heritage contributors and maintainers pledge to making participation in our
 project and our community a harassment-free experience for everyone, regardless
 of age, body size, disability, ethnicity, sex characteristics, gender identity
-and expression, level of experience, education, socio-economic status,
+and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, religion, or sexual identity and
 orientation.
 

CONTRIBUTORS

+Antoine Eiche
+Léo Andrès
+Franck Bret

MANIFEST.in

-include Makefile
-include requirements*.txt
-include version.txt
-include README.md
-include conftest.py
-recursive-include swh/loader/package/tests/ *.tar.gz
-recursive-include swh py.typed
-recursive-include swh/loader/package/tests/data/ *
-recursive-include swh/loader/package/*/tests/data/ *

README.md

-SWH-loader-core
-===============
-
-The Software Heritage Core Loader is a low-level loading utilities and
-helpers used by other loaders.
-
-The main entry points are classes:
-- :class:`swh.loader.core.loader.SWHLoader` for stateful loaders
-- :class:`swh.loader.core.loader.SWHStatelessLoader` for stateless loaders

README.rst

+Software Heritage - Loader foundations
+======================================
+
+The Software Heritage Loader Core is a low-level loading utilities and
+helpers used by `loaders <https://docs.softwareheritage.org/devel/glossary.html#term-loader>`_.
+
+The main entry points are classes:
+
+- `swh.loader.core.loader.BaseLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.BaseLoader>`_
+  for VCS loaders (e.g. git, svn, ...)
+
+- `swh.loader.core.loader.ContentLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.ContentLoader>`_
+  for Content loader
+
+- `swh.loader.core.loader.BaseDirectoryLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.BaseDirectoryLoader>`_
+  for Directory loaders
+
+- `swh.loader.package.loader.PackageLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.package.loader.html#swh.loader.package.loader.PackageLoader>`_
+  for Package loaders (e.g. PyPI, Npm, ...)
+
+Package loaders
+---------------
+
+This package also implements many package loaders directly, out of convenience,
+as they usually are quite similar and each fits in a single file.
+
+They all roughly follow these steps, explained in the
+`swh.loader.package.loader.PackageLoader.load <https://docs.softwareheritage.org/devel/apidoc/swh.loader.package.loader.html#swh.loader.package.loader.PackageLoader.load>`_
+documentation.
+See the `Package Loader tutorial <https://docs.softwareheritage.org/devel/swh-loader-core/package-loader-tutorial.html#package-loader-tutorial>`_
+for details.
+
+VCS loaders
+-----------
+
+Unlike package loaders, VCS loaders remain in separate packages,
+as they often need more advanced conversions and very VCS-specific operations.
+
+This usually involves getting the branches of a repository and recursively loading
+revisions in the history (and directory trees in these revisions),
+until a known revision is found

conftest.py

-# Copyright (C) 2019  The Software Heritage developers
+# Copyright (C) 2019-2024  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-import os
 import pytest
-import yaml
 
-from typing import Any, Dict
-
-from swh.storage.tests.conftest import * # noqa
-from swh.scheduler.tests.conftest import *  # noqa
-
-
-@pytest.fixture
-def swh_loader_config(swh_storage_postgresql) -> Dict[str, Any]:
-    return {
-        'storage': {
-            'cls': 'local',
-            'args': {
-                'db': swh_storage_postgresql.dsn,
-                'objstorage': {
-                    'cls': 'memory',
-                    'args': {}
-                },
-            },
-        },
-        'deposit': {
-            'url': 'https://deposit.softwareheritage.org/1/private',
-            'auth': {
-                'username': 'user',
-                'password': 'pass',
-            }
-        },
-    }
-
-
-@pytest.fixture
-def swh_config(swh_loader_config, monkeypatch, tmp_path):
-    conffile = os.path.join(str(tmp_path), 'loader.yml')
-    with open(conffile, 'w') as f:
-        f.write(yaml.dump(swh_loader_config))
-    monkeypatch.setenv('SWH_CONFIG_FILENAME', conffile)
-    return conffile
-
-
-@pytest.fixture(autouse=True, scope='session')
-def swh_proxy():
-    """Automatically inject this fixture in all tests to ensure no outside
-       connection takes place.
-
-    """
-    os.environ['http_proxy'] = 'http://localhost:999'
-    os.environ['https_proxy'] = 'http://localhost:999'
-
-
-@pytest.fixture(scope='session')  # type: ignore  # expected redefinition
-def celery_includes():
-    return [
-        'swh.loader.package.archive.tasks',
-        'swh.loader.package.debian.tasks',
-        'swh.loader.package.deposit.tasks',
-        'swh.loader.package.npm.tasks',
-        'swh.loader.package.pypi.tasks',
+pytest_plugins = [
+    "swh.scheduler.pytest_plugin",
+    "swh.storage.pytest_plugin",
+    "swh.loader.pytest_plugin",
+]
+
+
+@pytest.fixture(scope="session")
+def swh_scheduler_celery_includes(swh_scheduler_celery_includes):
+    return swh_scheduler_celery_includes + [
+        "swh.loader.package.archive.tasks",
+        "swh.loader.package.cran.tasks",
+        "swh.loader.package.debian.tasks",
+        "swh.loader.package.deposit.tasks",
+        "swh.loader.package.npm.tasks",
+        "swh.loader.package.pypi.tasks",
+        "swh.loader.package.maven.tasks",
     ]

docs/Makefile

-include ../../swh-docs/Makefile.sphinx
-
-html: copy_md
-
-copy_md:
-	cp ../README.md ./
+include Makefile.sphinx

docs/README.md

-../README.md
\ No newline at end of file

docs/README.rst

+../README.rst
\ No newline at end of file

docs/cli.rst

+.. _swh-loader-cli:
+
+Command-line interface
+======================
+
+.. click:: swh.loader.cli:loader
+  :prog: swh loader
+  :nested: full

docs/index.rst

 .. _swh-loader-core:
 
-Software Heritage - Loader foundations
-======================================
-
-Low-level loading utilities and helpers used by other loaders.
-
+.. include:: README.rst
 
 .. toctree::
    :maxdepth: 2
    :caption: Contents:
 
-   README
+   vcs-loader-overview
+   package-loader-tutorial
+   package-loader-specifications
 
 
 Reference Documentation
@@ -19,4 +17,13 @@ Reference Documentation
 .. toctree::
    :maxdepth: 2
 
-   /apidoc/swh.loader.core
+   cli
+
+.. only:: standalone_package_doc
+
+   Indices and tables
+   ------------------
+
+   * :ref:`genindex`
+   * :ref:`modindex`
+   * :ref:`search`

docs/package-loader-specifications.rst

+:html_theme.sidebar_secondary.remove:
+
+.. _package-loader-specifications:
+
+Package loader specifications
+=============================
+
+Release fields
+--------------
+
+Here is an overview of the fields (+ internal version name + branch name) used by each package loader, after D6616:
+
+.. raw:: html
+
+   <style>
+   .bd-article-container {
+     /* Otherwise our expanded table will result in a scrollbar in the parent container */
+     overflow-x: visible !important;
+   }
+   .expand-right {
+     /* 60em is defined in `pydata_sphinx_theme/assets/styles/sections/_article.scss` */
+     margin-right: min(0rem, calc(-1 * (100vw - 60em) / 2 + 1rem)) !important;
+   }
+   </style>
+
+.. container:: table-responsive expand-right
+
+   .. list-table:: Fields used by each package loader
+      :header-rows: 1
+      :stub-columns: 1
+      :class: table-striped
+
+      * - Loader
+        - internal version
+        - branch name
+        - name
+        - message
+        - synthetic
+        - author
+        - date
+        - Notes
+      * - arch
+        - ``p_info.​version``
+        - ``release_name(​version, filename)``
+        - =version
+        - Synthetic release for Arch Linux source package {p_info.name} version {p_info.version} {description}
+        - true
+        - from intrinsic metadata
+        - from extra_loader_arguments['arch_metadata']
+        - Intrinsic metadata extracted from .PKGINFO file of the package
+      * - archive
+        - passed as arg
+        - ``release_name(​version)``
+        - =version
+        - "Synthetic release for archive at {p_info.url}\n"
+        - true
+        - ""
+        - passed as arg
+        -
+      * - aur
+        - ``p_info.​version``
+        - ``release_name(​version, filename)``
+        - =version
+        - Synthetic release for Aur source package {p_info.name} version {p_info.version} {description}
+        - true
+        - ""
+        - from extra_loader_arguments['aur_metadata']
+        - Intrinsic metadata extracted from .SRCINFO file of the package
+      * - cpan
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Perl source package {name} version {version} {description}
+        - true
+        - from intrinsic metadata if any else from extrinsic
+        - from extrinsic metadata
+        - name, version and description from intrinsic metadata
+      * - cran
+        - ``metadata.get(​"Version", passed as arg)``
+        - ``release_name(​version)``
+        - =version
+        - standard message
+        - true
+        - ``metadata.get(​"Maintainer", "")``
+        - ``metadata.get(​"Date")``
+        - metadata is intrinsic
+      * - conda
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Conda source package {p_info.name} version {p_info.version}
+        - true
+        - from intrinsic metadata
+        - from extrinsic metadata
+        - ""
+      * - crates
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Crate source package {p_info.name} version {p_info.version}
+        - true
+        - from intrinsic metadata
+        - from extrinsic metadata
+        - ""
+      * - debian
+        - =``version``
+        - ``release_name(​version)``
+        - =``i_version``
+        - standard message (using ``i_version``)
+        - true
+        - ``metadata​.changelog​.person``
+        - ``metadata​.changelog​.date``
+        - metadata is intrinsic. Old revisions have ``dsc`` as type
+          ``i_version`` is the intrinsic version (eg. ``0.7.2-3``) while ``version``
+          contains the debian suite name (eg. ``stretch/contrib/0.7.2-3``) and is
+          passed as arg
+      * - golang
+        - ``p_info.​version``
+        - ``release_name(version)``
+        - =version
+        - Synthetic release for Golang source package {p_info.name} version {p_info.version}
+        - true
+        - ""
+        - from ext metadata
+        - Golang offers basically no metadata outside of version and timestamp
+      * - deposit
+        - HEAD
+        - only HEAD
+        - HEAD
+        - "{client}: Deposit {id} in collection {collection}\n"
+        - true
+        - original author
+        - ``<codemeta: dateCreated>`` from SWORD XML
+        - revisions had parents
+      * - hackage
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Haskell source package {p_info.name} version {p_info.version}
+        - true
+        - intrinsic metadata if any else from extrinsic metadata
+        - from extrinsic metadata
+        - ""
+      * - hex
+        - ``p_info.version``
+        - ``release_name(version)``
+        - =version
+        - standard message
+        - true
+        - from extrinsic metadata
+        - from extrinsic metadata
+        - Source code is extracted from a nested tarball
+      * - maven-loader
+        - passed as arg
+        - HEAD
+        - ``release_name(version)``
+        - "Synthetic release for archive at {p_info.url}\n"
+        - true
+        - ""
+        - passed as arg
+        - Only one artefact per url (jar/zip src)
+      * - npm
+        - ``metadata​["version"]``
+        - ``release_name(​version)``
+        - =version
+        - standard message
+        - true
+        - from int metadata or ""
+        - from ext metadata or None
+        -
+      * - opam
+        - as given by opam
+        - "{opam_package}​.{version}"
+        - =version
+        - standard message
+        - true
+        - from metadata
+        - None
+        - "{self.opam_package}​.{version}" matches the version names used by opam's backend. metadata is extrinsic
+      * - pubdev
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for pub.dev source package {p_info.name} version {p_info.version}
+        - true
+        - from extrinsic metadata
+        - from extrinsic metadata
+        - name and version from extrinsic metadata
+      * - puppet
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Puppet source package {p_info.name} version {version} {description}
+        - true
+        - from intrinsic metadata
+        - from extrinsic metadata
+        - version and description from intrinsic metadata
+      * - pypi
+        - ``metadata​["version"]``
+        - ``release_name(​version)`` or ``release_name(​version, filename)``
+        - =version
+        - ``metadata[​'comment_text']}`` or standard message
+        - true
+        - from int metadata or ""
+        - from ext metadata or None
+        - metadata is intrinsic
+      * - rubygems
+        - ``p_info.version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for RubyGems source package {p_info.name} version {p_info.version}
+        - true
+        - from ext metadata
+        - from ext metadata
+        - The source code is extracted from a tarball nested within the gem file
+
+using this function::
+
+    def release_name(version: str, filename: Optional[str] = None) -> str:
+        if filename:
+            return "releases/%s/%s" % (version, filename)
+        return "releases/%s" % version
+
+and "standard message" being::
+
+    msg = (
+        f"Synthetic release for {PACKAGE_MANAGER} source package {name} "
+        f"version {version}\n"
+    )
+
+
+The ``target_type`` field is always ``dir``, and the target the id of a directory
+loaded by unpacking a tarball/zip file/...

docs/vcs-loader-overview.rst

+.. _vcs-loader-tutorial:
+
+VCS Loader Overview
+===================
+
+In this overview, we will see how to write a loader for |swh| that loads
+:term:`artifacts <artifact>` from a Version Control System, such as Git,
+Mercurial, or Subversion
+
+First, you should be familiar with Python, unit-testing,
+|swh|'s :ref:`data-model` and :ref:`architecture`,
+and go through the :ref:`developer-setup`.
+
+As seen in the :ref:`swh-loader-core homepage <swh-loader-core>`,
+SWH loaders can be sorted into two large categories:
+Package Loaders and VCS loaders.
+
+This page is an overview of how to write a VCS loader. This is not a tutorial,
+because VCS loaders are hooked deeply into their respective VCS' internals;
+unlike :ref:`Package Loaders <package-loader-tutorial>` which are somewhat uniform
+(list tarballs, download tarballs, load content of tarball, done).
+
+
+Architecture
+------------
+
+A loader is a Python package, usually a subpackage of ``swh.loader``
+but in its own directory (eg. ``swh-loader-git/swh/loader/git``, as ``swh.loader``
+is a :pep:`namespace package <420>`), based on the `swh-py-template`_ repository.
+
+It has at least one `entrypoint`_, declared in :file:`setup.py` to be recognized
+by ``swh-loader-core``::
+
+    entry_points="""
+        [swh.workers]
+        loader.newloader=swh.loader.newloader:register
+    """,
+
+This entrypoint declares the task name (to be run by SWH Celery workers) and the
+loader class. For example, for the Subversion loader::
+
+   from typing import Any, Dict
+
+   def register() -> Dict[str, Any]:
+       from swh.loader.svn.loader import SvnLoader
+
+       return {
+           "task_modules": ["%s.tasks" % __name__],
+           "loader": SvnLoader,
+       }
+
+The bulk of the work is done by the returned ``loader`` class: it loads
+artifacts from the upstream VCS and writes them to the |swh| archive.
+Because of the heterogeneity of VCS loaders, it has a lot of freedom in how to
+achieve this. Once the initial setup is done (see the next section), its ``load``
+method is called, and it is expected to do all this work as a black box.
+
+.. _swh-py-template: https://forge.softwareheritage.org/source/swh-py-template/
+.. _entrypoint: https://setuptools.readthedocs.io/en/latest/userguide/entry_point.html
+
+Base classes
+------------
+
+All loaders inherit from :class:`swh.loader.core.loader.BaseLoader`, which takes care of
+all the SWH-specific setup and finalization:
+
+* Reading the configuration
+* Connecting to the :term:`storage database`
+* Storing :term:`origin` and :term:`visit` objects
+
+It also provides a default implementation of the ``load`` method, which takes care of:
+
+* calling its ``fetch_data`` (from the VCS) and ``store_data`` (to SWH) in a loop
+* on error, notifies swh-storage the loading failed, reports the error to
+  the monitoring infrastructure (Sentry), and cleanup
+* on success, cleanup and notify swh-storage the loading succeeded
+
+See :meth:`its documentation <swh.loader.core.loader.BaseLoader.load>` for details.
+
+Distributed VCS loaders will usually want to inherit from its child,
+:class:`swh.loader.core.DVCSLoader`, which takes care of implementing ``store_data``.
+Classes inheriting from ``DVCSLoader`` only need to implement ``fetch_data``, and
+a method for each object type: ``get_contents``, ``get_directories``, ``get_revisions``,
+``get_releases``, and ``get_snapshot``, each returning an iterable of the corresponding
+object from :mod:`swh.model.model`
+(except ``get_snapshot``, which returns a single one).
+
+If you are writing a DVCS loader, this allows your loader to fetch all the objects
+locally, then return them lazily on demand.
+
+
+Incremental loading
+-------------------
+
+Loading a repository from scratch can be costly, so ``swh-storage`` provides
+ways to remember what objects in the repository were already loaded,
+through :term:`extids <extid>`.
+They are represented by :class:`swh.model.model.ExtID`,
+which is essentially a 3-tuple that contains a SWHID, an id internal to the VCS type,
+(which is the actual "extid" itself), and the type of this id (eg. ``hg-nodeid``).
+
+When your loader is done loading, it can store extids for some of its objects
+(eg. the heads/tips of each branch of the :term:`snapshot` and some intermediate
+revisions in the history),
+with :meth:`swh.storage.interface.StorageInterface.extid_add`.
+
+And when it starts loading a known repository, fetches the previous snapshot
+using :func:`swh.storage.algos.snapshot.snapshot_get_latest`, then the extids
+it stores using :meth:`swh.storage.interface.StorageInterface.extid_get_from_target`
+for each of the branch targets.
+This way, it can find which objects from the origin were already loaded,
+without having to download them first.
+
+.. note::
+
+   For legacy reasons, the Subversion loader uses an alternative to ExtID,
+   which is to encode the repository UUID and the revision ID (an incremental integer)
+   directly in :attr:`swh.model.model.Revision.extra_headers`.
+
+   This is discouraged because it prevents deduplication across repositories,
+   and ``extra_headers`` does not have a well-defined schema.
+
+Integrity
+---------
+
+Loaders may be interrupted at any point, for various reasons (unhandled crash,
+out of memory, hardware failure, blocking IO, system or daemon restart, etc.)
+
+Therefore, they must take great care that if a load was interrupted, the next load
+will finish loading all objects. If they don't, this may happen:
+
+1. loader loads revision ``R``, pointing to directory ``D``
+2. loader starts loading ``D``, but crashes before it does
+3. [loader restarts]
+4. loader sees ``R`` is already loaded, so it doesn't load its children
+
+And ``D`` will never be loaded.
+
+The solution to this is to load objects in topological order of the DAG.
+
+Another reason to load objects in topological order is that it avoid having "holes"
+in the graph (aka. dangling references), even temporarily.
+Holes in the graph cause bad user experiences, when users click a link from
+an existing object and get a "not found" error.

mypy.ini

-[mypy]
-namespace_packages = True
-warn_unused_ignores = True
-
-# 3rd party libraries without stubs (yet)
-
-[mypy-celery.*]
-ignore_missing_imports = True
-
-[mypy-pkg_resources.*]
-ignore_missing_imports = True
-
-[mypy-psutil.*]
-ignore_missing_imports = True
-
-[mypy-psycopg2.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-requests_mock.*]
-ignore_missing_imports = True
-
-[mypy-retrying.*]
-ignore_missing_imports = True
-
-[mypy-chardet.*]
-ignore_missing_imports = True
-
-[mypy-debian.*]
-ignore_missing_imports = True
-
-[mypy-iso8601.*]
-ignore_missing_imports = True
-
-[mypy-pkginfo.*]
-ignore_missing_imports = True
-
-[mypy-swh.deposit.*]
-ignore_missing_imports = True
-

pyproject.toml

+[project]
+name = "swh.loader.core"
+authors = [
+    {name="Software Heritage developers", email="swh-devel@inria.fr"},
+]
+
+description = "Software Heritage core and package loaders"
+readme = {file = "README.rst", content-type = "text/x-rst"}
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+    "Development Status :: 5 - Production/Stable",
+]
+dynamic = ["version", "dependencies", "optional-dependencies"]
+
+[tool.setuptools.packages.find]
+include = ["swh.*"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt", "requirements-swh.txt"]}
+
+[tool.setuptools.dynamic.optional-dependencies]
+testing = {file = ["requirements-test.txt"]}
+
+[project.entry-points."swh.cli.subcommands"]
+"swh.loader.core" = "swh.loader.cli"
+"swh.loader.core.nar" = "swh.loader.core.nar"
+
+[project.entry-points."swh.workers"]
+"loader.content" = "swh.loader.core:register_content"
+"loader.directory" = "swh.loader.core:register_directory"
+"loader.arch" = "swh.loader.package.arch:register"
+"loader.archive" = "swh.loader.package.archive:register"
+"loader.aur" = "swh.loader.package.aur:register"
+"loader.conda" = "swh.loader.package.conda:register"
+"loader.cpan" = "swh.loader.package.cpan:register"
+"loader.cran" = "swh.loader.package.cran:register"
+"loader.crates" = "swh.loader.package.crates:register"
+"loader.debian" = "swh.loader.package.debian:register"
+"loader.deposit" = "swh.loader.package.deposit:register"
+"loader.golang" = "swh.loader.package.golang:register"
+"loader.hackage" = "swh.loader.package.hackage:register"
+"loader.hex" = "swh.loader.package.hex:register"
+"loader.npm" = "swh.loader.package.npm:register"
+"loader.opam" = "swh.loader.package.opam:register"
+"loader.pubdev" = "swh.loader.package.pubdev:register"
+"loader.puppet" = "swh.loader.package.puppet:register"
+"loader.pypi" = "swh.loader.package.pypi:register"
+"loader.maven" = "swh.loader.package.maven:register"
+"loader.rubygems" = "swh.loader.package.rubygems:register"
+"loader.rpm" = "swh.loader.package.rpm:register"
+"loader.bioconductor" = "swh.loader.package.bioconductor:register"
+
+[project.urls]
+"Homepage" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core"
+"Bug Reports" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core/-/issues"
+"Funding" = "https://www.softwareheritage.org/donate"
+"Documentation" = "https://docs.softwareheritage.org/devel/swh-loader-core/"
+"Source" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core.git"
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+fallback_version = "0.0.1"
+
+[tool.black]
+target-version = ['py39', 'py310', 'py311', 'py312']
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+ensure_newline_before_comments = true
+line_length = 88
+force_sort_within_sections = true
+known_first_party = ['swh']
+
+[tool.codespell]
+ignore-words-list = "iff,crate"
+
+[tool.mypy]
+namespace_packages = true
+warn_unused_ignores = true
+explicit_package_bases = true
+# ^ Needed for mypy to detect py.typed from swh packages installed
+# in editable mode
+
+plugins = []
+
+# 3rd party libraries without stubs (yet)
+# [[tool.mypy.overrides]]
+# module = [
+#     "package1.*",
+#     "package2.*",
+# ]
+# ignore_missing_imports = true
+
+[tool.flake8]
+select = ["C", "E", "F", "W", "B950"]
+ignore = [
+    "E203", # whitespaces before ':' <https://github.com/psf/black/issues/315>
+    "E231", # missing whitespace after ','
+    "E501", # line too long, use B950 warning from flake8-bugbear instead
+    "W503", # line break before binary operator <https://github.com/psf/black/issues/52>
+    "E704" # multiple statements on one line
+]
+max-line-length = 88
+
+[tool.pytest.ini_options]
+norecursedirs = "build docs .*"
+asyncio_mode = "strict"
+consider_namespace_packages = true