Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • ardumont/swh-loader-core
  • vlorentz/swh-loader-core
  • franckbret/swh-loader-core
  • KShivendu/swh-loader-core
  • lunar/swh-loader-core
  • anlambert/swh-loader-core
  • olasd/swh-loader-core
  • swh/devel/swh-loader-core
  • Alphare/swh-loader-core
  • douardda/swh-loader-core
  • marmoute/swh-loader-core
11 results
Show changes
Commits on Source (595)
# Changes here will be overwritten by Copier
_commit: v0.3.3
_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
description: Software Heritage core and package loaders
distribution_name: swh-loader-core
have_cli: true
have_workers: true
package_root: swh/loader
project_name: swh.loader.core
python_minimal_version: '3.7'
readme_format: rst
# python: Reformat code with black
cf496e18440a073ec3d2b65657882e1bdb69a4d2
9c8a00123c37db204c59f6e6d17af520d29c7b65
8148efe14edafda15ce468e2ac2d5c9fbcae5623
*.egg-info/
*.pyc
*.sw?
*~
/.coverage
/.coverage.*
.coverage
.eggs/
.hypothesis
.mypy_cache
.tox
__pycache__
*.egg-info/
build/
dist/
version.txt
.tox/
.mypy_cache/
# these are symlinks created by a hook in swh-docs' main sphinx conf.py
docs/README.rst
docs/README.md
# this should be a symlink for people who want to build the sphinx doc
# without using tox, generally created by the swh-env/bin/update script
docs/Makefile.sphinx
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.4.0
hooks:
- id: trailing-whitespace
- id: flake8
- id: check-json
- id: check-yaml
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: check-json
- id: check-yaml
- repo: https://github.com/codespell-project/codespell
rev: v1.16.0
hooks:
- id: codespell
exclude: ^(swh/loader/package/.*[/]+tests/data/.*)$
- repo: https://github.com/python/black
rev: 25.1.0
hooks:
- id: black
- repo: local
hooks:
- id: mypy
name: mypy
entry: mypy
args: [swh]
pass_filenames: false
language: system
types: [python]
- repo: https://github.com/PyCQA/isort
rev: 6.0.0
hooks:
- id: isort
# unfortunately, we are far from being able to enable this...
# - repo: https://github.com/PyCQA/pydocstyle.git
# rev: 4.0.0
# hooks:
# - id: pydocstyle
# name: pydocstyle
# description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions.
# entry: pydocstyle --convention=google
# language: python
# types: [python]
- repo: https://github.com/pycqa/flake8
rev: 7.1.1
hooks:
- id: flake8
additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
# black requires py3.6+
#- repo: https://github.com/python/black
# rev: 19.3b0
# hooks:
# - id: black
# language_version: python3
#- repo: https://github.com/asottile/blacken-docs
# rev: v1.0.0-1
# hooks:
# - id: blacken-docs
# additional_dependencies: [black==19.3b0]
- repo: https://github.com/codespell-project/codespell
rev: v2.4.1
hooks:
- id: codespell
name: Check source code spelling
additional_dependencies:
- tomli
exclude: ^(swh/loader/.*/tests/data/.*)$
stages: [pre-commit]
- id: codespell
name: Check commit message spelling
additional_dependencies:
- tomli
stages: [commit-msg]
- repo: local
hooks:
- id: mypy
name: mypy
entry: mypy
args: [swh]
pass_filenames: false
language: system
types: [python]
- id: twine-check
name: twine check
description: call twine check when pushing an annotated release tag
entry: bash -c "ref=$(git describe) &&
[[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
(python3 -m build --sdist && twine check $(ls -t dist/* | head -1)) || true"
pass_filenames: false
stages: [pre-push]
language: python
additional_dependencies: [twine, build]
......@@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as Software
Heritage contributors and maintainers pledge to making participation in our
project and our community a harassment-free experience for everyone, regardless
of age, body size, disability, ethnicity, sex characteristics, gender identity
and expression, level of experience, education, socio-economic status,
and expression, level of experience, education, socioeconomic status,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
......
Antoine Eiche
Léo Andrès
Franck Bret
include Makefile
include requirements*.txt
include version.txt
include README.md
include conftest.py
recursive-include swh/loader/package/tests/ *.tar.gz
recursive-include swh py.typed
recursive-include swh/loader/package/tests/data/ *
recursive-include swh/loader/package/*/tests/data/ *
SWH-loader-core
===============
The Software Heritage Core Loader is a low-level loading utilities and
helpers used by other loaders.
The main entry points are classes:
- :class:`swh.loader.core.loader.SWHLoader` for stateful loaders
- :class:`swh.loader.core.loader.SWHStatelessLoader` for stateless loaders
Software Heritage - Loader foundations
======================================
The Software Heritage Loader Core is a low-level loading utilities and
helpers used by `loaders <https://docs.softwareheritage.org/devel/glossary.html#term-loader>`_.
The main entry points are classes:
- `swh.loader.core.loader.BaseLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.BaseLoader>`_
for VCS loaders (e.g. git, svn, ...)
- `swh.loader.core.loader.ContentLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.ContentLoader>`_
for Content loader
- `swh.loader.core.loader.BaseDirectoryLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.BaseDirectoryLoader>`_
for Directory loaders
- `swh.loader.package.loader.PackageLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.package.loader.html#swh.loader.package.loader.PackageLoader>`_
for Package loaders (e.g. PyPI, Npm, ...)
Package loaders
---------------
This package also implements many package loaders directly, out of convenience,
as they usually are quite similar and each fits in a single file.
They all roughly follow these steps, explained in the
`swh.loader.package.loader.PackageLoader.load <https://docs.softwareheritage.org/devel/apidoc/swh.loader.package.loader.html#swh.loader.package.loader.PackageLoader.load>`_
documentation.
See the `Package Loader tutorial <https://docs.softwareheritage.org/devel/swh-loader-core/package-loader-tutorial.html#package-loader-tutorial>`_
for details.
VCS loaders
-----------
Unlike package loaders, VCS loaders remain in separate packages,
as they often need more advanced conversions and very VCS-specific operations.
This usually involves getting the branches of a repository and recursively loading
revisions in the history (and directory trees in these revisions),
until a known revision is found
# Copyright (C) 2019 The Software Heritage developers
# Copyright (C) 2019-2024 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
import pytest
import yaml
from typing import Any, Dict
from swh.storage.tests.conftest import * # noqa
from swh.scheduler.tests.conftest import * # noqa
@pytest.fixture
def swh_loader_config(swh_storage_postgresql) -> Dict[str, Any]:
return {
'storage': {
'cls': 'local',
'args': {
'db': swh_storage_postgresql.dsn,
'objstorage': {
'cls': 'memory',
'args': {}
},
},
},
'deposit': {
'url': 'https://deposit.softwareheritage.org/1/private',
'auth': {
'username': 'user',
'password': 'pass',
}
},
}
@pytest.fixture
def swh_config(swh_loader_config, monkeypatch, tmp_path):
conffile = os.path.join(str(tmp_path), 'loader.yml')
with open(conffile, 'w') as f:
f.write(yaml.dump(swh_loader_config))
monkeypatch.setenv('SWH_CONFIG_FILENAME', conffile)
return conffile
@pytest.fixture(autouse=True, scope='session')
def swh_proxy():
"""Automatically inject this fixture in all tests to ensure no outside
connection takes place.
"""
os.environ['http_proxy'] = 'http://localhost:999'
os.environ['https_proxy'] = 'http://localhost:999'
@pytest.fixture(scope='session') # type: ignore # expected redefinition
def celery_includes():
return [
'swh.loader.package.archive.tasks',
'swh.loader.package.debian.tasks',
'swh.loader.package.deposit.tasks',
'swh.loader.package.npm.tasks',
'swh.loader.package.pypi.tasks',
pytest_plugins = [
"swh.scheduler.pytest_plugin",
"swh.storage.pytest_plugin",
"swh.loader.pytest_plugin",
]
@pytest.fixture(scope="session")
def swh_scheduler_celery_includes(swh_scheduler_celery_includes):
return swh_scheduler_celery_includes + [
"swh.loader.package.archive.tasks",
"swh.loader.package.cran.tasks",
"swh.loader.package.debian.tasks",
"swh.loader.package.deposit.tasks",
"swh.loader.package.npm.tasks",
"swh.loader.package.pypi.tasks",
"swh.loader.package.maven.tasks",
]
include ../../swh-docs/Makefile.sphinx
html: copy_md
copy_md:
cp ../README.md ./
include Makefile.sphinx
../README.md
\ No newline at end of file
../README.rst
\ No newline at end of file
.. _swh-loader-cli:
Command-line interface
======================
.. click:: swh.loader.cli:loader
:prog: swh loader
:nested: full
.. _swh-loader-core:
Software Heritage - Loader foundations
======================================
Low-level loading utilities and helpers used by other loaders.
.. include:: README.rst
.. toctree::
:maxdepth: 2
:caption: Contents:
README
vcs-loader-overview
package-loader-tutorial
package-loader-specifications
Reference Documentation
......@@ -19,4 +17,13 @@ Reference Documentation
.. toctree::
:maxdepth: 2
/apidoc/swh.loader.core
cli
.. only:: standalone_package_doc
Indices and tables
------------------
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
:html_theme.sidebar_secondary.remove:
.. _package-loader-specifications:
Package loader specifications
=============================
Release fields
--------------
Here is an overview of the fields (+ internal version name + branch name) used by each package loader, after D6616:
.. raw:: html
<style>
.bd-article-container {
/* Otherwise our expanded table will result in a scrollbar in the parent container */
overflow-x: visible !important;
}
.expand-right {
/* 60em is defined in `pydata_sphinx_theme/assets/styles/sections/_article.scss` */
margin-right: min(0rem, calc(-1 * (100vw - 60em) / 2 + 1rem)) !important;
}
</style>
.. container:: table-responsive expand-right
.. list-table:: Fields used by each package loader
:header-rows: 1
:stub-columns: 1
:class: table-striped
* - Loader
- internal version
- branch name
- name
- message
- synthetic
- author
- date
- Notes
* - arch
- ``p_info.​version``
- ``release_name(​version, filename)``
- =version
- Synthetic release for Arch Linux source package {p_info.name} version {p_info.version} {description}
- true
- from intrinsic metadata
- from extra_loader_arguments['arch_metadata']
- Intrinsic metadata extracted from .PKGINFO file of the package
* - archive
- passed as arg
- ``release_name(​version)``
- =version
- "Synthetic release for archive at {p_info.url}\n"
- true
- ""
- passed as arg
-
* - aur
- ``p_info.​version``
- ``release_name(​version, filename)``
- =version
- Synthetic release for Aur source package {p_info.name} version {p_info.version} {description}
- true
- ""
- from extra_loader_arguments['aur_metadata']
- Intrinsic metadata extracted from .SRCINFO file of the package
* - cpan
- ``p_info.​version``
- ``release_name(​version)``
- =version
- Synthetic release for Perl source package {name} version {version} {description}
- true
- from intrinsic metadata if any else from extrinsic
- from extrinsic metadata
- name, version and description from intrinsic metadata
* - cran
- ``metadata.get(​"Version", passed as arg)``
- ``release_name(​version)``
- =version
- standard message
- true
- ``metadata.get(​"Maintainer", "")``
- ``metadata.get(​"Date")``
- metadata is intrinsic
* - conda
- ``p_info.​version``
- ``release_name(​version)``
- =version
- Synthetic release for Conda source package {p_info.name} version {p_info.version}
- true
- from intrinsic metadata
- from extrinsic metadata
- ""
* - crates
- ``p_info.​version``
- ``release_name(​version)``
- =version
- Synthetic release for Crate source package {p_info.name} version {p_info.version}
- true
- from intrinsic metadata
- from extrinsic metadata
- ""
* - debian
- =``version``
- ``release_name(​version)``
- =``i_version``
- standard message (using ``i_version``)
- true
- ``metadata​.changelog​.person``
- ``metadata​.changelog​.date``
- metadata is intrinsic. Old revisions have ``dsc`` as type
``i_version`` is the intrinsic version (eg. ``0.7.2-3``) while ``version``
contains the debian suite name (eg. ``stretch/contrib/0.7.2-3``) and is
passed as arg
* - golang
- ``p_info.​version``
- ``release_name(version)``
- =version
- Synthetic release for Golang source package {p_info.name} version {p_info.version}
- true
- ""
- from ext metadata
- Golang offers basically no metadata outside of version and timestamp
* - deposit
- HEAD
- only HEAD
- HEAD
- "{client}: Deposit {id} in collection {collection}\n"
- true
- original author
- ``<codemeta: dateCreated>`` from SWORD XML
- revisions had parents
* - hackage
- ``p_info.​version``
- ``release_name(​version)``
- =version
- Synthetic release for Haskell source package {p_info.name} version {p_info.version}
- true
- intrinsic metadata if any else from extrinsic metadata
- from extrinsic metadata
- ""
* - hex
- ``p_info.version``
- ``release_name(version)``
- =version
- standard message
- true
- from extrinsic metadata
- from extrinsic metadata
- Source code is extracted from a nested tarball
* - maven-loader
- passed as arg
- HEAD
- ``release_name(version)``
- "Synthetic release for archive at {p_info.url}\n"
- true
- ""
- passed as arg
- Only one artefact per url (jar/zip src)
* - npm
- ``metadata​["version"]``
- ``release_name(​version)``
- =version
- standard message
- true
- from int metadata or ""
- from ext metadata or None
-
* - opam
- as given by opam
- "{opam_package}​.{version}"
- =version
- standard message
- true
- from metadata
- None
- "{self.opam_package}​.{version}" matches the version names used by opam's backend. metadata is extrinsic
* - pubdev
- ``p_info.​version``
- ``release_name(​version)``
- =version
- Synthetic release for pub.dev source package {p_info.name} version {p_info.version}
- true
- from extrinsic metadata
- from extrinsic metadata
- name and version from extrinsic metadata
* - puppet
- ``p_info.​version``
- ``release_name(​version)``
- =version
- Synthetic release for Puppet source package {p_info.name} version {version} {description}
- true
- from intrinsic metadata
- from extrinsic metadata
- version and description from intrinsic metadata
* - pypi
- ``metadata​["version"]``
- ``release_name(​version)`` or ``release_name(​version, filename)``
- =version
- ``metadata[​'comment_text']}`` or standard message
- true
- from int metadata or ""
- from ext metadata or None
- metadata is intrinsic
* - rubygems
- ``p_info.version``
- ``release_name(​version)``
- =version
- Synthetic release for RubyGems source package {p_info.name} version {p_info.version}
- true
- from ext metadata
- from ext metadata
- The source code is extracted from a tarball nested within the gem file
using this function::
def release_name(version: str, filename: Optional[str] = None) -> str:
if filename:
return "releases/%s/%s" % (version, filename)
return "releases/%s" % version
and "standard message" being::
msg = (
f"Synthetic release for {PACKAGE_MANAGER} source package {name} "
f"version {version}\n"
)
The ``target_type`` field is always ``dir``, and the target the id of a directory
loaded by unpacking a tarball/zip file/...
This diff is collapsed.
.. _vcs-loader-tutorial:
VCS Loader Overview
===================
In this overview, we will see how to write a loader for |swh| that loads
:term:`artifacts <artifact>` from a Version Control System, such as Git,
Mercurial, or Subversion
First, you should be familiar with Python, unit-testing,
|swh|'s :ref:`data-model` and :ref:`architecture`,
and go through the :ref:`developer-setup`.
As seen in the :ref:`swh-loader-core homepage <swh-loader-core>`,
SWH loaders can be sorted into two large categories:
Package Loaders and VCS loaders.
This page is an overview of how to write a VCS loader. This is not a tutorial,
because VCS loaders are hooked deeply into their respective VCS' internals;
unlike :ref:`Package Loaders <package-loader-tutorial>` which are somewhat uniform
(list tarballs, download tarballs, load content of tarball, done).
Architecture
------------
A loader is a Python package, usually a subpackage of ``swh.loader``
but in its own directory (eg. ``swh-loader-git/swh/loader/git``, as ``swh.loader``
is a :pep:`namespace package <420>`), based on the `swh-py-template`_ repository.
It has at least one `entrypoint`_, declared in :file:`setup.py` to be recognized
by ``swh-loader-core``::
entry_points="""
[swh.workers]
loader.newloader=swh.loader.newloader:register
""",
This entrypoint declares the task name (to be run by SWH Celery workers) and the
loader class. For example, for the Subversion loader::
from typing import Any, Dict
def register() -> Dict[str, Any]:
from swh.loader.svn.loader import SvnLoader
return {
"task_modules": ["%s.tasks" % __name__],
"loader": SvnLoader,
}
The bulk of the work is done by the returned ``loader`` class: it loads
artifacts from the upstream VCS and writes them to the |swh| archive.
Because of the heterogeneity of VCS loaders, it has a lot of freedom in how to
achieve this. Once the initial setup is done (see the next section), its ``load``
method is called, and it is expected to do all this work as a black box.
.. _swh-py-template: https://forge.softwareheritage.org/source/swh-py-template/
.. _entrypoint: https://setuptools.readthedocs.io/en/latest/userguide/entry_point.html
Base classes
------------
All loaders inherit from :class:`swh.loader.core.loader.BaseLoader`, which takes care of
all the SWH-specific setup and finalization:
* Reading the configuration
* Connecting to the :term:`storage database`
* Storing :term:`origin` and :term:`visit` objects
It also provides a default implementation of the ``load`` method, which takes care of:
* calling its ``fetch_data`` (from the VCS) and ``store_data`` (to SWH) in a loop
* on error, notifies swh-storage the loading failed, reports the error to
the monitoring infrastructure (Sentry), and cleanup
* on success, cleanup and notify swh-storage the loading succeeded
See :meth:`its documentation <swh.loader.core.loader.BaseLoader.load>` for details.
Distributed VCS loaders will usually want to inherit from its child,
:class:`swh.loader.core.DVCSLoader`, which takes care of implementing ``store_data``.
Classes inheriting from ``DVCSLoader`` only need to implement ``fetch_data``, and
a method for each object type: ``get_contents``, ``get_directories``, ``get_revisions``,
``get_releases``, and ``get_snapshot``, each returning an iterable of the corresponding
object from :mod:`swh.model.model`
(except ``get_snapshot``, which returns a single one).
If you are writing a DVCS loader, this allows your loader to fetch all the objects
locally, then return them lazily on demand.
Incremental loading
-------------------
Loading a repository from scratch can be costly, so ``swh-storage`` provides
ways to remember what objects in the repository were already loaded,
through :term:`extids <extid>`.
They are represented by :class:`swh.model.model.ExtID`,
which is essentially a 3-tuple that contains a SWHID, an id internal to the VCS type,
(which is the actual "extid" itself), and the type of this id (eg. ``hg-nodeid``).
When your loader is done loading, it can store extids for some of its objects
(eg. the heads/tips of each branch of the :term:`snapshot` and some intermediate
revisions in the history),
with :meth:`swh.storage.interface.StorageInterface.extid_add`.
And when it starts loading a known repository, fetches the previous snapshot
using :func:`swh.storage.algos.snapshot.snapshot_get_latest`, then the extids
it stores using :meth:`swh.storage.interface.StorageInterface.extid_get_from_target`
for each of the branch targets.
This way, it can find which objects from the origin were already loaded,
without having to download them first.
.. note::
For legacy reasons, the Subversion loader uses an alternative to ExtID,
which is to encode the repository UUID and the revision ID (an incremental integer)
directly in :attr:`swh.model.model.Revision.extra_headers`.
This is discouraged because it prevents deduplication across repositories,
and ``extra_headers`` does not have a well-defined schema.
Integrity
---------
Loaders may be interrupted at any point, for various reasons (unhandled crash,
out of memory, hardware failure, blocking IO, system or daemon restart, etc.)
Therefore, they must take great care that if a load was interrupted, the next load
will finish loading all objects. If they don't, this may happen:
1. loader loads revision ``R``, pointing to directory ``D``
2. loader starts loading ``D``, but crashes before it does
3. [loader restarts]
4. loader sees ``R`` is already loaded, so it doesn't load its children
And ``D`` will never be loaded.
The solution to this is to load objects in topological order of the DAG.
Another reason to load objects in topological order is that it avoid having "holes"
in the graph (aka. dangling references), even temporarily.
Holes in the graph cause bad user experiences, when users click a link from
an existing object and get a "not found" error.
[mypy]
namespace_packages = True
warn_unused_ignores = True
# 3rd party libraries without stubs (yet)
[mypy-celery.*]
ignore_missing_imports = True
[mypy-pkg_resources.*]
ignore_missing_imports = True
[mypy-psutil.*]
ignore_missing_imports = True
[mypy-psycopg2.*]
ignore_missing_imports = True
[mypy-pytest.*]
ignore_missing_imports = True
[mypy-requests_mock.*]
ignore_missing_imports = True
[mypy-retrying.*]
ignore_missing_imports = True
[mypy-chardet.*]
ignore_missing_imports = True
[mypy-debian.*]
ignore_missing_imports = True
[mypy-iso8601.*]
ignore_missing_imports = True
[mypy-pkginfo.*]
ignore_missing_imports = True
[mypy-swh.deposit.*]
ignore_missing_imports = True
[project]
name = "swh.loader.core"
authors = [
{name="Software Heritage developers", email="swh-devel@inria.fr"},
]
description = "Software Heritage core and package loaders"
readme = {file = "README.rst", content-type = "text/x-rst"}
requires-python = ">=3.7"
classifiers = [
"Programming Language :: Python :: 3",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Operating System :: OS Independent",
"Development Status :: 5 - Production/Stable",
]
dynamic = ["version", "dependencies", "optional-dependencies"]
[tool.setuptools.packages.find]
include = ["swh.*"]
[tool.setuptools.dynamic]
dependencies = {file = ["requirements.txt", "requirements-swh.txt"]}
[tool.setuptools.dynamic.optional-dependencies]
testing = {file = ["requirements-test.txt"]}
[project.entry-points."swh.cli.subcommands"]
"swh.loader.core" = "swh.loader.cli"
"swh.loader.core.nar" = "swh.loader.core.nar"
[project.entry-points."swh.workers"]
"loader.content" = "swh.loader.core:register_content"
"loader.directory" = "swh.loader.core:register_directory"
"loader.arch" = "swh.loader.package.arch:register"
"loader.archive" = "swh.loader.package.archive:register"
"loader.aur" = "swh.loader.package.aur:register"
"loader.conda" = "swh.loader.package.conda:register"
"loader.cpan" = "swh.loader.package.cpan:register"
"loader.cran" = "swh.loader.package.cran:register"
"loader.crates" = "swh.loader.package.crates:register"
"loader.debian" = "swh.loader.package.debian:register"
"loader.deposit" = "swh.loader.package.deposit:register"
"loader.golang" = "swh.loader.package.golang:register"
"loader.hackage" = "swh.loader.package.hackage:register"
"loader.hex" = "swh.loader.package.hex:register"
"loader.npm" = "swh.loader.package.npm:register"
"loader.opam" = "swh.loader.package.opam:register"
"loader.pubdev" = "swh.loader.package.pubdev:register"
"loader.puppet" = "swh.loader.package.puppet:register"
"loader.pypi" = "swh.loader.package.pypi:register"
"loader.maven" = "swh.loader.package.maven:register"
"loader.rubygems" = "swh.loader.package.rubygems:register"
"loader.rpm" = "swh.loader.package.rpm:register"
"loader.bioconductor" = "swh.loader.package.bioconductor:register"
[project.urls]
"Homepage" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core"
"Bug Reports" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core/-/issues"
"Funding" = "https://www.softwareheritage.org/donate"
"Documentation" = "https://docs.softwareheritage.org/devel/swh-loader-core/"
"Source" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core.git"
[build-system]
requires = ["setuptools", "setuptools-scm"]
build-backend = "setuptools.build_meta"
[tool.setuptools_scm]
fallback_version = "0.0.1"
[tool.black]
target-version = ['py39', 'py310', 'py311', 'py312']
[tool.isort]
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
use_parentheses = true
ensure_newline_before_comments = true
line_length = 88
force_sort_within_sections = true
known_first_party = ['swh']
[tool.codespell]
ignore-words-list = "iff,crate"
[tool.mypy]
namespace_packages = true
warn_unused_ignores = true
explicit_package_bases = true
# ^ Needed for mypy to detect py.typed from swh packages installed
# in editable mode
plugins = []
# 3rd party libraries without stubs (yet)
# [[tool.mypy.overrides]]
# module = [
# "package1.*",
# "package2.*",
# ]
# ignore_missing_imports = true
[tool.flake8]
select = ["C", "E", "F", "W", "B950"]
ignore = [
"E203", # whitespaces before ':' <https://github.com/psf/black/issues/315>
"E231", # missing whitespace after ','
"E501", # line too long, use B950 warning from flake8-bugbear instead
"W503", # line break before binary operator <https://github.com/psf/black/issues/52>
"E704" # multiple statements on one line
]
max-line-length = 88
[tool.pytest.ini_options]
norecursedirs = "build docs .*"
asyncio_mode = "strict"
consider_namespace_packages = true