.copier-answers.yml

+# Changes here will be overwritten by Copier
+_commit: v0.3.3
+_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
+description: Software Heritage core and package loaders
+distribution_name: swh-loader-core
+have_cli: true
+have_workers: true
+package_root: swh/loader
+project_name: swh.loader.core
+python_minimal_version: '3.7'
+readme_format: rst

.git-blame-ignore-revs

-# Enable black
+# python: Reformat code with black
 cf496e18440a073ec3d2b65657882e1bdb69a4d2
-
-# python: Reformat code with black 22.3.0
 9c8a00123c37db204c59f6e6d17af520d29c7b65
+8148efe14edafda15ce468e2ac2d5c9fbcae5623

.gitignore

+*.egg-info/
 *.pyc
-*.sw?
-*~
-/.coverage
-/.coverage.*
+.coverage
 .eggs/
+.hypothesis
+.mypy_cache
+.tox
 __pycache__
-*.egg-info/
 build/
 dist/
-version.txt
-.tox/
-.mypy_cache/
+# these are symlinks created by a hook in swh-docs' main sphinx conf.py
+docs/README.rst
+docs/README.md
+# this should be a symlink for people who want to build the sphinx doc
+# without using tox, generally created by the swh-env/bin/update script
+docs/Makefile.sphinx

.pre-commit-config.yaml

 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: check-json
       - id: check-yaml
 
+  - repo: https://github.com/python/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.0
+    hooks:
+      - id: isort
+
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
+    rev: 7.1.1
     hooks:
       - id: flake8
-        additional_dependencies: [flake8-bugbear==22.9.23]
+        additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.4.1
     hooks:
       - id: codespell
         name: Check source code spelling
+        additional_dependencies:
+          - tomli
         exclude: ^(swh/loader/.*/tests/data/.*)$
-        args: [-L crate]
-        entry: codespell --ignore-words-list=iff
-        stages: [commit]
+        stages: [pre-commit]
+      - id: codespell
+        name: Check commit message spelling
+        additional_dependencies:
+          - tomli
+        stages: [commit-msg]
 
   - repo: local
     hooks:
@@ -31,13 +46,13 @@ repos:
         pass_filenames: false
         language: system
         types: [python]
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.11.5
-    hooks:
-      - id: isort
-
-  - repo: https://github.com/python/black
-    rev: 22.10.0
-    hooks:
-      - id: black
+      - id: twine-check
+        name: twine check
+        description: call twine check when pushing an annotated release tag
+        entry: bash -c "ref=$(git describe) &&
+          [[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+          (python3 -m build --sdist && twine check $(ls -t dist/* | head -1)) || true"
+        pass_filenames: false
+        stages: [pre-push]
+        language: python
+        additional_dependencies: [twine, build]

CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as Software
 Heritage contributors and maintainers pledge to making participation in our
 project and our community a harassment-free experience for everyone, regardless
 of age, body size, disability, ethnicity, sex characteristics, gender identity
-and expression, level of experience, education, socio-economic status,
+and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, religion, or sexual identity and
 orientation.
 

MANIFEST.in

-include Makefile
-include requirements*.txt
-include version.txt
-include README.md
-include conftest.py
-recursive-include swh py.typed
-recursive-include swh/loader/tests/data/ *
-recursive-include swh/loader/package/*/tests/data/ *

README.rst

-docs/README.rst
\ No newline at end of file

README.rst

+Software Heritage - Loader foundations
+======================================
+
+The Software Heritage Loader Core is a low-level loading utilities and
+helpers used by `loaders <https://docs.softwareheritage.org/devel/glossary.html#term-loader>`_.
+
+The main entry points are classes:
+
+- `swh.loader.core.loader.BaseLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.BaseLoader>`_
+  for VCS loaders (e.g. git, svn, ...)
+
+- `swh.loader.core.loader.ContentLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.ContentLoader>`_
+  for Content loader
+
+- `swh.loader.core.loader.BaseDirectoryLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.BaseDirectoryLoader>`_
+  for Directory loaders
+
+- `swh.loader.package.loader.PackageLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.package.loader.html#swh.loader.package.loader.PackageLoader>`_
+  for Package loaders (e.g. PyPI, Npm, ...)
+
+Package loaders
+---------------
+
+This package also implements many package loaders directly, out of convenience,
+as they usually are quite similar and each fits in a single file.
+
+They all roughly follow these steps, explained in the
+`swh.loader.package.loader.PackageLoader.load <https://docs.softwareheritage.org/devel/apidoc/swh.loader.package.loader.html#swh.loader.package.loader.PackageLoader.load>`_
+documentation.
+See the `Package Loader tutorial <https://docs.softwareheritage.org/devel/swh-loader-core/package-loader-tutorial.html#package-loader-tutorial>`_
+for details.
+
+VCS loaders
+-----------
+
+Unlike package loaders, VCS loaders remain in separate packages,
+as they often need more advanced conversions and very VCS-specific operations.
+
+This usually involves getting the branches of a repository and recursively loading
+revisions in the history (and directory trees in these revisions),
+until a known revision is found

conftest.py

-# Copyright (C) 2019-2020  The Software Heritage developers
+# Copyright (C) 2019-2024  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -21,6 +21,5 @@ def swh_scheduler_celery_includes(swh_scheduler_celery_includes):
         "swh.loader.package.deposit.tasks",
         "swh.loader.package.npm.tasks",
         "swh.loader.package.pypi.tasks",
-        "swh.loader.package.nixguix.tasks",
         "swh.loader.package.maven.tasks",
     ]

docs/Makefile

-include ../../swh-docs/Makefile.sphinx
+include Makefile.sphinx

docs/README.rst

-Software Heritage - Loader foundations
-======================================
-
-The Software Heritage Loader Core is a low-level loading utilities and
-helpers used by :term:`loaders <loader>`.
-
-The main entry points are classes:
-- :class:`swh.loader.core.loader.BaseLoader` for VCS loaders (e.g. svn)
-- :class:`swh.loader.core.loader.DVCSLoader` for DVCS loaders (e.g. git, ...)
-- :class:`swh.loader.core.loader.ContentLoader` for Content loader
-- :class:`swh.loader.core.loader.DirectoryLoader` for Directory loader
-- :class:`swh.loader.package.loader.PackageLoader` for Package loaders (e.g. PyPI, Npm, ...)
-- ...
-
-Package loaders
----------------
-
-This package also implements many package loaders directly, out of convenience,
-as they usually are quite similar and each fits in a single file.
-
-They all roughly follow these steps, explained in the
-:py:meth:`swh.loader.package.loader.PackageLoader.load` documentation.
-See the :ref:`package-loader-tutorial` for details.
-
-VCS loaders
------------
-
-Unlike package loaders, VCS loaders remain in separate packages,
-as they often need more advanced conversions and very VCS-specific operations.
-
-This usually involves getting the branches of a repository and recursively loading
-revisions in the history (and directory trees in these revisions),
-until a known revision is found

docs/README.rst

+../README.rst
\ No newline at end of file

docs/package-loader-specifications.rst

+:html_theme.sidebar_secondary.remove:
+
 .. _package-loader-specifications:
 
 Package loader specifications
@@ -8,192 +10,209 @@ Release fields
 
 Here is an overview of the fields (+ internal version name + branch name) used by each package loader, after D6616:
 
-.. list-table:: Fields used by each package loader
-   :header-rows: 1
+.. raw:: html
+
+   <style>
+   .bd-article-container {
+     /* Otherwise our expanded table will result in a scrollbar in the parent container */
+     overflow-x: visible !important;
+   }
+   .expand-right {
+     /* 60em is defined in `pydata_sphinx_theme/assets/styles/sections/_article.scss` */
+     margin-right: min(0rem, calc(-1 * (100vw - 60em) / 2 + 1rem)) !important;
+   }
+   </style>
+
+.. container:: table-responsive expand-right
+
+   .. list-table:: Fields used by each package loader
+      :header-rows: 1
+      :stub-columns: 1
+      :class: table-striped
 
-   * - Loader
-     - internal version
-     - branch name
-     - name
-     - message
-     - synthetic
-     - author
-     - date
-     - Notes
-   * - arch
-     - ``p_info.​version``
-     - ``release_name(​version, filename)``
-     - =version
-     - Synthetic release for Arch Linux source package {p_info.name} version {p_info.version} {description}
-     - true
-     - from intrinsic metadata
-     - from extra_loader_arguments['arch_metadata']
-     - Intrinsic metadata extracted from .PKGINFO file of the package
-   * - archive
-     - passed as arg
-     - ``release_name(​version)``
-     - =version
-     - "Synthetic release for archive at {p_info.url}\n"
-     - true
-     - ""
-     - passed as arg
-     -
-   * - aur
-     - ``p_info.​version``
-     - ``release_name(​version, filename)``
-     - =version
-     - Synthetic release for Aur source package {p_info.name} version {p_info.version} {description}
-     - true
-     - ""
-     - from extra_loader_arguments['aur_metadata']
-     - Intrinsic metadata extracted from .SRCINFO file of the package
-   * - cpan
-     - ``p_info.​version``
-     - ``release_name(​version)``
-     - =version
-     - Synthetic release for Perl source package {name} version {version} {description}
-     - true
-     - from intrinsic metadata if any else from extrinsic
-     - from extrinsic metadata
-     - name, version and description from intrinsic metadata
-   * - cran
-     - ``metadata.get(​"Version", passed as arg)``
-     - ``release_name(​version)``
-     - =version
-     - standard message
-     - true
-     - ``metadata.get(​"Maintainer", "")``
-     - ``metadata.get(​"Date")``
-     - metadata is intrinsic
-   * - conda
-     - ``p_info.​version``
-     - ``release_name(​version)``
-     - =version
-     - Synthetic release for Conda source package {p_info.name} version {p_info.version}
-     - true
-     - from intrinsic metadata
-     - from extrinsic metadata
-     - ""
-   * - crates
-     - ``p_info.​version``
-     - ``release_name(​version, filename) + "\n\n" + i_metadata.description + "\n"``
-     - =version
-     - Synthetic release for Crate source package {p_info.name} version {p_info.version} {description}
-     - true
-     - from int metadata
-     - from ext metadata
-     - ``i_metadata`` for intrinsic metadata, ``e_metadata`` for extrinsic metadata
-   * - debian
-     - =``version``
-     - ``release_name(​version)``
-     - =``i_version``
-     - standard message (using ``i_version``)
-     - true
-     - ``metadata​.changelog​.person``
-     - ``metadata​.changelog​.date``
-     - metadata is intrinsic. Old revisions have ``dsc`` as type
-       ``i_version`` is the intrinsic version (eg. ``0.7.2-3``) while ``version``
-       contains the debian suite name (eg. ``stretch/contrib/0.7.2-3``) and is
-       passed as arg
-   * - golang
-     - ``p_info.​version``
-     - ``release_name(version)``
-     - =version
-     - Synthetic release for Golang source package {p_info.name} version {p_info.version}
-     - true
-     - ""
-     - from ext metadata
-     - Golang offers basically no metadata outside of version and timestamp
-   * - deposit
-     - HEAD
-     - only HEAD
-     - HEAD
-     - "{client}: Deposit {id} in collection {collection}\n"
-     - true
-     - original author
-     - ``<codemeta: dateCreated>`` from SWORD XML
-     - revisions had parents
-   * - hackage
-     - ``p_info.​version``
-     - ``release_name(​version)``
-     - =version
-     - Synthetic release for Haskell source package {p_info.name} version {p_info.version}
-     - true
-     - intrinsic metadata if any else from extrinsic metadata
-     - from extrinsic metadata
-     - ""
-   * - maven-loader
-     - passed as arg
-     - HEAD
-     - ``release_name(version)``
-     - "Synthetic release for archive at {p_info.url}\n"
-     - true
-     - ""
-     - passed as arg
-     - Only one artefact per url (jar/zip src)
-   * - nixguix
-     - URL
-     - URL
-     - URL
-     - None
-     - true
-     - ""
-     - None
-     - it's the URL of the artifact referenced by the derivation
-   * - npm
-     - ``metadata​["version"]``
-     - ``release_name(​version)``
-     - =version
-     - standard message
-     - true
-     - from int metadata or ""
-     - from ext metadata or None
-     -
-   * - opam
-     - as given by opam
-     - "{opam_package}​.{version}"
-     - =version
-     - standard message
-     - true
-     - from metadata
-     - None
-     - "{self.opam_package}​.{version}" matches the version names used by opam's backend. metadata is extrinsic
-   * - pubdev
-     - ``p_info.​version``
-     - ``release_name(​version)``
-     - =version
-     - Synthetic release for pub.dev source package {p_info.name} version {p_info.version}
-     - true
-     - from extrinsic metadata
-     - from extrinsic metadata
-     - name and version from extrinsic metadata
-   * - puppet
-     - ``p_info.​version``
-     - ``release_name(​version)``
-     - =version
-     - Synthetic release for Puppet source package {p_info.name} version {version} {description}
-     - true
-     - from intrinsic metadata
-     - from extrinsic metadata
-     - version and description from intrinsic metadata
-   * - pypi
-     - ``metadata​["version"]``
-     - ``release_name(​version)`` or ``release_name(​version, filename)``
-     - =version
-     - ``metadata[​'comment_text']}`` or standard message
-     - true
-     - from int metadata or ""
-     - from ext metadata or None
-     - metadata is intrinsic
-   * - rubygems
-     - ``p_info.version``
-     - ``release_name(​version)``
-     - =version
-     - Synthetic release for RubyGems source package {p_info.name} version {p_info.version}
-     - true
-     - from ext metadata
-     - from ext metadata
-     - The source code is extracted from a tarball nested within the gem file
+      * - Loader
+        - internal version
+        - branch name
+        - name
+        - message
+        - synthetic
+        - author
+        - date
+        - Notes
+      * - arch
+        - ``p_info.​version``
+        - ``release_name(​version, filename)``
+        - =version
+        - Synthetic release for Arch Linux source package {p_info.name} version {p_info.version} {description}
+        - true
+        - from intrinsic metadata
+        - from extra_loader_arguments['arch_metadata']
+        - Intrinsic metadata extracted from .PKGINFO file of the package
+      * - archive
+        - passed as arg
+        - ``release_name(​version)``
+        - =version
+        - "Synthetic release for archive at {p_info.url}\n"
+        - true
+        - ""
+        - passed as arg
+        -
+      * - aur
+        - ``p_info.​version``
+        - ``release_name(​version, filename)``
+        - =version
+        - Synthetic release for Aur source package {p_info.name} version {p_info.version} {description}
+        - true
+        - ""
+        - from extra_loader_arguments['aur_metadata']
+        - Intrinsic metadata extracted from .SRCINFO file of the package
+      * - cpan
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Perl source package {name} version {version} {description}
+        - true
+        - from intrinsic metadata if any else from extrinsic
+        - from extrinsic metadata
+        - name, version and description from intrinsic metadata
+      * - cran
+        - ``metadata.get(​"Version", passed as arg)``
+        - ``release_name(​version)``
+        - =version
+        - standard message
+        - true
+        - ``metadata.get(​"Maintainer", "")``
+        - ``metadata.get(​"Date")``
+        - metadata is intrinsic
+      * - conda
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Conda source package {p_info.name} version {p_info.version}
+        - true
+        - from intrinsic metadata
+        - from extrinsic metadata
+        - ""
+      * - crates
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Crate source package {p_info.name} version {p_info.version}
+        - true
+        - from intrinsic metadata
+        - from extrinsic metadata
+        - ""
+      * - debian
+        - =``version``
+        - ``release_name(​version)``
+        - =``i_version``
+        - standard message (using ``i_version``)
+        - true
+        - ``metadata​.changelog​.person``
+        - ``metadata​.changelog​.date``
+        - metadata is intrinsic. Old revisions have ``dsc`` as type
+          ``i_version`` is the intrinsic version (eg. ``0.7.2-3``) while ``version``
+          contains the debian suite name (eg. ``stretch/contrib/0.7.2-3``) and is
+          passed as arg
+      * - golang
+        - ``p_info.​version``
+        - ``release_name(version)``
+        - =version
+        - Synthetic release for Golang source package {p_info.name} version {p_info.version}
+        - true
+        - ""
+        - from ext metadata
+        - Golang offers basically no metadata outside of version and timestamp
+      * - deposit
+        - HEAD
+        - only HEAD
+        - HEAD
+        - "{client}: Deposit {id} in collection {collection}\n"
+        - true
+        - original author
+        - ``<codemeta: dateCreated>`` from SWORD XML
+        - revisions had parents
+      * - hackage
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Haskell source package {p_info.name} version {p_info.version}
+        - true
+        - intrinsic metadata if any else from extrinsic metadata
+        - from extrinsic metadata
+        - ""
+      * - hex
+        - ``p_info.version``
+        - ``release_name(version)``
+        - =version
+        - standard message
+        - true
+        - from extrinsic metadata
+        - from extrinsic metadata
+        - Source code is extracted from a nested tarball
+      * - maven-loader
+        - passed as arg
+        - HEAD
+        - ``release_name(version)``
+        - "Synthetic release for archive at {p_info.url}\n"
+        - true
+        - ""
+        - passed as arg
+        - Only one artefact per url (jar/zip src)
+      * - npm
+        - ``metadata​["version"]``
+        - ``release_name(​version)``
+        - =version
+        - standard message
+        - true
+        - from int metadata or ""
+        - from ext metadata or None
+        -
+      * - opam
+        - as given by opam
+        - "{opam_package}​.{version}"
+        - =version
+        - standard message
+        - true
+        - from metadata
+        - None
+        - "{self.opam_package}​.{version}" matches the version names used by opam's backend. metadata is extrinsic
+      * - pubdev
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for pub.dev source package {p_info.name} version {p_info.version}
+        - true
+        - from extrinsic metadata
+        - from extrinsic metadata
+        - name and version from extrinsic metadata
+      * - puppet
+        - ``p_info.​version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for Puppet source package {p_info.name} version {version} {description}
+        - true
+        - from intrinsic metadata
+        - from extrinsic metadata
+        - version and description from intrinsic metadata
+      * - pypi
+        - ``metadata​["version"]``
+        - ``release_name(​version)`` or ``release_name(​version, filename)``
+        - =version
+        - ``metadata[​'comment_text']}`` or standard message
+        - true
+        - from int metadata or ""
+        - from ext metadata or None
+        - metadata is intrinsic
+      * - rubygems
+        - ``p_info.version``
+        - ``release_name(​version)``
+        - =version
+        - Synthetic release for RubyGems source package {p_info.name} version {p_info.version}
+        - true
+        - from ext metadata
+        - from ext metadata
+        - The source code is extracted from a tarball nested within the gem file
 
 using this function::
 

docs/package-loader-tutorial.rst

@@ -20,8 +20,8 @@ It can be either a subdirectory of ``swh-loader-core/swh/loader/package/`` like
 the other package loaders, or it can be in its own package.
 
 If you choose the latter, you should also create the base file of any Python
-package (such as ``setup.py``), you should import them from the `swh-py-template`_
-repository.
+package (such as ``pyproject.toml``), you should bootstrap your project using
+`copier`_ and the `swh-py-template`_ repository.
 
 In the rest of this tutorial, we will assume you chose the former and
 your loader is named "New Loader", so your package loader is in
@@ -33,17 +33,15 @@ copy them from an existing package, such as
 ``swh-loader-core/swh/loader/package/pypi/``, and replace the names in those
 with your loader's.
 
-Finally, create an `entrypoint`_ in :file:`setup.py`, so your loader can be discovered
-by the SWH Celery workers::
+Finally, create an `entrypoint`_ in :file:`pyproject.toml`, so your loader can
+be discovered by the SWH Celery workers::
 
-    entry_points="""
-        [swh.workers]
-        loader.newloader=swh.loader.package.newloader:register
-    """,
+    [project.entry-points."swh.workers"]
+    "loader.newloader" = "swh.loader.package.newloader:register"
 
-.. _swh-py-template: https://forge.softwareheritage.org/source/swh-py-template/
+.. _swh-py-template: https://gitlab.softwareheritage.org/swh/devel/swh-py-template/
 .. _entrypoint: https://setuptools.readthedocs.io/en/latest/userguide/entry_point.html
-
+.. _copier: https://copier.readthedocs.io/
 
 Writing a minimal loader
 ------------------------
@@ -116,7 +114,7 @@ interactions with the package repository.
 
 
 Building a release
-+++++++++++++++++++
+++++++++++++++++++
 
 The final step for your minimal loader to work, is to implement ``build_release``.
 This is a very important part, as it will create a release object that will be
@@ -185,7 +183,7 @@ With Docker
 We recommend you use our `Docker environment`_ to test your loader.
 
 In short, install Docker, ``cd`` to ``swh-environment/docker/``,
-then `edit docker-compose.override.yml`_ to insert your new loader in the Docker
+then `edit compose.override.yml`_ to insert your new loader in the Docker
 environment, something like this will do::
 
    version: '2'
@@ -197,24 +195,24 @@ environment, something like this will do::
 
 Then start the Docker environment::
 
-   docker-compose start
+   docker compose start
 
 Then, you can run your loader::
 
-   docker-compose exec swh-loader swh loader run newloader "https://example.org/~jdoe/project/"
+   docker compose exec swh-loader swh loader run newloader "https://example.org/~jdoe/project/"
 
-where ``newloader`` is the name you registered as an entrypoint in ``setup.py`` and
+where ``newloader`` is the name you registered as an entrypoint in ``pyproject.toml`` and
 ``https://example.org/~jdoe/project/`` is the origin URL, that will be set as the
 ``self.url`` attribute of your loader.
 
 
 For example, to run the PyPI loader, the command would be::
 
-   docker-compose exec swh-loader swh loader run pypi "https://pypi.org/project/requests/"
+   docker compose exec swh-loader swh loader run pypi "https://pypi.org/project/requests/"
 
 
 If you get this error, make sure you properly configured
-``docker-compose.override.yml``::
+``compose.override.yml``::
 
    Error: Invalid value for '[...]': invalid choice: newloader
 
@@ -256,9 +254,23 @@ For example, with PyPI::
 
    swh loader -C loader.yml run pypi "https://pypi.org/project/requests/"
 
+For loaders that require complex arguments, you can pass them encoded as a yaml string::
+
+  swh loader -C swh_inmemory.yml run conda \
+      https://anaconda.org/conda-forge/lifetimes \
+      artifacts='[{
+        "url": "https://conda.anaconda.org/conda-forge/linux-64/lifetimes-0.11.1-py36h9f0ad1d_1.tar.bz2",  # noqa: B950
+        "date": "2020-07-06T12:19:36.425000+00:00",
+        "version": "linux-64/0.11.1-py36h9f0ad1d_1",
+        "filename": "lifetimes-0.11.1-py36h9f0ad1d_1.tar.bz2",
+        "checksums": {
+            "md5": "faa398f7ba0d60ce44aa6eeded490cee",
+            "sha256": "f82a352dfae8abceeeaa538b220fd9c5e4aa4e59092a6a6cea70b9ec0581ea03",
+        },
+      }]'
 
-.. _Docker environment: https://forge.softwareheritage.org/source/swh-environment/browse/master/docker/
-.. _edit docker-compose.override.yml: https://forge.softwareheritage.org/source/swh-environment/browse/master/docker/#install-a-swh-package-from
+.. _Docker environment: https://gitlab.softwareheritage.org/swh/devel
+.. _edit compose.override.yml: https://docs.softwareheritage.org/devel/getting-started/using-docker.html#install-a-swh-package-from-sources-in-a-container
 
 
 Testing your loader
@@ -318,11 +330,11 @@ are used in this test::
            branches={
                b"releases/1.1.0/nexter-1.1.0.zip": SnapshotBranch(
                    target=hash_to_bytes("f7d43faeb65b64d3faa67e4f46559db57d26b9a4"),
-                   target_type=TargetType.RELEASE,
+                   target_type=SnapshotTargetType.RELEASE,
                ),
                b"releases/1.1.0/nexter-1.1.0.tar.gz": SnapshotBranch(
                    target=hash_to_bytes("732bb9dc087e6015884daaebb8b82559be729b5a"),
-                   target_type=TargetType.RELEASE,
+                   target_type=SnapshotTargetType.RELEASE,
                ),
            },
        )
@@ -692,6 +704,8 @@ Most of them are a reminder of other sections above.
   original format unchanged, instead of converting it to a JSON/msgpack/... format)
   and :ref:`their format is documented <extrinsic-metadata-formats>`.
   They must tested as well.
+* The forge is present in the `swh-docs software-origins list <https://gitlab.softwareheritage.org/swh/devel/swh-docs/-/blob/master/docs/software-origins-support.yml>`_
+  and the `software-origins user documentation <https://gitlab.softwareheritage.org/swh/devel/swh-docs/-/tree/master/docs/user/software-origins>`_
 * There is no risk of extid clashes, even across instances (if relevant),
   even in presence of malicious actors (as far as reasonably possible)
 

mypy.ini

-[mypy]
-namespace_packages = True
-warn_unused_ignores = True
-
-# 3rd party libraries without stubs (yet)
-
-[mypy-celery.*]
-ignore_missing_imports = True
-
-[mypy-pkg_resources.*]
-ignore_missing_imports = True
-
-[mypy-psutil.*]
-ignore_missing_imports = True
-
-[mypy-psycopg2.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-requests_mock.*]
-ignore_missing_imports = True
-
-[mypy-retrying.*]
-ignore_missing_imports = True
-
-[mypy-chardet.*]
-ignore_missing_imports = True
-
-[mypy-debian.*]
-ignore_missing_imports = True
-
-[mypy-iso8601.*]
-ignore_missing_imports = True
-
-[mypy-pkginfo.*]
-ignore_missing_imports = True
-
-[mypy-swh.deposit.*]
-ignore_missing_imports = True
-
-[mypy-swh.model.*]
-ignore_missing_imports = True
-

pyproject.toml

+[project]
+name = "swh.loader.core"
+authors = [
+    {name="Software Heritage developers", email="swh-devel@inria.fr"},
+]
+
+description = "Software Heritage core and package loaders"
+readme = {file = "README.rst", content-type = "text/x-rst"}
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+    "Development Status :: 5 - Production/Stable",
+]
+dynamic = ["version", "dependencies", "optional-dependencies"]
+
+[tool.setuptools.packages.find]
+include = ["swh.*"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt", "requirements-swh.txt"]}
+
+[tool.setuptools.dynamic.optional-dependencies]
+testing = {file = ["requirements-test.txt"]}
+
+[project.entry-points."swh.cli.subcommands"]
+"swh.loader.core" = "swh.loader.cli"
+"swh.loader.core.nar" = "swh.loader.core.nar"
+
+[project.entry-points."swh.workers"]
+"loader.content" = "swh.loader.core:register_content"
+"loader.directory" = "swh.loader.core:register_directory"
+"loader.arch" = "swh.loader.package.arch:register"
+"loader.archive" = "swh.loader.package.archive:register"
+"loader.aur" = "swh.loader.package.aur:register"
+"loader.conda" = "swh.loader.package.conda:register"
+"loader.cpan" = "swh.loader.package.cpan:register"
+"loader.cran" = "swh.loader.package.cran:register"
+"loader.crates" = "swh.loader.package.crates:register"
+"loader.debian" = "swh.loader.package.debian:register"
+"loader.deposit" = "swh.loader.package.deposit:register"
+"loader.golang" = "swh.loader.package.golang:register"
+"loader.hackage" = "swh.loader.package.hackage:register"
+"loader.hex" = "swh.loader.package.hex:register"
+"loader.npm" = "swh.loader.package.npm:register"
+"loader.opam" = "swh.loader.package.opam:register"
+"loader.pubdev" = "swh.loader.package.pubdev:register"
+"loader.puppet" = "swh.loader.package.puppet:register"
+"loader.pypi" = "swh.loader.package.pypi:register"
+"loader.maven" = "swh.loader.package.maven:register"
+"loader.rubygems" = "swh.loader.package.rubygems:register"
+"loader.rpm" = "swh.loader.package.rpm:register"
+"loader.bioconductor" = "swh.loader.package.bioconductor:register"
+
+[project.urls]
+"Homepage" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core"
+"Bug Reports" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core/-/issues"
+"Funding" = "https://www.softwareheritage.org/donate"
+"Documentation" = "https://docs.softwareheritage.org/devel/swh-loader-core/"
+"Source" = "https://gitlab.softwareheritage.org/swh/devel/swh-loader-core.git"
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+fallback_version = "0.0.1"
+
 [tool.black]
-target-version = ['py37']
+target-version = ['py39', 'py310', 'py311', 'py312']
 
 [tool.isort]
 multi_line_output = 3
@@ -9,3 +79,40 @@ use_parentheses = true
 ensure_newline_before_comments = true
 line_length = 88
 force_sort_within_sections = true
+known_first_party = ['swh']
+
+[tool.codespell]
+ignore-words-list = "iff,crate"
+
+[tool.mypy]
+namespace_packages = true
+warn_unused_ignores = true
+explicit_package_bases = true
+# ^ Needed for mypy to detect py.typed from swh packages installed
+# in editable mode
+
+plugins = []
+
+# 3rd party libraries without stubs (yet)
+# [[tool.mypy.overrides]]
+# module = [
+#     "package1.*",
+#     "package2.*",
+# ]
+# ignore_missing_imports = true
+
+[tool.flake8]
+select = ["C", "E", "F", "W", "B950"]
+ignore = [
+    "E203", # whitespaces before ':' <https://github.com/psf/black/issues/315>
+    "E231", # missing whitespace after ','
+    "E501", # line too long, use B950 warning from flake8-bugbear instead
+    "W503", # line break before binary operator <https://github.com/psf/black/issues/52>
+    "E704" # multiple statements on one line
+]
+max-line-length = 88
+
+[tool.pytest.ini_options]
+norecursedirs = "build docs .*"
+asyncio_mode = "strict"
+consider_namespace_packages = true

pytest.ini

-[pytest]
-
-norecursedirs = build docs .*
-
-markers =
-    db: marks tests as using a db (deselect with '-m "not db"')
-    fs: marks tests as using the filesystem (deselect with '-m "not fs"')

requirements-swh.txt

-swh.core >= 2.12
-swh.model >= 6.5.1
+swh.core >= 2.23.0
+swh.model >= 6.13.0
 swh.objstorage >= 0.2.2
-swh.scheduler >= 0.4.0
-swh.storage >= 0.29.0
+swh.scheduler >= 2.3.0
+swh.storage >= 2.4.1

requirements-test.txt

+celery-types
 pytest
 pytest-mock
-requests_mock
+requests_mock >= 1.11.0
+urllib3
 swh-core[testing]
-swh-scheduler[testing] >= 0.5.0
-swh-storage[testing] >= 0.10.6
+swh.scheduler[testing] >= 2.3.0
+swh-storage[testing] >= 2.4.1
+swh-vault >= 1.12.2
 types-click
 types-python-dateutil
+types-psutil
 types-pyyaml
 types-requests

requirements.txt

@@ -2,12 +2,14 @@
 # should match https://pypi.python.org/pypi names. For the full spec or
 # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html
 
+chardet
 psutil
 requests
 iso8601
+looseversion
 pkginfo
 python-debian
 python-dateutil
+tenacity >= 8.4.2
 typing-extensions
 toml
-packaging