.copier-answers.yml

+# Changes here will be overwritten by Copier
+_commit: v0.3.3
+_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
+description: Software Heritage lister
+distribution_name: swh-lister
+have_cli: true
+have_workers: true
+package_root: swh/lister
+project_name: swh.lister
+python_minimal_version: '3.7'
+readme_format: rst

.git-blame-ignore-revs

-# Enable black
+# python: Reformat code with black
 93a4d8b784dabec76b65633a9e5a3a11b338e7f2
-
-# python: Reformat code with black 22.3.0
 d38e05cff7253de9259b3a0b6cdb59856f2afb92
+714fccc3c7f3e7548028de5fa70594accda2649d

.gitignore

+*.egg-info/
 *.pyc
-*.sw?
-*~
-/.coverage
-/.coverage.*
+.coverage
 .eggs/
+.hypothesis
+.mypy_cache
+.tox
 __pycache__
 build/
 dist/
-*.egg-info
-version.txt
-.tox/
-.mypy_cache/
+# these are symlinks created by a hook in swh-docs' main sphinx conf.py
+docs/README.rst
+docs/README.md
+# this should be a symlink for people who want to build the sphinx doc
+# without using tox, generally created by the swh-env/bin/update script
+docs/Makefile.sphinx

.pre-commit-config.yaml

 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: check-json
       - id: check-yaml
 
+  - repo: https://github.com/python/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.0
+    hooks:
+      - id: isort
+
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
+    rev: 7.1.1
     hooks:
       - id: flake8
-        additional_dependencies: [flake8-bugbear==22.9.23]
+        additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.4.1
     hooks:
       - id: codespell
         name: Check source code spelling
         exclude: ^(swh/lister/.*/tests/data/.*)$
         args: [-L crate]
-        stages: [commit]
+        stages: [pre-commit]
+      - id: codespell
+        name: Check commit message spelling
+        args: [-L crate]
+        stages: [commit-msg]
 
   - repo: local
     hooks:
@@ -30,13 +44,13 @@ repos:
         pass_filenames: false
         language: system
         types: [python]
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.11.5
-    hooks:
-      - id: isort
-
-  - repo: https://github.com/python/black
-    rev: 22.10.0
-    hooks:
-      - id: black
+      - id: twine-check
+        name: twine check
+        description: call twine check when pushing an annotated release tag
+        entry: bash -c "ref=$(git describe) &&
+          [[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+          (python3 -m build --sdist && twine check $(ls -t dist/* | head -1)) || true"
+        pass_filenames: false
+        stages: [pre-push]
+        language: python
+        additional_dependencies: [twine, build]

AUTHORS

+Copyright (C) 2019-2023 The Software Heritage developers
+
+See http://www.softwareheritage.org/ for more information.

CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as Software
 Heritage contributors and maintainers pledge to making participation in our
 project and our community a harassment-free experience for everyone, regardless
 of age, body size, disability, ethnicity, sex characteristics, gender identity
-and expression, level of experience, education, socio-economic status,
+and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, religion, or sexual identity and
 orientation.
 

MANIFEST.in

-include Makefile
-include README
-include requirements*.txt
-include version.txt
-include conftest.py
-include swh/lister/cran/list_all_packages.R
-recursive-include swh/lister/*/tests/data/ *
-recursive-include swh py.typed

README.md

-swh-lister
-==========
-
-This component from the Software Heritage stack aims to produce listings
-of software origins and their urls hosted on various public developer platforms
-or package managers. As these operations are quite similar, it provides a set of
-Python modules abstracting common software origins listing behaviors.
-
-It also provides several lister implementations, contained in the
-following Python modules:
-
-- `swh.lister.bitbucket`
-- `swh.lister.cgit`
-- `swh.lister.cran`
-- `swh.lister.debian`
-- `swh.liser.fedora`
-- `swh.lister.gitea`
-- `swh.lister.github`
-- `swh.lister.gitlab`
-- `swh.lister.gnu`
-- `swh.lister.gogs`
-- `swh.lister.golang`
-- `swh.lister.hex`
-- `swh.lister.launchpad`
-- `swh.lister.maven`
-- `swh.lister.npm`
-- `swh.lister.packagist`
-- `swh.lister.phabricator`
-- `swh.lister.pypi`
-- `swh.lister.tuleap`
-
-Dependencies
-------------
-
-All required dependencies can be found in the `requirements*.txt` files located
-at the root of the repository.
-
-Local deployment
-----------------
-
-## lister configuration
-
-Each lister implemented so far by Software Heritage (`bitbucket`, `cgit`, `cran`, `debian`,
-`gitea`, `github`, `gitlab`, `gnu`, `golang`, `launchpad`, `npm`, `packagist`, `phabricator`, `pypi`, `tuleap`, `maven`)
-must be configured by following the instructions below (please note that you have to replace
-`<lister_name>` by one of the lister name introduced above).
-
-### Preparation steps
-
-1. `mkdir ~/.config/swh/`
-2. create configuration file `~/.config/swh/listers.yml`
-
-### Configuration file sample
-
-Minimalistic configuration shared by all listers to add in file `~/.config/swh/listers.yml`:
-
-```lang=yml
-scheduler:
-  cls: 'remote'
-  args:
-    url: 'http://localhost:5008/'
-
-credentials: {}
-```
-
-Note: This expects scheduler (5008) service to run locally
-
-## Executing a lister
-
-Once configured, a lister can be executed by using the `swh` CLI tool with the
-following options and commands:
-
-```
-$ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister <lister_name> [lister_parameters]
-```
-
-Examples:
-
-```
-$ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister bitbucket
-
-$ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister cran
-
-$ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister gitea url=https://codeberg.org/api/v1/
-
-$ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister gitlab url=https://salsa.debian.org/api/v4/
-
-$ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister npm
-
-$ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister pypi
-```
-
-Licensing
----------
-
-This program is free software: you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free Software
-Foundation, either version 3 of the License, or (at your option) any later
-version.
-
-This program is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-
-See top-level LICENSE file for the full text of the GNU General Public License
-along with this program.

README.rst

+Software Heritage - Listers
+===========================
+
+Collection of listers for source code distribution places like development
+forges, FOSS distributions, package managers, etc. Each lister is in charge to
+enumerate the software origins (e.g., VCS, packages, etc.) available at a
+source code distribution place.
+
+A lister is a component from the Software Heritage stack aims to produce
+listings of software origins and their urls hosted on various public developer
+platforms or package managers. As these operations are quite similar, this
+package provides a set of Python modules abstracting common software origins
+listing behaviors.
+
+It also provides several lister implementations, contained in the Python
+``swh.lister.*`` modules. See `this documentation
+<https://docs.softwareheritage.org/user/listers.html>`_ for the list of
+supported listers.
+
+
+Dependencies
+------------
+
+All required python dependencies can be found in the ``requirements*.txt`` files
+located at the root of the repository.
+
+In order to be able to run all the listers (and thus execute the tests), some
+tools must be available on your system, namely:
+
+- ``opam``
+- ``tar``
+- ``psql``
+
+On a Debian-like system, you may use:
+
+.. code-block: console
+
+   $ sudo apt update
+   $ sudo apt install opam tar postgresql-client-common
+
+
+Local deployment
+----------------
+
+Lister configuration
+++++++++++++++++++++
+
+Each lister implemented so far by Software Heritage (``bitbucket``, ``cgit``,
+``cran``, ``debian``, ``gitea``, ``github``, ``gitlab``, ``gnu``, ``golang``,
+``launchpad``, ``npm``, ``packagist``, ``phabricator``, ``pypi``, ``tuleap``,
+``maven``) must be configured by following the instructions below (please note
+that you have to replace ``<lister_name>`` by one of the lister name introduced
+above).
+
+Preparation steps
+~~~~~~~~~~~~~~~~~
+
+1. ``mkdir ~/.config/swh/``
+2. create configuration file ``~/.config/swh/listers.yml``
+
+Configuration file sample
++++++++++++++++++++++++++
+
+Minimalistic configuration shared by all listers to add in file
+``~/.config/swh/listers.yml``:
+
+.. code-block:: yaml
+
+   scheduler:
+     cls: 'remote'
+     args:
+       url: 'http://localhost:5008/'
+
+   credentials: {}
+
+Note: This expects scheduler (5008) service to run locally
+
+Executing a lister
+------------------
+
+Once configured, a lister can be executed by using the ``swh`` CLI tool with
+the following options and commands:
+
+.. code-block:: shell
+
+   $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister <lister_name> [lister_parameters]
+
+
+Examples:
+
+.. code-block:: shell
+
+   $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister bitbucket
+
+   $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister cran
+
+   $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister gitea url=https://codeberg.org/api/v1/
+
+   $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister gitlab url=https://salsa.debian.org/api/v4/
+
+   $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister npm
+
+   $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister pypi
+
+
+Licensing
+---------
+
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+See top-level LICENSE file for the full text of the GNU General Public License
+along with this program.

conftest.py

-# Copyright (C) 2020-2022  The Software Heritage developers
+# Copyright (C) 2020-2024  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -13,8 +13,6 @@ os.environ["LC_ALL"] = "C.UTF-8"
 
 
 @pytest.fixture(autouse=True)
-def tenacity_wait(mocker):
-    # Stops tenacity from blocking lister tests for 50x errors
-    from swh.lister.pattern import Lister
-
-    mocker.patch.object(Lister.http_request.retry, "sleep")
+def mock_sleep(mocker):
+    # Stops tenacity from blocking lister tests when retrying
+    return mocker.patch("time.sleep")

docs/Makefile

-include ../../swh-docs/Makefile.sphinx
+include Makefile.sphinx

docs/index.rst

 .. _swh-lister:
 
-Software Heritage - Listers
-===========================
-
-Collection of listers for source code distribution places like development
-forges, FOSS distributions, package managers, etc. Each lister is in charge to
-enumerate the software origins (e.g., VCS, packages, etc.) available at a
-source code distribution place.
+.. include:: README.rst
 
 
 Overview
@@ -16,7 +10,6 @@ Overview
    :maxdepth: 2
    :titlesonly:
 
-
    tutorial
    run_a_new_lister
    save_forge

docs/new_lister_template.py

@@ -99,7 +99,6 @@ class NewForgeLister(Lister[NewForgeListerState, NewForgeListerPage]):
         assert self.lister_obj.id is not None
 
         for element in page:
-
             yield ListedOrigin(
                 # Required. Should use this value.
                 lister_id=self.lister_obj.id,

docs/tutorial.rst

@@ -374,7 +374,7 @@ More about listers
 
 See current implemented listers as examples (GitHub_, Bitbucket_, CGit_, GitLab_ ).
 
-.. _GitHub: https://forge.softwareheritage.org/source/swh-lister/browse/master/swh/lister/github/lister.py
-.. _Bitbucket: https://forge.softwareheritage.org/source/swh-lister/browse/master/swh/lister/bitbucket/lister.py
-.. _CGit: https://forge.softwareheritage.org/source/swh-lister/browse/master/swh/lister/cgit/lister.py
-.. _GitLab: https://forge.softwareheritage.org/source/swh-lister/browse/master/swh/lister/gitlab/lister.py
+.. _GitHub: https://gitlab.softwareheritage.org/swh/devel/swh-lister/-/blob/master/swh/lister/github/lister.py
+.. _Bitbucket: https://gitlab.softwareheritage.org/swh/devel/swh-lister/-/blob/master/swh/lister/bitbucket/lister.py
+.. _CGit: https://gitlab.softwareheritage.org/swh/devel/swh-lister/-/blob/master/swh/lister/cgit/lister.py
+.. _GitLab: https://gitlab.softwareheritage.org/swh/devel/swh-lister/-/blob/master/swh/lister/gitlab/lister.py

mypy.ini

-[mypy]
-namespace_packages = True
-warn_unused_ignores = True
-
-# 3rd party libraries without stubs (yet)
-
-[mypy-bs4.*]
-ignore_missing_imports = True
-
-[mypy-celery.*]
-ignore_missing_imports = True
-
-[mypy-debian.*]
-ignore_missing_imports = True
-
-[mypy-iso8601.*]
-ignore_missing_imports = True
-
-[mypy-launchpadlib.*]
-ignore_missing_imports = True
-
-[mypy-lazr.*]
-ignore_missing_imports = True
-
-[mypy-lxml.*]
-ignore_missing_imports = True
-
-[mypy-pkg_resources.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-pytest_postgresql.*]
-ignore_missing_imports = True
-
-[mypy-requests_mock.*]
-ignore_missing_imports = True
-
-[mypy-urllib3.util.*]
-ignore_missing_imports = True
-
-[mypy-dulwich.*]
-ignore_missing_imports = True
-
-[mypy-dateparser.*]
-ignore_missing_imports = True
-
-[mypy-testing.postgresql.*]
-ignore_missing_imports = True
-
-[mypy-psycopg2.*]
-ignore_missing_imports = True
-
-[mypy-repomd.*]
-ignore_missing_imports = True
-
-[mypy-defusedxml.*]
-ignore_missing_imports = True

pyproject.toml

+[project]
+name = "swh.lister"
+authors = [
+    {name="Software Heritage developers", email="swh-devel@inria.fr"},
+]
+
+description = "Software Heritage lister"
+readme = {file = "README.rst", content-type = "text/x-rst"}
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+    "Development Status :: 5 - Production/Stable",
+]
+dynamic = ["version", "dependencies", "optional-dependencies"]
+
+[tool.setuptools.packages.find]
+include = ["swh.*"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt", "requirements-swh.txt"]}
+
+[tool.setuptools.dynamic.optional-dependencies]
+testing = {file = ["requirements-test.txt"]}
+
+[project.entry-points."swh.cli.subcommands"]
+"swh.lister" = "swh.lister.cli"
+
+[project.entry-points."swh.workers"]
+"lister.arch" = "swh.lister.arch:register"
+"lister.aur" = "swh.lister.aur:register"
+"lister.bioconductor" = "swh.lister.bioconductor:register"
+"lister.bitbucket" = "swh.lister.bitbucket:register"
+"lister.bower" = "swh.lister.bower:register"
+"lister.save-bulk" = "swh.lister.save_bulk:register"
+"lister.cgit" = "swh.lister.cgit:register"
+"lister.conda" = "swh.lister.conda:register"
+"lister.cpan" = "swh.lister.cpan:register"
+"lister.cran" = "swh.lister.cran:register"
+"lister.crates" = "swh.lister.crates:register"
+"lister.debian" = "swh.lister.debian:register"
+"lister.dlang" = "swh.lister.dlang:register"
+"lister.elm" = "swh.lister.elm:register"
+"lister.gitea" = "swh.lister.gitea:register"
+"lister.github" = "swh.lister.github:register"
+"lister.gitiles" = "swh.lister.gitiles:register"
+"lister.gitlab" = "swh.lister.gitlab:register"
+"lister.gitweb" = "swh.lister.gitweb:register"
+"lister.gnu" = "swh.lister.gnu:register"
+"lister.gogs" = "swh.lister.gogs:register"
+"lister.golang" = "swh.lister.golang:register"
+"lister.hackage" = "swh.lister.hackage:register"
+"lister.hex" = "swh.lister.hex:register"
+"lister.julia" = "swh.lister.julia:register"
+"lister.launchpad" = "swh.lister.launchpad:register"
+"lister.maven" = "swh.lister.maven:register"
+"lister.nixguix" = "swh.lister.nixguix:register"
+"lister.npm" = "swh.lister.npm:register"
+"lister.nuget" = "swh.lister.nuget:register"
+"lister.opam" = "swh.lister.opam:register"
+"lister.packagist" = "swh.lister.packagist:register"
+"lister.pagure" = "swh.lister.pagure:register"
+"lister.phabricator" = "swh.lister.phabricator:register"
+"lister.pubdev" = "swh.lister.pubdev:register"
+"lister.puppet" = "swh.lister.puppet:register"
+"lister.pypi" = "swh.lister.pypi:register"
+"lister.rpm" = "swh.lister.rpm:register"
+"lister.rubygems" = "swh.lister.rubygems:register"
+"lister.sourceforge" = "swh.lister.sourceforge:register"
+"lister.stagit" = "swh.lister.stagit:register"
+"lister.tuleap" = "swh.lister.tuleap:register"
+
+[project.urls]
+"Homepage" = "https://gitlab.softwareheritage.org/swh/devel/swh-lister"
+"Bug Reports" = "https://gitlab.softwareheritage.org/swh/devel/swh-lister/-/issues"
+"Funding" = "https://www.softwareheritage.org/donate"
+"Documentation" = "https://docs.softwareheritage.org/devel/swh-lister/"
+"Source" = "https://gitlab.softwareheritage.org/swh/devel/swh-lister.git"
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+fallback_version = "0.0.1"
+
 [tool.black]
-target-version = ['py37']
+target-version = ['py39', 'py310', 'py311', 'py312']
 
 [tool.isort]
 multi_line_output = 3
@@ -9,3 +97,41 @@ use_parentheses = true
 ensure_newline_before_comments = true
 line_length = 88
 force_sort_within_sections = true
+known_first_party = ['swh']
+
+[tool.mypy]
+namespace_packages = true
+warn_unused_ignores = true
+explicit_package_bases = true
+# ^ Needed for mypy to detect py.typed from swh packages installed
+# in editable mode
+
+plugins = []
+
+# 3rd party libraries without stubs (yet)
+[[tool.mypy.overrides]]
+module = [
+    "launchpadlib.*",
+    "lazr.*",
+    "mercurial.*",
+    "pyreadr.*",
+    "subvertpy.*",
+    "repomd.*",
+    "testing.postgresql.*",
+]
+ignore_missing_imports = true
+
+[tool.flake8]
+select = ["C", "E", "F", "W", "B950"]
+ignore = [
+    "E203", # whitespaces before ':' <https://github.com/psf/black/issues/315>
+    "E231", # missing whitespace after ','
+    "E501", # line too long, use B950 warning from flake8-bugbear instead
+    "W503" # line break before binary operator <https://github.com/psf/black/issues/52>
+]
+max-line-length = 88
+
+[tool.pytest.ini_options]
+norecursedirs = "build docs .*"
+asyncio_mode = "strict"
+consider_namespace_packages = true

pytest.ini

-[pytest]
-# We need this to avoid conflicts between the autoloaded fixture, and the
-# manually loaded one.
-addopts = -p no:pytest_swh_scheduler
-norecursedirs = build docs .*
-
-asyncio_mode = strict

requirements-swh.txt

-swh.core[db,github] >= 2.22.0
-swh.scheduler >= 0.8
+swh.core[db] >= 4.0.0
+swh.scheduler >= 3.0.0

requirements-test.txt

-pytest
+celery-types
+pandas
+pandas-stubs
+pytest >= 8.1
 pytest-mock
 requests_mock
+swh-scheduler[testing] >= 3.0.0
+types-beautifulsoup4
 types-click
+types-dateparser
 types-pyyaml
 types-requests
+types-toml
+types-lxml

requirements.txt

-python_debian
-requests
-setuptools
+beautifulsoup4 >= 4.13.3
+breezy >= 3.3.1, < 3.3.5 # use versions with available binary wheels
+dateparser
+dulwich
 iso8601
-beautifulsoup4
 launchpadlib
-tenacity >= 6.2
+looseversion
 lxml
-dateparser
-dulwich
-testing.postgresql
-psycopg2
+mercurial
+psycopg
+pyreadr
+python_debian
 repomd
+requests
+setuptools
+subvertpy
+tenacity >= 8.4.2
+testing.postgresql
+toml