.copier-answers.yml

+# Changes here will be overwritten by Copier
+_commit: v0.3.3
+_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
+description: Software Heritage core and package loaders
+distribution_name: swh-loader-core
+have_cli: true
+have_workers: true
+package_root: swh/loader
+project_name: swh.loader.core
+python_minimal_version: '3.7'
+readme_format: rst

.git-blame-ignore-revs

+# python: Reformat code with black
+cf496e18440a073ec3d2b65657882e1bdb69a4d2
+9c8a00123c37db204c59f6e6d17af520d29c7b65
+8148efe14edafda15ce468e2ac2d5c9fbcae5623

.gitignore

+*.egg-info/
 *.pyc
-*.sw?
-*~
 .coverage
 .eggs/
+.hypothesis
+.mypy_cache
+.tox
 __pycache__
-*.egg-info/
-version.txt
\ No newline at end of file
+build/
+dist/
+# these are symlinks created by a hook in swh-docs' main sphinx conf.py
+docs/README.rst
+docs/README.md
+# this should be a symlink for people who want to build the sphinx doc
+# without using tox, generally created by the swh-env/bin/update script
+docs/Makefile.sphinx

.pre-commit-config.yaml

+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-json
+      - id: check-yaml
+
+  - repo: https://github.com/python/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.0
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        name: Check source code spelling
+        additional_dependencies:
+          - tomli
+        exclude: ^(swh/loader/.*/tests/data/.*)$
+        stages: [pre-commit]
+      - id: codespell
+        name: Check commit message spelling
+        additional_dependencies:
+          - tomli
+        stages: [commit-msg]
+
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        args: [swh]
+        pass_filenames: false
+        language: system
+        types: [python]
+      - id: twine-check
+        name: twine check
+        description: call twine check when pushing an annotated release tag
+        entry: bash -c "ref=$(git describe) &&
+          [[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+          (python3 -m build --sdist && twine check $(ls -t dist/* | head -1)) || true"
+        pass_filenames: false
+        stages: [pre-push]
+        language: python
+        additional_dependencies: [twine, build]

CODE_OF_CONDUCT.md

+# Software Heritage Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as Software
+Heritage contributors and maintainers pledge to making participation in our
+project and our community a harassment-free experience for everyone, regardless
+of age, body size, disability, ethnicity, sex characteristics, gender identity
+and expression, level of experience, education, socioeconomic status,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at `conduct@softwareheritage.org`. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an
+incident.  Further details of specific enforcement policies may be posted
+separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTORS

+Antoine Eiche
+Léo Andrès
+Franck Bret

MANIFEST.in

-include Makefile
-include requirements.txt
-include version.txt

README.rst

+Software Heritage - Loader foundations
+======================================
+
+The Software Heritage Loader Core is a low-level loading utilities and
+helpers used by `loaders <https://docs.softwareheritage.org/devel/glossary.html#term-loader>`_.
+
+The main entry points are classes:
+
+- `swh.loader.core.loader.BaseLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.BaseLoader>`_
+  for VCS loaders (e.g. git, svn, ...)
+
+- `swh.loader.core.loader.ContentLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.ContentLoader>`_
+  for Content loader
+
+- `swh.loader.core.loader.BaseDirectoryLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.core.loader.html#swh.loader.core.loader.BaseDirectoryLoader>`_
+  for Directory loaders
+
+- `swh.loader.package.loader.PackageLoader <https://docs.softwareheritage.org/devel/apidoc/swh.loader.package.loader.html#swh.loader.package.loader.PackageLoader>`_
+  for Package loaders (e.g. PyPI, Npm, ...)
+
+Package loaders
+---------------
+
+This package also implements many package loaders directly, out of convenience,
+as they usually are quite similar and each fits in a single file.
+
+They all roughly follow these steps, explained in the
+`swh.loader.package.loader.PackageLoader.load <https://docs.softwareheritage.org/devel/apidoc/swh.loader.package.loader.html#swh.loader.package.loader.PackageLoader.load>`_
+documentation.
+See the `Package Loader tutorial <https://docs.softwareheritage.org/devel/swh-loader-core/package-loader-tutorial.html#package-loader-tutorial>`_
+for details.
+
+VCS loaders
+-----------
+
+Unlike package loaders, VCS loaders remain in separate packages,
+as they often need more advanced conversions and very VCS-specific operations.
+
+This usually involves getting the branches of a repository and recursively loading
+revisions in the history (and directory trees in these revisions),
+until a known revision is found

conftest.py

+# Copyright (C) 2019-2024  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+
+pytest_plugins = [
+    "swh.scheduler.pytest_plugin",
+    "swh.storage.pytest_plugin",
+    "swh.loader.pytest_plugin",
+]
+
+
+@pytest.fixture(scope="session")
+def swh_scheduler_celery_includes(swh_scheduler_celery_includes):
+    return swh_scheduler_celery_includes + [
+        "swh.loader.package.archive.tasks",
+        "swh.loader.package.cran.tasks",
+        "swh.loader.package.debian.tasks",
+        "swh.loader.package.deposit.tasks",
+        "swh.loader.package.npm.tasks",
+        "swh.loader.package.pypi.tasks",
+        "swh.loader.package.maven.tasks",
+    ]

debian/changelog

-<module-name> (0.0.1-1) unstable; urgency=low
-
-  *
-
- -- AUTHOR-NAME <AUTHOR-EMAIL>  <UTC-DATE>

debian/compat

-9

debian/control

-Source: swh-loader-core
-Maintainer: Software Heritage developers <swh-devel@inria.fr>
-Section: python
-Priority: optional
-Build-Depends: debhelper (>= 9),
-               dh-python,
-               python3-all,
-               python3-nose,
-               python3-setuptools,
-               python3-swh.core,
-               python3-swh.storage,
-               python3-retrying,
-               python3-vcversioner
-Standards-Version: 3.9.6
-Homepage: https://forge.softwareheritage.org/diffusion/60/
-
-Package: python3-swh.loader.core
-Architecture: all
-Depends: ${misc:Depends}, ${python3:Depends}
-Description: Software Heritage <module-description>

debian/copyright

-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-
-Files: *
-Copyright: 2015 The Software Heritage developers
-License: GPL-3+
-
-License: GPL-3+
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License, or
- (at your option) any later version.
- .
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- GNU General Public License for more details.
- .
- You should have received a copy of the GNU General Public License
- along with this program.  If not, see <http://www.gnu.org/licenses/>.
- .
- On Debian systems, the complete text of the GNU General Public
- License version 3 can be found in `/usr/share/common-licenses/GPL-3'.

debian/rules

-#!/usr/bin/make -f
-
-export PYBUILD_NAME=<dash-separated-module-name>
-
-%:
-	dh $@ --with python3 --buildsystem=pybuild
-
-
-override_dh_auto_test:

debian/source/format

-3.0 (quilt)

docs/.gitignore

+_build/
+apidoc/
+*-stamp
+README.md

docs/Makefile

+include Makefile.sphinx

docs/README.rst

+../README.rst
\ No newline at end of file