.copier-answers.yml

+# Changes here will be overwritten by Copier
+_commit: v0.3.3
+_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
+description: Software Heritage deposit server
+distribution_name: swh-deposit
+have_cli: true
+have_workers: true
+package_root: swh/deposit
+project_name: swh.deposit
+python_minimal_version: '3.7'
+readme_format: rst

.git-blame-ignore-revs

+# python: Reformat code with black
+f5426d6722826972e2d611d4e7040abbf40c49a1
+8a006aeebf7d0cf52abc71b07cd560cbd098349e
+7b0fac22d29db6ad27cb650f835cae2f8786ad70
+# isort
+9c0d0496369828c8fad882d5d676978fb76105f8

.gitignore

+*.egg-info/
 *.pyc
-*.sw?
-*~
 .coverage
 .eggs/
+.hypothesis
+.mypy_cache
+.tox
 __pycache__
-*.egg-info/
-version.txt
-/analysis.org
-/swh/deposit/fixtures/private_data.yaml
-/swh/deposit.json
-/test.json
-/swh/test
-db.sqlite3
-/.noseids
+build/
+dist/
+# these are symlinks created by a hook in swh-docs' main sphinx conf.py
+docs/README.rst
+docs/README.md
+# this should be a symlink for people who want to build the sphinx doc
+# without using tox, generally created by the swh-env/bin/update script
+docs/Makefile.sphinx

.pre-commit-config.yaml

+exclude: ^swh/deposit/tests/data/atom/.*$
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-json
+      - id: check-yaml
+
+  - repo: https://github.com/python/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.0
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        name: Check source code spelling
+        args: [-L sur]
+        stages: [pre-commit]
+      - id: codespell
+        name: Check commit message spelling
+        stages: [commit-msg]
+
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: env DJANGO_SETTINGS_MODULE=swh.deposit.settings.testing mypy
+        args: [swh]
+        pass_filenames: false
+        language: system
+        types: [python]
+      - id: twine-check
+        name: twine check
+        description: call twine check when pushing an annotated release tag
+        entry: bash -c "ref=$(git describe) &&
+          [[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+          (python3 -m build --sdist && twine check $(ls -t dist/* | head -1)) || true"
+        pass_filenames: false
+        stages: [pre-push]
+        language: python
+        additional_dependencies: [twine, build]

CODE_OF_CONDUCT.md

+# Software Heritage Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as Software
+Heritage contributors and maintainers pledge to making participation in our
+project and our community a harassment-free experience for everyone, regardless
+of age, body size, disability, ethnicity, sex characteristics, gender identity
+and expression, level of experience, education, socioeconomic status,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at `conduct@softwareheritage.org`. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an
+incident.  Further details of specific enforcement policies may be posted
+separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTORS

+Ishan Bhanuka

MANIFEST.in

-include Makefile
-include requirements.txt
-include requirements-swh.txt
-include version.txt
-recursive-include swh/deposit/static *
-recursive-include swh/deposit/fixtures *
-recursive-include swh/deposit/templates *

Makefile.local

-FLAKEFLAGS='--exclude=swh/manage.py,swh/deposit/settings.py,swh/deposit/migrations/'
+FLAKEFLAGS='--exclude=swh/deposit/manage.py,swh/deposit/settings.py,swh/deposit/migrations/'
 
-MANAGE=python3 -m swh.manage
+MANAGE=python3 -m swh.deposit.manage
 
 db-drop:
 	dropdb swh-deposit-dev || return 0
@@ -26,5 +26,9 @@ run-dev:
 run:
 	gunicorn3 -b 127.0.0.1:5006 swh.deposit.wsgi
 
-test:
-	./swh/manage.py test
+# Override default rule to make sure DJANGO env var is properly set. It
+# *should* work without any override thanks to the mypy django-stubs plugin,
+# but it currently doesn't; see
+# https://github.com/typeddjango/django-stubs/issues/166
+check-mypy:
+	DJANGO_SETTINGS_MODULE=swh.deposit.settings.testing $(MYPY) $(MYPYFLAGS) swh

README.md

-# swh-deposit
-
-This is [Software Heritage](https://www.softwareheritage.org)'s
-[SWORD 2.0](http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html) Server
-implementation.
-
-**S.W.O.R.D** (**S**imple **W**eb-Service **O**ffering **R**epository
-**D**eposit) is an interoperability standard for digital file deposit.
-
-This implementation will permit interaction between a client (a
-repository) and a server (SWH repository) to permit deposits of
-software source code archives and associated metadata.
-
-The documentation is at ./docs/README-specification.md

README.rst

+Software Heritage - Deposit
+===========================
+
+Simple Web-Service Offering Repository Deposit (S.W.O.R.D) is an interoperability
+standard for digital file deposit.
+
+This repository is both the `SWORD v2`_ Server and a deposit command-line client
+implementations.
+
+This implementation allows interaction between a client (a repository) and a server (SWH
+repository) to deposit software source code archives and associated metadata.
+
+Description
+-----------
+
+Most of the software source code artifacts present in the SWH Archive are gathered by
+the mean of `loader`_ workers run by the SWH project from source code
+origins identified by `lister`_ workers. This is a pull mechanism: it's
+the responsibility of the SWH project to gather and collect source code artifacts that
+way.
+
+Alternatively, SWH allows its partners to push source code artifacts and metadata
+directly into the Archive with a push-based mechanism. By using this possibility
+different actors, holding software artifacts or metadata, can preserve their assets
+without having to pass through an intermediate collaborative development platform, which
+is already harvested by SWH (e.g GitHub, Gitlab, etc.).
+
+This mechanism is the ``deposit``.
+
+The main idea is the deposit is an authenticated access to an API allowing the user to
+provide source code artifacts -- with metadata -- to be ingested in the SWH Archive. The
+result of that is a `SWHID`_ that can be used to uniquely
+and persistently identify that very piece of source code.
+
+This unique identifier can then be used to `reference the source code
+<https://hal.archives-ouvertes.fr/hal-02446202>`_ (e.g. in a `scientific paper
+<https://www.softwareheritage.org/2020/05/26/citing-software-with-style/>`_) and
+retrieve it using the `vault`_ feature of the SWH Archive platform.
+
+The differences between a piece of code uploaded using the deposit rather than simply
+asking SWH to archive a repository using the `save code now`_ feature
+are:
+
+- a deposited artifact is provided from one of the SWH partners which is regarded as a
+  trusted authority,
+- a deposited artifact requires metadata properties describing the source code artifact,
+- a deposited artifact has a codemeta_ metadata entry attached to it,
+- a deposited artifact has the same visibility on the SWH Archive than a collected
+  repository,
+- a deposited artifact can be searched with its provided url property on the SWH
+  Archive,
+- the deposit API uses the `SWORD v2`_ API, thus requires some tooling to send deposits
+  to SWH. These tools are provided with this repository.
+
+
+See the `User Manual`_ page for more details on how to use the deposit client
+command line tools to push a deposit in the SWH Archive.
+
+See the `API Documentation`_ reference pages of the SWORDv2 API implementation
+in ``swh.deposit`` if you want to do upload deposits using HTTP requests.
+
+Read the `Deposit metadata`_ chapter to get more details on what metadata
+are supported when doing a deposit.
+
+See `Running swh-deposit locally`_ if you want to hack the code of the ``swh.deposit`` module.
+
+See `Production deployment`_ if you want to deploy your own copy of the
+`swh.deposit` stack.
+
+
+.. _codemeta: https://codemeta.github.io/
+.. _SWORD v2: http://swordapp.org/sword-v2/
+.. _loader: https://docs.softwareheritage.org/devel/glossary.html#term-loader
+.. _lister: https://docs.softwareheritage.org/devel/glossary.html#term-lister
+.. _SWHID: https://docs.softwareheritage.org/devel/swh-model/persistent-identifiers.html#persistent-identifiers
+.. _vault: https://docs.softwareheritage.org/devel/swh-vault/index.html#swh-vault
+.. _save code now: https://archive.softwareheritage.org/save/
+.. _User Manual: https://docs.softwareheritage.org/devel/swh-deposit/api/user-manual.html#deposit-user-manual
+.. _API Documentation: https://docs.softwareheritage.org/devel/swh-deposit/api/api-documentation.html#deposit-api-specifications
+.. _Deposit metadata: https://docs.softwareheritage.org/devel/swh-deposit/api/metadata.html#deposit-metadata
+.. _Running swh-deposit locally: https://docs.softwareheritage.org/devel/swh-deposit/internals/dev-environment.html#swh-deposit-dev-env
+.. _Production deployment: https://docs.softwareheritage.org/devel/swh-deposit/internals/prod-environment.html#swh-deposit-prod-env
\ No newline at end of file

bin/Makefile

 DEPOSIT_ID=1
 ARCHIVE=../../swh-deposit.zip
 ARCHIVE2=../../swh-model.zip
-STATUS=false
-PARTIAL_STATUS=true
-UPDATE_STATUS='success'
+STATUS=--no-partial
+PARTIAL_STATUS=--partial
+UPDATE_STATUS='done'
 ATOM_ENTRY=../../atom-entry.xml
 EXTERNAL_ID='external-id'
 

bin/create_deposit.sh

@@ -4,18 +4,13 @@
 
 ARCHIVE=${1-'../../deposit.zip'}
 
-NAME=$(basename ${ARCHIVE})
-MD5=$(md5sum ${ARCHIVE} | cut -f 1 -d' ')
+STATUS=${2-'--no-partial'}
 
-PROGRESS=${2-'false'}
-
-curl -i -u "$CREDS" \
-     -X POST \
-     --data-binary @${ARCHIVE} \
-     -H "In-Progress: ${PROGRESS}" \
-     -H "Content-MD5: ${MD5}" \
-     -H "Content-Disposition: attachment; filename=${NAME}" \
-     -H 'Slug: external-id' \
-     -H 'Packaging: http://purl.org/net/sword/package/SimpleZip' \
-     -H 'Content-type: application/zip' \
-     ${SERVER}/1/${COLLECTION}/
+./swh-deposit \
+        --username ${USER} \
+        --password ${PASSWORD} \
+        --collection ${COLLECTION} \
+        --archive-deposit \
+        --archive ${ARCHIVE} \
+        ${STATUS} \
+        --url ${SERVER}/1

bin/create_deposit_with_metadata.sh

@@ -5,16 +5,17 @@
 ARCHIVE=${1-'../../swh-deposit.zip'}
 ATOM_ENTRY=${2-'../../atom-entry.xml'}
 
-NAME=$(basename $ARCHIVE)
-MD5=$(md5sum $ARCHIVE | cut -f 1 -d' ')
-
-PROGRESS=${3-'false'}
+STATUS=${3-'--no-partial'}
 EXTERNAL_ID=${4-'external-id'}
 
-set -x
-curl -i --user "${CREDS}" \
-     -H "In-Progress: ${PROGRESS}" \
-     -H "Slug: ${EXTERNAL_ID}" \
-     -F "file=@${ARCHIVE};type=application/zip" \
-     -F "atom=@${ATOM_ENTRY};type=application/atom+xml;type=entry" \
-     -XPOST ${SERVER}/1/${COLLECTION}/
+./swh-deposit \
+        --username ${USER} \
+        --password ${PASSWORD} \
+        --collection ${COLLECTION} \
+        --archive-deposit \
+        --archive ${ARCHIVE} \
+        --metadata-deposit \
+        --metadata ${ATOM_ENTRY} \
+        --slug ${EXTERNAL_ID} \
+        ${STATUS} \
+        --url ${SERVER}/1

bin/default-setup

 SERVER=http://127.0.0.1:5006
-CREDS='hal:hal'
+USER='hal'
+PASSWORD='hal'
 COLLECTION=hal
+CREDS="$USER:$PASSWORD"

bin/status.sh

@@ -4,4 +4,10 @@
 
 DEPOSIT_ID=${1-1}
 
-curl -i -u "${CREDS}" ${SERVER}/1/${COLLECTION}/${DEPOSIT_ID}/status/
+./swh-deposit \
+        --username ${USER} \
+        --password ${PASSWORD} \
+        --collection ${COLLECTION} \
+        --status \
+        --deposit-id ${DEPOSIT_ID} \
+        --url ${SERVER}/1

bin/update-status.sh

@@ -3,7 +3,7 @@
 . ./default-setup
 
 DEPOSIT_ID=${1-1}
-UPDATE_STATUS=${2-'success'}
+UPDATE_STATUS=${2-'done'}
 
 curl -i \
      -X PUT \

conftest.py

+# Copyright (C) 2020  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+
+pytest_plugins = [
+    "swh.scheduler.pytest_plugin",
+    "swh.storage.pytest_plugin",
+    "swh.core.pytest_plugin",
+]
+
+
+@pytest.fixture(scope="session")
+def swh_scheduler_celery_includes(swh_scheduler_celery_includes):
+    return swh_scheduler_celery_includes + [
+        "swh.deposit.loader.tasks",
+    ]

debian/changelog

-swh-deposit (0.0.1-1) unstable; urgency=low
-
-  * Bootstrap version
-
- -- Antoine R. Dumont (@ardumont) <antoine.romain.dumont@gmail.com>  Mon, 22 May 2017 20:14:03 +0200

debian/compat

-9

debian/control

-Source: swh-deposit
-Maintainer: Software Heritage developers <swh-devel@inria.fr>
-Section: python
-Priority: optional
-Build-Depends: debhelper (>= 9),
-               dh-python (>= 2),
-               python3-setuptools,
-               python3-all,
-               python3-nose,
-               python3-django-nose,
-               python3-vcversioner,
-               python3-swh.core (>= 0.0.14~),
-               python3-swh.loader.core (>= 0.0.26~),
-               python3-swh.loader.tar (>= 0.0.31~),
-               python3-swh.scheduler (>= 0.0.19~),
-               python3-django,
-               python3-click,
-               python3-vcversioner,
-               python3-djangorestframework,
-               python3-djangorestframework-xml,
-               python3-requests
-Standards-Version: 3.9.6
-Homepage: https://forge.softwareheritage.org/source/swh-deposit/
-
-Package: python3-swh.deposit
-Architecture: all
-Depends: python3-swh.core (>= 0.0.14~),
-         python3-swh.loader.tar (>= 0.0.31~),
-         python3-swh.scheduler (>= 0.0.19~),
-         ${misc:Depends}, ${python3:Depends}
-Description: Software Heritage Deposit Server
-
-Package: python3-swh.deposit.loader
-Conflict: python3-swh.deposit.injection
-Architecture: all
-Depends: python3-swh.core (>= 0.0.14~),
-         python3-swh.loader.core (>= 0.0.26~),
-         python3-swh.loader.tar (>= 0.0.31~),
-         python3-swh.scheduler (>= 0.0.19~),
-         python3-requests,
-         ${misc:Depends}, ${python3:Depends}
-Description: Software Heritage Deposit Loader