cypress/e2e/origin-save.cy.js

@@ -22,8 +22,8 @@ const saveCodeMsg = {
   'csrfError': 'CSRF Failed: Referrer checking failed - no Referrer.'
 };
 
-const anonymousVisitTypes = ['bzr', 'cvs', 'git', 'hg', 'svn'];
-const allVisitTypes = ['archives', 'bzr', 'cvs', 'git', 'hg', 'svn'];
+const anonymousVisitTypes = ['bzr', 'cvs', 'git', 'hg', 'svn', 'tarball'];
+const allVisitTypes = ['archives', 'bzr', 'cvs', 'git', 'hg', 'svn', 'tarball'];
 
 function makeOriginSaveRequest(originType, originUrl) {
   cy.get('#swh-input-origin-url')

requirements-swh.txt

 swh.auth[django] >= 0.6.7
-swh.core >= 3.7.0
+swh.core >= 4.0.0
 swh.counters >= 0.5.1
-swh.indexer >= 3.7.0
+swh.indexer >= 4.0.0
 swh.model >= 6.16.0
 swh.provenance >= 0.1.1
-swh.scheduler >= 2.7.0
+swh.scheduler >= 3.0.0
 swh.search >= 0.16.0
-swh.storage >= 2.7.0
-swh-vault >= 1.12.2
+swh.storage >= 3.0.0
+swh-vault >= 2.0.0
 swh.webhooks >= 0.1.1

requirements-test.txt

@@ -10,15 +10,14 @@ pytest-django
 pytest-mock
 pytest-postgresql
 requests-mock != 1.9.0, != 1.9.1
-swh.core[http] >= 3.0.0
+swh.core[http] >= 4.0.0
 swh.graph >= 5.1.1
 swh.loader.git >= 0.8.0
-swh-scheduler[testing] >= 2.7.0
-swh.storage >= 0.1.1
+swh-scheduler[testing] >= 3.0.0
+swh.storage >= 3.0.0
 types-beautifulsoup4
 types-cryptography
 types-docutils
-types-psycopg2
 types-Pygments
 types-pyyaml
-types-requests
\ No newline at end of file
+types-requests

requirements.txt

@@ -19,7 +19,7 @@ iso8601
 looseversion
 msgpack
 prometheus-client
-psycopg2
+psycopg
 pybadges >= 2.2.1
 pygments
 pymemcache

swh/web/conftest.py

@@ -1219,6 +1219,22 @@ def swh_scheduler(config_updater):
         )
     )
 
+    # create load-bzr task type
+    swh_scheduler.create_task_type(
+        TaskType(
+            type="load-tarball-directory",
+            description="Load a tarball into the archive",
+            backend_name="swh.loader.core.tasks.LoadTarballDirectory",
+            default_interval=timedelta(days=64),
+            min_interval=timedelta(hours=12),
+            max_interval=timedelta(days=64),
+            backoff_factor=2.0,
+            max_queue_length=None,
+            num_retries=7,
+            retry_delay=timedelta(hours=2),
+        )
+    )
+
     # add method to add load-archive-files task type during tests
     def add_load_archive_task_type():
         swh_scheduler.create_task_type(

swh/web/mailmap/management/commands/sync_mailmaps.py

@@ -5,9 +5,9 @@
 
 from __future__ import annotations
 
-import psycopg2
-import psycopg2.extensions
-from psycopg2.extras import execute_values
+from typing import Any
+
+import psycopg
 
 from django.core.management.base import BaseCommand
 from django.db import transaction
@@ -20,14 +20,14 @@ from swh.web.mailmap.models import UserMailmap
 DISABLE_MAILMAPS_QUERY = """\
 UPDATE person
    SET displayname = NULL
-   FROM (VALUES %s) AS emails (email)
+   FROM (VALUES (%s)) AS emails (email)
    WHERE person.email = emails.email
 """
 
 REFRESH_MAILMAPS_QUERY = """\
 UPDATE person
    SET displayname = displaynames.displayname
-   FROM (VALUES %s) AS displaynames (email, displayname)
+   FROM (VALUES (%s, %s)) AS displaynames (email, displayname)
    WHERE
      person.email = displaynames.email
      AND person.displayname IS DISTINCT FROM displaynames.displayname
@@ -47,24 +47,22 @@ class Command(BaseCommand):
 
     def disable_mailmaps(
         self,
-        storage_db: psycopg2.extensions.connection,
+        storage_db: psycopg.Connection[Any],
         mailmaps: QuerySet[UserMailmap, UserMailmap],
     ):
         """Return the SQL to disable a set of mailmaps"""
 
-        execute_values(
-            storage_db.cursor(),
+        storage_db.cursor().executemany(
             DISABLE_MAILMAPS_QUERY,
             ((mailmap.from_email.encode("utf-8"),) for mailmap in mailmaps),
         )
 
     def refresh_mailmaps(
         self,
-        storage_db: psycopg2.extensions.connection,
+        storage_db: psycopg.Connection[Any],
         mailmaps: QuerySet[UserMailmap, UserMailmap],
     ):
-        execute_values(
-            storage_db.cursor(),
+        storage_db.cursor().executemany(
             REFRESH_MAILMAPS_QUERY,
             (
                 (
@@ -99,7 +97,7 @@ class Command(BaseCommand):
                 )
             )
 
-            with psycopg2.connect(options["storage_dbconn"]) as db:
+            with psycopg.connect(options["storage_dbconn"]) as db:
                 self.disable_mailmaps(db, to_disable.select_for_update())
                 self.refresh_mailmaps(db, to_refresh.select_for_update())
                 if not options["perform"]:

swh/web/mailmap/tests/test_mailmap.py

@@ -6,8 +6,6 @@
 import datetime
 import json
 
-import psycopg2
-from psycopg2.extras import execute_values
 import pytest
 
 from swh.model.model import Person
@@ -385,28 +383,29 @@ MAILMAP_KNOWN_PEOPLE = tuple(
 
 
 def init_stub_storage_db(postgresql):
-    if not isinstance(postgresql, psycopg2.extensions.connection):
-        # wrap pytest-postgresql >= 4's psycopg3 connection into a psycopg2 one
-        postgresql = psycopg2.connect(postgresql.info.dsn)
-    cur = postgresql.cursor()
-    cur.execute(
-        """
-        CREATE TABLE person (
-          fullname bytea PRIMARY KEY,
-          name bytea,
-          email bytea,
-          displayname bytea
+    with postgresql.cursor() as cur:
+        cur.execute(
+            """
+            CREATE TABLE person (
+              fullname bytea PRIMARY KEY,
+              name bytea,
+              email bytea,
+              displayname bytea
+            )
+            """
         )
-        """
-    )
-    execute_values(
-        cur,
-        "INSERT INTO person (fullname, name, email) VALUES %s",
-        (p.to_dict() for p in MAILMAP_KNOWN_PEOPLE),
-        template="(%(fullname)s, %(name)s, %(email)s)",
-    )
-    cur.execute("CREATE INDEX ON person (email)")
-    postgresql.commit()
-    cur.close()
+        cur.executemany(
+            "INSERT INTO person (fullname, name, email) VALUES (%s, %s, %s)",
+            [
+                (
+                    d["fullname"],
+                    d["name"],
+                    d["email"],
+                )
+                for d in (p.to_dict() for p in MAILMAP_KNOWN_PEOPLE)
+            ],
+        )
+        cur.execute("CREATE INDEX ON person (email)")
+        postgresql.commit()
 
-    return postgresql.dsn
+    return postgresql.info.dsn

swh/web/save_code_now/origin_save.py

-# Copyright (C) 2018-2024  The Software Heritage developers
+# Copyright (C) 2018-2025  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU Affero General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -126,6 +126,7 @@ _visit_type_task = {
     "svn": "load-svn",
     "cvs": "load-cvs",
     "bzr": "load-bzr",
+    "tarball": "load-tarball-directory",
 }
 
 
@@ -527,6 +528,8 @@ def create_save_origin_request(
                     }
                 )
             task_kwargs = dict(**task_kwargs, artifacts=artifacts, snapshot_append=True)
+        elif visit_type == "tarball":
+            task_kwargs["checksums"] = {}
         sor = None
         # get list of previously submitted save requests (most recent first)
         current_sors = list(

swh/web/save_code_now/templates/origin-save-help.html

 {% extends "./origin-save.html" %}
 
 {% comment %}
-Copyright (C) 2018-2023  The Software Heritage developers
+Copyright (C) 2018-2025  The Software Heritage developers
 See the AUTHORS file at the top-level directory of this distribution
 License: GNU Affero General Public License version 3, or any later version
 See top-level LICENSE file for more information
@@ -12,7 +12,7 @@ See top-level LICENSE file for more information
     <p style="margin-top: 1rem;">A "Save code now" request takes the following parameters:</p>
     <ul>
       <li>
-        <b>Origin type:</b> the type of version control system the software origin is using.
+        <b>Origin type:</b> the type of software origin.
         Currently, the supported types are:
         <ul>
           <li>
@@ -34,12 +34,19 @@ See top-level LICENSE file for more information
               <code>bzr</code>, for origins using <a href="https://bazaar.canonical.com/en/">Bazaar</a>
             </li>
           {% endif %}
+          {% if "tarball" in visit_types %}
+            <li>
+              <code>tarball</code>, for tarball origins
+            </li>
+          {% endif %}
         </ul>
       </li>
       <li>
-        <b>Origin url:</b> the url of the remote repository for the software origin.
+        <b>Origin URL:</b> the URL of the remote repository for the software origin
+        {% if "tarball" in visit_types %}or the URL for downloading a tarball{% endif %}
+        .
         <br />
-        In order to avoid saving errors from Software Heritage, you should provide the clone/checkout url
+        In order to avoid saving errors from Software Heritage, you should provide the clone/checkout URL
         as given by the provider hosting the software origin.
         <br />
         It can easily be found in the
@@ -58,7 +65,7 @@ See top-level LICENSE file for more information
         load its content into the archive as soon as possible
       </li>
       <li>
-        <b>rejected:</b> the provided origin url is blacklisted and no visit will be scheduled
+        <b>rejected:</b> the provided origin URL is blacklisted and no visit will be scheduled
       </li>
       <li>
         put in <b>pending</b> state: a manual review will then be performed in order to determine if the

swh/web/settings/production.py

@@ -57,16 +57,14 @@ REST_FRAMEWORK["NUM_PROXIES"] = 2
 db_conf = swh_web_config["production_db"]
 if db_conf.get("name", "").startswith("postgresql://"):
     # poor man's support for dsn connection string...
-    import psycopg2
-
-    with psycopg2.connect(db_conf.get("name")) as cnx:
-        dsn_dict = cnx.get_dsn_parameters()
-
-    db_conf["name"] = dsn_dict.get("dbname")
-    db_conf["host"] = dsn_dict.get("host")
-    db_conf["port"] = dsn_dict.get("port")
-    db_conf["user"] = dsn_dict.get("user")
-    db_conf["password"] = dsn_dict.get("password")
+    import psycopg
+
+    with psycopg.connect(db_conf.get("name")) as cnx:
+        db_conf["name"] = cnx.info.dbname
+        db_conf["host"] = cnx.info.host
+        db_conf["port"] = cnx.info.port
+        db_conf["user"] = cnx.info.user
+        db_conf["password"] = cnx.info.password
 
 
 # https://docs.djangoproject.com/en/1.10/ref/settings/#databases