Makefile.local

@@ -2,6 +2,7 @@ TEST_DIRS := ./swh/web/
 TESTFLAGS += --hypothesis-profile=swh-web-fast
 TESTFULL_FLAGS = --hypothesis-profile=swh-web
 YARN ?= yarn
+SETTINGS_CYPRESS ?= swh.web.settings.cypress
 SETTINGS_TEST ?= swh.web.settings.tests
 SETTINGS_DEV ?= swh.web.settings.development
 SETTINGS_PROD = swh.web.settings.production
@@ -95,7 +96,7 @@ test-full:
 test-frontend-cmd: build-webpack-test add-users-test
 	bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT && \
 	jobs -p | xargs -r kill' SIGINT SIGTERM ERR EXIT; \
-	python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_TEST) & \
+	python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_CYPRESS) & \
 	sleep 10 && $(YARN) run cypress run --config numTestsKeptInMemory=0 && \
 	$(YARN) mochawesome && $(YARN) nyc-report"
 
@@ -113,7 +114,7 @@ test-frontend-ui-cmd: add-users-test yarn-install
 	bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT && \
 	ps -o pgid= $$$$ | grep -o [0-9]* | xargs pkill -g' SIGINT SIGTERM ERR EXIT; \
 	$(YARN) start-dev & \
-	python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_TEST) & \
+	python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_CYPRESS) & \
 	sleep 10 && $(YARN) run cypress open"
 
 test-frontend-ui: export CYPRESS_SKIP_SLOW_TESTS=1

cypress/e2e/add-forge-now-request-create.cy.js

@@ -293,8 +293,8 @@ describe('Test add-forge-request creation', function() {
     populateForm('bitbucket', 'https://gitlab.example.com/', 'test', 'test@example.com', 'on', 'test comment');
     submitForm();
 
-    cy.visit(this.addForgeNowUrl);
-    cy.get('#swh-add-forge-requests-list-tab').click();
+    // click the link to the list in the form footer
+    cy.get('#swh-show-forge-add-requests-list').click();
 
     cy.wait('@addForgeRequestsList');
 

cypress/e2e/layout.cy.js

@@ -6,7 +6,8 @@
  */
 
 const url = '/browse/help/';
-const statusUrl = 'https://status.softwareheritage.org';
+// statusUrl must match what is configured in settings/test.py
+const statusUrl = 'https://status.example.org';
 
 describe('Test top-bar', function() {
   beforeEach(function() {

cypress/plugins/index.js

@@ -175,7 +175,7 @@ module.exports = (on, config) => {
     processAddForgeNowInboundEmail(emailSrc) {
       try {
         execFileSync('django-admin',
-                     ['process_inbound_email', '--settings=swh.web.settings.tests'],
+                     ['process_inbound_email', '--settings=swh.web.settings.cypress'],
                      {input: emailSrc});
         return true;
       } catch (_) {

requirements-swh.txt

 swh.auth[django] >= 0.6.7
-swh.core >= 3.0.1
+swh.core >= 3.7.0
 swh.counters >= 0.5.1
 swh.indexer >= 3.7.0
 swh.model >= 6.16.0

swh/web/add_forge_now/templates/add-forge-creation-form.html

@@ -114,8 +114,9 @@ See top-level LICENSE file for more information
         </div>
       </form>
       <p>
-        Once an add-forge-request is submitted, its status can be viewed in
-        the <a id="swh-show-forge-add-requests-list" href="#browse-requests">submitted requests list</a>.
+        Once an add-forge-request is submitted, its status can be viewed in the
+        <a id="swh-show-forge-add-requests-list"
+           href="{% url 'forge-add-list' %}">submitted requests list</a>.
         This process involves a moderator approval and might take a few days to handle (it primarily
         depends on the response time from the forge).
       </p>

swh/web/admin/urls.py

@@ -9,16 +9,14 @@ from django.conf import settings
 from django.contrib import admin
 from django.urls import path as url
 
-from swh.web.config import get_config
+from swh.web.config import get_config, oidc_enabled
 
 # prevent code execution when building documentation to avoid errors
 # with autodoc processing
 if "SWH_DOC_BUILD" not in os.environ:
     config = get_config()
 
-    oidc_enabled = bool(config["keycloak"]["server_url"])
-
-    if oidc_enabled:
+    if oidc_enabled():
         # use swh-auth views in admin site for login/logout when webapp
         # uses Keycloak for authentication
         from swh.auth.django.views import oidc_login, oidc_logout

swh/web/api/tests/views/test_graph.py

@@ -15,11 +15,24 @@ from django.http.response import StreamingHttpResponse
 from swh.model.hashutil import hash_to_bytes
 from swh.model.swhids import ExtendedObjectType, ExtendedSWHID
 from swh.web.api.views.graph import API_GRAPH_PERM
-from swh.web.config import SWH_WEB_INTERNAL_SERVER_NAMES, get_config
+from swh.web.config import get_config
 from swh.web.tests.helpers import check_http_get_response
 from swh.web.utils import reverse
 
 
+@pytest.fixture(autouse=True)
+def graph_config(config_updater):
+    config_updater(
+        {
+            "graph": {
+                "server_url": "http://example.org/graph/",
+                "max_edges": {"staff": 0, "user": 100000, "anonymous": 1000},
+            },
+            "unauthenticated_api_hosts": ["local.network"],
+        }
+    )
+
+
 def test_graph_endpoint_no_authentication_for_vpn_users(api_client, requests_mock):
     graph_query = "stats"
     url = reverse("api-1-graph", url_args={"graph_query": graph_query})
@@ -29,13 +42,25 @@ def test_graph_endpoint_no_authentication_for_vpn_users(api_client, requests_moc
         headers={"Content-Type": "application/json"},
     )
     check_http_get_response(
-        api_client, url, status_code=200, server_name=SWH_WEB_INTERNAL_SERVER_NAMES[0]
+        api_client,
+        url,
+        status_code=200,
+        server_name="localhost",
+    )
+    check_http_get_response(
+        api_client,
+        url,
+        status_code=200,
+        server_name="local.network",
     )
 
 
 def test_graph_endpoint_needs_authentication(api_client):
     url = reverse("api-1-graph", url_args={"graph_query": "stats"})
     check_http_get_response(api_client, url, status_code=401)
+    check_http_get_response(
+        api_client, url, status_code=401, server_name="public.network"
+    )
 
 
 def _authenticate_graph_user(api_client, keycloak_oidc, is_staff=False):
@@ -365,9 +390,7 @@ def test_graph_endpoint_max_edges_settings(api_client, keycloak_oidc, requests_m
 
     # currently unauthenticated user can only use the graph endpoint from
     # Software Heritage VPN
-    check_http_get_response(
-        api_client, url, status_code=200, server_name=SWH_WEB_INTERNAL_SERVER_NAMES[0]
-    )
+    check_http_get_response(api_client, url, status_code=200, server_name="localhost")
     assert (
         f"max_edges={graph_config['max_edges']['anonymous']}"
         in requests_mock.request_history[0].url

swh/web/api/tests/views/test_origin.py

@@ -607,10 +607,10 @@ def test_api_origin_search_words(api_client):
 
 
 @pytest.mark.parametrize("backend", ["swh-search", "swh-storage"])
-def test_api_origin_search_visit_type(api_client, mocker, backend):
+def test_api_origin_search_visit_type(api_client, config_updater, backend):
     if backend != "swh-search":
         # equivalent to not configuring search in the config
-        mocker.patch("swh.web.utils.archive.search", None)
+        config_updater({"search": None})
 
     expected_origins = {
         "https://github.com/wcoder/highlightjs-line-numbers.js",
@@ -638,18 +638,20 @@ def test_api_origin_search_visit_type(api_client, mocker, backend):
     assert rv.data == []
 
 
-def test_api_origin_search_use_ql(api_client, mocker):
+def test_api_origin_search_use_ql(api_client, patch_backend):
     expected_origins = {
         "https://github.com/wcoder/highlightjs-line-numbers.js",
         "https://github.com/memononen/libtess2",
     }
 
     ORIGINS = [{"url": origin} for origin in expected_origins]
-
-    mock_archive_search = mocker.patch("swh.web.utils.archive.search")
-    mock_archive_search.origin_search.return_value = PagedResult(
-        results=ORIGINS,
-        next_page_token=None,
+    mock_origin_search = patch_backend(
+        "search",
+        "origin_search",
+        return_value=PagedResult(
+            results=ORIGINS,
+            next_page_token=None,
+        ),
     )
 
     query = "origin : 'github.com'"
@@ -662,17 +664,15 @@ def test_api_origin_search_use_ql(api_client, mocker):
     rv = check_api_get_responses(api_client, url, status_code=200)
     assert {origin["url"] for origin in rv.data} == expected_origins
 
-    mock_archive_search.origin_search.assert_called_with(
+    mock_origin_search.assert_called_with(
         query=query, page_token=None, with_visit=False, visit_types=["git"], limit=70
     )
 
 
-def test_api_origin_search_ql_syntax_error(api_client, mocker):
-    mock_archive_search = mocker.patch("swh.web.utils.archive.search")
-    mock_archive_search.origin_search.side_effect = SearchQuerySyntaxError(
-        "Invalid syntax"
+def test_api_origin_search_ql_syntax_error(api_client, patch_backend):
+    mock_origin_search = patch_backend(
+        "search", "origin_search", side_effect=SearchQuerySyntaxError("Invalid syntax")
     )
-
     query = "this is not a valid query"
 
     url = reverse(
@@ -686,17 +686,18 @@ def test_api_origin_search_ql_syntax_error(api_client, mocker):
         "reason": "Syntax error in search query: Invalid syntax",
     }
 
-    mock_archive_search.origin_search.assert_called_with(
+    mock_origin_search.assert_called_with(
         query=query, page_token=None, with_visit=False, visit_types=["git"], limit=70
     )
 
 
 @pytest.mark.parametrize("backend", ["swh-search", "swh-storage"])
 @pytest.mark.parametrize("limit", [1, 2, 3, 10])
-def test_api_origin_search_scroll(api_client, archive_data, mocker, limit, backend):
+def test_api_origin_search_scroll(
+    api_client, archive_data, mocker, config_updater, limit, backend
+):
     if backend != "swh-search":
-        # equivalent to not configuring search in the config
-        mocker.patch("swh.web.utils.archive.search", None)
+        config_updater({"search": None})
 
     expected_origins = {
         "https://github.com/wcoder/highlightjs-line-numbers.js",
@@ -715,14 +716,15 @@ def test_api_origin_search_scroll(api_client, archive_data, mocker, limit, backe
 
 
 @pytest.mark.parametrize("backend", ["swh-search", "swh-storage"])
-def test_api_origin_search_limit(api_client, archive_data, tests_data, mocker, backend):
+def test_api_origin_search_limit(
+    api_client, archive_data, tests_data, config_updater, backend
+):
     if backend == "swh-search":
         tests_data["search"].origin_update(
             [{"url": "http://foobar/{}".format(i)} for i in range(2000)]
         )
     else:
-        # equivalent to not configuring search in the config
-        mocker.patch("swh.web.utils.archive.search", None)
+        config_updater({"search": None})
 
         archive_data.origin_add(
             [Origin(url="http://foobar/{}".format(i)) for i in range(2000)]
@@ -738,11 +740,10 @@ def test_api_origin_search_limit(api_client, archive_data, tests_data, mocker, b
 
 
 @pytest.mark.parametrize("backend", ["swh-search", "swh-indexer-storage"])
-def test_api_origin_metadata_search(api_client, mocker, tests_data, backend):
-    mock_config = mocker.patch("swh.web.utils.archive.config")
-    mock_config.get_config.return_value = {
-        "search_config": {"metadata_backend": backend}
-    }
+def test_api_origin_metadata_search_1(
+    api_client, mocker, tests_data, config_updater, backend
+):
+    config_updater({"search_config": {"metadata_backend": backend}})
 
     url = reverse(
         "api-1-origin-metadata-search", query_params={"fulltext": ORIGIN_METADATA_VALUE}
@@ -787,22 +788,24 @@ def test_api_origin_metadata_search(api_client, mocker, tests_data, backend):
         assert response == expected
 
 
-def test_api_origin_metadata_search_not_in_idx_storage(api_client, mocker, tests_data):
+def test_api_origin_metadata_search_not_in_idx_storage(
+    api_client, mocker, tests_data, config_updater
+):
     """Tests the origin search for results present in swh-search but not
     returned by ``origin_intrinsic_metadata_get`` (which happens when results
     come from extrinsic metadata).
     """
-
-    mock_idx_storage = mocker.patch("swh.web.utils.archive.idx_storage")
+    mock_idx_storage = mocker.MagicMock()
     mock_idx_storage.origin_intrinsic_metadata_get.return_value = []
     mock_idx_storage.origin_intrinsic_metadata_search_fulltext.side_effect = (
         AssertionError("origin_intrinsic_metadata_search_fulltext was called")
     )
-
-    mock_config = mocker.patch("swh.web.utils.archive.config")
-    mock_config.get_config.return_value = {
-        "search_config": {"metadata_backend": "swh-search"}
-    }
+    config_updater(
+        {
+            "search_config": {"metadata_backend": "swh-search"},
+            "indexer_storage": mock_idx_storage,
+        }
+    )
 
     url = reverse(
         "api-1-origin-metadata-search",
@@ -832,19 +835,18 @@ def test_api_origin_metadata_search_not_in_idx_storage(api_client, mocker, tests
     "backend,fields",
     itertools.product(["swh-search", "swh-indexer-storage"], ["url", "url,foobar"]),
 )
-def test_api_origin_metadata_search_url_only(api_client, mocker, backend, fields):
+def test_api_origin_metadata_search_url_only(
+    api_client, patch_backend, config_updater, backend, fields
+):
     """Checks that idx_storage.origin_intrinsic_metadata_get is not called when
     its results are not needed"""
-    mocker.patch(
-        "swh.web.utils.archive.idx_storage.origin_intrinsic_metadata_get",
+    config_updater({"search_config": {"metadata_backend": backend}})
+    patch_backend(
+        "indexer_storage",
+        "origin_intrinsic_metadata_get",
         side_effect=AssertionError("origin_intrinsic_metadata_get was called"),
     )
 
-    mock_config = mocker.patch("swh.web.utils.archive.config")
-    mock_config.get_config.return_value = {
-        "search_config": {"metadata_backend": backend}
-    }
-
     url = reverse(
         "api-1-origin-metadata-search",
         query_params={"fulltext": ORIGIN_METADATA_VALUE, "fields": fields},
@@ -861,19 +863,16 @@ def test_api_origin_metadata_search_url_only(api_client, mocker, backend, fields
 
 
 @pytest.mark.parametrize("backend", ["swh-search", "swh-indexer-storage"])
-def test_api_origin_metadata_search_limit(api_client, mocker, backend):
-    mock_config = mocker.patch("swh.web.utils.archive.config")
-    mock_config.get_config.return_value = {
-        "search_config": {"metadata_backend": backend}
-    }
+def test_api_origin_metadata_search_limit(api_client, mocker, config_updater, backend):
+    config_updater({"search_config": {"metadata_backend": backend}})
 
-    from swh.web.utils import archive
+    from swh.web import config
 
     if backend == "swh-search":
-        spied_search = mocker.spy(archive.search, "origin_search")
+        spied_search = mocker.spy(config.search(), "origin_search")
     else:
         spied_search = mocker.spy(
-            archive.idx_storage, "origin_intrinsic_metadata_search_fulltext"
+            config.indexer_storage(), "origin_intrinsic_metadata_search_fulltext"
         )
 
     def _check_search_call(limit):
@@ -941,16 +940,17 @@ def test_api_origin_extrinsic_metadata(api_client, origin):
 
 
 def test_api_origin_metadata_search_invalid(api_client, mocker):
-    mock_idx_storage = mocker.patch("swh.web.utils.archive.idx_storage")
+    from swh.web import config
+
+    spied_idx_storage = mocker.spy(config, "indexer_storage")
     url = reverse("api-1-origin-metadata-search")
     check_api_get_responses(api_client, url, status_code=400)
-    mock_idx_storage.assert_not_called()
+    spied_idx_storage.assert_not_called()
 
 
 @pytest.mark.parametrize("backend", ["swh-counters", "swh-storage"])
-def test_api_stat_counters(api_client, mocker, backend):
-    mock_config = mocker.patch("swh.web.utils.archive.config")
-    mock_config.get_config.return_value = {"counters_backend": backend}
+def test_api_stat_counters(api_client, config_updater, backend):
+    config_updater({"counters_backend": backend})
 
     url = reverse("api-1-stat-counters")
     rv = check_api_get_responses(api_client, url, status_code=200)

swh/web/api/views/graph.py

@@ -9,7 +9,9 @@ from urllib.parse import unquote, urlparse, urlunparse
 
 import requests
 
+from django.conf import settings
 from django.http import QueryDict
+from django.http.request import split_domain_port, validate_host
 from django.http.response import StreamingHttpResponse
 from rest_framework.decorators import renderer_classes
 from rest_framework.renderers import JSONRenderer
@@ -22,7 +24,7 @@ from swh.model.swhids import ExtendedObjectType, ExtendedSWHID
 from swh.web.api.apidoc import api_doc
 from swh.web.api.apiurls import api_route
 from swh.web.api.renderers import PlainTextRenderer
-from swh.web.config import SWH_WEB_INTERNAL_SERVER_NAMES, get_config
+from swh.web.config import get_config
 from swh.web.utils import archive, strtobool
 
 API_GRAPH_PERM = "swh.web.api.graph"
@@ -128,7 +130,11 @@ def api_graph(request: Request) -> None:
 def api_graph_proxy(
     request: Request, graph_query: str
 ) -> Union[Response, StreamingHttpResponse]:
-    if request.get_host() not in SWH_WEB_INTERNAL_SERVER_NAMES:
+    domain, port = split_domain_port(request.get_host())
+    if not (domain and validate_host(domain, settings.UNAUTHENTICATED_HOSTS)):
+        # request does not come from an identified allowed host/network (see
+        # 'unauthenticated_api_hosts' config entry), check for proper auth and
+        # permission
         if not bool(request.user and request.user.is_authenticated):
             return Response("Authentication credentials were not provided.", status=401)
         if not request.user.has_perm(API_GRAPH_PERM):

swh/web/api/views/metadata.py

@@ -15,6 +15,7 @@ from rest_framework.request import Request
 
 from swh.model import hashutil, swhids
 from swh.model.model import MetadataAuthority, MetadataAuthorityType, Origin
+from swh.web import config
 from swh.web.api.apidoc import api_doc, format_docstring
 from swh.web.api.apiurls import api_route
 from swh.web.utils import archive, converters, reverse
@@ -127,7 +128,7 @@ def api_raw_extrinsic_metadata_swhid(request: Request, target: str):
     else:
         page_token = None
 
-    result_page = archive.storage.raw_extrinsic_metadata_get(
+    result_page = config.storage().raw_extrinsic_metadata_get(
         target=parsed_target,
         authority=authority,
         after=after,
@@ -195,7 +196,7 @@ def api_raw_extrinsic_metadata_swhid(request: Request, target: str):
 def api_raw_extrinsic_metadata_get(request: Request, id: str):
     # This is an internal endpoint that should only be accessed via URLs given
     # by /raw-extrinsic-metadata/swhid/; so it is not documented.
-    metadata = archive.storage.raw_extrinsic_metadata_get_by_ids(
+    metadata = config.storage().raw_extrinsic_metadata_get_by_ids(
         [hashutil.hash_to_bytes(id)]
     )
     if not metadata:
@@ -263,7 +264,7 @@ def api_raw_extrinsic_metadata_swhid_authorities(request: Request, target: str):
     except swhids.ValidationError as e:
         raise BadInputExc(f"Invalid target SWHID: {e}") from None
 
-    authorities = archive.storage.raw_extrinsic_metadata_get_authorities(
+    authorities = config.storage().raw_extrinsic_metadata_get_authorities(
         target=parsed_target
     )
     results = [

swh/web/api/views/raw.py

@@ -18,9 +18,9 @@ from swh.model.hashutil import hash_to_hex
 from swh.model.swhids import ObjectType
 from swh.storage.algos.directory import directory_get
 from swh.storage.algos.snapshot import snapshot_get_all_branches
+from swh.web import config
 from swh.web.api.apidoc import api_doc, format_docstring
 from swh.web.api.apiurls import api_route
-from swh.web.utils import archive
 from swh.web.utils.exc import NotFoundExc
 from swh.web.utils.identifiers import parse_core_swhid
 
@@ -60,18 +60,20 @@ def api_raw_object(request: Request, swhid: str):
     object_id = parsed_swhid.object_id
     object_type = parsed_swhid.object_type
 
+    storage = config.storage()
+
     def not_found():
         return NotFoundExc(f"Object with id {swhid} not found.")
 
     if object_type == ObjectType.CONTENT:
-        results = archive.storage.content_find({"sha1_git": object_id})
+        results = storage.content_find({"sha1_git": object_id})
         if len(results) == 0:
             raise not_found()
         cnt = results[0]
         # `cnt.with_data()` unfortunately doesn't seem to work.
         if cnt.data is None:
             d = cnt.to_dict()
-            d["data"] = archive.storage.content_get_data({"sha1": cnt.sha1})
+            d["data"] = storage.content_get_data({"sha1": cnt.sha1})
             cnt = model.Content.from_dict(d)
             assert (
                 cnt.data is not None
@@ -79,25 +81,25 @@ def api_raw_object(request: Request, swhid: str):
         result = content_git_object(cnt)
 
     elif object_type == ObjectType.DIRECTORY:
-        dir_ = directory_get(archive.storage, object_id)
+        dir_ = directory_get(storage, object_id)
         if dir_ is None:
             raise not_found()
         result = directory_git_object(dir_)
 
     elif object_type == ObjectType.REVISION:
-        rev = archive.storage.revision_get([object_id])[0]
+        rev = storage.revision_get([object_id])[0]
         if rev is None:
             raise not_found()
         result = revision_git_object(rev)
 
     elif object_type == ObjectType.RELEASE:
-        rel = archive.storage.release_get([object_id])[0]
+        rel = storage.release_get([object_id])[0]
         if rel is None:
             raise not_found()
         result = release_git_object(rel)
 
     elif object_type == ObjectType.SNAPSHOT:
-        snp = snapshot_get_all_branches(archive.storage, object_id)
+        snp = snapshot_get_all_branches(storage, object_id)
         if snp is None:
             raise not_found()
         result = snapshot_git_object(snp)

swh/web/archive_coverage/tests/test_coverage.py

@@ -21,7 +21,6 @@ from swh.web.archive_coverage.views import (
     legacy_origins,
     listed_origins,
 )
-from swh.web.config import SWH_WEB_SERVER_NAMES
 from swh.web.tests.django_asserts import assert_contains, assert_not_contains
 from swh.web.tests.helpers import check_html_get_response, check_http_get_response
 from swh.web.utils import reverse
@@ -142,9 +141,7 @@ def test_coverage_view_with_metrics(client, mocker, swh_scheduler):
         assert_contains(resp, f"<td>{visit_type}</td>")
 
     # check request as in production with cache enabled
-    check_http_get_response(
-        client, url, status_code=200, server_name=SWH_WEB_SERVER_NAMES[0]
-    )
+    check_http_get_response(client, url, status_code=200, server_name="localhost")
 
 
 def test_coverage_view_with_focus(client, mocker, swh_scheduler):
@@ -253,7 +250,7 @@ def test_coverage_view_filter_out_non_visited_origins(
         url,
         status_code=200,
         template_used="archive-coverage.html",
-        server_name=SWH_WEB_SERVER_NAMES[0],
+        server_name="not-web-dev",
     )
 
     for i, origins in enumerate(origins["origins"]):

swh/web/archive_coverage/views.py

@@ -17,14 +17,7 @@ from django.views.decorators.clickjacking import xframe_options_exempt
 
 from swh.scheduler.model import SchedulerMetrics
 from swh.web.config import scheduler
-from swh.web.utils import (
-    archive,
-    django_cache,
-    get_deposits_list,
-    is_swh_web_development,
-    is_swh_web_production,
-    reverse,
-)
+from swh.web.utils import archive, django_cache, get_deposits_list, reverse
 
 _swh_arch_overview_doc = (
     "https://docs.softwareheritage.org/devel/architecture/overview.html"
@@ -551,8 +544,7 @@ def _search_url(query: str, visit_type: str) -> str:
 @xframe_options_exempt
 @never_cache
 def swh_coverage(request: HttpRequest) -> HttpResponse:
-    use_cache = is_swh_web_production(request)
-    listers_metrics = _get_listers_metrics(use_cache)
+    listers_metrics = _get_listers_metrics(cache_metrics=True)
 
     for origins in listed_origins["origins"]:
         origins["count"] = "0"
@@ -574,7 +566,7 @@ def swh_coverage(request: HttpRequest) -> HttpResponse:
                 )
             # visit type from legacy nixguix lister
             visit_type_counts["nixguix"] = _get_nixguix_origins_count(
-                manifest_url, cache_count=use_cache
+                manifest_url, cache_count=True
             )
 
             count = sum(visit_type_counts.values())
@@ -634,11 +626,10 @@ def swh_coverage(request: HttpRequest) -> HttpResponse:
                     search_url = _search_url(search_pattern, visit_type)
                 visit_types[visit_type]["search_url"] = search_url
 
-    # filter out origin types without archived origins on production and staging
-    if not is_swh_web_development(request):
-        listed_origins["origins"] = list(
-            filter(lambda o: o["count"] != "0", listed_origins["origins"])
-        )
+    # filter out origin types without archived origins
+    listed_origins["origins"] = list(
+        filter(lambda o: o["count"] != "0", listed_origins["origins"])
+    )
 
     for origins in legacy_origins["origins"]:
         origins["search_urls"] = {}
@@ -647,7 +638,7 @@ def swh_coverage(request: HttpRequest) -> HttpResponse:
                 origins["search_pattern"], visit_type
             )
 
-    deposits_counts = _get_deposits_netloc_counts(use_cache)
+    deposits_counts = _get_deposits_netloc_counts(cache_counts=True)
 
     for origins in deposited_origins["origins"]:
         origins["count"] = "0"

swh/web/auth/urls.py

@@ -18,15 +18,12 @@ from swh.web.auth.views import (
     oidc_profile_view,
     oidc_revoke_bearer_tokens,
 )
-from swh.web.config import get_config
+from swh.web.config import get_config, oidc_enabled
 
 config = get_config()
-
-oidc_enabled = bool(config["keycloak"]["server_url"])
-
 urlpatterns = []
 
-if not oidc_enabled:
+if not oidc_enabled():
     urlpatterns = [
         url(
             r"^login/$",
@@ -35,7 +32,7 @@ if not oidc_enabled:
         )
     ]
 
-if oidc_enabled or config["e2e_tests_mode"]:
+if oidc_enabled() or config["e2e_tests_mode"]:
     urlpatterns += auth_urlpatterns + [
         url(
             r"^oidc/generate-bearer-token/$",

swh/web/banners/tests/test_app.py

@@ -14,10 +14,11 @@ from swh.web.utils import reverse
 
 
 @pytest.mark.django_db
-def test_banners_deactivate(client, django_settings):
+def test_banners_deactivate(client, django_settings, config_updater):
     """Check banners feature is deactivated when the swh.web.banners django
     application is not in installed apps."""
 
+    config_updater({"show_corner_ribbon": True, "corner_ribbon_text": "Tests"})
     banners_app = "swh.web.banners"
     if banners_app not in django_settings.SWH_DJANGO_APPS:
         django_settings.SWH_DJANGO_APPS = django_settings.SWH_DJANGO_APPS + [

swh/web/browse/browseurls.py

@@ -5,6 +5,9 @@
 
 from typing import List, Optional
 
+# call this early to ensure all the 'swhid' url path converted is registered
+# so any url registration will work as intended
+import swh.web.utils.url_path_converters  # noqa: F401
 from swh.web.utils.urlsindex import UrlsIndex
 
 browse_urls = UrlsIndex()

swh/web/browse/templates/includes/readme-display.html

@@ -5,6 +5,7 @@ License: GNU Affero General Public License version 3, or any later version
 See top-level LICENSE file for more information
 {% endcomment %}
 
+{% load static %}
 {% load swh_templatetags %}
 
 {% if readme_name %}
@@ -12,7 +13,12 @@ See top-level LICENSE file for more information
     <div class="card-header swh-background-gray">
       <h4>{{ readme_name }}</h4>
     </div>
-    <div class="swh-readme card-body"></div>
+    <div class="swh-readme card-body">
+      <div class="text-center">
+        <img src="{% static 'img/swh-spinner.gif' %}" alt="swh spinner" />
+        <p>Loading {{ readme_name }} ...</p>
+      </div>
+    </div>
   </div>
   {% if readme_html %}
     <script>

swh/web/browse/urls.py

@@ -16,8 +16,7 @@ import swh.web.browse.views.origin  # noqa
 import swh.web.browse.views.release  # noqa
 import swh.web.browse.views.revision  # noqa
 import swh.web.browse.views.snapshot  # noqa
-from swh.web.utils import is_swh_web_production, origin_visit_types, reverse
-from swh.web.utils.url_path_converters import register_url_path_converters
+from swh.web.utils import origin_visit_types, reverse
 
 
 def _browse_help_view(request: HttpRequest) -> HttpResponse:
@@ -32,7 +31,7 @@ def _browse_search_view(request: HttpRequest) -> HttpResponse:
         "browse-search.html",
         {
             "heading": "Search software origins to browse",
-            "visit_types": origin_visit_types(use_cache=is_swh_web_production(request)),
+            "visit_types": origin_visit_types(use_cache=True),
         },
     )
 
@@ -45,8 +44,6 @@ def _browse_swhid_iframe_legacy(request: HttpRequest, swhid: str) -> HttpRespons
     return redirect(reverse("browse-swhid-iframe", url_args={"swhid": swhid}))
 
 
-register_url_path_converters()
-
 urlpatterns = [
     url("browse/", _browse_search_view),
     url("browse/help/", _browse_help_view, name="browse-help"),

swh/web/config.py

-# Copyright (C) 2017-2024  The Software Heritage developers
+# Copyright (C) 2017-2025  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU Affero General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
 from __future__ import annotations
 
+import importlib
 import os
-from typing import TYPE_CHECKING, Any, Dict
+from typing import TYPE_CHECKING, Any
 
 from swh.core import config
-from swh.counters import get_counters
-from swh.indexer.storage import get_indexer_storage
-from swh.scheduler import get_scheduler
-from swh.search import get_search
-from swh.storage import get_storage
-from swh.vault import get_vault
 from swh.web import settings
 
 if TYPE_CHECKING:
@@ -25,66 +20,30 @@ if TYPE_CHECKING:
     from swh.storage.interface import StorageInterface
     from swh.vault.interface import VaultInterface
 
-SWH_WEB_INTERNAL_SERVER_NAMES = [
-    "archive.internal.softwareheritage.org",
-    "webapp1.internal.softwareheritage.org",
-]
-SWH_WEB_SERVER_NAMES = ["archive.softwareheritage.org"] + SWH_WEB_INTERNAL_SERVER_NAMES
+SETTINGS_DIR = os.path.dirname(settings.__file__)
 
-SWH_WEB_STAGING_SERVER_NAMES = [
-    "webapp.staging.swh.network",
-    "webapp.internal.staging.swh.network",
-]
 
-SETTINGS_DIR = os.path.dirname(settings.__file__)
+class ConfigurationError(Exception):
+    pass
+
+
+class SWHWebConfig(dict[str, Any]):
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return super().__getitem__(key)
+        except KeyError:
+            raise ConfigurationError(f"Missing '{key}' configuration")
+
 
 DEFAULT_CONFIG = {
     "allowed_hosts": ("list", []),
-    "storage": (
-        "dict",
-        {
-            "cls": "remote",
-            "url": "http://127.0.0.1:5002/",
-            "timeout": 10,
-        },
-    ),
-    "indexer_storage": (
-        "dict",
-        {
-            "cls": "remote",
-            "url": "http://127.0.0.1:5007/",
-            "timeout": 1,
-        },
-    ),
-    "counters": (
-        "dict",
-        {
-            "cls": "remote",
-            "url": "http://127.0.0.1:5011/",
-            "timeout": 1,
-        },
-    ),
-    "search": (
-        "dict",
-        {
-            "cls": "remote",
-            "url": "http://127.0.0.1:5010/",
-            "timeout": 10,
-        },
-    ),
+    "unauthenticated_api_hosts": ("list", []),
     "search_config": (
         "dict",
         {
             "metadata_backend": "swh-search",
         },  # or "swh-search"
     ),
-    "provenance": (
-        "dict",
-        {
-            "cls": "graph",
-            "url": "granet.internal.softwareheritage.org:50091",
-        },
-    ),
     "log_dir": ("string", "/tmp/swh/log"),
     "debug": ("bool", False),
     "serve_assets": ("bool", False),
@@ -128,52 +87,13 @@ DEFAULT_CONFIG = {
             },
         },
     ),
-    "vault": (
-        "dict",
-        {
-            "cls": "remote",
-            "url": "http://127.0.0.1:5005/",
-        },
-    ),
-    "scheduler": ("dict", {"cls": "remote", "url": "http://127.0.0.1:5008/"}),
     "development_db": ("string", os.path.join(SETTINGS_DIR, "db.sqlite3")),
     "test_db": ("dict", {"name": "swh-web-test"}),
     "production_db": ("dict", {"name": "swh-web"}),
-    "deposit": (
-        "dict",
-        {
-            "private_api_url": "https://deposit.softwareheritage.org/1/private/",
-            "private_api_user": "swhworker",
-            "private_api_password": "some-password",
-        },
-    ),
     "e2e_tests_mode": ("bool", False),
-    "history_counters_url": (
-        "string",
-        (
-            "http://counters1.internal.softwareheritage.org:5011"
-            "/counters_history/history.json"
-        ),
-    ),
     "client_config": ("dict", {}),
     "keycloak": ("dict", {"server_url": "", "realm_name": ""}),
-    "graph": (
-        "dict",
-        {
-            "server_url": "http://graph.internal.softwareheritage.org:5009/graph/",
-            "max_edges": {"staff": 0, "user": 100000, "anonymous": 1000},
-        },
-    ),
-    "status": (
-        "dict",
-        {
-            "server_url": "https://status.softwareheritage.org/",
-            "json_path": "1.0/status/578e5eddcdc0cc7951000520",
-        },
-    ),
     "counters_backend": ("string", "swh-storage"),  # or "swh-counters"
-    "staging_server_names": ("list", SWH_WEB_STAGING_SERVER_NAMES),
-    "production_server_names": ("list", SWH_WEB_SERVER_NAMES),
     "instance_name": ("str", "archive-test.softwareheritage.org"),
     "give": ("dict", {"public_key": "", "token": ""}),
     "features": ("dict", {"add_forge_now": True}),
@@ -220,7 +140,8 @@ DEFAULT_CONFIG = {
         },
     ),
     "matomo": ("dict", {}),
-    "show_corner_ribbon": ("bool", True),
+    "show_corner_ribbon": ("bool", False),
+    "corner_ribbon_text": ("str", ""),
     "save_code_now_webhook_secret": ("str", ""),
     "inbound_email": ("dict", {"shared_key": "shared_key"}),
     "browse_content_rate_limit": ("dict", {"enabled": True, "rate": "60/m"}),
@@ -228,10 +149,10 @@ DEFAULT_CONFIG = {
     "datatables_max_page_size": ("int", 1000),
 }
 
-swhweb_config: Dict[str, Any] = {}
+swhweb_config: SWHWebConfig = SWHWebConfig()
 
 
-def get_config(config_file: str = "web/web") -> Dict[str, Any]:
+def get_config(config_file: str = "web/web") -> SWHWebConfig:
     """Read the configuration file `config_file`.
 
     If an environment variable SWH_CONFIG_FILENAME is defined, this
@@ -251,21 +172,28 @@ def get_config(config_file: str = "web/web") -> Dict[str, Any]:
         cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
         swhweb_config.update(cfg)
         config.prepare_folders(swhweb_config, "log_dir")
-        if swhweb_config.get("search"):
-            swhweb_config["search"] = get_search(**swhweb_config["search"])
-        else:
-            swhweb_config["search"] = None
-        swhweb_config["storage"] = get_storage(**swhweb_config["storage"])
-        swhweb_config["vault"] = get_vault(**swhweb_config["vault"])
-        swhweb_config["indexer_storage"] = get_indexer_storage(
-            **swhweb_config["indexer_storage"]
-        )
-        swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"])
-        swhweb_config["counters"] = get_counters(**swhweb_config["counters"])
-
+    for service, modname in (
+        ("search", "search"),
+        ("storage", "storage"),
+        ("vault", "vault"),
+        ("indexer_storage", "indexer.storage"),
+        ("scheduler", "scheduler"),
+        ("counters", "counters"),
+    ):
+        if isinstance(swhweb_config.get(service), dict):
+            mod = importlib.import_module(f"swh.{modname}")
+            getter = getattr(mod, f"get_{service}")
+            swhweb_config[service] = getter(**swhweb_config[service])
     return swhweb_config
 
 
+def oidc_enabled() -> bool:
+    try:
+        return bool(get_config()["keycloak"]["server_url"])
+    except:  # noqa: E722
+        return False
+
+
 def search() -> SearchInterface:
     """Return the current application's search."""
     return get_config()["search"]