tests: Ensure failing test with random fixture data can be reproduced

Some tests input data returned by swh-web fixtures are sampled from
filtered objects stored in a static test archive.

In order to reproduce any failing test with the same fixture data:

- add a new autouse function scope fixture setting the random seed
  to current time before each test execution and putting that seed
  in cache

- implement custom pytest reporting which adds a new section detailing
  the random seeds used for each failing test and the pytests commands
  to reproduce the failures

- add a new pytest option --swh-web-random-seed enabling to explicitly
  specify the random seed that will be used to run the tests

pytest.ini

 [pytest]
-addopts = -p no:flask -p no:pytest_swh_storage
+addopts = -p no:flask -p no:pytest_swh_storage --ignore-glob=*random_fixtures_test.py
 norecursedirs = docs node_modules .tox
 DJANGO_SETTINGS_MODULE = swh.web.settings.tests
 filterwarnings =

swh/web/tests/conftest.py

@@ -12,6 +12,7 @@ import random
 import shutil
 from subprocess import PIPE, run
 import sys
+import time
 from typing import Any, Dict, List, Optional
 
 from _pytest.python import Function
@@ -85,6 +86,10 @@ settings.register_profile(
 )
 
 
+def pytest_addoption(parser):
+    parser.addoption("--swh-web-random-seed", action="store", default=None)
+
+
 def pytest_configure(config):
     # Use fast hypothesis profile by default if none has been
     # explicitly specified in pytest option
@@ -137,6 +142,47 @@ def pytest_configure(config):
         json.dump(mock_webpack_stats, outfile)
 
 
+_swh_web_custom_section = "swh-web custom section"
+_random_seed_cache_key = "swh-web/random-seed"
+
+
+@pytest.fixture(scope="function", autouse=True)
+def random_seed(pytestconfig):
+    state = random.getstate()
+    seed = pytestconfig.getoption("--swh-web-random-seed")
+    if seed is None:
+        seed = time.time()
+    seed = int(seed)
+    cache.set(_random_seed_cache_key, seed)
+    random.seed(seed)
+    yield seed
+    random.setstate(state)
+
+
+def pytest_report_teststatus(report, config):
+    if report.when == "call" and report.outcome == "failed":
+        seed = cache.get(_random_seed_cache_key, None)
+        line = (
+            f'FAILED {report.nodeid}: Use "pytest --swh-web-random-seed={seed} '
+            f'{report.nodeid}" to reproduce that test failure with same inputs'
+        )
+        report.sections.append((_swh_web_custom_section, line))
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    reports = terminalreporter.getreports("failed")
+    content = os.linesep.join(
+        text
+        for report in reports
+        for secname, text in report.sections
+        if secname == _swh_web_custom_section
+    )
+    if content:
+        terminalreporter.ensure_newline()
+        terminalreporter.section(_swh_web_custom_section, sep="-", blue=True, bold=True)
+        terminalreporter.line(content)
+
+
 # Clear Django cache before each test
 @pytest.fixture(autouse=True)
 def django_cache_cleared():

swh/web/tests/data.py

@@ -345,7 +345,7 @@ def _init_tests_data():
             rev_id = rev_log[0]
             revisions.add(rev_id)
 
-        for rev in storage.revision_get(origin_revisions):
+        for rev in storage.revision_get(sorted(origin_revisions)):
             if rev is None:
                 continue
             dir_id = rev.directory
@@ -464,11 +464,11 @@ def _init_tests_data():
         "idx_storage": idx_storage,
         "counters": counters,
         "origins": _TEST_ORIGINS,
-        "contents": contents,
-        "directories": list(directories),
-        "releases": list(releases),
-        "revisions": list(map(hash_to_hex, revisions)),
-        "snapshots": list(snapshots),
+        "contents": list(sorted(contents, key=lambda c: c["sha1"])),
+        "directories": list(sorted(directories)),
+        "releases": list(sorted(releases)),
+        "revisions": list(sorted(map(hash_to_hex, revisions))),
+        "snapshots": list(sorted(snapshots)),
         "swhids": swhids,
     }
 

swh/web/tests/random_fixtures_test.py

+# Copyright (C) 2021  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import sys
+
+
+def test_random_fixture_values(
+    sha1,
+    invalid_sha1,
+    sha256,
+    content,
+    contents,
+    unknown_content,
+    unknown_contents,
+    content_text,
+    content_text_non_utf8,
+    content_application_no_highlight,
+    content_text_no_highlight,
+    content_image_type,
+    content_unsupported_image_type_rendering,
+    content_utf8_detected_as_binary,
+    directory,
+    directory_with_subdirs,
+    directory_with_files,
+    unknown_directory,
+    release,
+    releases,
+    unknown_release,
+    revision,
+    revisions,
+    revisions_list,
+    unknown_revision,
+    ancestor_revisions,
+    non_ancestor_revisions,
+    snapshot,
+    unknown_snapshot,
+    origin,
+    origin_with_multiple_visits,
+    origin_with_releases,
+    origin_with_pull_request_branches,
+    content_swhid,
+    directory_swhid,
+    release_swhid,
+    revision_swhid,
+    snapshot_swhid,
+):
+    """Special test to print values of swh-web fixtures returning random data.
+
+    It is not integrated in swh-web test suite but will be executed by explicitly
+    invoking pytest in tests located in swh/web/tests/test_random_fixtures.py.
+    """
+    print(
+        "\n".join(
+            [
+                sha1,
+                invalid_sha1,
+                sha256,
+                content["sha1"],
+                str([c["sha1"] for c in contents]),
+                unknown_content["sha1"],
+                str([c["sha1"] for c in unknown_contents]),
+                content_text["sha1"],
+                content_text_non_utf8["sha1"],
+                content_application_no_highlight["sha1"],
+                content_text_no_highlight["sha1"],
+                content_image_type["sha1"],
+                content_unsupported_image_type_rendering["sha1"],
+                content_utf8_detected_as_binary["sha1"],
+                directory,
+                directory_with_subdirs,
+                directory_with_files,
+                unknown_directory,
+                release,
+                str(releases),
+                unknown_release,
+                revision,
+                str(revisions),
+                str(revisions_list(size=3)),
+                unknown_revision,
+                str(ancestor_revisions),
+                str(non_ancestor_revisions),
+                snapshot,
+                unknown_snapshot,
+                origin["url"],
+                origin_with_multiple_visits["url"],
+                origin_with_releases["url"],
+                origin_with_pull_request_branches.url,
+                str(content_swhid),
+                str(directory_swhid),
+                str(release_swhid),
+                str(revision_swhid),
+                str(snapshot_swhid),
+            ]
+        ),
+        file=sys.stderr,
+    )
+    assert False

swh/web/tests/test_random_fixtures.py

+# Copyright (C) 2021  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import os
+import subprocess
+
+
+def test_random_fixtures():
+    """Check random fixture values will be different when random seed
+    is not explicitly provided.
+    """
+    result_first = subprocess.run(
+        ["pytest", "-s", "random_fixtures_test.py"],
+        capture_output=True,
+        cwd=os.path.dirname(__file__),
+    )
+    result_second = subprocess.run(
+        ["pytest", "-s", "random_fixtures_test.py"],
+        capture_output=True,
+        cwd=os.path.dirname(__file__),
+    )
+    assert result_first.stderr != result_second.stderr
+    assert b'Use "pytest --swh-web-random-seed=' in result_first.stdout
+
+
+def test_random_fixtures_with_seed():
+    """Check random fixture values will be the same when random seed
+    is explicitly provided through a custom pytest option.
+    """
+    result_first = subprocess.run(
+        ["pytest", "-s", "--swh-web-random-seed=2021", "random_fixtures_test.py"],
+        capture_output=True,
+        cwd=os.path.dirname(__file__),
+    )
+    result_second = subprocess.run(
+        ["pytest", "-s", "--swh-web-random-seed=2021", "random_fixtures_test.py"],
+        capture_output=True,
+        cwd=os.path.dirname(__file__),
+    )
+    assert result_first.stderr == result_second.stderr
+    assert b'Use "pytest --swh-web-random-seed=2021' in result_first.stdout