Prefix task types used in tests with 'test-'

so that tests do not depend on a lucky guess on what the scheduler db
state actually is. DB initialization scripts do create task types for
git, hg and svn (used in tests) but these tests depends on the fact the
db fixture has been called already once before, so tables are
truncated (especially the task and task_type ones).

For example running a single test involved in task-type creation was
failing (eg. 'pytest swh -k test_create_task_type_idempotence').

This commit does make tests not collide with any existing task or task
type initialization scripts may create.

Note that this also means that there is actually no test dealing with
the scheduler db state after initialization, which is not grat and
should be addressed.

swh/scheduler/simulator/origins.py

@@ -54,7 +54,7 @@ def generate_listed_origin(
         now = datetime.now(tz=timezone.utc)
 
     url = f"https://example.com/{_nb_generated_origins:06d}.git"
-    visit_type = "git"
+    visit_type = "test-git"
     origin = OriginModel(visit_type, url)
 
     return ListedOrigin(

swh/scheduler/tests/common.py

@@ -8,13 +8,13 @@ import datetime
 from typing import Dict, List, Optional
 
 TEMPLATES = {
-    "git": {
-        "type": "load-git",
+    "test-git": {
+        "type": "load-test-git",
         "arguments": {"args": [], "kwargs": {},},
         "next_run": None,
     },
-    "hg": {
-        "type": "load-hg",
+    "test-hg": {
+        "type": "load-test-hg",
         "arguments": {"args": [], "kwargs": {},},
         "next_run": None,
         "policy": "oneshot",
@@ -23,8 +23,8 @@ TEMPLATES = {
 
 
 TASK_TYPES = {
-    "git": {
-        "type": "load-git",
+    "test-git": {
+        "type": "load-test-git",
         "description": "Update a git repository",
         "backend_name": "swh.loader.git.tasks.UpdateGitRepository",
         "default_interval": datetime.timedelta(days=64),
@@ -35,8 +35,8 @@ TASK_TYPES = {
         "num_retries": 7,
         "retry_delay": datetime.timedelta(hours=2),
     },
-    "hg": {
-        "type": "load-hg",
+    "test-hg": {
+        "type": "load-test-hg",
         "description": "Update a mercurial repository",
         "backend_name": "swh.loader.mercurial.tasks.UpdateHgRepository",
         "default_interval": datetime.timedelta(days=64),

swh/scheduler/tests/conftest.py

@@ -31,7 +31,7 @@ def stored_lister(swh_scheduler) -> Lister:
 @pytest.fixture
 def visit_types() -> List[str]:
     """Possible visit types in `ListedOrigin`s"""
-    return ["git", "svn"]
+    return ["test-git", "test-svn"]
 
 
 @pytest.fixture

swh/scheduler/tests/test_cli_origin.py

@@ -60,6 +60,8 @@ def test_grab_next(swh_scheduler, listed_origins_by_type):
     NUM_RESULTS = 10
     # Strict inequality to check that grab_next_visits doesn't return more
     # results than requested
+
+    # XXX: should test all of 'listed_origins_by_type' here...
     visit_type = next(iter(listed_origins_by_type))
     assert len(listed_origins_by_type[visit_type]) > NUM_RESULTS
 

swh/scheduler/tests/test_common.py

@@ -10,7 +10,7 @@ from .common import TEMPLATES, tasks_from_template
 
 def test_tasks_from_template_no_priority():
     nb_tasks = 3
-    template = TEMPLATES["git"]
+    template = TEMPLATES["test-git"]
     next_run = datetime.datetime.utcnow()
     tasks = tasks_from_template(template, next_run, nb_tasks)
 
@@ -27,7 +27,7 @@ def test_tasks_from_template_no_priority():
 
 
 def test_tasks_from_template_priority():
-    template = TEMPLATES["hg"]
+    template = TEMPLATES["test-hg"]
     num_priorities = {
         None: 3,
         "high": 5,

swh/scheduler/tests/test_recurrent_visits.py

@@ -6,7 +6,7 @@
 from datetime import timedelta
 import logging
 from queue import Queue
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import pytest
 
@@ -34,7 +34,7 @@ def _compute_backend_name(visit_type: str) -> str:
 @pytest.fixture
 def swh_scheduler(swh_scheduler):
     """Override default fixture of the scheduler to install some more task types."""
-    for visit_type in ["git", "hg", "svn"]:
+    for visit_type in ["test-git", "test-hg", "test-svn"]:
         task_type = f"load-{visit_type}"
         swh_scheduler.create_task_type(
             {
@@ -58,7 +58,13 @@ def test_cli_schedule_recurrent_unknown_visit_type(swh_scheduler):
         invoke(
             swh_scheduler,
             False,
-            ["schedule-recurrent", "--visit-type", "unknown", "--visit-type", "git"],
+            [
+                "schedule-recurrent",
+                "--visit-type",
+                "unknown",
+                "--visit-type",
+                "test-git",
+            ],
         )
 
 
@@ -108,7 +114,7 @@ def test_recurrent_visit_scheduling(
         # we'll limit the orchestrator to the origins' type we know
         task_types.append(task_type)
 
-    for visit_type in ["git", "svn"]:
+    for visit_type in ["test-git", "test-svn"]:
         task_type = f"load-{visit_type}"
         send_visits_for_visit_type(
             swh_scheduler, mock_celery_app, visit_type, all_task_types[task_type]
@@ -134,9 +140,12 @@ def test_recurrent_visit_scheduling(
         assert expected_record in set(records)
 
 
+@patch.dict(
+    POLICY_ADDITIONAL_PARAMETERS, {"test-git": POLICY_ADDITIONAL_PARAMETERS["git"]}
+)
 @pytest.mark.parametrize(
     "visit_type, tablesamples",
-    [("hg", {}), ("git", POLICY_ADDITIONAL_PARAMETERS["git"])],
+    [("test-hg", {}), ("test-git", POLICY_ADDITIONAL_PARAMETERS["git"])],
 )
 def test_recurrent_visit_additional_parameters(
     swh_scheduler, mocker, visit_type, tablesamples

swh/scheduler/tests/test_scheduler.py

@@ -86,22 +86,22 @@ class TestScheduler:
         assert missing_methods == []
 
     def test_add_task_type(self, swh_scheduler):
-        tt = TASK_TYPES["git"]
+        tt = TASK_TYPES["test-git"]
         swh_scheduler.create_task_type(tt)
         assert tt == swh_scheduler.get_task_type(tt["type"])
-        tt2 = TASK_TYPES["hg"]
+        tt2 = TASK_TYPES["test-hg"]
         swh_scheduler.create_task_type(tt2)
         assert tt == swh_scheduler.get_task_type(tt["type"])
         assert tt2 == swh_scheduler.get_task_type(tt2["type"])
 
     def test_create_task_type_idempotence(self, swh_scheduler):
-        tt = TASK_TYPES["git"]
+        tt = TASK_TYPES["test-git"]
         swh_scheduler.create_task_type(tt)
         swh_scheduler.create_task_type(tt)
         assert tt == swh_scheduler.get_task_type(tt["type"])
 
     def test_get_task_types(self, swh_scheduler):
-        tt, tt2 = TASK_TYPES["git"], TASK_TYPES["hg"]
+        tt, tt2 = TASK_TYPES["test-git"], TASK_TYPES["test-hg"]
         swh_scheduler.create_task_type(tt)
         swh_scheduler.create_task_type(tt2)
         actual_task_types = swh_scheduler.get_task_types()
@@ -111,9 +111,9 @@ class TestScheduler:
     def test_create_tasks(self, swh_scheduler):
         self._create_task_types(swh_scheduler)
         num_git = 100
-        tasks_1 = tasks_from_template(TEMPLATES["git"], utcnow(), num_git)
+        tasks_1 = tasks_from_template(TEMPLATES["test-git"], utcnow(), num_git)
         tasks_2 = tasks_from_template(
-            TEMPLATES["hg"], utcnow(), num_priorities=NUM_PRIORITY_TASKS
+            TEMPLATES["test-hg"], utcnow(), num_priorities=NUM_PRIORITY_TASKS
         )
         tasks = tasks_1 + tasks_2
 
@@ -134,7 +134,7 @@ class TestScheduler:
 
         for task, orig_task in zip(ret, tasks):
             task = copy.deepcopy(task)
-            task_type = TASK_TYPES[orig_task["type"].split("-")[-1]]
+            task_type = TASK_TYPES[orig_task["type"].split("-", 1)[-1]]
             assert task["id"] not in ids
             assert task["status"] == "next_run_not_scheduled"
             assert task["current_interval"] == task_type["default_interval"]
@@ -161,8 +161,8 @@ class TestScheduler:
     def test_peek_ready_tasks_no_priority(self, swh_scheduler):
         self._create_task_types(swh_scheduler)
         t = utcnow()
-        task_type = TEMPLATES["git"]["type"]
-        tasks = tasks_from_template(TEMPLATES["git"], t, 100)
+        task_type = TEMPLATES["test-git"]["type"]
+        tasks = tasks_from_template(TEMPLATES["test-git"], t, 100)
         random.shuffle(tasks)
         swh_scheduler.create_tasks(tasks)
 
@@ -203,10 +203,10 @@ class TestScheduler:
         """Peek ready tasks only return standard tasks (no priority)"""
         self._create_task_types(swh_scheduler)
         t = utcnow()
-        task_type = TEMPLATES["git"]["type"]
+        task_type = TEMPLATES["test-git"]["type"]
         # Create tasks with and without priorities
         tasks = tasks_from_template(
-            TEMPLATES["git"], t, num_priorities=NUM_PRIORITY_TASKS,
+            TEMPLATES["test-git"], t, num_priorities=NUM_PRIORITY_TASKS,
         )
 
         count_priority = 0
@@ -230,10 +230,10 @@ class TestScheduler:
     def test_grab_ready_tasks(self, swh_scheduler):
         self._create_task_types(swh_scheduler)
         t = utcnow()
-        task_type = TEMPLATES["git"]["type"]
+        task_type = TEMPLATES["test-git"]["type"]
         # Create tasks with and without priorities
         tasks = tasks_from_template(
-            TEMPLATES["git"], t, num_priorities=NUM_PRIORITY_TASKS
+            TEMPLATES["test-git"], t, num_priorities=NUM_PRIORITY_TASKS
         )
         random.shuffle(tasks)
         swh_scheduler.create_tasks(tasks)
@@ -257,17 +257,17 @@ class TestScheduler:
         """check the grab and peek priority tasks endpoint behave as expected"""
         self._create_task_types(swh_scheduler)
         t = utcnow()
-        task_type = TEMPLATES["git"]["type"]
+        task_type = TEMPLATES["test-git"]["type"]
         num_tasks = 100
         # Create tasks with and without priorities
         tasks0 = tasks_with_priority_from_template(
-            TEMPLATES["git"], t, num_tasks, "high",
+            TEMPLATES["test-git"], t, num_tasks, "high",
         )
         tasks1 = tasks_with_priority_from_template(
-            TEMPLATES["hg"], t, num_tasks, "low",
+            TEMPLATES["test-hg"], t, num_tasks, "low",
         )
         tasks2 = tasks_with_priority_from_template(
-            TEMPLATES["hg"], t, num_tasks, "normal",
+            TEMPLATES["test-hg"], t, num_tasks, "normal",
         )
         tasks = tasks0 + tasks1 + tasks2
 
@@ -291,7 +291,7 @@ class TestScheduler:
     def test_get_tasks(self, swh_scheduler):
         self._create_task_types(swh_scheduler)
         t = utcnow()
-        tasks = tasks_from_template(TEMPLATES["git"], t, 100)
+        tasks = tasks_from_template(TEMPLATES["test-git"], t, 100)
         tasks = swh_scheduler.create_tasks(tasks)
         random.shuffle(tasks)
         while len(tasks) > 1:
@@ -311,7 +311,7 @@ class TestScheduler:
 
         self._create_task_types(swh_scheduler)
         t = utcnow()
-        tasks = tasks_from_template(TEMPLATES["git"], t, 100)
+        tasks = tasks_from_template(TEMPLATES["test-git"], t, 100)
         tasks = swh_scheduler.create_tasks(tasks)
         assert make_real_dicts(swh_scheduler.search_tasks()) == make_real_dicts(tasks)
 
@@ -336,8 +336,8 @@ class TestScheduler:
         """
         self._create_task_types(swh_scheduler)
         _time = utcnow()
-        recurring = tasks_from_template(TEMPLATES["git"], _time, 12)
-        oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12)
+        recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12)
+        oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12)
         total_tasks = len(recurring) + len(oneshots)
 
         # simulate scheduling tasks
@@ -458,8 +458,8 @@ class TestScheduler:
     def test_delete_archived_tasks(self, swh_scheduler):
         self._create_task_types(swh_scheduler)
         _time = utcnow()
-        recurring = tasks_from_template(TEMPLATES["git"], _time, 12)
-        oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12)
+        recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12)
+        oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12)
         total_tasks = len(recurring) + len(oneshots)
         pending_tasks = swh_scheduler.create_tasks(recurring + oneshots)
         backend_tasks = [
@@ -505,8 +505,8 @@ class TestScheduler:
         """
         self._create_task_types(swh_scheduler)
         _time = utcnow()
-        recurring = tasks_from_template(TEMPLATES["git"], _time, 12)
-        oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12)
+        recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12)
+        oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12)
         swh_scheduler.create_tasks(recurring + oneshots)
 
         assert not swh_scheduler.get_task_runs(task_ids=())
@@ -520,8 +520,8 @@ class TestScheduler:
         """
         self._create_task_types(swh_scheduler)
         _time = utcnow()
-        recurring = tasks_from_template(TEMPLATES["git"], _time, 12)
-        oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12)
+        recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12)
+        oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12)
         total_tasks = len(recurring) + len(oneshots)
         pending_tasks = swh_scheduler.create_tasks(recurring + oneshots)
         backend_tasks = [
@@ -574,8 +574,8 @@ class TestScheduler:
         """
         self._create_task_types(swh_scheduler)
         _time = utcnow()
-        recurring = tasks_from_template(TEMPLATES["git"], _time, 12)
-        oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12)
+        recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12)
+        oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12)
         pending_tasks = swh_scheduler.create_tasks(recurring + oneshots)
         backend_tasks = [
             {