Skip to content
Snippets Groups Projects
Unverified Commit 54ae8663 authored by Antoine R. Dumont's avatar Antoine R. Dumont
Browse files

runner: Schedule task types whose queue exists

To avoid the churn of hitting the scheduler too often for noops.

Refs. swh/infra/sysadm-environment#5512
parent b477d563
No related branches found
No related tags found
No related merge requests found
Pipeline #13977 failed
# Copyright (C) 2015-2024 The Software Heritage developers
# Copyright (C) 2015-2025 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
......@@ -20,7 +20,7 @@ from kombu.utils.uuid import uuid
from swh.core.statsd import statsd
from swh.scheduler import get_scheduler
from swh.scheduler.celery_backend.config import get_available_slots
from swh.scheduler.celery_backend.config import get_available_slots, get_queue_stats
from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import TaskRun, TaskType
from swh.scheduler.utils import utcnow
......@@ -73,11 +73,32 @@ def run_ready_tasks(
"""
all_backend_tasks: List[TaskRun] = []
while True:
# Only the task types whose queue exits in rabbitmq will be scheduled
task_types_to_schedule = []
# Keep the ignore task types during the loop
task_types_to_skip = []
# If no task type filter provided, we use all scheduler task types registered
if not task_types:
task_types = backend.get_task_types()
# Only schedule task types whose queue exists in rabbitmq
for _task_type in task_types:
task_type_name = _task_type.type if isinstance(TaskType) else _task_type
task_type = backend.get_task_type(task_type_name)
queue_stats = get_queue_stats(app, task_type.backend_name)
lst = task_types_to_schedule if queue_stats else task_types_to_skip
lst.append(task_type)
if task_types_to_skip:
logger.info(
"Task types skipped because their queue is non-existent: [%s]",
", ".join(tt.type for tt in task_types_to_skip),
)
task_types_d = {}
pending_tasks = []
for task_type in task_types:
for task_type in task_types_to_schedule:
task_type_name = task_type.type
task_types_d[task_type_name] = task_type
max_queue_length = task_type.max_queue_length
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment