Skip to content
Snippets Groups Projects
Commit 23260277 authored by Raphaël Gomès's avatar Raphaël Gomès
Browse files

Use billiard instead of stdlib multiprocessing

This circumvents a few celery-related issues, and is consistent with
what the rest of the codebase does.

stdlib multiprocessing is not able to spawn children from daemonic
processes, and even says so plainly if you try:

`AssertionError: daemonic processes are not allowed to have children`

This is incompatible with the SWH infrastructure which needs to do this
exactly. Fortunately, we're already using billiard and celery. I'm
assuming that there could be other blocking or annoying differences
between stdlib and billiard, but we will save ourselves the trouble of
finding out.
parent 504ee123
No related branches found
No related tags found
No related merge requests found
......@@ -4,10 +4,14 @@
# See top-level LICENSE file for more information
import io
from multiprocessing import Process, Queue
import os
import signal
import time
import traceback
from typing import Dict, NewType
from billiard import Process, Queue
# The internal Mercurial API is not guaranteed to be stable.
from mercurial import context, error, hg, smartset, util # type: ignore
import mercurial.ui # type: ignore
......@@ -68,7 +72,7 @@ def _clone_task(src: str, dest: str, errors: Queue) -> None:
raise e
def clone(src: str, dest: str, timeout: int) -> None:
def clone(src: str, dest: str, timeout: float) -> None:
"""Clone a repository with timeout.
Args:
......@@ -83,10 +87,18 @@ def clone(src: str, dest: str, timeout: int) -> None:
if process.is_alive():
process.terminate()
process.join(1)
if process.is_alive():
process.kill()
raise CloneTimeout(src, timeout)
# Give it a second (literally), then kill it
# Can't use `process.join(1)` here, billiard appears to be bugged
# https://github.com/celery/billiard/issues/270
killed = False
for _ in range(10):
time.sleep(0.1)
if not process.is_alive():
break
else:
killed = True
os.kill(process.pid, signal.SIGKILL)
raise CloneTimeout(src, timeout, killed)
if not errors.empty():
raise CloneFailure(src, dest, errors.get())
......@@ -2,7 +2,7 @@
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import signal
import time
import traceback
......@@ -15,16 +15,19 @@ from .. import hgutil
def test_clone_timeout(monkeypatch):
src = "https://www.mercurial-scm.org/repo/hello"
dest = "/dev/null"
timeout = 1
timeout = 0.1
def clone(*args, **kwargs):
time.sleep(5)
# ignore SIGTERM to force sigkill
signal.signal(signal.SIGTERM, lambda signum, frame: None)
time.sleep(2)
monkeypatch.setattr(hg, "clone", clone)
with pytest.raises(hgutil.CloneTimeout) as e:
hgutil.clone(src, dest, timeout)
assert e.value.args == (src, timeout)
killed = True
assert e.value.args == (src, timeout, killed)
def test_clone_error(caplog, tmp_path, monkeypatch):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment