Skip to content
Snippets Groups Projects
Commit 031e1aa6 authored by Jenkins for Software Heritage's avatar Jenkins for Software Heritage
Browse files

Update upstream source from tag 'debian/upstream/2.1.0'

Update to upstream version '2.1.0'
with Debian dir 0e0206897fd866fbbe27b1b465fb800687815846
parents 25209125 4a851484
No related branches found
No related tags found
No related merge requests found
Metadata-Version: 2.1
Name: swh.loader.mercurial
Version: 2.0.0
Version: 2.1.0
Summary: Software Heritage Mercurial Loader
Home-page: https://forge.softwareheritage.org/diffusion/DLDHG/
Author: Software Heritage developers
......
Metadata-Version: 2.1
Name: swh.loader.mercurial
Version: 2.0.0
Version: 2.1.0
Summary: Software Heritage Mercurial Loader
Home-page: https://forge.softwareheritage.org/diffusion/DLDHG/
Author: Software Heritage developers
......
......@@ -47,7 +47,7 @@ FLAG_PERMS = {
TEMPORARY_DIR_PREFIX_PATTERN = "swh.loader.mercurial.from_disk"
EXTID_TYPE = "hg-nodeid"
EXTID_VERSION: int = 1
T = TypeVar("T")
......@@ -268,13 +268,13 @@ class HgLoaderFromDisk(BaseLoader):
)
def _get_extids_for_targets(self, targets: List[bytes]) -> List[ExtID]:
# Get all Mercurial ExtIDs for the targets in the latest snapshot
"""Get all Mercurial ExtIDs for the targets in the latest snapshot"""
extids = [
extid
for extid in self.storage.extid_get_from_target(
identifiers.ObjectType.REVISION, targets
)
if extid.extid_type == EXTID_TYPE
if extid.extid_type == EXTID_TYPE and extid.extid_version == EXTID_VERSION
]
if extids:
......@@ -461,7 +461,11 @@ class HgLoaderFromDisk(BaseLoader):
if from_cache is not None:
return from_cache
# The parent was not loaded in this run, get it from storage
from_storage = self.storage.extid_get_from_extid(EXTID_TYPE, ids=[hg_nodeid])
from_storage = [
extid
for extid in self.storage.extid_get_from_extid(EXTID_TYPE, ids=[hg_nodeid])
if extid.extid_version == EXTID_VERSION
]
msg = "Expected 1 match from storage for hg node %r, got %d"
assert len(from_storage) == 1, msg % (hg_nodeid.hex(), len(from_storage))
......@@ -546,7 +550,14 @@ class HgLoaderFromDisk(BaseLoader):
object_type=identifiers.ObjectType.REVISION, object_id=revision.id,
)
self.storage.extid_add(
[ExtID(extid_type=EXTID_TYPE, extid=hg_nodeid, target=revision_swhid)]
[
ExtID(
extid_type=EXTID_TYPE,
extid_version=EXTID_VERSION,
extid=hg_nodeid,
target=revision_swhid,
)
]
)
def store_release(self, name: bytes, target: Sha1Git) -> Sha1Git:
......
......@@ -2,6 +2,7 @@
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from collections import defaultdict
from dataclasses import dataclass
import io
......@@ -154,7 +155,7 @@ def clone(src: str, dest: str, timeout: float) -> None:
if process.is_alive():
process.terminate()
# Give it a second (literally), then kill it
# Give it literally a second (in successive steps of 0.1 second), then kill it.
# Can't use `process.join(1)` here, billiard appears to be bugged
# https://github.com/celery/billiard/issues/270
killed = False
......
......@@ -7,6 +7,7 @@ from hashlib import sha1
import os
from pathlib import Path
import subprocess
import unittest
import attr
import pytest
......@@ -26,7 +27,7 @@ from swh.model.model import RevisionType, Snapshot, SnapshotBranch, TargetType
from swh.storage import get_storage
from swh.storage.algos.snapshot import snapshot_get_latest
from ..from_disk import HgDirectory, HgLoaderFromDisk
from ..from_disk import EXTID_VERSION, HgDirectory, HgLoaderFromDisk
from .loader_checker import ExpectedSwhids, LoaderChecker
VISIT_DATE = parse_visit_date("2016-05-03 15:16:32+00")
......@@ -613,3 +614,59 @@ def test_load_repo_with_new_commits(swh_storage, datadir, tmp_path):
"skipped_content": 0,
"snapshot": 2,
}
def test_load_repo_check_extids_write_version(swh_storage, datadir, tmp_path):
"""ExtIDs should be stored with a given version when loading is done"""
archive_name = "hello"
archive_path = Path(datadir, f"{archive_name}.tgz")
repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path)
hg_strip(repo_url.replace("file://", ""), "tip")
loader = HgLoaderFromDisk(swh_storage, repo_url)
assert loader.load() == {"status": "eventful"}
# Ensure we write ExtIDs to a specific version.
snapshot = snapshot_get_latest(swh_storage, repo_url)
# First, filter out revisions from that snapshot
revision_ids = [
branch.target
for branch in snapshot.branches.values()
if branch.target_type == TargetType.REVISION
]
assert len(revision_ids) > 0
# Those revisions should have their associated ExtID version set to EXTID_VERSION
extids = swh_storage.extid_get_from_target(ObjectType.REVISION, revision_ids)
assert len(extids) == len(revision_ids)
for extid in extids:
assert extid.extid_version == EXTID_VERSION
def test_load_new_extid_should_be_eventful(swh_storage, datadir, tmp_path):
"""Changing the extid version should make loaders ignore existing extids,
and load the repo again."""
archive_name = "hello"
archive_path = os.path.join(datadir, f"{archive_name}.tgz")
repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path)
repo_path = repo_url.replace("file://", "")
with unittest.mock.patch("swh.loader.mercurial.from_disk.EXTID_VERSION", 0):
loader = HgLoaderFromDisk(swh_storage, repo_path)
assert loader.load() == {"status": "eventful"}
loader = HgLoaderFromDisk(swh_storage, repo_path)
assert loader.load() == {"status": "eventful"}
loader = HgLoaderFromDisk(swh_storage, repo_path)
assert loader.load() == {"status": "uneventful"}
with unittest.mock.patch("swh.loader.mercurial.from_disk.EXTID_VERSION", 10000):
loader = HgLoaderFromDisk(swh_storage, repo_path)
assert loader.load() == {"status": "eventful"}
loader = HgLoaderFromDisk(swh_storage, repo_path)
assert loader.load() == {"status": "uneventful"}
......@@ -15,12 +15,14 @@ from .. import hgutil
def test_clone_timeout(monkeypatch):
src = "https://www.mercurial-scm.org/repo/hello"
dest = "/dev/null"
timeout = 0.1
timeout = 1
sleepy_time = 100 * timeout
assert sleepy_time > timeout
def clone(*args, **kwargs):
# ignore SIGTERM to force sigkill
signal.signal(signal.SIGTERM, lambda signum, frame: None)
time.sleep(2)
time.sleep(sleepy_time) # we make sure we exceed the timeout
monkeypatch.setattr(hg, "clone", clone)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment