Skip to content
Snippets Groups Projects
Commit 3fe09968 authored by Antoine Lambert's avatar Antoine Lambert
Browse files

tests: Use mocker pytest fixture instead of importing from unittest

parent 7236daab
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
swh/loader/git/tests/loader_tests.py
+ 20
− 20
View file @ 3fe09968
......@@ -5,7 +5,6 @@
import copy
import os.path
from unittest.mock import patch
import dulwich.objects
import dulwich.porcelain
......@@ -207,7 +206,7 @@ class CommonGitLoaderTests:
snapshot=None,
)
def test_load_incremental_partial_history(self):
def test_load_incremental_partial_history(self, mocker):
"""Check that loading a partial snapshot, then negotiating a full snapshot, works."""
# We pick this branch because it contains the target of the refs/heads/master
......@@ -218,27 +217,28 @@ class CommonGitLoaderTests:
branches={interesting_branch: SNAPSHOT1.branches[interesting_branch]}
)
with (
patch.object(
utils,
"ignore_branch_name",
lambda name: name != interesting_branch,
),
patch.object(
utils,
"filter_refs",
lambda refs: {
ref_name: utils.HexBytes(target)
for ref_name, target in refs.items()
if ref_name == interesting_branch
},
),
):
# Ensure that only the interesting branch is loaded
res = self.loader.load()
ignore_branch_name = mocker.patch.object(
utils,
"ignore_branch_name",
lambda name: name != interesting_branch,
)
filter_refs = mocker.patch.object(
utils,
"filter_refs",
lambda refs: {
ref_name: utils.HexBytes(target)
for ref_name, target in refs.items()
if ref_name == interesting_branch
},
)
# Ensure that only the interesting branch is loaded
res = self.loader.load()
assert res == {"status": "eventful"}
mocker.stop(ignore_branch_name)
mocker.stop(filter_refs)
assert self.loader.storage.snapshot_get_branches(partial_snapshot.id)
res = self.loader.load()
......
swh/loader/git/tests/test_loader.py
+ 81
− 76
View file @ 3fe09968
......@@ -14,7 +14,6 @@ import sys
from tempfile import SpooledTemporaryFile
from threading import Thread
import time
from unittest.mock import MagicMock, call
import attr
from dulwich.errors import GitProtocolError, NotGitRepository, ObjectFormatException
......@@ -48,10 +47,6 @@ from .loader_tests import SNAPSHOT1, FullGitLoaderTests
class CommonGitLoaderNotFound:
@pytest.fixture(autouse=True)
def __inject_fixtures(self, mocker):
"""Inject required fixtures in unittest.TestCase class"""
self.mocker = mocker
@pytest.mark.parametrize(
"failure_exception",
......@@ -66,14 +61,14 @@ class CommonGitLoaderNotFound:
NotGitRepository("not a git repo"),
],
)
def test_load_visit_not_found(self, failure_exception):
def test_load_visit_not_found(self, failure_exception, mocker):
"""Ingesting an unknown url result in a visit with not_found status"""
# simulate an initial communication error (e.g no repository found, ...)
self.mocker.patch(
mocker.patch(
"swh.loader.git.loader.GitLoader.fetch_pack_from_origin"
).side_effect = failure_exception
self.mocker.patch("swh.loader.git.loader.dumb.check_protocol").side_effect = (
mocker.patch("swh.loader.git.loader.dumb.check_protocol").side_effect = (
HTTPError("404 not found")
)
......@@ -99,15 +94,13 @@ class CommonGitLoaderNotFound:
GitProtocolError(ConnectionResetError("Connection reset by peer")),
],
)
def test_load_visit_failure(self, failure_exception):
def test_load_visit_failure(self, failure_exception, mocker):
"""Failing during the fetch pack step result in failing visit"""
# simulate a fetch communication error after the initial connection
# server error (e.g IOError, ObjectFormatException, ...)
mock = self.mocker.patch(
mocker.patch(
"swh.loader.git.loader.GitLoader.fetch_pack_from_origin"
)
mock.side_effect = failure_exception
).side_effect = failure_exception
res = self.loader.load()
assert res["status"] == "failed"
......@@ -148,26 +141,26 @@ class TestGitLoader(FullGitLoaderTests, CommonGitLoaderNotFound):
# metrics are sent
statsd_calls = statsd_report.mock_calls
assert [c for c in statsd_calls if c[1][0].startswith("git_")] == [
call("git_total", "c", 1, {}, 1),
call("git_ignored_refs_percent", "h", 0.0, {}, 1),
call("git_known_refs_percent", "h", 0.0, {}, 1),
mocker.call("git_total", "c", 1, {}, 1),
mocker.call("git_ignored_refs_percent", "h", 0.0, {}, 1),
mocker.call("git_known_refs_percent", "h", 0.0, {}, 1),
]
total_sum_name = "filtered_objects_total_sum"
total_count_name = "filtered_objects_total_count"
percent_name = "filtered_objects_percent"
assert [c for c in statsd_calls if c[1][0].startswith("filtered_")] == [
call(percent_name, "h", 0.0, {"object_type": "content"}, 1),
call(total_sum_name, "c", 0, {"object_type": "content"}, 1),
call(total_count_name, "c", 4, {"object_type": "content"}, 1),
call(percent_name, "h", 0.0, {"object_type": "directory"}, 1),
call(total_sum_name, "c", 0, {"object_type": "directory"}, 1),
call(total_count_name, "c", 7, {"object_type": "directory"}, 1),
call(percent_name, "h", 0.0, {"object_type": "revision"}, 1),
call(total_sum_name, "c", 0, {"object_type": "revision"}, 1),
call(total_count_name, "c", 7, {"object_type": "revision"}, 1),
call(percent_name, "h", 0.0, {"object_type": "snapshot"}, 1),
call(total_sum_name, "c", 0, {"object_type": "snapshot"}, 1),
call(total_count_name, "c", 1, {"object_type": "snapshot"}, 1),
mocker.call(percent_name, "h", 0.0, {"object_type": "content"}, 1),
mocker.call(total_sum_name, "c", 0, {"object_type": "content"}, 1),
mocker.call(total_count_name, "c", 4, {"object_type": "content"}, 1),
mocker.call(percent_name, "h", 0.0, {"object_type": "directory"}, 1),
mocker.call(total_sum_name, "c", 0, {"object_type": "directory"}, 1),
mocker.call(total_count_name, "c", 7, {"object_type": "directory"}, 1),
mocker.call(percent_name, "h", 0.0, {"object_type": "revision"}, 1),
mocker.call(total_sum_name, "c", 0, {"object_type": "revision"}, 1),
mocker.call(total_count_name, "c", 7, {"object_type": "revision"}, 1),
mocker.call(percent_name, "h", 0.0, {"object_type": "snapshot"}, 1),
mocker.call(total_sum_name, "c", 0, {"object_type": "snapshot"}, 1),
mocker.call(total_count_name, "c", 1, {"object_type": "snapshot"}, 1),
]
assert self.loader.statsd.constant_tags == {
"visit_type": "git",
......@@ -215,26 +208,26 @@ class TestGitLoader(FullGitLoaderTests, CommonGitLoaderNotFound):
# metrics are sent
statsd_calls = statsd_report.mock_calls
assert [c for c in statsd_calls if c[1][0].startswith("git_")] == [
call("git_total", "c", 1, {}, 1),
call("git_ignored_refs_percent", "h", 0.0, {}, 1),
call("git_known_refs_percent", "h", 0.0, {}, 1),
mocker.call("git_total", "c", 1, {}, 1),
mocker.call("git_ignored_refs_percent", "h", 0.0, {}, 1),
mocker.call("git_known_refs_percent", "h", 0.0, {}, 1),
]
total_sum_name = "filtered_objects_total_sum"
total_count_name = "filtered_objects_total_count"
percent_name = "filtered_objects_percent"
assert [c for c in statsd_calls if c[1][0].startswith("filtered_")] == [
call(percent_name, "h", 1 / 4, {"object_type": "content"}, 1),
call(total_sum_name, "c", 1, {"object_type": "content"}, 1),
call(total_count_name, "c", 4, {"object_type": "content"}, 1),
call(percent_name, "h", 3 / 7, {"object_type": "directory"}, 1),
call(total_sum_name, "c", 3, {"object_type": "directory"}, 1),
call(total_count_name, "c", 7, {"object_type": "directory"}, 1),
call(percent_name, "h", 2 / 7, {"object_type": "revision"}, 1),
call(total_sum_name, "c", 2, {"object_type": "revision"}, 1),
call(total_count_name, "c", 7, {"object_type": "revision"}, 1),
call(percent_name, "h", 0.0, {"object_type": "snapshot"}, 1),
call(total_sum_name, "c", 0, {"object_type": "snapshot"}, 1),
call(total_count_name, "c", 1, {"object_type": "snapshot"}, 1),
mocker.call(percent_name, "h", 1 / 4, {"object_type": "content"}, 1),
mocker.call(total_sum_name, "c", 1, {"object_type": "content"}, 1),
mocker.call(total_count_name, "c", 4, {"object_type": "content"}, 1),
mocker.call(percent_name, "h", 3 / 7, {"object_type": "directory"}, 1),
mocker.call(total_sum_name, "c", 3, {"object_type": "directory"}, 1),
mocker.call(total_count_name, "c", 7, {"object_type": "directory"}, 1),
mocker.call(percent_name, "h", 2 / 7, {"object_type": "revision"}, 1),
mocker.call(total_sum_name, "c", 2, {"object_type": "revision"}, 1),
mocker.call(total_count_name, "c", 7, {"object_type": "revision"}, 1),
mocker.call(percent_name, "h", 0.0, {"object_type": "snapshot"}, 1),
mocker.call(total_sum_name, "c", 0, {"object_type": "snapshot"}, 1),
mocker.call(total_count_name, "c", 1, {"object_type": "snapshot"}, 1),
]
assert self.loader.statsd.constant_tags == {
"visit_type": "git",
......@@ -244,9 +237,9 @@ class TestGitLoader(FullGitLoaderTests, CommonGitLoaderNotFound):
"has_parent_origins": False,
}
def test_load_incremental_partial_history(self, caplog):
def test_load_incremental_partial_history(self, caplog, mocker):
with caplog.at_level(logging.DEBUG, logger="swh.loader.git.loader"):
super().test_load_incremental_partial_history()
super().test_load_incremental_partial_history(mocker)
# Check that we've indeed inferred the target type for one of the snapshot
# branches
......@@ -417,12 +410,16 @@ class TestGitLoader(FullGitLoaderTests, CommonGitLoaderNotFound):
key=lambda c: c[1][3]["type"],
)
) == [
call(statsd_metric, "c", 1, {"type": "content", "result": "found"}, 1),
call(
mocker.call(
statsd_metric, "c", 1, {"type": "content", "result": "found"}, 1
),
mocker.call(
statsd_metric, "c", 1, {"type": "directory", "result": "found"}, 1
),
call(statsd_metric, "c", 1, {"type": "release", "result": "found"}, 1),
call(
mocker.call(
statsd_metric, "c", 1, {"type": "release", "result": "found"}, 1
),
mocker.call(
statsd_metric, "c", 1, {"type": "unknown", "result": "not_found"}, 1
),
]
......@@ -446,12 +443,18 @@ class TestGitLoader(FullGitLoaderTests, CommonGitLoaderNotFound):
key=lambda c: c[1][3]["type"],
)
) == [
call(statsd_metric, "c", 1, {"type": "content", "result": "found"}, 1),
call(
mocker.call(
statsd_metric, "c", 1, {"type": "content", "result": "found"}, 1
),
mocker.call(
statsd_metric, "c", 1, {"type": "directory", "result": "found"}, 1
),
call(statsd_metric, "c", 1, {"type": "release", "result": "found"}, 1),
call(statsd_metric, "c", 1, {"type": "revision", "result": "found"}, 1),
mocker.call(
statsd_metric, "c", 1, {"type": "release", "result": "found"}, 1
),
mocker.call(
statsd_metric, "c", 1, {"type": "revision", "result": "found"}, 1
),
]
def test_load_pack_size_limit(self, sentry_events):
......@@ -485,12 +488,12 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
)
self.destination_path = os.path.join(tmp_path, archive_name)
self.fetcher = MagicMock()
self.fetcher = mocker.MagicMock()
self.fetcher.get_origin_metadata.return_value = []
self.fetcher.get_parent_origins.return_value = [
Origin(url=f"base://{self.repo_url}")
]
self.fetcher_cls = MagicMock(return_value=self.fetcher)
self.fetcher_cls = mocker.MagicMock(return_value=self.fetcher)
self.fetcher_cls.SUPPORTED_LISTERS = ["fake-lister"]
mocker.patch(
"swh.loader.core.metadata_fetchers._fetchers",
......@@ -498,7 +501,7 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
)
self.loader = GitLoader(
MagicMock(wraps=swh_storage),
mocker.MagicMock(wraps=swh_storage),
self.repo_url,
lister_name="fake-lister",
lister_instance_name="",
......@@ -520,14 +523,14 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
# First tries the same origin
assert self.loader.storage.origin_visit_get_latest.mock_calls == [
call(
mocker.call(
self.repo_url,
allowed_statuses=None,
require_snapshot=True,
type="git",
),
# As it does not already have a snapshot, fall back to the parent origin
call(
mocker.call(
f"base://{self.repo_url}",
allowed_statuses=None,
require_snapshot=True,
......@@ -538,9 +541,9 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
# TODO: assert "incremental" is added to constant tags before these
# metrics are sent
assert [c for c in statsd_report.mock_calls if c[1][0].startswith("git_")] == [
call("git_total", "c", 1, {}, 1),
call("git_ignored_refs_percent", "h", 0.0, {}, 1),
call("git_known_refs_percent", "h", 0.0, {}, 1),
mocker.call("git_total", "c", 1, {}, 1),
mocker.call("git_ignored_refs_percent", "h", 0.0, {}, 1),
mocker.call("git_known_refs_percent", "h", 0.0, {}, 1),
]
assert self.loader.statsd.constant_tags == {
"visit_type": "git",
......@@ -593,14 +596,14 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
# First tries the same origin
assert self.loader.storage.origin_visit_get_latest.mock_calls == [
call(
mocker.call(
self.repo_url,
allowed_statuses=None,
require_snapshot=True,
type="git",
),
# As it does not already have a snapshot, fall back to the parent origin
call(
mocker.call(
f"base://{self.repo_url}",
allowed_statuses=None,
require_snapshot=True,
......@@ -611,9 +614,9 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
# TODO: assert "incremental*" is added to constant tags before these
# metrics are sent
assert [c for c in statsd_report.mock_calls if c[1][0].startswith("git_")] == [
call("git_total", "c", 1, {}, 1),
call("git_ignored_refs_percent", "h", 0.0, {}, 1),
call("git_known_refs_percent", "h", 0.25, {}, 1),
mocker.call("git_total", "c", 1, {}, 1),
mocker.call("git_ignored_refs_percent", "h", 0.0, {}, 1),
mocker.call("git_known_refs_percent", "h", 0.25, {}, 1),
]
assert self.loader.statsd.constant_tags == {
"visit_type": "git",
......@@ -652,7 +655,7 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
assert self.loader.storage.origin_visit_get_latest.mock_calls == [
# Tries the same origin, and finds a snapshot
call(
mocker.call(
self.repo_url,
type="git",
allowed_statuses=None,
......@@ -660,7 +663,7 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
),
# also fetches the parent, in case the origin was rebased on the parent
# since the last visit
call(
mocker.call(
f"base://{self.repo_url}",
type="git",
allowed_statuses=None,
......@@ -671,9 +674,9 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
# TODO: assert "incremental*" is added to constant tags before these
# metrics are sent
assert [c for c in statsd_report.mock_calls if c[1][0].startswith("git_")] == [
call("git_total", "c", 1, {}, 1),
call("git_ignored_refs_percent", "h", 0.0, {}, 1),
call("git_known_refs_percent", "h", 1.0, {}, 1),
mocker.call("git_total", "c", 1, {}, 1),
mocker.call("git_ignored_refs_percent", "h", 0.0, {}, 1),
mocker.call("git_known_refs_percent", "h", 1.0, {}, 1),
]
assert self.loader.statsd.constant_tags == {
"visit_type": "git",
......@@ -776,14 +779,14 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
# First tries the same origin
assert self.loader.storage.origin_visit_get_latest.mock_calls == [
call(
mocker.call(
self.repo_url,
allowed_statuses=None,
require_snapshot=True,
type="git",
),
# As it does not already have a snapshot, fall back to the parent origin
call(
mocker.call(
f"base://{self.repo_url}",
allowed_statuses=None,
require_snapshot=True,
......@@ -799,9 +802,11 @@ class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound):
"has_parent_origins": True,
}
assert [c for c in statsd_report.mock_calls if c[1][0].startswith("git_")] == [
call("git_total", "c", 1, {}, 1),
call("git_ignored_refs_percent", "h", 0.0, {}, 1),
call("git_known_refs_percent", "h", expected_git_known_refs_percent, {}, 1),
mocker.call("git_total", "c", 1, {}, 1),
mocker.call("git_ignored_refs_percent", "h", 0.0, {}, 1),
mocker.call(
"git_known_refs_percent", "h", expected_git_known_refs_percent, {}, 1
),
]
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment