Skip to content
Snippets Groups Projects
Verified Commit 0f8f293f authored by Antoine R. Dumont's avatar Antoine R. Dumont
Browse files

nixguix: Deal with connection error with server

When that arises, we skip the origins.

Related to T3781
parent d92474bb
No related branches found
No related tags found
No related merge requests found
......@@ -25,7 +25,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
from urllib.parse import urlparse
import requests
from requests.exceptions import InvalidSchema, SSLError
from requests.exceptions import ConnectionError, InvalidSchema, SSLError
from swh.core.github.utils import GitHubSession
from swh.core.tarball import MIMETYPE_TO_ARCHIVE_FORMAT
......@@ -143,7 +143,7 @@ def is_tarball(urls: List[str], request: Optional[Any] = None) -> Tuple[bool, st
try:
response = request.head(url)
except (InvalidSchema, SSLError):
except (InvalidSchema, SSLError, ConnectionError):
raise ArtifactNatureUndetected(
f"Cannot determine artifact type from url <{url}>"
)
......
......@@ -20,6 +20,13 @@
],
"integrity": "sha256-bss09x9yOnuW+Q5BHHjf8nNcCNxCKMdl9/2/jKSFcrQ="
},
{
"type": "url",
"urls": [
"https://git-tails.immerda.ch/onioncircuits"
],
"integrity": "sha256-lV3xiWUZmSnt4LW0ni/sUyC/bbtaxkTzvFLFtJKLuI4="
},
{
"type": "url",
"urls": [ "unknown://example.org/wrong-scheme-so-skipped.txt" ],
......
......@@ -11,6 +11,7 @@ from typing import Dict, List
import pytest
import requests
from requests.exceptions import ConnectionError, InvalidSchema, SSLError
from swh.lister import TARBALL_EXTENSIONS
from swh.lister.nixguix.lister import (
......@@ -229,16 +230,21 @@ def test_lister_nixguix_mostly_noop(datadir, swh_scheduler, requests_mock):
"https://crates.io/api/v1/0.1.5/no-extension-and-head-404-so-skipped",
status_code=404,
)
# Will raise for that origin, this will get ignored as we cannot determine anything
# Invalid schema for that origin (and no extension), so skip origin
# from its name
requests_mock.head(
"ftp://ftp.ourproject.org/file-with-no-extension",
exc=requests.exceptions.InvalidSchema,
exc=InvalidSchema,
)
# Cannot communicate with an expired cert so skip
# Cannot communicate with an expired cert, so skip origin
requests_mock.head(
"https://code.9front.org/hg/plan9front",
exc=requests.exceptions.SSLError,
exc=SSLError,
)
# Cannot connect to the site, so skip origin
requests_mock.head(
"https://git-tails.immerda.ch/onioncircuits",
exc=ConnectionError,
)
listed_result = lister.run()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment