Skip to content
Snippets Groups Projects
Commit 8848bd31 authored by Nicolas Dandrimont's avatar Nicolas Dandrimont
Browse files

loader: add shortcuts for the connect and read timeouts

This sets the connect and read timeout for both the smart loader (via
urllib3/dulwich) and for the dumb loader (via requests).
parent c6b2b577
No related branches found
No related tags found
1 merge request!177Add support for passing extra arguments to urllib3 and requests (including timeouts and certificate verification)
......@@ -29,6 +29,7 @@ import dulwich.client
from dulwich.object_store import ObjectStoreGraphWalker
from dulwich.objects import Blob, Commit, ShaFile, Tag, Tree
from dulwich.pack import PackData, PackInflater
import urllib3.util
from swh.core.statsd import Statsd
from swh.loader.exception import NotFound
......@@ -179,6 +180,8 @@ class GitLoader(BaseGitLoader):
repo_representation: Type[RepoRepresentation] = RepoRepresentation,
pack_size_bytes: int = 4 * 1024 * 1024 * 1024,
temp_file_cutoff: int = 100 * 1024 * 1024,
connect_timeout: float = 120,
read_timeout: float = 60,
urllib3_extra_kwargs: Dict[str, Any] = {},
requests_extra_kwargs: Dict[str, Any] = {},
**kwargs: Any,
......@@ -207,7 +210,11 @@ class GitLoader(BaseGitLoader):
self.ext_refs: Dict[bytes, Optional[Tuple[int, bytes]]] = {}
self.repo_pack_size_bytes = 0
self.urllib3_extra_kwargs = urllib3_extra_kwargs
self.urllib3_extra_kwargs["timeout"] = urllib3.util.Timeout(
connect=connect_timeout, read=read_timeout
)
self.requests_extra_kwargs = requests_extra_kwargs
self.requests_extra_kwargs["timeout"] = (connect_timeout, read_timeout)
def fetch_pack_from_origin(
self,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment