From d4f3e91466c5511a2e98deaa4a1a480ab6c87897 Mon Sep 17 00:00:00 2001
From: "Antoine R. Dumont (@ardumont)" <ardumont@softwareheritage.org>
Date: Thu, 3 Aug 2023 11:01:01 +0200
Subject: [PATCH] packagist: Allow batch size records configuration in
constructor
This allows to configure smaller batch when testing from docker & cli.
---
swh/lister/packagist/lister.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/swh/lister/packagist/lister.py b/swh/lister/packagist/lister.py
index 1dd05379..171f1696 100644
--- a/swh/lister/packagist/lister.py
+++ b/swh/lister/packagist/lister.py
@@ -75,6 +75,7 @@ class PackagistLister(Lister[PackagistListerState, PackagistPageType]):
max_origins_per_page: Optional[int] = None,
max_pages: Optional[int] = None,
enable_origins: bool = True,
+ record_batch_size: int = 1000,
):
super().__init__(
scheduler=scheduler,
@@ -85,6 +86,7 @@ class PackagistLister(Lister[PackagistListerState, PackagistPageType]):
max_origins_per_page=max_origins_per_page,
max_pages=max_pages,
enable_origins=enable_origins,
+ record_batch_size=record_batch_size,
)
self.session.headers.update({"Accept": "application/json"})
--
GitLab