From b2ece7ca637e781cc885dce7296725aaf432a3c4 Mon Sep 17 00:00:00 2001
From: Antoine Lambert <anlambert@softwareheritage.org>
Date: Wed, 21 Aug 2024 13:46:03 +0200
Subject: [PATCH] crates: Bump csv field size limit
A size limit of 1000000 was not enough to properly process
all CSV crates data so bump to a higher value.
---
swh/lister/crates/lister.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/swh/lister/crates/lister.py b/swh/lister/crates/lister.py
index 57aea577..a17ce7d3 100644
--- a/swh/lister/crates/lister.py
+++ b/swh/lister/crates/lister.py
@@ -131,7 +131,7 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]):
members_to_extract.append(member)
tf.extractall(members=members_to_extract, path=db_dump_path)
- csv.field_size_limit(1000000)
+ csv.field_size_limit(10000000)
(crates_csv_path,) = list(db_dump_path.glob("*/data/crates.csv"))
(versions_csv_path,) = list(db_dump_path.glob("*/data/versions.csv"))
--
GitLab