diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ce247f184cbb17c7ee5dfd7592af5a3a22b0883c..5e130010fd4e375a80f041d2e4785bc277a74bd9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,6 +32,7 @@ repos: stages: [commit] - id: codespell name: Check commit message spelling + args: [-L crate] stages: [commit-msg] - repo: local diff --git a/requirements.txt b/requirements.txt index e8bf14cc263f3d68cd248c94f3f8814306281618..dead79e942580b9c2c472ffcac94ee247b111074 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ dateparser dulwich iso8601 launchpadlib +looseversion lxml psycopg2 pyreadr diff --git a/swh/lister/crates/lister.py b/swh/lister/crates/lister.py index a17ce7d307a4e2480c5349b1cf215bd18473cd54..e31756c53f926983fdf743de1e8740d0482cc739 100644 --- a/swh/lister/crates/lister.py +++ b/swh/lister/crates/lister.py @@ -15,7 +15,7 @@ from typing import Any, Dict, Iterator, List, Optional from urllib.parse import urlparse import iso8601 -from packaging.version import parse as parse_version +from looseversion import LooseVersion2 from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -201,7 +201,7 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]): for name, item in dataset.items(): page = [] # sort crate versions - versions: list = sorted(item["versions"].keys(), key=parse_version) + versions = sorted(item["versions"].keys(), key=LooseVersion2) for version in versions: v = item["versions"][version] diff --git a/swh/lister/crates/tests/data/fake_crates_repository_init.sh b/swh/lister/crates/tests/data/fake_crates_repository_init.sh index b58d195da09f3c99f0df3f867affe5658edb4cd3..8078fadce7e5b959073aaaed9ffdf2e4d5e9d66b 100755 --- a/swh/lister/crates/tests/data/fake_crates_repository_init.sh +++ b/swh/lister/crates/tests/data/fake_crates_repository_init.sh @@ -17,6 +17,7 @@ echo -e '''created_at,description,documentation,downloads,homepage,id,max_upload ''' > data/crates.csv echo -e '''checksum,crate_id,crate_size,created_at,downloads,features,id,license,links,num,published_by,updated_at,yanked +d879626d5babe4ca6c4ec953d712e28d939672b325a4f9352f28ca3c82568a15,1339,,2014-12-18 06:56:46.88489,845,{},1321,MIT/Apache-2.0,,0.1.3-experimental,,2017-11-30 05:24:37.146115,f 398952a2f6cd1d22bc1774fd663808e32cf36add0280dee5cdd84a8fff2db944,2233,,2015-05-27 23:19:16.848643,1961,{},10855,MIT/Apache-2.0,,0.1.0,,2017-11-30 03:37:17.449539,f 343bd0171ee23346506db6f4c64525de6d72f0e8cc533f83aea97f3e7488cbf9,545,,2014-12-18 06:56:46.88489,845,{},1321,MIT/Apache-2.0,,0.1.2,,2017-11-30 02:29:20.01125,f 6e229ed392842fa93c1d76018d197b7e1b74250532bafb37b0e1d121a92d4cf7,1339,,2015-02-03 11:15:19.001762,8211,{},4371,MIT/Apache-2.0,,0.1.2,,2017-11-30 03:14:27.545115,f diff --git a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz index bd74c75b0fffe577720c990446d756eed21118f8..12b3082d29757e19fbe127261bb86bec5cce6b1c 100644 Binary files a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz and b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz differ diff --git a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 index 0b7dd384a90ce172fa758391869b2d998bfd8b5c..7164da399f78a0a27ead124833eaf7ec7b30d344 100644 Binary files a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 and b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 differ diff --git a/swh/lister/crates/tests/test_lister.py b/swh/lister/crates/tests/test_lister.py index 387ca266a7e911c251b3a36040fa226d00181f67..ebc9220c4ddd75f78f87853aad3874aac9657d36 100644 --- a/swh/lister/crates/tests/test_lister.py +++ b/swh/lister/crates/tests/test_lister.py @@ -26,6 +26,14 @@ expected_origins = [ "filename": "rand-0.1.2.crate", "url": "https://static.crates.io/crates/rand/rand-0.1.2.crate", }, + { + "version": "0.1.3-experimental", + "checksums": { + "sha256": "d879626d5babe4ca6c4ec953d712e28d939672b325a4f9352f28ca3c82568a15", # noqa: B950 + }, + "filename": "rand-0.1.3-experimental.crate", + "url": "https://static.crates.io/crates/rand/rand-0.1.3-experimental.crate", + }, ], "crates_metadata": [ { @@ -38,6 +46,11 @@ expected_origins = [ "last_update": "2017-11-30 03:14:27.545115", "yanked": False, }, + { + "version": "0.1.3-experimental", + "last_update": "2017-11-30 05:24:37.146115", + "yanked": False, + }, ], }, {