Support the new paginated snapshot branch fetch function

debian/control

@@ -13,7 +13,7 @@ Build-Depends: debhelper (>= 9),
                python3-swh.core,
                python3-swh.loader.core (>= 0.0.34~),
                python3-swh.model (>= 0.0.27~),
-               python3-swh.storage,
+               python3-swh.storage (>= 0.0.108~),
                python3-swh.scheduler,
                python3-vcversioner
 Standards-Version: 3.9.6
@@ -24,6 +24,6 @@ Architecture: all
 Depends: python3-swh.core,
          python3-swh.loader.core (>= 0.0.34~),
          python3-swh.model (>= 0.0.27~),
-         python3-swh.storage,
+         python3-swh.storage (>= 0.0.108~),
          ${misc:Depends}, ${python3:Depends}
 Description: Software Heritage PyPI Loader

requirements-swh.txt

 swh.core
 swh.model >= 0.0.27
-swh.storage
+swh.storage >= 0.0.108
 swh.scheduler
 swh.loader.core >= 0.0.34

swh/loader/pypi/loader.py

@@ -16,6 +16,7 @@ from swh.model.identifiers import (
     revision_identifier, snapshot_identifier,
     identifier_to_bytes, normalize_timestamp
 )
+from swh.storage.algos.snapshot import snapshot_get_all_branches
 
 from .client import PyPIClient, PyPIProject
 
@@ -122,7 +123,9 @@ class PyPILoader(SWHLoader):
         """Retrieve the last snapshot
 
         """
-        return self.storage.snapshot_get_latest(self.origin_id)
+        snapshot = self.storage.snapshot_get_latest(self.origin_id)
+        if snapshot and snapshot.pop('next_branch', None):
+            return snapshot_get_all_branches(self.storage, snapshot['id'])
 
     def prepare(self, project_name, project_url,
                 project_metadata_url=None):