diff --git a/swh/lister/pattern.py b/swh/lister/pattern.py index 621b643ec796b00327289e74b318e911ec25f0ee..014fdf3e4175a710239d21db0709b43e33209234 100644 --- a/swh/lister/pattern.py +++ b/swh/lister/pattern.py @@ -182,17 +182,20 @@ class Lister(Generic[StateType, PageType]): try: for page in self.get_pages(): full_stats.pages += 1 - origins = list(self.get_origins_from_page(page)) - if ( - self.max_origins_per_page - and len(origins) > self.max_origins_per_page - ): - logger.info( - "Max origins per page set, truncated %s page results down to %s", - len(origins), - self.max_origins_per_page, - ) - origins = origins[: self.max_origins_per_page] + origins = [] + for origin in self.get_origins_from_page(page): + origins.append(origin) + if ( + self.max_origins_per_page + and len(origins) == self.max_origins_per_page + ): + logger.info( + "Max origins per page set to %s and reached, " + "aborting page processing", + self.max_origins_per_page, + ) + break + if not self.enable_origins: logger.info( "Disabling origins before sending them to the scheduler"