README.rst

@@ -12,36 +12,33 @@ platforms or package managers. As these operations are quite similar, this
 package provides a set of Python modules abstracting common software origins
 listing behaviors.
 
-It also provides several lister implementations, contained in the
-following Python modules:
-
-- ``swh.lister.bitbucket``
-- ``swh.lister.cgit``
-- ``swh.lister.cran``
-- ``swh.lister.debian``
-- ``swh.lister.gitea``
-- ``swh.lister.github``
-- ``swh.lister.gitlab``
-- ``swh.lister.gnu``
-- ``swh.lister.gogs``
-- ``swh.lister.golang``
-- ``swh.lister.hex``
-- ``swh.lister.launchpad``
-- ``swh.lister.maven``
-- ``swh.lister.npm``
-- ``swh.lister.packagist``
-- ``swh.lister.phabricator``
-- ``swh.lister.pypi``
-- ``swh.lister.rpm``
-- ``swh.lister.tuleap``
-- ``swh.lister.bioconductor``
+It also provides several lister implementations, contained in the Python
+``swh.lister.*`` modules. See `this documentation
+<https://docs.softwareheritage.org/user/listers.html>`_ for the list of
+supported listers.
+
 
 Dependencies
 ------------
 
-All required dependencies can be found in the ``requirements*.txt`` files
+All required python dependencies can be found in the ``requirements*.txt`` files
 located at the root of the repository.
 
+In order to be able to run all the listers (and thus execute the tests), some
+tools must be available on your system, namely:
+
+- ``opam``
+- ``tar``
+- ``psql``
+
+On a Debian-like system, you may use:
+
+.. code-block: console
+
+   $ sudo apt update
+   $ sudo apt install opam tar postgresql-client-common
+
+
 Local deployment
 ----------------
 

pyproject.toml

@@ -29,46 +29,47 @@ testing = {file = ["requirements-test.txt"]}
 "swh.lister" = "swh.lister.cli"
 
 [project.entry-points."swh.workers"]
-"swh.lister.arch" = "swh.lister.arch:register"
-"swh.lister.aur" = "swh.lister.aur:register"
-"swh.lister.bitbucket" = "swh.lister.bitbucket:register"
-"swh.lister.bower" = "swh.lister.bower:register"
-"swh.lister.cgit" = "swh.lister.cgit:register"
-"swh.lister.conda" = "swh.lister.conda:register"
-"swh.lister.cpan" = "swh.lister.cpan:register"
-"swh.lister.cran" = "swh.lister.cran:register"
-"swh.lister.crates" = "swh.lister.crates:register"
-"swh.lister.debian" = "swh.lister.debian:register"
-"swh.lister.dlang" = "swh.lister.dlang:register"
-"swh.lister.gitea" = "swh.lister.gitea:register"
-"swh.lister.github" = "swh.lister.github:register"
-"swh.lister.gitiles" = "swh.lister.gitiles:register"
-"swh.lister.gitlab" = "swh.lister.gitlab:register"
-"swh.lister.gitweb" = "swh.lister.gitweb:register"
-"swh.lister.gnu" = "swh.lister.gnu:register"
-"swh.lister.golang" = "swh.lister.golang:register"
-"swh.lister.gogs" = "swh.lister.gogs:register"
-"swh.lister.hackage" = "swh.lister.hackage:register"
-"swh.lister.hex" = "swh.lister.hex:register"
-"swh.lister.julia" = "swh.lister.julia:register"
-"swh.lister.launchpad" = "swh.lister.launchpad:register"
-"swh.lister.nixguix" = "swh.lister.nixguix:register"
-"swh.lister.npm" = "swh.lister.npm:register"
-"swh.lister.nuget" = "swh.lister.nuget:register"
-"swh.lister.opam" = "swh.lister.opam:register"
-"swh.lister.packagist" = "swh.lister.packagist:register"
-"swh.lister.pagure" = "swh.lister.pagure:register"
-"swh.lister.phabricator" = "swh.lister.phabricator:register"
-"swh.lister.pubdev" = "swh.lister.pubdev:register"
-"swh.lister.puppet" = "swh.lister.puppet:register"
-"swh.lister.pypi" = "swh.lister.pypi:register"
-"swh.lister.rpm" = "swh.lister.rpm:register"
-"swh.lister.rubygems" = "swh.lister.rubygems:register"
-"swh.lister.sourceforge" = "swh.lister.sourceforge:register"
-"swh.lister.stagit" = "swh.lister.stagit:register"
-"swh.lister.tuleap" = "swh.lister.tuleap:register"
-"swh.lister.maven" = "swh.lister.maven:register"
-"swh.lister.bioconductor" = "swh.lister.bioconductor:register"
+"lister.arch" = "swh.lister.arch:register"
+"lister.aur" = "swh.lister.aur:register"
+"lister.bioconductor" = "swh.lister.bioconductor:register"
+"lister.bitbucket" = "swh.lister.bitbucket:register"
+"lister.bower" = "swh.lister.bower:register"
+"lister.cgit" = "swh.lister.cgit:register"
+"lister.conda" = "swh.lister.conda:register"
+"lister.cpan" = "swh.lister.cpan:register"
+"lister.cran" = "swh.lister.cran:register"
+"lister.crates" = "swh.lister.crates:register"
+"lister.debian" = "swh.lister.debian:register"
+"lister.dlang" = "swh.lister.dlang:register"
+"lister.elm" = "swh.lister.elm:register"
+"lister.gitea" = "swh.lister.gitea:register"
+"lister.github" = "swh.lister.github:register"
+"lister.gitiles" = "swh.lister.gitiles:register"
+"lister.gitlab" = "swh.lister.gitlab:register"
+"lister.gitweb" = "swh.lister.gitweb:register"
+"lister.gnu" = "swh.lister.gnu:register"
+"lister.gogs" = "swh.lister.gogs:register"
+"lister.golang" = "swh.lister.golang:register"
+"lister.hackage" = "swh.lister.hackage:register"
+"lister.hex" = "swh.lister.hex:register"
+"lister.julia" = "swh.lister.julia:register"
+"lister.launchpad" = "swh.lister.launchpad:register"
+"lister.maven" = "swh.lister.maven:register"
+"lister.nixguix" = "swh.lister.nixguix:register"
+"lister.npm" = "swh.lister.npm:register"
+"lister.nuget" = "swh.lister.nuget:register"
+"lister.opam" = "swh.lister.opam:register"
+"lister.packagist" = "swh.lister.packagist:register"
+"lister.pagure" = "swh.lister.pagure:register"
+"lister.phabricator" = "swh.lister.phabricator:register"
+"lister.pubdev" = "swh.lister.pubdev:register"
+"lister.puppet" = "swh.lister.puppet:register"
+"lister.pypi" = "swh.lister.pypi:register"
+"lister.rpm" = "swh.lister.rpm:register"
+"lister.rubygems" = "swh.lister.rubygems:register"
+"lister.sourceforge" = "swh.lister.sourceforge:register"
+"lister.stagit" = "swh.lister.stagit:register"
+"lister.tuleap" = "swh.lister.tuleap:register"
 
 [project.urls]
 "Homepage" = "https://gitlab.softwareheritage.org/swh/devel/swh-lister"

swh/lister/bitbucket/tests/test_lister.py

@@ -78,7 +78,7 @@ def test_bitbucket_incremental_lister(
     url_params["after"] = last_repo_cdate
 
     lister.session.request.assert_called_once_with(
-        "GET", lister.API_URL, params=url_params
+        "GET", lister.API_URL, params=url_params, timeout=(120, 60)
     )
 
     all_origins = (

swh/lister/elm/__init__.py

+# Copyright (C) 2023  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+"""
+Elm lister
+==========
+
+`Elm`_ is a functional language that compiles to JavaScript.
+
+Additional packages for the language can be searched from the `Packages`_ website
+and installed with `elm install`_ command. The Elm packages website also provides a
+`Http Api endpoint`_ listing all available packages versions since a count of
+package versions.
+
+Elm origins are Git repositories hosted on GitHub. Each repository must provide its
+packaged releases using the GitHub release system.
+
+As of July 2023 `Packages`_ list 1746 packages.
+
+Origins retrieving strategy
+---------------------------
+
+To build a list of origins we make a GET request to the `Http Api endpoint`_ with a
+``since`` argument as  a sequential index in the history which returns a Json array
+of strings.
+Each string represents a new version for a package. The string is split to get the
+``name`` of the package.
+The origin url for each package is constructed with the information of corresponding
+``name`` entry which represents the suffix of GitHub repositories (*org*/*project_name*).
+
+Page listing
+------------
+
+There is only one page listing all origins url.
+
+Origins from page
+-----------------
+
+The lister is stateful and yields all new origins url from one page since the last run.
+It is a list of package repository url.
+
+Running tests
+-------------
+
+Activate the virtualenv and run from within swh-lister directory::
+
+   pytest -s -vv --log-cli-level=DEBUG swh/lister/elm/tests
+
+Testing with Docker
+-------------------
+
+Change directory to swh/docker then launch the docker environment::
+
+   docker compose up -d
+
+Then schedule a elm listing task::
+
+   docker compose exec swh-scheduler swh scheduler task add -p oneshot list-elm
+
+You can follow lister execution by displaying logs of swh-lister service::
+
+   docker compose logs -f swh-lister
+
+.. _Elm: https://elm-lang.org/
+.. _Packages: https://package.elm-lang.org/
+.. _elm install: https://guide.elm-lang.org/install/elm.html#elm-install
+.. _Http Api endpoint: https://package.elm-lang.org/all-packages/since/5000
+"""
+
+
+def register():
+    from .lister import ElmLister
+
+    return {
+        "lister": ElmLister,
+        "task_modules": ["%s.tasks" % __name__],
+    }

swh/lister/elm/lister.py

+# Copyright (C) 2023  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from dataclasses import asdict, dataclass
+import logging
+from typing import Any, Dict, Iterator, Optional, Set
+
+from swh.scheduler.interface import SchedulerInterface
+from swh.scheduler.model import ListedOrigin
+
+from ..pattern import CredentialsType, Lister
+
+logger = logging.getLogger(__name__)
+
+# Aliasing the page results returned by `get_pages` method from the lister.
+ElmListerPage = Set[str]
+
+
+@dataclass
+class ElmListerState:
+    """Store lister state for incremental mode operations"""
+
+    all_packages_count: Optional[int] = None
+    """Store the count of all existing packages, used as ``since`` argument of
+    API endpoint url.
+    """
+
+
+class ElmLister(Lister[ElmListerState, ElmListerPage]):
+    """List Elm packages origins"""
+
+    LISTER_NAME = "elm"
+    VISIT_TYPE = "git"  # Elm origins url are Git repositories
+    INSTANCE = "elm"
+
+    BASE_URL = "https://package.elm-lang.org"
+    ALL_PACKAGES_URL_PATTERN = "{base_url}/all-packages/since/{since}"
+    REPO_URL_PATTERN = "https://github.com/{name}"
+
+    def __init__(
+        self,
+        scheduler: SchedulerInterface,
+        credentials: Optional[CredentialsType] = None,
+        url: str = BASE_URL,
+        instance: str = INSTANCE,
+        max_origins_per_page: Optional[int] = None,
+        max_pages: Optional[int] = None,
+        enable_origins: bool = True,
+    ):
+        super().__init__(
+            scheduler=scheduler,
+            credentials=credentials,
+            url=url,
+            instance=instance,
+            max_origins_per_page=max_origins_per_page,
+            max_pages=max_pages,
+            enable_origins=enable_origins,
+        )
+        self.all_packages_count: int = 0
+        self.session.headers.update({"Accept": "application/json"})
+
+    def state_from_dict(self, d: Dict[str, Any]) -> ElmListerState:
+        return ElmListerState(**d)
+
+    def state_to_dict(self, state: ElmListerState) -> Dict[str, Any]:
+        return asdict(state)
+
+    def get_pages(self) -> Iterator[ElmListerPage]:
+        """Yield an iterator which returns 'page'
+
+        It uses the Http api endpoint ``https://package.elm-lang.org/all-packages/since/:since``
+        to get a list of packages versions from where we get names corresponding to GitHub
+        repository url suffixes.
+
+        There is only one page that list all origins urls.
+        """
+
+        if not self.state.all_packages_count:
+            since = 0
+        else:
+            since = self.state.all_packages_count
+
+        response = self.http_request(
+            self.ALL_PACKAGES_URL_PATTERN.format(base_url=self.url, since=since)
+        )
+        # We’ll save this to the state in finalize()
+        self.all_packages_count = len(response.json()) + since
+
+        res = set()
+        for entry in response.json():
+            res.add(entry.split("@")[0])
+
+        yield res
+
+    def get_origins_from_page(self, page: ElmListerPage) -> Iterator[ListedOrigin]:
+        """Iterate on all pages and yield ListedOrigin instances"""
+        assert self.lister_obj.id is not None
+
+        for name in page:
+            repo_url: str = self.REPO_URL_PATTERN.format(name=name)
+
+            yield ListedOrigin(
+                lister_id=self.lister_obj.id,
+                visit_type=self.VISIT_TYPE,
+                url=repo_url,
+                last_update=None,
+            )
+
+    def finalize(self) -> None:
+        if (
+            self.state.all_packages_count is None
+            or self.all_packages_count > self.state.all_packages_count
+        ):
+            self.state.all_packages_count = self.all_packages_count
+            self.updated = True

swh/lister/elm/tasks.py

+# Copyright (C) 2023  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from celery import shared_task
+
+from swh.lister.elm.lister import ElmLister
+
+
+@shared_task(name=__name__ + ".ElmListerTask")
+def list_elm(**lister_args):
+    """Lister task for Elm lang packages"""
+    return ElmLister.from_configfile(**lister_args).run().dict()
+
+
+@shared_task(name=__name__ + ".ping")
+def _ping():
+    return "OK"

swh/lister/elm/tests/data/https_package.elm-lang.org/all-packages_since_0

+["mercurymedia/elm-ag-grid@20.0.0","elm-toulouse/cbor@3.4.0","elm-toulouse/cbor@3.3.0"]

swh/lister/elm/tests/data/https_package.elm-lang.org/all-packages_since_3

+["miniBill/elm-avataaars@1.1.1"]

swh/lister/elm/tests/data/https_package.elm-lang.org/all-packages_since_4

+[]

swh/lister/elm/tests/test_lister.py

+# Copyright (C) 2023  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.lister.elm.lister import ElmLister
+
+expected_origins_since_0 = [
+    "https://github.com/elm-toulouse/cbor",
+    "https://github.com/mercurymedia/elm-ag-grid",
+]
+
+expected_origins_since_3 = [
+    "https://github.com/miniBill/elm-avataaars",
+]
+
+
+def test_elm_lister(datadir, requests_mock_datadir, swh_scheduler):
+    lister = ElmLister(scheduler=swh_scheduler)
+    res = lister.run()
+
+    assert res.pages == 1
+    # 2 of the 3 entries are related to the same package so the origins count is 2
+    assert res.origins == 2
+
+    scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
+
+    assert len(scheduler_origins) == len(expected_origins_since_0)
+    assert {
+        (
+            scheduled.visit_type,
+            scheduled.url,
+            scheduled.last_update,
+        )
+        for scheduled in scheduler_origins
+    } == {("git", expected, None) for expected in expected_origins_since_0}
+
+    # Check that all_packages_count is set
+    assert lister.state.all_packages_count == 3  # 3 entries
+
+
+def test_elm_lister_incremental(datadir, requests_mock_datadir, swh_scheduler):
+    # First run, since=0
+    lister = ElmLister(scheduler=swh_scheduler)
+    res = lister.run()
+
+    assert res.pages == 1
+    # 2 of the 3 entries are related to the same package so the origins count is 2
+    assert res.origins == 2
+
+    scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
+
+    assert len(scheduler_origins) == len(expected_origins_since_0)
+    assert {
+        (
+            scheduled.visit_type,
+            scheduled.url,
+            scheduled.last_update,
+        )
+        for scheduled in scheduler_origins
+    } == {("git", expected, None) for expected in expected_origins_since_0}
+
+    # Check that all_packages_count is set
+    assert lister.state.all_packages_count == 3  # 3 entries
+
+    # Second run, since=3
+    lister = ElmLister(scheduler=swh_scheduler)
+    res = lister.run()
+    assert res.pages == 1
+    assert res.origins == 1
+
+    scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
+    expected_origins = expected_origins_since_0 + expected_origins_since_3
+    assert len(scheduler_origins) == len(expected_origins)
+    assert {
+        (
+            scheduled.visit_type,
+            scheduled.url,
+            scheduled.last_update,
+        )
+        for scheduled in scheduler_origins
+    } == {("git", expected, None) for expected in expected_origins}
+    assert lister.state.all_packages_count == 4  # 4 entries
+
+    # Third run, since=4, nothing new
+    lister = ElmLister(scheduler=swh_scheduler)
+    res = lister.run()
+    assert res.pages == 1
+    assert res.origins == 0
+    assert lister.state.all_packages_count == 4  # 4 entries

swh/lister/elm/tests/test_tasks.py

+# Copyright (C) 2023  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.lister.pattern import ListerStats
+
+
+def test_elm_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
+    res = swh_scheduler_celery_app.send_task("swh.lister.elm.tasks.ping")
+    assert res
+    res.wait()
+    assert res.successful()
+    assert res.result == "OK"
+
+
+def test_elm_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker):
+    # setup the mocked ElmLister
+    lister = mocker.patch("swh.lister.elm.tasks.ElmLister")
+    lister.from_configfile.return_value = lister
+    stats = ListerStats(pages=42, origins=42)
+    lister.run.return_value = stats
+
+    res = swh_scheduler_celery_app.send_task("swh.lister.elm.tasks.ElmListerTask")
+    assert res
+    res.wait()
+    assert res.successful()
+    assert res.result == stats.dict()
+
+    lister.from_configfile.assert_called_once_with()
+    lister.run.assert_called_once_with()

swh/lister/gitlab/lister.py

@@ -156,7 +156,7 @@ class GitLabLister(Lister[GitLabListerState, PageResult]):
     )
     def get_page_result(self, url: str) -> PageResult:
         logger.debug("Fetching URL %s", url)
-        response = self.session.get(url)
+        response = self.session.get(url, **self.requests_kwargs())
         if response.status_code != 200:
             logger.warning(
                 "Unexpected HTTP status code %s on %s: %s",
@@ -175,7 +175,7 @@ class GitLabLister(Lister[GitLabListerState, PageResult]):
             while True:
                 next_id_after = id_after + self.per_page
                 url = url.replace(f"id_after={id_after}", f"id_after={next_id_after}")
-                response = self.session.get(url)
+                response = self.session.get(url, **self.requests_kwargs())
                 if response.status_code == 200:
                     break
                 else:

swh/lister/gogs/tests/test_lister.py

@@ -288,7 +288,10 @@ def test_gogs_incremental_lister(
     query_params["page"] = page_id
 
     lister.session.request.assert_called_once_with(
-        "GET", TRY_GOGS_URL + lister.REPO_LIST_PATH, params=query_params
+        "GET",
+        TRY_GOGS_URL + lister.REPO_LIST_PATH,
+        params=query_params,
+        timeout=(120, 60),
     )
 
     # All the 9 origins (3 pages) should be passed on to the scheduler:

swh/lister/julia/__init__.py

@@ -28,9 +28,9 @@ Origins retrieval strategy
 --------------------------
 
 To build a list of origins we clone the `Julia General registry`_ Git repository, then
-read the `Registry.toml`_ file to get the path to packages directories.
-Each directory have a `Package.toml` file from where we get the Git repository url for
-a package.
+walk through commits with the help of `Dulwich`_ to detect commit related to a new package
+or a new version of a package. For each of those commits we get the path to `Package.toml`
+file from where we get the Git repository url for a package.
 
 Page listing
 ------------
@@ -40,7 +40,12 @@ There is only one page listing all origins url.
 Origins from page
 -----------------
 
-The lister is stateless and yields all origins url from one page.
+The lister yields all origins url from one page.
+
+Each time the lister is executed, the HEAD commit id of `Julia General registry`_
+is stored as ``state.last_seen_commit`` and used on next run to retrieve new origins
+since the last commit.
+
 Each url corresponds to the Git url of the package repository.
 
 Running tests
@@ -71,6 +76,7 @@ You can follow lister execution by displaying logs of swh-lister service::
 .. _JuliaHub: https://juliahub.com/
 .. _Julia Packages: https://julialang.org/packages/
 .. _Registry.toml: https://github.com/JuliaRegistries/General/blob/master/Registry.toml
+.. _Dulwich: https://www.dulwich.io/
 """  # noqa: B950
 
 

swh/lister/julia/lister.py

@@ -3,27 +3,40 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+from dataclasses import asdict, dataclass
+import datetime
 import logging
 from pathlib import Path
 import shutil
 import tempfile
-from typing import Any, Iterator, List, Optional, Tuple
+from typing import Any, Dict, Iterator, Optional
 
 from dulwich import porcelain
+from dulwich.repo import Repo
+from dulwich.walk import WalkEntry
+import iso8601
 import toml
 
 from swh.scheduler.interface import SchedulerInterface
 from swh.scheduler.model import ListedOrigin
 
-from ..pattern import CredentialsType, StatelessLister
+from ..pattern import CredentialsType, Lister
 
 logger = logging.getLogger(__name__)
 
 # Aliasing the page results returned by `get_pages` method from the lister.
-JuliaListerPage = List[Tuple[str, Any]]
+JuliaListerPage = Dict[str, Any]
 
 
-class JuliaLister(StatelessLister[JuliaListerPage]):
+@dataclass
+class JuliaListerState:
+    """Store lister state for incremental mode operations"""
+
+    last_seen_commit: Optional[str] = None
+    """Hash of the latest Git commit when lister was executed"""
+
+
+class JuliaLister(Lister[JuliaListerState, JuliaListerPage]):
     """List Julia packages origins"""
 
     LISTER_NAME = "julia"
@@ -34,7 +47,6 @@ class JuliaLister(StatelessLister[JuliaListerPage]):
         "https://github.com/JuliaRegistries/General.git"  # Julia General Registry
     )
     REPO_PATH = Path(tempfile.mkdtemp(), "General")
-    REGISTRY_PATH = REPO_PATH / "Registry.toml"
 
     def __init__(
         self,
@@ -63,40 +75,111 @@ class JuliaLister(StatelessLister[JuliaListerPage]):
         except FileExistsError:
             porcelain.pull(self.REPO_PATH, remote_location=self.url)
 
+    def state_from_dict(self, d: Dict[str, Any]) -> JuliaListerState:
+        return JuliaListerState(**d)
+
+    def state_to_dict(self, state: JuliaListerState) -> Dict[str, Any]:
+        return asdict(state)
+
+    def get_origin_data(self, entry: WalkEntry) -> Dict[str, Any]:
+        """
+        Given an entry object parse its commit message and other attributes
+        to detect if the commit is valid to describe a new package or
+        a new package version.
+
+        Returns a dict with origin url as key and iso8601 commit date as value
+        """
+        assert entry
+
+        if (
+            entry.commit
+            and entry.changes()
+            and (
+                entry.commit.message.startswith(b"New package: ")
+                or entry.commit.message.startswith(b"New version: ")
+            )
+        ):
+            package_toml = None
+            for change in entry.changes():
+                if change and hasattr(change, "new"):
+                    if change.new.path.endswith(b"/Package.toml"):
+                        package_toml = self.REPO_PATH / change.new.path.decode()
+                        break
+                    elif change.new.path.endswith(b"/Versions.toml"):
+                        versions_path = self.REPO_PATH / change.new.path.decode()
+                        if versions_path.exists():
+                            package_path, _ = change.new.path.decode().split(
+                                "Versions.toml"
+                            )
+                            package_toml = (
+                                self.REPO_PATH / package_path / "Package.toml"
+                            )
+                            break
+
+            if package_toml and package_toml.exists():
+                origin = toml.load(package_toml)["repo"]
+                last_update = datetime.datetime.fromtimestamp(
+                    entry.commit.commit_time,
+                    tz=datetime.timezone.utc,
+                ).isoformat()
+                return {f"{origin}": last_update}
+
+        return {}
+
     def get_pages(self) -> Iterator[JuliaListerPage]:
         """Yield an iterator which returns 'page'
 
-        To build a list of origins the `Julia General registry` Git
-        repository is cloned to get a `Registry.toml` file, an index file of
-        packages directories.
+        To build a list of origins the ``Julia General registry`` Git
+        repository is cloned to look at commits history to discover new
+        package and new package versions.
+
+        Depending on ``last_seen_commit`` state it initiate a commit walker
+        since the last time the lister has been executed.
 
         There is only one page that list all origins urls.
         """
+        # Clone the repository
         self.get_registry_repository()
-        assert self.REGISTRY_PATH.exists()
-        registry = toml.load(self.REGISTRY_PATH)
-        yield registry["packages"].items()
+        assert self.REPO_PATH.exists()
+
+        repo = Repo(str(self.REPO_PATH))
+
+        # Detect commits related to new package and new versions since last_seen_commit
+        if not self.state.last_seen_commit:
+            walker = repo.get_walker()
+        else:
+            last = repo[self.state.last_seen_commit.encode()]
+            walker = repo.get_walker(since=last.commit_time, exclude=[last.id])
+
+        assert walker
+        packages = {}
+        for entry in walker:
+            packages.update(self.get_origin_data(entry=entry))
+
+        yield packages
 
     def get_origins_from_page(self, page: JuliaListerPage) -> Iterator[ListedOrigin]:
         """Iterate on all pages and yield ListedOrigin instances
 
-        Each directory of the Git repository have a `Package.toml` file from
-        where we get the Git repository url for each package.
+        Each directory of the Git repository have a ``Package.toml`` file from
+        where we get the Git repository url as an origin for each package.
         """
         assert self.lister_obj.id is not None
-        assert self.REPO_PATH.exists()
 
-        for uuid, info in page:
-            package_info_path = self.REPO_PATH / info["path"] / "Package.toml"
-            package_info = toml.load(package_info_path)
+        for origin, last_update in page.items():
+            last_update = iso8601.parse_date(last_update)
             yield ListedOrigin(
                 lister_id=self.lister_obj.id,
                 visit_type=self.VISIT_TYPE,
-                url=package_info["repo"],
-                last_update=None,
+                url=origin,
+                last_update=last_update,
             )
 
     def finalize(self) -> None:
+        # Get Git HEAD commit hash
+        repo = Repo(str(self.REPO_PATH))
+        self.state.last_seen_commit = repo.head().decode("ascii")
+        self.updated = True
         # Rm tmp directory REPO_PATH
         if self.REPO_PATH.exists():
             shutil.rmtree(self.REPO_PATH)

swh/lister/julia/tests/data/fake_julia_registry_repository.sh

@@ -27,6 +27,9 @@ some amount of consideration when choosing package names.
 [packages]' > Registry.toml
 
 # Init as a git repository
+# Force author and commit date to be the same
+export GIT_AUTHOR_DATE='2001-01-01T17:18:19+00:00'
+export GIT_COMMITTER_DATE=$GIT_AUTHOR_DATE
 git init
 git add .
 git commit -m "Init fake Julia registry repository for tests purpose"
@@ -50,6 +53,8 @@ git-tree-sha1 = "65301af3ab06b04cf8a52cd43b06222bab5249c2"
 
 echo 'a3ea4736-0a3b-4c29-ac8a-20364318a635 = { name = "Fable", path = "F/Fable" }' >> Registry.toml
 
+export GIT_AUTHOR_DATE='2001-01-02T17:18:19+00:00'
+export GIT_COMMITTER_DATE=$GIT_AUTHOR_DATE
 git add .
 git commit -m "New package: Fable v0.0.2"
 
@@ -132,16 +137,60 @@ git-tree-sha1 = "59619a31c56c9e61b5dabdbd339e30c227c5d13d"
 
 echo 'f1435218-dba5-11e9-1e4d-f1a5fab5fc13 = { name = "Oscar", path = "O/Oscar" }' >> Registry.toml
 
+export GIT_AUTHOR_DATE='2001-01-03T17:18:19+00:00'
+export GIT_COMMITTER_DATE=$GIT_AUTHOR_DATE
 git add .
 git commit -m "New package: Oscar v0.12.1"
 
 # Save some space
 rm .git/hooks/*.sample
 
-# Archive
+# First Archive
 cd ../
-tar -czf fake-julia-registry-repository.tar.gz General
-mv fake-julia-registry-repository.tar.gz ../
+tar -czf fake-julia-registry-repository_0.tar.gz General
+mv fake-julia-registry-repository_0.tar.gz ../
+
+# Add some more commits and build a second archive for incremental tests purpose
+cd General
+echo '
+
+["0.13.0"]
+git-tree-sha1 = "c090495f818a063ed23d2d911fe74cc4358b5351"
+' >> O/Oscar/Versions.toml
+
+# New version, replace previous uuid with a new one
+sed -i -e 's/f1435218-dba5-11e9-1e4d-f1a5fab5fc13/a3ea4736-0a3b-4c29-ac8a-20364318a635/g' Registry.toml
+
+export GIT_AUTHOR_DATE='2001-01-04T17:18:19+00:00'
+export GIT_COMMITTER_DATE=$GIT_AUTHOR_DATE
+git add .
+git commit -m "New version: Oscar v0.13.0"
+
+mkdir -p V/VulkanSpec
+
+touch V/VulkanSpec/Package.toml
+touch V/VulkanSpec/Versions.toml
+
+echo 'name = "VulkanSpec"
+uuid = "99a7788f-8f0f-454f-8f6c-c6cf389551ae"
+repo = "https://github.com/serenity4/VulkanSpec.jl.git"
+' > V/VulkanSpec/Package.toml
+
+echo '["0.1.0"]
+git-tree-sha1 = "b5fef67130191c797007a1484f4dc6bfc840caa2"
+' > V/VulkanSpec/Versions.toml
+
+echo '99a7788f-8f0f-454f-8f6c-c6cf389551ae = { name = "VulkanSpec", path = "V/VulkanSpec" }' >> Registry.toml
+
+export GIT_AUTHOR_DATE='2001-01-05T17:18:19+00:00'
+export GIT_COMMITTER_DATE=$GIT_AUTHOR_DATE
+git add .
+git commit -m "New package: VulkanSpec v0.1.0"
+
+# Second Archive
+cd ../
+tar -czf fake-julia-registry-repository_1.tar.gz General
+mv fake-julia-registry-repository_1.tar.gz ../
 
 # Clean up tmp_dir
 cd ../