Add Gitweb lister

Depending on some instances, we have some specific heuristics, some instances:
- have summary pages which do not not list metadata_url (so some
  computation happens to list git:// origins which are cloneable)
- have summary page which reference metadata_url as a multiple comma separated urls
- lists relative urls of the repository so we need to join it with the main instance url
  to have a complete cloneable origins (or summary page)
- lists "down" http origins (cloning those won't work) so lists those as cloneable https
  ones (when the main url is behind https).

Refs. swh/devel/swh-lister#1800

mypy.ini

@@ -43,6 +43,9 @@ ignore_missing_imports = True
 [mypy-dulwich.*]
 ignore_missing_imports = True
 
+[mypy-dateparser.*]
+ignore_missing_imports = True
+
 [mypy-testing.postgresql.*]
 ignore_missing_imports = True
 

requirements.txt

@@ -6,6 +6,7 @@ beautifulsoup4
 launchpadlib
 tenacity >= 6.2
 lxml
+dateparser
 dulwich
 testing.postgresql
 psycopg2

setup.py

 #!/usr/bin/env python3
-# Copyright (C) 2015-2020  The Software Heritage developers
+# Copyright (C) 2015-2023  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -69,6 +69,7 @@ setup(
         lister.gitea=swh.lister.gitea:register
         lister.github=swh.lister.github:register
         lister.gitlab=swh.lister.gitlab:register
+        lister.gitweb=swh.lister.gitweb:register
         lister.gnu=swh.lister.gnu:register
         lister.golang=swh.lister.golang:register
         lister.gogs=swh.lister.gogs:register

swh/lister/gitweb/__init__.py

+# Copyright (C) 2023 The Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+def register():
+    from .lister import GitwebLister
+
+    return {
+        "lister": GitwebLister,
+        "task_modules": [f"{__name__}.tasks"],
+    }

swh/lister/gitweb/lister.py

+# Copyright (C) 2023 The Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from datetime import datetime, timezone
+import logging
+import re
+from typing import Any, Dict, Iterator, List, Optional
+from urllib.parse import parse_qs, urljoin, urlparse
+
+from bs4 import BeautifulSoup
+from dateparser import parse
+from requests.exceptions import HTTPError
+
+from swh.lister.pattern import CredentialsType, StatelessLister
+from swh.scheduler.interface import SchedulerInterface
+from swh.scheduler.model import ListedOrigin
+
+logger = logging.getLogger(__name__)
+
+Repositories = List[Dict[str, Any]]
+
+
+class GitwebLister(StatelessLister[Repositories]):
+    """Lister class for Gitweb repositories.
+
+    This lister will retrieve the list of published git repositories by
+    parsing the HTML page(s) of the index retrieved at `url`.
+
+    """
+
+    LISTER_NAME = "gitweb"
+
+    def __init__(
+        self,
+        scheduler: SchedulerInterface,
+        url: Optional[str] = None,
+        instance: Optional[str] = None,
+        credentials: Optional[CredentialsType] = None,
+        max_origins_per_page: Optional[int] = None,
+        max_pages: Optional[int] = None,
+        enable_origins: bool = True,
+    ):
+        """Lister class for Gitweb repositories.
+
+        Args:
+            url: (Optional) Root URL of the Gitweb instance, i.e. url of the index of
+                published git repositories on this instance. Defaults to
+                :file:`https://{instance}` if unset.
+            instance: Name of gitweb instance. Defaults to url's network location
+                if unset.
+
+        """
+        super().__init__(
+            scheduler=scheduler,
+            url=url,
+            instance=instance,
+            credentials=credentials,
+            max_origins_per_page=max_origins_per_page,
+            max_pages=max_pages,
+            enable_origins=enable_origins,
+        )
+
+        self.session.headers.update({"Accept": "application/html"})
+        self.instance_scheme = urlparse(url).scheme
+
+    def _get_and_parse(self, url: str) -> BeautifulSoup:
+        """Get the given url and parse the retrieved HTML using BeautifulSoup"""
+        response = self.http_request(url)
+        return BeautifulSoup(response.text, features="html.parser")
+
+    def get_pages(self) -> Iterator[Repositories]:
+        """Generate git 'project' URLs found on the current Gitweb server."""
+        bs_idx = self._get_and_parse(self.url)
+
+        page_results = []
+
+        for tr in bs_idx.find("table", {"class": re.compile("project_list")}).find_all(
+            "tr"
+        ):
+            link = tr.find("a")
+            if not link:
+                continue
+
+            repo_url = urljoin(self.url, link["href"]).strip("/")
+
+            # Skip this description page which is listed but won't yield any origins to list
+            if repo_url.endswith("?o=descr"):
+                continue
+
+            # This retrieves the date interval in natural language (e.g. '9 years ago')
+            # to actual python datetime interval so we can derive last update
+            span = tr.find("td", {"class": re.compile("age.*")})
+            page_results.append(
+                {"url": repo_url, "last_update_interval": span.text if span else None}
+            )
+
+        yield page_results
+
+    def get_origins_from_page(
+        self, repositories: Repositories
+    ) -> Iterator[ListedOrigin]:
+        """Convert a page of gitweb repositories into a list of ListedOrigins."""
+        assert self.lister_obj.id is not None
+
+        for repo in repositories:
+            origin_url = self._get_origin_from_repository_url(repo["url"])
+            if origin_url is None:
+                continue
+
+            yield ListedOrigin(
+                lister_id=self.lister_obj.id,
+                url=origin_url,
+                visit_type="git",
+                last_update=parse_last_update(repo.get("last_update_interval")),
+            )
+
+    def _get_origin_from_repository_url(self, repository_url: str) -> Optional[str]:
+        """Extract the git url from the repository page"""
+        try:
+            bs = self._get_and_parse(repository_url)
+        except HTTPError as e:
+            logger.warning(
+                "Unexpected HTTP status code %s on %s",
+                e.response.status_code,
+                e.response.url,
+            )
+            return None
+
+        urls = []
+        for row in bs.find_all("tr", {"class": "metadata_url"}):
+            url = row.contents[-1].string.strip()
+
+            if "," in url:
+                urls_ = [s.strip() for s in url.split(",") if s]
+                urls.extend(urls_)
+            else:
+                urls.append(url)
+
+        if not urls:
+            repo = try_to_determine_git_repository(repository_url)
+            if not repo:
+                logger.debug("No git urls found on %s", repository_url)
+            return repo
+
+        # look for the http/https url, if any, and use it as origin_url
+        for url in urls:
+            parsed_url = urlparse(url)
+            if parsed_url.scheme == "https":
+                origin_url = url
+                break
+            elif parsed_url.scheme == "http" and self.instance_scheme == "https":
+                # workaround for non-working listed http origins
+                origin_url = url.replace("http://", "https://")
+                break
+        else:
+            # otherwise, choose the first one
+            origin_url = urls[0]
+        return origin_url
+
+
+def try_to_determine_git_repository(repository_url: str) -> Optional[str]:
+    """Some gitweb instances does not advertise the git urls.
+
+    This heuristic works on instances demonstrating this behavior.
+
+    """
+    result = None
+    parsed_url = urlparse(repository_url)
+    params = parse_qs(parsed_url.query).get("p")
+    if params:
+        repo = params[0]
+        if repo and repo.endswith(";a=summary"):
+            repo = repo.rstrip(";a=summary")
+
+        result = f"git://{parsed_url.netloc}/{repo}"
+    return result
+
+
+def parse_last_update(last_update_interval: Optional[str]) -> Optional[datetime]:
+    """Parse the last update string into a datetime."""
+    if not last_update_interval:
+        return None
+    last_update_date = parse(last_update_interval)
+    last_update = None
+    if last_update_date is not None:
+        last_update = last_update_date.replace(tzinfo=timezone.utc)
+    return last_update

swh/lister/gitweb/tasks.py

+# Copyright (C) 2023 The Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from typing import Dict
+
+from celery import shared_task
+
+from .lister import GitwebLister
+
+
+@shared_task(name=f"{__name__}.GitwebListerTask")
+def list_gitweb(**lister_args) -> Dict[str, str]:
+    """Lister task for Gitweb instances"""
+    lister = GitwebLister.from_configfile(**lister_args)
+    return lister.run().dict()

swh/lister/gitweb/tests/data/https_git.distorted.org.uk/README

+These files are a partial dump of https://git.distorted.org.uk/~mdw/.
+
+To ease testing, the page is named index.html. It does not represent the reality of
+those gitweb instances.
+

swh/lister/gitweb/tests/data/https_git.distorted.org.uk/foobar

+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US" lang="en-US">
+<!-- git web interface version 2.11.0, (C) 2005-2006, Kay Sievers <kay.sievers@vrfy.org>, Christian Gierke -->
+<!-- git core binaries version 2.11.0 -->
+<head>
+<meta http-equiv="content-type" content="application/xhtml+xml; charset=utf-8"/>
+<meta name="generator" content="gitweb/2.11.0 git/2.11.0"/>
+<meta name="robots" content="index, nofollow"/>
+<title>mdw@git.distorted.org.uk Git</title>
+<base href="https://git.distorted.org.uk" />
+<link rel="stylesheet" type="text/css" href="/gitweb/gitweb.css"/>
+<link rel="stylesheet" type="text/css" href="/local/gitweb.css"/>
+<link rel="alternate" title="mdw@git.distorted.org.uk Git projects list" href="https://git.distorted.org.uk/~mdw/?a=project_index" type="text/plain; charset=utf-8" />
+<link rel="alternate" title="mdw@git.distorted.org.uk Git projects feeds" href="https://git.distorted.org.uk/~mdw/?a=opml" type="text/x-opml" />
+<link rel="shortcut icon" href="/gitweb/git-favicon.png" type="image/png" />
+</head>
+<body>
+<div class="page_header">
+<a href="http://git-scm.com/" title="git homepage"><img alt="git" class="logo" height="27" src="/gitweb/git-logo.png" width="72" /></a><a href="/~mdw/">~mdw</a> / </div>
+<div class="index_include">
+<!-- -*-html-*- -->
+  hello
+</div>
+<div class="projsearch">
+<form method="get" action="https://git.distorted.org.uk/~mdw/" enctype="multipart/form-data"><input type="hidden" name="a" value="project_list"  />
+<input type="text" name="s"  size="60" title="Search project by name and description" />
+<span title="Extended regular expression"><label><input type="checkbox" name="sr" value="1" />re</label></span>
+<input type="submit" name="btnS" value="Search" />
+</form>
+<a href="https://git.distorted.org.uk/~mdw/">List all projects</a><br />
+</div>
+<table class="project_list">
+<tr>
+<th>Project</th>
+<th><a class="header" href="https://git.distorted.org.uk/~mdw/?o=descr">Description</a></th>
+<th><a class="header" href="https://git.distorted.org.uk/~mdw/?o=owner">Owner</a></th>
+<th><a class="header" href="https://git.distorted.org.uk/~mdw/?o=age">Last Change</a></th>
+<th></th>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/adns">adns</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/adns" title="GNU ADNS, an asynchronous DNS stub resolver -- local hacking">GNU ADNS, an asynchronous... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">6 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/adns">summary</a> | <a href="https://git.distorted.org.uk/~mdw/adns/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/adns/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/adns/tree">tree</a></td>
+</tr>
+<tr class="light">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/anag">anag</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/anag" title="Simple word-game solver">Simple word-game solver</a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">3 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/anag">summary</a> | <a href="https://git.distorted.org.uk/~mdw/anag/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/anag/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/anag/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/atoms">atoms</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/atoms" title="Amusing computer-mediated board game.  In Common Lisp, using GTK; hard to set up.">Amusing computer-mediated... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">10 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/atoms">summary</a> | <a href="https://git.distorted.org.uk/~mdw/atoms/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/atoms/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/atoms/tree">tree</a></td>
+</tr>
+<tr class="light">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/firewall">firewall</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/firewall" title="Firewall scripts for distorted.org.uk.">Firewall scripts for distorted... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">3 months ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/firewall">summary</a> | <a href="https://git.distorted.org.uk/~mdw/firewall/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/firewall/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/firewall/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/doc/ips">doc/ips</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/doc/ips" title="Introduction to Provable Security slides and notes">Introduction to Provable Secur... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">16 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/doc/ips">summary</a> | <a href="https://git.distorted.org.uk/~mdw/doc/ips/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/doc/ips/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/doc/ips/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/mdwtools">mdwtools</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/mdwtools" title="Various LaTeX packages">Various LaTeX packages</a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">7 weeks ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/mdwtools">summary</a> | <a href="https://git.distorted.org.uk/~mdw/mdwtools/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/mdwtools/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/mdwtools/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/scad">scad</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/scad" title="OpenSCAD models that I've designed.">OpenSCAD models that I&#39;ve... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">3 months ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/scad">summary</a> | <a href="https://git.distorted.org.uk/~mdw/scad/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/scad/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/scad/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/strayman">strayman</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/strayman" title="LaTeX document class for various documents">LaTeX document class for vario... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">2 months ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/strayman">summary</a> | <a href="https://git.distorted.org.uk/~mdw/strayman/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/strayman/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/strayman/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/udpkey">udpkey</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/udpkey" title="Transmit and receive cryptographic keys over UDP; useful during boot.">Transmit and receive cryptogra... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">7 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/udpkey">summary</a> | <a href="https://git.distorted.org.uk/~mdw/udpkey/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/udpkey/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/udpkey/tree">tree</a></td>
+</tr>
+<tr class="light">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/vmctl">vmctl</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/vmctl" title="Constrained VM management, via SSH.">Constrained VM management... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">8 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/vmctl">summary</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/tree">tree</a></td>
+</tr>
+</table>
+<div class="page_footer">
+<a class="rss_logo" href="https://git.distorted.org.uk/~mdw/?a=opml">OPML</a> <a class="rss_logo" href="https://git.distorted.org.uk/~mdw/?a=project_index">TXT</a>
+</div>
+<script type="text/javascript" src="/gitweb/gitweb.js"></script>
+<script type="text/javascript">
+window.onload = function () {
+	var tz_cookie = { name: 'gitweb_tz', expires: 14, path: '/' };
+	onloadTZSetup('local', tz_cookie, 'datetime');
+};
+</script>
+</body>
+</html>

swh/lister/gitweb/tests/data/https_git.distorted.org.uk/~mdw

+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US" lang="en-US">
+<!-- git web interface version 2.11.0, (C) 2005-2006, Kay Sievers <kay.sievers@vrfy.org>, Christian Gierke -->
+<!-- git core binaries version 2.11.0 -->
+<head>
+<meta http-equiv="content-type" content="application/xhtml+xml; charset=utf-8"/>
+<meta name="generator" content="gitweb/2.11.0 git/2.11.0"/>
+<meta name="robots" content="index, nofollow"/>
+<title>mdw@git.distorted.org.uk Git</title>
+<base href="https://git.distorted.org.uk" />
+<link rel="stylesheet" type="text/css" href="/gitweb/gitweb.css"/>
+<link rel="stylesheet" type="text/css" href="/local/gitweb.css"/>
+<link rel="alternate" title="mdw@git.distorted.org.uk Git projects list" href="https://git.distorted.org.uk/~mdw/?a=project_index" type="text/plain; charset=utf-8" />
+<link rel="alternate" title="mdw@git.distorted.org.uk Git projects feeds" href="https://git.distorted.org.uk/~mdw/?a=opml" type="text/x-opml" />
+<link rel="shortcut icon" href="/gitweb/git-favicon.png" type="image/png" />
+</head>
+<body>
+<div class="page_header">
+<a href="http://git-scm.com/" title="git homepage"><img alt="git" class="logo" height="27" src="/gitweb/git-logo.png" width="72" /></a><a href="/~mdw/">~mdw</a> / </div>
+<div class="index_include">
+<!-- -*-html-*- -->
+  <p>These are the GIT repositories for some of my various free software
+    projects, and some other projects I hack on or just find useful to
+    have local copies of.  Feel free to browse them here.</p>
+
+  <p>The primary source for browsing these projects is
+    <a href="https://git.distorted.org.uk/~mdw/"><tt>https://git.distorted.org.uk/~mdw/</tt></a>.
+    There's a similar browser at <a href="https://www.chiark.greenend.org.uk/ucgi/~mdw/git/"><tt>https://www.chiark.greenend.org.uk/ucgi/~mdw/git/</tt></a>
+    which might be faster, or more available, but very slightly less
+    up-to-date.</p>
+
+  <p>Project <i>foo</i> can be cloned using any of the following URLs:
+    <ul>
+      <li><tt>https://git.distorted.org.uk/~mdw/git/</tt><i>foo</i></li>
+      <li><tt>git://git.distorted.org.uk/~mdw/</tt><i>foo</i></li>
+      <li><tt>https://www.chiark.greenend.org.uk/ucgi/~mdw/git/</tt><i>foo</i></li>
+      <li><tt>git://git.chiark.greenend.org.uk/~mdw/</tt><i>foo</i></li>
+    </ul>
+    The <tt>https://</tt>&hellip; URLs are recommended if you can use
+    them, because they provide a measure of authenticity (as well as the
+    obvious privacy benefits).
+  </p>
+
+  <p>In order to build many of these projects, you'll need to build and
+    install <tt>cfd</tt>, and quite possibly one or more of the
+    libraries <tt>mLib</tt> and <tt>catacomb</tt>.  You'll also need
+    recent-ish Autoconf, Automake and Libtool, and the Autoconf archive.
+    General procedure is as follows:
+    <ul>
+      <li>Run <tt>mdw-setup</tt>.  This will run the appropriate
+	autotools.</li>
+      <li>Say <tt>mkdir build</tt> to make a build directory.</li>
+      <li>Say <tt>cd build</tt> to change to the build directory.</li>
+      <li>Say <tt>../configure</tt>, maybe with some options to control
+	the configuration process.</li>
+      <li>Say <tt>make</tt>.</li>
+      <li>Now start hacking on things.</li>
+    </ul>
+    If you wanted to build Debian packages, run <tt>mdw-setup
+    &ndash;d</tt> instead.  This will skip making a <tt>build</tt>
+    directory, which is good because otherwise it interferes with the
+    Debian build process.  The various <tt>debian/rules</tt> targets
+    should work OK after that.</p>
+
+  <p>Please <a href="mailto:mdw@distorted.org.uk">mail me</a> patches!</p>
+</div>
+<div class="projsearch">
+<form method="get" action="https://git.distorted.org.uk/~mdw/" enctype="multipart/form-data"><input type="hidden" name="a" value="project_list"  />
+<input type="text" name="s"  size="60" title="Search project by name and description" />
+<span title="Extended regular expression"><label><input type="checkbox" name="sr" value="1" />re</label></span>
+<input type="submit" name="btnS" value="Search" />
+</form>
+<a href="https://git.distorted.org.uk/~mdw/">List all projects</a><br />
+</div>
+<table class="project_list">
+<tr>
+<th>Project</th>
+<th><a class="header" href="https://git.distorted.org.uk/~mdw/?o=descr">Description</a></th>
+<th><a class="header" href="https://git.distorted.org.uk/~mdw/?o=owner">Owner</a></th>
+<th><a class="header" href="https://git.distorted.org.uk/~mdw/?o=age">Last Change</a></th>
+<th></th>
+</tr>
+<tr class="light">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/firewall">firewall</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/firewall" title="Firewall scripts for distorted.org.uk.">Firewall scripts for distorted... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">3 months ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/firewall">summary</a> | <a href="https://git.distorted.org.uk/~mdw/firewall/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/firewall/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/firewall/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/doc/ips">doc/ips</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/doc/ips" title="Introduction to Provable Security slides and notes">Introduction to Provable Secur... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">16 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/doc/ips">summary</a> | <a href="https://git.distorted.org.uk/~mdw/doc/ips/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/doc/ips/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/doc/ips/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/mdwtools">mdwtools</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/mdwtools" title="Various LaTeX packages">Various LaTeX packages</a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">7 weeks ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/mdwtools">summary</a> | <a href="https://git.distorted.org.uk/~mdw/mdwtools/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/mdwtools/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/mdwtools/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/scad">scad</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/scad" title="OpenSCAD models that I've designed.">OpenSCAD models that I&#39;ve... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">3 months ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/scad">summary</a> | <a href="https://git.distorted.org.uk/~mdw/scad/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/scad/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/scad/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/strayman">strayman</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/strayman" title="LaTeX document class for various documents">LaTeX document class for vario... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">2 months ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/strayman">summary</a> | <a href="https://git.distorted.org.uk/~mdw/strayman/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/strayman/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/strayman/tree">tree</a></td>
+</tr>
+<tr class="dark">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/udpkey">udpkey</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/udpkey" title="Transmit and receive cryptographic keys over UDP; useful during boot.">Transmit and receive cryptogra... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">7 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/udpkey">summary</a> | <a href="https://git.distorted.org.uk/~mdw/udpkey/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/udpkey/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/udpkey/tree">tree</a></td>
+</tr>
+<tr class="light">
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/vmctl">vmctl</a></td>
+<td><a class="list" href="https://git.distorted.org.uk/~mdw/vmctl" title="Constrained VM management, via SSH.">Constrained VM management... </a></td>
+<td><i>Mark Wooding</i></td>
+<td class="age2">8 years ago</td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/vmctl">summary</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/tree">tree</a></td>
+</tr>
+</table>
+<div class="page_footer">
+<a class="rss_logo" href="https://git.distorted.org.uk/~mdw/?a=opml">OPML</a> <a class="rss_logo" href="https://git.distorted.org.uk/~mdw/?a=project_index">TXT</a>
+</div>
+<script type="text/javascript" src="/gitweb/gitweb.js"></script>
+<script type="text/javascript">
+window.onload = function () {
+	var tz_cookie = { name: 'gitweb_tz', expires: 14, path: '/' };
+	onloadTZSetup('local', tz_cookie, 'datetime');
+};
+</script>
+</body>
+</html>

swh/lister/gitweb/tests/data/https_git.distorted.org.uk/~mdw_scad

+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US" lang="en-US">
+<!-- git web interface version 2.11.0, (C) 2005-2006, Kay Sievers <kay.sievers@vrfy.org>, Christian Gierke -->
+<!-- git core binaries version 2.11.0 -->
+<head>
+<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+<meta name="generator" content="gitweb/2.11.0 git/2.11.0"/>
+<meta name="robots" content="index, nofollow"/>
+<title>mdw@git.distorted.org.uk Git - scad/summary</title>
+<base href="https://git.distorted.org.uk" />
+<link rel="stylesheet" type="text/css" href="/gitweb/gitweb.css"/>
+<link rel="stylesheet" type="text/css" href="/local/gitweb.css"/>
+<link rel="alternate" title="scad - log - RSS feed" href="https://git.distorted.org.uk/~mdw/scad/rss" type="application/rss+xml" />
+<link rel="alternate" title="scad - log - RSS feed (no merges)" href="https://git.distorted.org.uk/~mdw/scad/rss?opt=--no-merges" type="application/rss+xml" />
+<link rel="alternate" title="scad - log - Atom feed" href="https://git.distorted.org.uk/~mdw/scad/atom" type="application/atom+xml" />
+<link rel="alternate" title="scad - log - Atom feed (no merges)" href="https://git.distorted.org.uk/~mdw/scad/atom?opt=--no-merges" type="application/atom+xml" />
+<link rel="shortcut icon" href="/gitweb/git-favicon.png" type="image/png" />
+</head>
+<body>
+<div class="page_header">
+<a href="http://git-scm.com/" title="git homepage"><img alt="git" class="logo" height="27" src="/gitweb/git-logo.png" width="72" /></a><a href="/~mdw/">~mdw</a> / <a href="https://git.distorted.org.uk/~mdw/scad">scad</a> / summary
+</div>
+<form method="get" action="https://git.distorted.org.uk/~mdw//scad" enctype="multipart/form-data"><div class="search">
+<input name="a" type="hidden" value="search" />
+<input name="h" type="hidden" value="HEAD" />
+<select name="st" >
+<option selected="selected" value="commit">commit</option>
+<option value="grep">grep</option>
+<option value="author">author</option>
+<option value="committer">committer</option>
+<option value="pickaxe">pickaxe</option>
+</select> <a href="https://git.distorted.org.uk/~mdw/scad/search_help" title="search help">?</a> search:
+<input type="text" name="s"  />
+<span title="Extended regular expression"><label><input type="checkbox" name="sr" value="1" />re</label></span></div>
+</form>
+<div class="page_nav">
+summary | <a href="https://git.distorted.org.uk/~mdw/scad/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/scad/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/scad/commit/e05980e366e948132d10acde1abd0de29b6dae8a">commit</a> | <a href="https://git.distorted.org.uk/~mdw/scad/commitdiff/e05980e366e948132d10acde1abd0de29b6dae8a">commitdiff</a> | <a href="https://git.distorted.org.uk/~mdw/scad/tree">tree</a><br/>
+<br/>
+</div>
+<div class="title">&nbsp;</div>
+<table class="projects_list">
+<tr id="metadata_desc"><td>description</td><td>OpenSCAD models that I&#39;ve designed.</td></tr>
+<tr id="metadata_owner"><td>owner</td><td>Mark Wooding</td></tr>
+<tr id="metadata_lchange"><td>last change</td><td><span class="datetime">Wed, 15 Mar 2023 00:57:55 +0000</span> (<span class="atnight">00:57</span> +0000)</td></tr>
+<tr class="metadata_url"><td>URL</td><td>https://git.distorted.org.uk/~mdw/scad</td></tr>
+<tr class="metadata_url"><td></td><td>git://git.distorted.org.uk/~mdw/scad</td></tr>
+</table>
+<div class="header">
+<a class="title" href="https://git.distorted.org.uk/~mdw/scad/shortlog">shortlog</a>
+</div>
+<table class="shortlog">
+<tr class="dark">
+<td title="3 months ago"><i>2023-03-15</i></td>
+<td class="author"><a class="list" href="https://git.distorted.org.uk/~mdw/scad/search?s=Mark+Wooding;st=author" title="Search for commits authored by Mark Wooding">Mark Wooding</a></td><td><a class="list subject" href="https://git.distorted.org.uk/~mdw/scad/commit/e05980e366e948132d10acde1abd0de29b6dae8a" title="discpick-tensioner.scad: Improved tensioning component with longer prongs.">discpick-tensioner.scad: Improved tensioning component... </a> <span class="refs"> <span class="head" title="heads/master"><a href="https://git.distorted.org.uk/~mdw/scad/shortlog/refs/heads/master">master</a></span></span></td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/scad/commit/e05980e366e948132d10acde1abd0de29b6dae8a">commit</a> | <a href="https://git.distorted.org.uk/~mdw/scad/commitdiff/e05980e366e948132d10acde1abd0de29b6dae8a">commitdiff</a> | <a href="https://git.distorted.org.uk/~mdw/scad/tree/e05980e366e948132d10acde1abd0de29b6dae8a">tree</a> | snapshot (<a href="https://git.distorted.org.uk/~mdw/scad/snapshot/e05980e366e948132d10acde1abd0de29b6dae8a.tar.gz">tar.gz</a> <a href="https://git.distorted.org.uk/~mdw/scad/snapshot/e05980e366e948132d10acde1abd0de29b6dae8a.zip">zip</a>)</td>
+</tr>
+<tr class="light">
+<td title="8 months ago"><i>2022-10-28</i></td>
+<td class="author"><a class="list" href="https://git.distorted.org.uk/~mdw/scad/search?s=Mark+Wooding;st=author" title="Search for commits authored by Mark Wooding">Mark Wooding</a></td><td><a class="list subject" href="https://git.distorted.org.uk/~mdw/scad/commit/5f1872e84d2cb5198dfc6b9b6214ae7836cd9e4a">Add a build system.</a></td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/scad/commit/5f1872e84d2cb5198dfc6b9b6214ae7836cd9e4a">commit</a> | <a href="https://git.distorted.org.uk/~mdw/scad/commitdiff/5f1872e84d2cb5198dfc6b9b6214ae7836cd9e4a">commitdiff</a> | <a href="https://git.distorted.org.uk/~mdw/scad/tree/5f1872e84d2cb5198dfc6b9b6214ae7836cd9e4a">tree</a> | snapshot (<a href="https://git.distorted.org.uk/~mdw/scad/snapshot/5f1872e84d2cb5198dfc6b9b6214ae7836cd9e4a.tar.gz">tar.gz</a> <a href="https://git.distorted.org.uk/~mdw/scad/snapshot/5f1872e84d2cb5198dfc6b9b6214ae7836cd9e4a.zip">zip</a>)</td>
+</tr>
+<tr class="dark">
+<td title="9 months ago"><i>2022-09-30</i></td>
+<td class="author"><a class="list" href="https://git.distorted.org.uk/~mdw/scad/search?s=Mark+Wooding;st=author" title="Search for commits authored by Mark Wooding">Mark Wooding</a></td><td><a class="list subject" href="https://git.distorted.org.uk/~mdw/scad/commit/bfe485ff99b7430ceb074c89e2aa2f5bfd256b2a">discpick-collar.scad: Successful print.</a></td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/scad/commit/bfe485ff99b7430ceb074c89e2aa2f5bfd256b2a">commit</a> | <a href="https://git.distorted.org.uk/~mdw/scad/commitdiff/bfe485ff99b7430ceb074c89e2aa2f5bfd256b2a">commitdiff</a> | <a href="https://git.distorted.org.uk/~mdw/scad/tree/bfe485ff99b7430ceb074c89e2aa2f5bfd256b2a">tree</a> | snapshot (<a href="https://git.distorted.org.uk/~mdw/scad/snapshot/bfe485ff99b7430ceb074c89e2aa2f5bfd256b2a.tar.gz">tar.gz</a> <a href="https://git.distorted.org.uk/~mdw/scad/snapshot/bfe485ff99b7430ceb074c89e2aa2f5bfd256b2a.zip">zip</a>)</td>
+</tr>
+</table>
+<div class="header">
+<a class="title" href="https://git.distorted.org.uk/~mdw/scad/heads">heads</a>
+</div>
+<table class="heads">
+<tr class="dark">
+<td><i>3 months ago</i></td>
+<td class="current_head"><a class="list name" href="https://git.distorted.org.uk/~mdw/scad/shortlog/refs/heads/master">master</a></td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/scad/shortlog/refs/heads/master">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/scad/log/refs/heads/master">log</a> | <a href="https://git.distorted.org.uk/~mdw/scad/tree/refs/heads/master">tree</a></td>
+</tr></table>
+<div class="page_footer">
+<div class="page_footer_text">OpenSCAD models that I&#39;ve designed.</div>
+<a class="rss_logo" href="https://git.distorted.org.uk/~mdw/scad/rss" title="log RSS feed">RSS</a>
+<a class="rss_logo" href="https://git.distorted.org.uk/~mdw/scad/atom" title="log Atom feed">Atom</a>
+</div>
+<script type="text/javascript" src="/gitweb/gitweb.js"></script>
+<script type="text/javascript">
+window.onload = function () {
+	var tz_cookie = { name: 'gitweb_tz', expires: 14, path: '/' };
+	onloadTZSetup('local', tz_cookie, 'datetime');
+};
+</script>
+</body>
+</html>
\ No newline at end of file

swh/lister/gitweb/tests/data/https_git.distorted.org.uk/~mdw_vmctl

+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US" lang="en-US">
+<!-- git web interface version 2.11.0, (C) 2005-2006, Kay Sievers <kay.sievers@vrfy.org>, Christian Gierke -->
+<!-- git core binaries version 2.11.0 -->
+<head>
+<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+<meta name="generator" content="gitweb/2.11.0 git/2.11.0"/>
+<meta name="robots" content="index, nofollow"/>
+<title>mdw@git.distorted.org.uk Git - vmctl/summary</title>
+<base href="https://git.distorted.org.uk" />
+<link rel="stylesheet" type="text/css" href="/gitweb/gitweb.css"/>
+<link rel="stylesheet" type="text/css" href="/local/gitweb.css"/>
+<link rel="alternate" title="vmctl - log - RSS feed" href="https://git.distorted.org.uk/~mdw/vmctl/rss" type="application/rss+xml" />
+<link rel="alternate" title="vmctl - log - RSS feed (no merges)" href="https://git.distorted.org.uk/~mdw/vmctl/rss?opt=--no-merges" type="application/rss+xml" />
+<link rel="alternate" title="vmctl - log - Atom feed" href="https://git.distorted.org.uk/~mdw/vmctl/atom" type="application/atom+xml" />
+<link rel="alternate" title="vmctl - log - Atom feed (no merges)" href="https://git.distorted.org.uk/~mdw/vmctl/atom?opt=--no-merges" type="application/atom+xml" />
+<link rel="shortcut icon" href="/gitweb/git-favicon.png" type="image/png" />
+</head>
+<body>
+<div class="page_header">
+<a href="http://git-scm.com/" title="git homepage"><img alt="git" class="logo" height="27" src="/gitweb/git-logo.png" width="72" /></a><a href="/~mdw/">~mdw</a> / <a href="https://git.distorted.org.uk/~mdw/vmctl">vmctl</a> / summary
+</div>
+<form method="get" action="https://git.distorted.org.uk/~mdw//vmctl" enctype="multipart/form-data"><div class="search">
+<input name="a" type="hidden" value="search" />
+<input name="h" type="hidden" value="HEAD" />
+<select name="st" >
+<option selected="selected" value="commit">commit</option>
+<option value="grep">grep</option>
+<option value="author">author</option>
+<option value="committer">committer</option>
+<option value="pickaxe">pickaxe</option>
+</select> <a href="https://git.distorted.org.uk/~mdw/vmctl/search_help" title="search help">?</a> search:
+<input type="text" name="s"  />
+<span title="Extended regular expression"><label><input type="checkbox" name="sr" value="1" />re</label></span></div>
+</form>
+<div class="page_nav">
+summary | <a href="https://git.distorted.org.uk/~mdw/vmctl/shortlog">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/log">log</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/commit/95ec7b4d9cb014f12349fc193684baab0903fbf7">commit</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/commitdiff/95ec7b4d9cb014f12349fc193684baab0903fbf7">commitdiff</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/tree">tree</a><br/>
+<br/>
+</div>
+<div class="title">&nbsp;</div>
+<table class="projects_list">
+<tr id="metadata_desc"><td>description</td><td>Constrained VM management, via SSH.</td></tr>
+<tr id="metadata_owner"><td>owner</td><td>Mark Wooding</td></tr>
+<tr id="metadata_lchange"><td>last change</td><td><span class="datetime">Sat, 4 Apr 2015 12:36:59 +0000</span> (13:36 +0100)</td></tr>
+<tr class="metadata_url"><td>URL</td><td>http://git.distorted.org.uk/~mdw/vmctl</td></tr>
+<tr class="metadata_url"><td></td><td>git://git.distorted.org.uk/~mdw/vmctl</td></tr>
+</table>
+<div class="header">
+<a class="title" href="https://git.distorted.org.uk/~mdw/vmctl/shortlog">shortlog</a>
+</div>
+<table class="shortlog">
+<tr class="dark">
+<td title="8 years ago"><i>2015-04-04</i></td>
+<td class="author"><a class="list" href="https://git.distorted.org.uk/~mdw/vmctl/search?s=Mark+Wooding;st=author" title="Search for commits authored by Mark Wooding">Mark Wooding</a></td><td><a class="list subject" href="https://git.distorted.org.uk/~mdw/vmctl/commit/95ec7b4d9cb014f12349fc193684baab0903fbf7">.ssh/Makefile: Add dependency on sshsvc.conf.</a> <span class="refs"> <span class="head" title="heads/master"><a href="https://git.distorted.org.uk/~mdw/vmctl/shortlog/refs/heads/master">master</a></span></span></td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/vmctl/commit/95ec7b4d9cb014f12349fc193684baab0903fbf7">commit</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/commitdiff/95ec7b4d9cb014f12349fc193684baab0903fbf7">commitdiff</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/tree/95ec7b4d9cb014f12349fc193684baab0903fbf7">tree</a> | snapshot (<a href="https://git.distorted.org.uk/~mdw/vmctl/snapshot/95ec7b4d9cb014f12349fc193684baab0903fbf7.tar.gz">tar.gz</a> <a href="https://git.distorted.org.uk/~mdw/vmctl/snapshot/95ec7b4d9cb014f12349fc193684baab0903fbf7.zip">zip</a>)</td>
+</tr>
+<tr class="light">
+<td title="9 years ago"><i>2013-09-03</i></td>
+<td class="author"><a class="list" href="https://git.distorted.org.uk/~mdw/vmctl/search?s=Mark+Wooding;st=author" title="Search for commits authored by Mark Wooding">Mark Wooding</a></td><td><a class="list subject" href="https://git.distorted.org.uk/~mdw/vmctl/commit/1724d7d6ca67cc3c484db19e23b3e27fa64bb3a5">.ssh: Generate the awful authorized_keys file.</a></td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/vmctl/commit/1724d7d6ca67cc3c484db19e23b3e27fa64bb3a5">commit</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/commitdiff/1724d7d6ca67cc3c484db19e23b3e27fa64bb3a5">commitdiff</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/tree/1724d7d6ca67cc3c484db19e23b3e27fa64bb3a5">tree</a> | snapshot (<a href="https://git.distorted.org.uk/~mdw/vmctl/snapshot/1724d7d6ca67cc3c484db19e23b3e27fa64bb3a5.tar.gz">tar.gz</a> <a href="https://git.distorted.org.uk/~mdw/vmctl/snapshot/1724d7d6ca67cc3c484db19e23b3e27fa64bb3a5.zip">zip</a>)</td>
+</tr>
+<tr class="dark">
+<td title="10 years ago"><i>2012-08-26</i></td>
+<td class="author"><a class="list" href="https://git.distorted.org.uk/~mdw/vmctl/search?s=Mark+Wooding;st=author" title="Search for commits authored by Mark Wooding">Mark Wooding</a></td><td><a class="list subject" href="https://git.distorted.org.uk/~mdw/vmctl/commit/da13c8714dc32c5d493952b38825d3ad58e38635">Initial version.</a></td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/vmctl/commit/da13c8714dc32c5d493952b38825d3ad58e38635">commit</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/commitdiff/da13c8714dc32c5d493952b38825d3ad58e38635">commitdiff</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/tree/da13c8714dc32c5d493952b38825d3ad58e38635">tree</a> | snapshot (<a href="https://git.distorted.org.uk/~mdw/vmctl/snapshot/da13c8714dc32c5d493952b38825d3ad58e38635.tar.gz">tar.gz</a> <a href="https://git.distorted.org.uk/~mdw/vmctl/snapshot/da13c8714dc32c5d493952b38825d3ad58e38635.zip">zip</a>)</td>
+</tr>
+</table>
+<div class="header">
+<a class="title" href="https://git.distorted.org.uk/~mdw/vmctl/heads">heads</a>
+</div>
+<table class="heads">
+<tr class="dark">
+<td><i>8 years ago</i></td>
+<td class="current_head"><a class="list name" href="https://git.distorted.org.uk/~mdw/vmctl/shortlog/refs/heads/master">master</a></td>
+<td class="link"><a href="https://git.distorted.org.uk/~mdw/vmctl/shortlog/refs/heads/master">shortlog</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/log/refs/heads/master">log</a> | <a href="https://git.distorted.org.uk/~mdw/vmctl/tree/refs/heads/master">tree</a></td>
+</tr></table>
+<div class="page_footer">
+<div class="page_footer_text">Constrained VM management, via SSH.</div>
+<a class="rss_logo" href="https://git.distorted.org.uk/~mdw/vmctl/rss" title="log RSS feed">RSS</a>
+<a class="rss_logo" href="https://git.distorted.org.uk/~mdw/vmctl/atom" title="log Atom feed">Atom</a>
+</div>
+<script type="text/javascript" src="/gitweb/gitweb.js"></script>
+<script type="text/javascript">
+window.onload = function () {
+	var tz_cookie = { name: 'gitweb_tz', expires: 14, path: '/' };
+	onloadTZSetup('local', tz_cookie, 'datetime');
+};
+</script>
+</body>
+</html>

swh/lister/gitweb/tests/test_lister.py

+# Copyright (C) 2023 The Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import os
+from typing import List
+
+import pytest
+
+from swh.lister import __version__
+from swh.lister.gitweb.lister import (
+    GitwebLister,
+    parse_last_update,
+    try_to_determine_git_repository,
+)
+from swh.lister.pattern import ListerStats
+
+MAIN_INSTANCE = "git.distorted.org.uk"
+MAIN_INSTANCE_URL = f"https://{MAIN_INSTANCE}/~mdw"
+
+
+def test_lister_gitweb_instantiate(swh_scheduler):
+    """Build a lister with either an url or an instance is supported."""
+    url = MAIN_INSTANCE_URL
+    lister = GitwebLister(swh_scheduler, url=url)
+    assert lister is not None
+    assert lister.url == url
+
+    assert GitwebLister(swh_scheduler, instance=MAIN_INSTANCE) is not None
+    assert lister is not None
+    assert lister.url == url
+
+
+def test_lister_gitweb_fail_to_instantiate(swh_scheduler):
+    """Build a lister without its url nor its instance should raise"""
+    # ... It will raise without any of those
+    with pytest.raises(ValueError, match="'url' or 'instance'"):
+        GitwebLister(swh_scheduler)
+
+
+def test_lister_gitweb_get_pages(requests_mock_datadir, swh_scheduler):
+    """Computing the number of pages scrapped during a listing."""
+    url = MAIN_INSTANCE_URL
+    lister_gitweb = GitwebLister(swh_scheduler, url=url)
+
+    expected_nb_origins = 7
+
+    repos: List[List[str]] = list(lister_gitweb.get_pages())
+    flattened_repos = sum(repos, [])
+    assert len(flattened_repos) == expected_nb_origins
+
+    for listed_url in flattened_repos:
+        assert listed_url["url"].startswith(url)
+
+
+def test_lister_gitweb_run(requests_mock_datadir, swh_scheduler):
+    """Gitweb lister nominal listing case."""
+
+    url = MAIN_INSTANCE_URL
+    lister_gitweb = GitwebLister(swh_scheduler, url=url)
+
+    stats = lister_gitweb.run()
+
+    expected_nb_origins = 7  # main page will get filtered out
+    assert stats == ListerStats(pages=1, origins=expected_nb_origins)
+
+    # test page parsing
+    scheduler_origins = swh_scheduler.get_listed_origins(
+        lister_gitweb.lister_obj.id
+    ).results
+    assert len(scheduler_origins) == expected_nb_origins
+
+    assert url.startswith("https://")
+
+    # test listed repositories
+    for listed_origin in scheduler_origins:
+        assert listed_origin.visit_type == "git"
+        assert listed_origin.url.startswith(url)
+        assert listed_origin.url.startswith("https://")
+        assert listed_origin.last_update is not None
+        assert "," not in listed_origin.url
+
+    # test user agent content
+    for request in requests_mock_datadir.request_history:
+        assert "User-Agent" in request.headers
+        user_agent = request.headers["User-Agent"]
+        assert "Software Heritage gitweb lister" in user_agent
+        assert __version__ in user_agent
+
+
+def test_lister_gitweb_get_pages_with_pages_and_retry(
+    requests_mock_datadir, requests_mock, datadir, mocker, swh_scheduler
+):
+    """Rate limited page are tested back after some time so ingestion can proceed."""
+    url = MAIN_INSTANCE_URL
+    with open(os.path.join(datadir, f"https_{MAIN_INSTANCE}/~mdw"), "rb") as page:
+        requests_mock.get(
+            url,
+            [
+                {"content": None, "status_code": 429},
+                {"content": None, "status_code": 429},
+                {"content": page.read(), "status_code": 200},
+            ],
+        )
+
+        lister_gitweb = GitwebLister(swh_scheduler, url=url)
+
+        mocker.patch.object(lister_gitweb.http_request.retry, "sleep")
+
+        pages: List[List[str]] = list(lister_gitweb.get_pages())
+        flattened_repos = sum(pages, [])
+        assert len(pages) == 1
+        assert len(flattened_repos) == 7
+
+
+def test_lister_gitweb_get_origin_from_repo_failing(
+    swh_scheduler, requests_mock_datadir
+):
+    """Instances whose summary does not return anything are filtered out."""
+    # This instance has some more origins which no longer returns their summary
+    lister_gitweb = GitwebLister(swh_scheduler, url=f"https://{MAIN_INSTANCE}/foobar")
+
+    stats = lister_gitweb.run()
+
+    # so they are filtered out, only the 7 we know are thus listed
+    expected_nb_origins = 7
+    assert stats == ListerStats(pages=1, origins=expected_nb_origins)
+
+
+@pytest.mark.parametrize(
+    "url,expected_repo",
+    [
+        (
+            "https://git.shadowcat.co.uk?p=urisagit/gitosis-admin.git",
+            "git://git.shadowcat.co.uk/urisagit/gitosis-admin.git",
+        ),
+        (
+            "https://git.shadowcat.co.uk?p=File-Slurp.git;a=summary",
+            "git://git.shadowcat.co.uk/File-Slurp.git",
+        ),
+        ("https://domain.org/foobar", None),
+    ],
+)
+def test_try_to_determine_git_repository(url, expected_repo):
+    assert try_to_determine_git_repository(url) == expected_repo
+
+
+def test_parse_last_update():
+    assert parse_last_update(None) is None
+    assert parse_last_update("No commits") is None
+
+    date = parse_last_update("6 months ago")
+    assert date is not None
+    assert date.tzinfo is not None

swh/lister/gitweb/tests/test_tasks.py

+# Copyright (C) 2023 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.lister.pattern import ListerStats
+
+
+def test_gitweb_lister_task(
+    swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
+):
+    # setup the mocked GitwebLister
+    lister = mocker.patch("swh.lister.gitweb.tasks.GitwebLister")
+    lister.from_configfile.return_value = lister
+    lister.run.return_value = ListerStats(pages=10, origins=500)
+
+    kwargs = dict(
+        url="https://git.gentoo.org/", instance="kernel", base_git_url=None, max_pages=1
+    )
+
+    res = swh_scheduler_celery_app.send_task(
+        "swh.lister.gitweb.tasks.GitwebListerTask",
+        kwargs=kwargs,
+    )
+    assert res
+    res.wait()
+    assert res.successful()
+
+    lister.from_configfile.assert_called_once_with(**kwargs)
+    lister.run.assert_called_once_with()

swh/lister/tests/test_cli.py

@@ -42,6 +42,9 @@ lister_args = {
         "url": "https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/",
     },
     "pagure": {"instance": "pagure.io"},
+    "gitweb": {
+        "url": "https://git.distorted.org.uk/~mdw/",
+    },
 }