directory: Align process to checkout a remote git ref with guix one

It has been observed that the process used by SWH to checkout a remote git reference can lead to different recursive nar hash values compared to those computed by guix. This seems related to CR/LF normalization.

So prefer to align the process to checkout a remote git reference with the one used by guix. It seems also faster than the previous approach.

Also refine the not found repository detection process as previously some non related git errors could be missed.

Related to #4751 (comment 150664).

Before:

$ time swh loader run git-checkout https://github.com/apache/groovy ref=GROOVY_3_0_5 checksum_layout=nar checksums='{"sha256": "d5a653ce9c1f4662cdcaa2fe2d0eb7359936f807d67807fb50c3b22a3b36a500"}'
WARNING:swh.loader.cli:No storage configuration detected, using an in-memory storage instead.
INFO:swh.loader.git.directory.GitCheckoutLoader:Load origin 'https://github.com/apache/groovy' with type 'git-checkout'
Cloning into '/tmp/tmpa6661kuh/groovy'...
remote: Enumerating objects: 37577, done.
remote: Counting objects: 100% (621/621), done.
remote: Compressing objects: 100% (579/579), done.
remote: Total 37577 (delta 66), reused 563 (delta 42), pack-reused 36956
Receiving objects: 100% (37577/37577), 8.43 MiB | 3.15 MiB/s, done.
Resolving deltas: 100% (1613/1613), done.
remote: Enumerating objects: 1186, done.
remote: Counting objects: 100% (420/420), done.
remote: Compressing objects: 100% (330/330), done.
remote: Total 1186 (delta 1), reused 243 (delta 0), pack-reused 766
Receiving objects: 100% (1186/1186), 210.31 KiB | 3.75 MiB/s, done.
Resolving deltas: 100% (2/2), done.
remote: Enumerating objects: 4965, done.
remote: Counting objects: 100% (4440/4440), done.
remote: Compressing objects: 100% (3520/3520), done.
remote: Total 4965 (delta 2531), reused 1132 (delta 920), pack-reused 525
Receiving objects: 100% (4965/4965), 6.85 MiB | 2.81 MiB/s, done.
Resolving deltas: 100% (2748/2748), done.
Updating files: 100% (4988/4988), done.
remote: Enumerating objects: 1116, done.
remote: Counting objects: 100% (230/230), done.
remote: Compressing objects: 100% (230/230), done.
remote: Total 1116 (delta 0), reused 0 (delta 0), pack-reused 886
Receiving objects: 100% (1116/1116), 199.34 KiB | 3.44 MiB/s, done.
remote: Enumerating objects: 2394, done.
remote: Counting objects: 100% (2275/2275), done.
remote: Compressing objects: 100% (2052/2052), done.
remote: Total 2394 (delta 1003), reused 223 (delta 223), pack-reused 119
Receiving objects: 100% (2394/2394), 4.14 MiB | 1.68 MiB/s, done.
Resolving deltas: 100% (1024/1024), done.
Updating files: 100% (3762/3762), done.
HEAD is now at 0be35bb1 Release 3.0.5: update versions
ERROR:swh.loader.git.directory.GitCheckoutLoader:Loading failure, updating to `failed` status
Traceback (most recent call last):
  File "/home/anlambert/swh/swh-environment/swh-loader-core/swh/loader/core/loader.py", line 433, in load
    more_data_to_fetch = self.fetch_data()
                         ^^^^^^^^^^^^^^^^^
  File "/home/anlambert/swh/swh-environment/swh-loader-core/swh/loader/core/loader.py", line 818, in fetch_data
    raise errors[0]
ValueError: Checksum mismatched on <https://github.com/apache/groovy>: {'sha256': 'cbfc7055d6baa0011c0aa5d8e2fa79ea96148851f09ab203b5d064c7cec0900c'} != {'sha256': 'd5a653ce9c1f4662cdcaa2fe2d0eb7359936f807d67807fb50c3b22a3b36a500'}
{'status': 'failed'} for origin 'https://github.com/apache/groovy'

real    0m15,583s
user    0m5,440s
sys     0m2,389s

After:

$ time swh loader run git-checkout https://github.com/apache/groovy ref=GROOVY_3_0_5 checksum_layout=nar checksums='{"sha256": "d5a653ce9c1f4662cdcaa2fe2d0eb7359936f807d67807fb50c3b22a3b36a500"}'
WARNING:swh.loader.cli:No storage configuration detected, using an in-memory storage instead.
INFO:swh.loader.git.directory.GitCheckoutLoader:Load origin 'https://github.com/apache/groovy' with type 'git-checkout'
{'status': 'eventful'} for origin 'https://github.com/apache/groovy'

real    0m6,735s
user    0m3,672s
sys     0m1,405s

swh/loader/git/tests/test_directory.py

@@ -9,13 +9,18 @@ from subprocess import run
 import threading
 from typing import Tuple
 
+from dulwich.client import LocalGitClient
 from dulwich.repo import Repo
 from dulwich.server import DictBackend, TCPGitServer
 import pytest
 
 from swh.loader.core.nar import Nar
 from swh.loader.exception import NotFound
-from swh.loader.git.directory import GitCheckoutLoader, clone_repository, list_git_tree
+from swh.loader.git.directory import (
+    GitCheckoutLoader,
+    checkout_repository_ref,
+    list_git_tree,
+)
 from swh.loader.tests import (
     assert_last_visit_matches,
     fetch_nar_extids_from_checksums,
@@ -28,7 +33,7 @@ def test_list_git_tree(datadir, tmp_path):
     """Listing a git tree should not list any .git paths nor empty folders."""
     archive_name = "testrepo"
     archive_path = os.path.join(datadir, f"{archive_name}.tgz")
-    repo, repo_url = prepare_test_git_clone(
+    repo, repo_url = prepare_test_git_checkout(
         archive_path, archive_name, tmp_path, "branch2-after-delete"
     )
 
@@ -86,13 +91,13 @@ def compute_nar_hash_for_ref(
     """Compute the nar from a git checked out by git."""
     tmp_path = Path(os.path.join(temp_dir, "compute-nar"))
     tmp_path.mkdir(exist_ok=True)
-    git_repo_path = clone_repository(repo_url, ref, tmp_path)
+    git_repo_path = checkout_repository_ref(repo_url, ref, tmp_path)
     nar = Nar(hash_names=[hash_name], exclude_vcs=True)
     nar.serialize(git_repo_path)
     return nar.hexdigest()[hash_name]
 
 
-def prepare_test_git_clone(
+def prepare_test_git_checkout(
     archive_path: str, archive_name: str, tmp_path: str, ref: str
 ) -> Tuple:
     repo_url = prepare_repository_from_archive(
@@ -101,7 +106,7 @@ def prepare_test_git_clone(
 
     temp_dir = Path(tmp_path) / "checkout"
     os.makedirs(temp_dir)
-    repo_path = clone_repository(repo_url, ref, temp_dir)
+    repo_path = checkout_repository_ref(repo_url, ref, temp_dir)
     assert repo_path and repo_path.exists()
     expected_path = temp_dir / os.path.basename(repo_url)
     assert str(repo_path) == str(expected_path)
@@ -116,27 +121,30 @@ def prepare_test_git_clone(
         ("commit", "bd746cd1913721b269b395a56a97baf6755151c2"),
     ],
 )
-def test_clone_repository_from(datadir, tmp_path, reference, reference_type):
+def test_checkout_repository_ref_from(datadir, tmp_path, reference, reference_type):
     """Cloning a repository from a branch, tag or commit should be ok"""
     archive_name = "testrepo"
     archive_path = os.path.join(datadir, f"{archive_name}.tgz")
     # reference is either: branch, tags, or commit
-    repo, _ = prepare_test_git_clone(archive_path, archive_name, tmp_path, reference)
+    checkout, repo_url = prepare_test_git_checkout(
+        archive_path, archive_name, tmp_path, reference
+    )
+    refs = LocalGitClient().get_refs(repo_url.replace("file://", ""))
     # Ensure the repository exists and is at the branch_name required
     if reference_type == "branch":
         ref = f"refs/heads/{reference}".encode()
-        expected_head = repo[ref].id
+        expected_head = refs[ref]
     elif reference_type == "tag":
         ref = f"refs/tags/{reference}".encode()
-        expected_head = repo[ref].id
+        expected_head = refs[ref]
     else:
         expected_head = reference.encode()
-    assert repo.head() == expected_head
+    assert checkout.head() == expected_head
 
 
-def test_clone_directory_not_found(tmp_path):
-    with pytest.raises(NotFound, match="Repository <file://"):
-        clone_repository(
+def test_checkout_repository_ref_not_found(tmp_path):
+    with pytest.raises(NotFound, match=r"Repository <file://.*not found"):
+        checkout_repository_ref(
             "file:///home/origin/does/not/exist", "not-important", tmp_path
         )
 
@@ -153,7 +161,7 @@ def test_git_loader_directory(swh_storage, datadir, tmp_path, reference):
     """Loading a git directory should be eventful"""
     archive_name = "testrepo"
     archive_path = os.path.join(datadir, f"{archive_name}.tgz")
-    _, repo_url = prepare_test_git_clone(
+    _, repo_url = prepare_test_git_checkout(
         archive_path, archive_name, tmp_path, reference
     )
 
@@ -197,7 +205,7 @@ def test_loader_git_directory_hash_mismatch(swh_storage, datadir, tmp_path):
     archive_name = "testrepo"
     archive_path = os.path.join(datadir, f"{archive_name}.tgz")
     release_name = "branch2-before-delete"
-    _, repo_url = prepare_test_git_clone(
+    _, repo_url = prepare_test_git_checkout(
         archive_path, archive_name, tmp_path, release_name
     )
 
@@ -303,7 +311,9 @@ def test_loader_git_directory_without_or_with_submodule(
     archive_name = "testrepo"
     archive_path = os.path.join(datadir, f"{archive_name}.tgz")
     release_name = "branch2-before-delete"
-    repo, _ = prepare_test_git_clone(archive_path, archive_name, tmp_path, release_name)
+    repo, _ = prepare_test_git_checkout(
+        archive_path, archive_name, tmp_path, release_name
+    )
     repo_url = f"file://{repo.path}"
 
     if with_submodule: