Daniele Serafini · Phabricator Migration user
--- a/swh/scanner/cli.py 0 → 100644
+++ b/swh/scanner/cli.py 0 → 100644
+from swh.core.cli import CONTEXT_SETTINGS
+
+@click.group(name='scanner', context_settings=CONTEXT_SETTINGS)
+@click.pass_context
+def scanner(ctx):
+    '''Software Heritage Scanner tools.'''
+    pass
+
+
+def parse_url(url):
+    if not url.startswith('http://') or not url.startswith('https://'):
+        url = 'https://' + url
+    if not url.endswith('/'):
+        url += '/'
+    return url
+
--- a/swh/scanner/scanner.py 0 → 100644
+++ b/swh/scanner/scanner.py 0 → 100644
+from .model import Tree
+
+from swh.model.cli import pid_of_file, pid_of_dir
+from swh.model.identifiers import (
+        parse_persistent_identifier,
+        DIRECTORY, CONTENT
+)
+
+
+async def pids_discovery(
+        pids: List[str], session: aiohttp.ClientSession, api_url: str,
+        ) -> Dict[str, Dict[str, bool]]:
+    """API Request to get information about the persistent identifiers given in
+    input.
+
+    Args:
--- a/swh/scanner/scanner.py 0 → 100644
+++ b/swh/scanner/scanner.py 0 → 100644
+
+            return await resp.json()
+
+    if len(pids) > chunk_size:
+        for pids_chunk in get_chunk(pids):
+            requests.append(asyncio.create_task(
+                make_request(pids_chunk)))
+
+        res = await asyncio.gather(*requests)
+        # concatenate list of dictionaries
+        return dict(itertools.chain.from_iterable(e.items() for e in res))
+    else:
+        return await make_request(pids)
+
+
+def get_subpaths(
--- a/swh/scanner/scanner.py 0 → 100644
+++ b/swh/scanner/scanner.py 0 → 100644
+    """Find the persistent identifier of the directories and files under a
+    given path.
+
+    Args:
+        path: the root path
+
+    Yields:
+        pairs of: path, the relative persistent identifier
+
+    """
+    def pid_of(path):
+        if path.is_dir():
+            return pid_of_dir(bytes(path))
+        elif path.is_file():
+            return pid_of_file(bytes(path))
+
--- a/swh/foo/bar.py → swh/scanner/exceptions.py
+++ b/swh/foo/bar.py → swh/scanner/exceptions.py
-# Copyright (C) 2019  The Software Heritage developers
+# Copyright (C) 2020  The Software Heritage developers
--- a/swh/scanner/model.py 0 → 100644
+++ b/swh/scanner/model.py 0 → 100644
+        """
+        relative_path = path.relative_to(self.path)
+
+        if relative_path == PosixPath('.'):
+            if pid is not None:
+                self.pid = pid
+            return
+
+        new_path = self.path.joinpath(relative_path.parts[0])
+        if new_path not in self.children:
+            self.children[new_path] = Tree(new_path, self)
+
+        self.children[new_path].addNode(path, pid)
+
+    def show(self) -> None:
+        """Print all the tree"""
--- a/swh/scanner/scanner.py 0 → 100644
+++ b/swh/scanner/scanner.py 0 → 100644
+from swh.model.cli import pid_of_file, pid_of_dir
+from swh.model.identifiers import (
+        parse_persistent_identifier,
+        DIRECTORY, CONTENT
+)
+
+
+async def pids_discovery(
+        pids: List[str], session: aiohttp.ClientSession, api_url: str,
+        ) -> Dict[str, Dict[str, bool]]:
+    """API Request to get information about the persistent identifiers given in
+    input.
+
+    Args:
+        pids: a list of persistent identifier
+        api_url: url for the API request
--- a/swh/scanner/scanner.py 0 → 100644
+++ b/swh/scanner/scanner.py 0 → 100644
+
+    if len(pids) > chunk_size:
+        for pids_chunk in get_chunk(pids):
+            requests.append(asyncio.create_task(
+                make_request(pids_chunk)))
+
+        res = await asyncio.gather(*requests)
+        # concatenate list of dictionaries
+        return dict(itertools.chain.from_iterable(e.items() for e in res))
+    else:
+        return await make_request(pids)
+
+
+def get_subpaths(
+        path: PosixPath) -> Iterator[Tuple[PosixPath, str]]:
+    """Find the persistent identifier of the directories and files under a
--- a/swh/scanner/scanner.py 0 → 100644
+++ b/swh/scanner/scanner.py 0 → 100644
+    parsed_pids = await pids_discovery(
+        list(parsed_paths.values()), session, api_url)
+
+    def unpack(tup):
+        subpath, pid = tup
+        return (subpath, pid, parsed_pids[pid]['known'])
+
+    return map(unpack, parsed_paths.items())
+
+
+async def run(
+        root: PosixPath, api_url: str, source_tree: Tree) -> None:
+    """Start scanning from the given root.
+
+    It fills the source tree with the path discovered.
+
--- a/swh/scanner/cli.py 0 → 100644
+++ b/swh/scanner/cli.py 0 → 100644
+    pass
+
+
+def parse_url(url):
+    if not url.startswith('http://') or not url.startswith('https://'):
+        url = 'https://' + url
+    if not url.endswith('/'):
+        url += '/'
+    return url
+
+
+@scanner.command(name='scan')
+@click.argument('path', required=True, type=click.Path(exists=True))
+@click.option('--api-url', default='https://archive.softwareheritage.org/api/1',
+              metavar='API_URL', show_default=True,
+              help="url for the api request")