Generated commit for differential D2854

swh/loader/package/functional/loader.py

@@ -5,13 +5,15 @@
 
 import json
 import requests
+import tempfile
 
-from typing import Dict, Optional, Any, Mapping
+from typing import Dict, Optional, Any, Mapping, Tuple
 
 from swh.model import hashutil
+from swh.model import from_disk
 
 from swh.model.model import (
-    Sha1Git, Revision, RevisionType
+    Sha1Git, Revision, RevisionType, Release, ObjectType
 )
 
 from swh.loader.package.utils import EMPTY_AUTHOR
@@ -113,3 +115,45 @@ class FunctionalLoader(PackageLoader):
                 },
             }
         )
+
+    def _load_target(
+            self, p_info, origin) \
+            -> Tuple[str, Optional[Sha1Git], bool]:
+
+        revision_id, loaded = self._load_revision(p_info, origin)
+
+        if loaded is True:
+            return 'revision', revision_id, loaded
+
+        import logging
+        logger = logging.getLogger(__name__)
+        logger.error("fILE\n\n")
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            try:
+                dl_artifacts = self.download_package(p_info, tmpdir)
+            except Exception:
+                logger.exception('Unable to retrieve %s',
+                                 p_info)
+                return ('', None, False)
+
+            content = from_disk.Content.from_file(
+                path=dl_artifacts[0][0].encode('utf-8'),
+                max_content_length=self.max_content_size)
+
+            self.storage.content_add([content.to_model()])
+
+            release = Release(
+                name=b'iop',
+                message=b'iop',
+                author=EMPTY_AUTHOR,
+                date=None,
+                target_type=ObjectType.CONTENT,
+                target=content.hash,
+                synthetic=True
+            )
+            self.storage.release_add([release])
+
+            return 'release', release.id, True
+
+        return '', None, False

swh/loader/package/functional/tests/test_functional.py

@@ -40,12 +40,12 @@ def test_loader_one_visit(swh_config, requests_mock_datadir):
 
     stats = get_stats(loader.storage)
     assert {
-        'content': 1,
+        'content': 2,
         'directory': 3,
         'origin': 1,
         'origin_visit': 1,
         'person': 1,
-        'release': 0,
+        'release': 1,
         'revision': 1,
         'skipped_content': 0,
         'snapshot': 1
@@ -54,7 +54,7 @@ def test_loader_one_visit(swh_config, requests_mock_datadir):
     origin_visit = next(loader.storage.origin_visit_get(sources_url))
     # The visit is partial because urls pointing to non tarball file
     # are not handled yet
-    assert origin_visit['status'] == 'partial'
+    assert origin_visit['status'] == 'full'
     assert origin_visit['type'] == 'functional'
 
 

swh/loader/package/loader.py

@@ -172,6 +172,13 @@ class PackageLoader:
         """
         return None
 
+    def resolve_target_from(
+            self, known_artifacts: Dict, artifact_metadata: Dict) \
+            -> Tuple[str, Optional[bytes]]:
+        return 'revision', self.resolve_revision_from(
+            known_artifacts,
+            artifact_metadata)
+
     def download_package(self, p_info: Mapping[str, Any],
                          tmpdir: str) -> List[Tuple[str, Mapping]]:
         """Download artifacts for a specific package. All downloads happen in
@@ -290,19 +297,23 @@ class PackageLoader:
                 # `p_` stands for `package_`
                 for branch_name, p_info in self.get_package_info(version):
                     logger.debug('package_info: %s', p_info)
-                    revision_id = self.resolve_revision_from(
+                    target_type, target_id = self.resolve_target_from(
                         known_artifacts, p_info['raw'])
-                    if revision_id is None:
-                        (revision_id, loaded) = \
-                            self._load_revision(p_info, origin)
+                    if target_id is None:
+                        (target_type, target_id, loaded) = \
+                            self._load_target(p_info, origin)
+
                         if loaded:
                             status_load = 'eventful'
                         else:
                             status_visit = 'partial'
-                        if revision_id is None:
+                        if target_id is None:
                             continue
 
-                    tmp_revisions[version].append((branch_name, revision_id))
+                    tmp_revisions[version].append((
+                        branch_name,
+                        target_type,
+                        target_id))
 
             logger.debug('tmp_revisions: %s', tmp_revisions)
             # Build and load the snapshot
@@ -320,9 +331,9 @@ class PackageLoader:
                             'target': branch_name.encode('utf-8'),
                         }
 
-                for branch_name, target in branch_name_revisions:
+                for branch_name, target_type, target in branch_name_revisions:
                     branches[branch_name.encode('utf-8')] = {
-                        'target_type': 'revision',
+                        'target_type': target_type,
                         'target': target,
                     }
 
@@ -436,3 +447,9 @@ class PackageLoader:
         self.storage.revision_add([revision])
 
         return (revision.id, True)
+
+    def _load_target(
+            self, p_info, origin) \
+            -> Tuple[str, Optional[Sha1Git], bool]:
+        revision_id, loaded = self._load_revision(p_info, origin)
+        return 'revision', revision_id, loaded