From f51fec34302beb8b431f54ab9609609ab55c8910 Mon Sep 17 00:00:00 2001
From: Antoine Eiche <lewo@abesis.fr>
Date: Fri, 6 Jan 2023 16:44:56 +0000
Subject: [PATCH] Generated commit for differential D2854
---
swh/loader/package/functional/loader.py | 48 ++++++++++++++++++-
.../functional/tests/test_functional.py | 6 +--
swh/loader/package/loader.py | 33 +++++++++----
3 files changed, 74 insertions(+), 13 deletions(-)
diff --git a/swh/loader/package/functional/loader.py b/swh/loader/package/functional/loader.py
index 381f190b..d861d985 100644
--- a/swh/loader/package/functional/loader.py
+++ b/swh/loader/package/functional/loader.py
@@ -5,13 +5,15 @@
import json
import requests
+import tempfile
-from typing import Dict, Optional, Any, Mapping
+from typing import Dict, Optional, Any, Mapping, Tuple
from swh.model import hashutil
+from swh.model import from_disk
from swh.model.model import (
- Sha1Git, Revision, RevisionType
+ Sha1Git, Revision, RevisionType, Release, ObjectType
)
from swh.loader.package.utils import EMPTY_AUTHOR
@@ -113,3 +115,45 @@ class FunctionalLoader(PackageLoader):
},
}
)
+
+ def _load_target(
+ self, p_info, origin) \
+ -> Tuple[str, Optional[Sha1Git], bool]:
+
+ revision_id, loaded = self._load_revision(p_info, origin)
+
+ if loaded is True:
+ return 'revision', revision_id, loaded
+
+ import logging
+ logger = logging.getLogger(__name__)
+ logger.error("fILE\n\n")
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ try:
+ dl_artifacts = self.download_package(p_info, tmpdir)
+ except Exception:
+ logger.exception('Unable to retrieve %s',
+ p_info)
+ return ('', None, False)
+
+ content = from_disk.Content.from_file(
+ path=dl_artifacts[0][0].encode('utf-8'),
+ max_content_length=self.max_content_size)
+
+ self.storage.content_add([content.to_model()])
+
+ release = Release(
+ name=b'iop',
+ message=b'iop',
+ author=EMPTY_AUTHOR,
+ date=None,
+ target_type=ObjectType.CONTENT,
+ target=content.hash,
+ synthetic=True
+ )
+ self.storage.release_add([release])
+
+ return 'release', release.id, True
+
+ return '', None, False
diff --git a/swh/loader/package/functional/tests/test_functional.py b/swh/loader/package/functional/tests/test_functional.py
index 812384fa..f2e52258 100644
--- a/swh/loader/package/functional/tests/test_functional.py
+++ b/swh/loader/package/functional/tests/test_functional.py
@@ -40,12 +40,12 @@ def test_loader_one_visit(swh_config, requests_mock_datadir):
stats = get_stats(loader.storage)
assert {
- 'content': 1,
+ 'content': 2,
'directory': 3,
'origin': 1,
'origin_visit': 1,
'person': 1,
- 'release': 0,
+ 'release': 1,
'revision': 1,
'skipped_content': 0,
'snapshot': 1
@@ -54,7 +54,7 @@ def test_loader_one_visit(swh_config, requests_mock_datadir):
origin_visit = next(loader.storage.origin_visit_get(sources_url))
# The visit is partial because urls pointing to non tarball file
# are not handled yet
- assert origin_visit['status'] == 'partial'
+ assert origin_visit['status'] == 'full'
assert origin_visit['type'] == 'functional'
diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py
index 1f3cd18f..8dba84ea 100644
--- a/swh/loader/package/loader.py
+++ b/swh/loader/package/loader.py
@@ -172,6 +172,13 @@ class PackageLoader:
"""
return None
+ def resolve_target_from(
+ self, known_artifacts: Dict, artifact_metadata: Dict) \
+ -> Tuple[str, Optional[bytes]]:
+ return 'revision', self.resolve_revision_from(
+ known_artifacts,
+ artifact_metadata)
+
def download_package(self, p_info: Mapping[str, Any],
tmpdir: str) -> List[Tuple[str, Mapping]]:
"""Download artifacts for a specific package. All downloads happen in
@@ -290,19 +297,23 @@ class PackageLoader:
# `p_` stands for `package_`
for branch_name, p_info in self.get_package_info(version):
logger.debug('package_info: %s', p_info)
- revision_id = self.resolve_revision_from(
+ target_type, target_id = self.resolve_target_from(
known_artifacts, p_info['raw'])
- if revision_id is None:
- (revision_id, loaded) = \
- self._load_revision(p_info, origin)
+ if target_id is None:
+ (target_type, target_id, loaded) = \
+ self._load_target(p_info, origin)
+
if loaded:
status_load = 'eventful'
else:
status_visit = 'partial'
- if revision_id is None:
+ if target_id is None:
continue
- tmp_revisions[version].append((branch_name, revision_id))
+ tmp_revisions[version].append((
+ branch_name,
+ target_type,
+ target_id))
logger.debug('tmp_revisions: %s', tmp_revisions)
# Build and load the snapshot
@@ -320,9 +331,9 @@ class PackageLoader:
'target': branch_name.encode('utf-8'),
}
- for branch_name, target in branch_name_revisions:
+ for branch_name, target_type, target in branch_name_revisions:
branches[branch_name.encode('utf-8')] = {
- 'target_type': 'revision',
+ 'target_type': target_type,
'target': target,
}
@@ -436,3 +447,9 @@ class PackageLoader:
self.storage.revision_add([revision])
return (revision.id, True)
+
+ def _load_target(
+ self, p_info, origin) \
+ -> Tuple[str, Optional[Sha1Git], bool]:
+ revision_id, loaded = self._load_revision(p_info, origin)
+ return 'revision', revision_id, loaded
--
GitLab