diff --git a/swh/loader/pypi/loader.py b/swh/loader/pypi/loader.py
index 423c7ab33ce6be228451362caf0a99320d0464e0..0824234cd603f5b9389ba546d481387ab87905ff 100644
--- a/swh/loader/pypi/loader.py
+++ b/swh/loader/pypi/loader.py
@@ -3,6 +3,7 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import arrow
import hashlib
import logging
import os
@@ -14,8 +15,10 @@ from swh.loader.core.utils import clean_dangling_folders
from swh.loader.core.loader import SWHStatelessLoader
from swh.model import hashutil
from swh.model.from_disk import Directory
-from swh.model.identifiers import (release_identifier, revision_identifier,
- snapshot_identifier, identifier_to_bytes)
+from swh.model.identifiers import (
+ release_identifier, revision_identifier, snapshot_identifier,
+ identifier_to_bytes, normalize_timestamp
+)
from .model import PyPiProject
@@ -89,8 +92,7 @@ class PyPiClient:
"""
import gzip
from json import dumps
- from arrow import utcnow
- datepath = utcnow().isoformat()
+ datepath = arrow.utcnow().isoformat()
fname = os.path.join(self.cache_dir, datepath + '.gz')
with gzip.open(fname, 'w') as f:
f.write(bytes(
@@ -115,13 +117,18 @@ class PyPiClient:
"""Given a dictionary of releases, retrieve them locally.
"""
- _releases = releases.copy()
- for version, release in releases.items():
- logging.debug('version: %s' % version)
+ # order the release in time order
+ _release_versions = list(releases.keys())
+ _release_versions.sort()
+
+ for version in _release_versions:
+ release = releases[version]
+ _release = release.copy()
+ logging.debug('Release version: %s' % version)
path = os.path.join(self.temp_directory, project, version)
os.makedirs(path, exist_ok=True)
filepath = os.path.join(path, release['filename'])
- logging.debug('filepath to write: %s' % filepath)
+ logging.debug('Release local path: %s' % filepath)
r = self.session.get(release['url'])
if not r.ok:
@@ -150,14 +157,14 @@ class PyPiClient:
os.makedirs(uncompress_path, exist_ok=True)
nature = tarball.uncompress(filepath, uncompress_path)
- _releases[version]['directory'] = uncompress_path
+ _release['directory'] = uncompress_path
artifact = convert_to_hex(hashutil.hash_path(filepath))
artifact['archive_type'] = nature
for key, value in artifact.items():
- _releases[version][key] = value
+ _release[key] = value
- return _releases
+ yield version, _release
class PyPiLoader(SWHStatelessLoader):
@@ -222,22 +229,27 @@ class PyPiLoader(SWHStatelessLoader):
self.origin_metadata_url = origin_metadata_url
def get_contents(self):
- return self.contents
+ return self._contents
def get_directories(self):
- return self.directories()
+ return self._directories
def get_revisions(self):
- return self.revisions
+ return self._revisions
def get_releases(self):
- return self.releases
+ return self._releases
def get_snapshot(self):
- return self.snapshot
+ return self._snapshot
def fetch_data(self):
- """(override) Retrieve the pypi origin's information
+ """(override) Compute pypi data:
+
+ - 1. Retrieve project information
+ - 2. Fetch the releases and uncompress them
+ - 3. Collection object information (contents, directories,
+ revisions, releases, snapshot)
"""
project_info = self.pypi_client.info(self.origin_metadata_url)
@@ -255,16 +267,22 @@ class PyPiLoader(SWHStatelessLoader):
'branches': {}
}
- # for each
- for version, release in releases.items():
+ _last_rev = None
+
+ for version, release in releases:
_dir_path = release.pop('directory')
- directory = Directory.from_disk(path=_dir_path.encode('utf-8'),
- save_path=True)
+ _dir_path = _dir_path.encode('utf-8')
+ directory = Directory.from_disk(path=_dir_path, data=True)
_objects = directory.collect()
- _contents.append(_objects['content'].values())
- _directories.append(_objects['directory'].values())
+ _contents.extend(_objects['content'].values())
+ _directories.extend(_objects['directory'].values())
+ date = normalize_timestamp(
+ int(arrow.get(release['date']).timestamp))
+
+ name = release['name'].encode('utf-8')
+ message = release['message'].encode('utf-8')
_revision = {
'synthetic': True,
'metadata': {
@@ -272,44 +290,43 @@ class PyPiLoader(SWHStatelessLoader):
'project': info,
},
'author': author,
- 'date': release['date'],
+ 'date': date,
'committer': author,
- 'committer_date': release['date'],
- 'name': release['name'],
- 'message': release['message'],
+ 'committer_date': date,
+ 'name': name,
+ 'message': message,
'directory': directory.hash,
- 'parents': [],
+ 'parents': [] if _last_rev is None else [_last_rev['id']],
'type': 'tar',
}
_revision['id'] = identifier_to_bytes(
revision_identifier(_revision))
_revisions.append(_revision)
+ _last_rev = _revision
_release = {
- 'name': release['name'],
+ 'name': name,
'author': author,
- 'date': release['date'],
- 'message': release['message'],
+ 'date': date,
+ 'message': message,
'target_type': 'revision',
'target': _revision['id'],
+ 'synthetic': False,
}
_release['id'] = identifier_to_bytes(
release_identifier(_release))
_releases.append(_release)
- _snapshot['branches'][release['name']] = {
+ _snapshot['branches'][name] = {
'target': _release['id'],
'target_type': 'release',
}
- logging.debug('version: %s' % version)
- logging.debug('release: %s' % release['directory'])
-
_snapshot['id'] = identifier_to_bytes(
snapshot_identifier(_snapshot))
- self.contents = _contents
- self.directories = _directories
- self.revisions = _revisions
- self.releases = _releases
- self.snapshot = _snapshot
+ self._contents = _contents
+ self._directories = _directories
+ self._revisions = _revisions
+ self._releases = _releases
+ self._snapshot = _snapshot
diff --git a/swh/loader/pypi/model.py b/swh/loader/pypi/model.py
index 365255c6aaf230b24ff61d4917bbab7585916072..89213cdbb061b4fd0fed56ff333f391bf20161ee 100644
--- a/swh/loader/pypi/model.py
+++ b/swh/loader/pypi/model.py
@@ -4,6 +4,9 @@
# See top-level LICENSE file for more information
+import logging
+
+
class PyPiProject:
"""PyPi project representation
@@ -23,15 +26,20 @@ class PyPiProject:
}
def author(self):
+ name = self.data['info']['author'].encode('utf-8')
+ email = self.data['info']['author_email'].encode('utf-8')
return {
- 'fullname': self.data['info']['author'],
- 'name': self.data['info']['author'],
- 'email': self.data['info']['author_email']
+ 'fullname': name,
+ 'name': name,
+ 'email': email,
}
def releases(self):
releases = {}
for version, release in self.data['releases'].items():
+ logging.debug('version: %s, release: %s' % (version, release))
+ # FIXME: there can be multiple 'package_type' here:
+ # sdist, bdist_egg, bdist_wheel
if isinstance(release, list):
if len(release) > 1:
raise ValueError( # unexpected so fail so that we
diff --git a/swh/loader/pypi/tests/test_model.py b/swh/loader/pypi/tests/test_model.py
index 6e2444eb2138fa42590031077d0c77c1a0b33414..4f143a0853d4a7ae1307515a744922120dc11ae2 100644
--- a/swh/loader/pypi/tests/test_model.py
+++ b/swh/loader/pypi/tests/test_model.py
@@ -38,10 +38,12 @@ class ModelTest(TestCase):
def author(self):
actual_author = self.project.author()
+ name = self.data['info']['author'].encode('utf-8')
+ email = self.data['info']['author_email'].encode('utf-8')
expected_author = {
- 'fullname': self.data['info']['author'],
- 'name': self.data['info']['author'],
- 'email': self.data['info']['author_email'],
+ 'fullname': name,
+ 'name': name,
+ 'email': email,
}
self.assertEqual(expected_author, actual_author)