diff --git a/swh/loader/pypi/loader.py b/swh/loader/pypi/loader.py index ba97c4761ae216aebb6e3a5bbf4d6da34e393dca..e6a35457f1f7def16edbe57b69379903bc70deb2 100644 --- a/swh/loader/pypi/loader.py +++ b/swh/loader/pypi/loader.py @@ -9,7 +9,7 @@ import os import shutil from swh.loader.core.utils import clean_dangling_folders -from swh.loader.core.loader import SWHStatelessLoader +from swh.loader.core.loader import SWHLoader from swh.model.from_disk import Directory from swh.model.identifiers import ( release_identifier, revision_identifier, snapshot_identifier, @@ -23,7 +23,7 @@ from .model import PyPiProject TEMPORARY_DIR_PREFIX_PATTERN = 'swh.loader.pypi.' -class PyPiLoader(SWHStatelessLoader): +class PyPiLoader(SWHLoader): CONFIG_BASE_FILENAME = 'loader/pypi' ADDITIONAL_CONFIG = { 'temp_directory': ('str', '/tmp/swh.loader.pypi/'), @@ -86,43 +86,29 @@ class PyPiLoader(SWHStatelessLoader): self.project = PyPiProject(self.pypi_client, self.project_name, self.origin_metadata_url) - def get_contents(self): - return self._contents - - def get_directories(self): - return self._directories - - def get_revisions(self): - return self._revisions - - def get_releases(self): - return self._releases - - def get_snapshot(self): - return self._snapshot - def fetch_data(self): - """(override) Compute pypi data: - - - 1. Retrieve project information - - 2. Fetch the releases and uncompress them - - 3. Collection object information (contents, directories, - revisions, releases, snapshot) + """(override) This will fetch and prepare the needed releases. """ - pypi_releases = self.project.releases() + self.pypi_releases = self.project.releases() - _contents = [] - _directories = [] - _revisions = [] - _releases = [] + def store_data(self): + """(override) This collects the necessary objects information and send + them to storage. + + """ _snapshot = { 'branches': {} } _last_rev = None - for version, _release in pypi_releases: + _contents = [] + _directories = [] + _revisions = [] + _releases = [] + + for version, _release in self.pypi_releases: info = self.project.info(version) author = self.project.author(version) logging.debug('author: %s' % author) @@ -159,6 +145,7 @@ class PyPiLoader(SWHStatelessLoader): _revision['id'] = identifier_to_bytes( revision_identifier(_revision)) _revisions.append(_revision) + _last_rev = _revision _release = { @@ -182,13 +169,8 @@ class PyPiLoader(SWHStatelessLoader): _snapshot['id'] = identifier_to_bytes( snapshot_identifier(_snapshot)) - logging.debug('contents: %s' % len(_contents)) - logging.debug('directories: %s' % len(_directories)) - logging.debug('revisions: %s' % len(_revisions)) - logging.debug('releases: %s' % len(_releases)) - - self._contents = _contents - self._directories = _directories - self._revisions = _revisions - self._releases = _releases - self._snapshot = _snapshot + self.maybe_load_contents(_contents) + self.maybe_load_directories(_directories) + self.maybe_load_revisions(_revisions) + self.maybe_load_releases(_releases) + self.maybe_load_snapshot(_snapshot)