swh.loader.pypi: Use the default loader which filter existing data

Related T421

swh/loader/pypi/loader.py

@@ -9,7 +9,7 @@ import os
 import shutil
 
 from swh.loader.core.utils import clean_dangling_folders
-from swh.loader.core.loader import SWHStatelessLoader
+from swh.loader.core.loader import SWHLoader
 from swh.model.from_disk import Directory
 from swh.model.identifiers import (
     release_identifier, revision_identifier, snapshot_identifier,
@@ -23,7 +23,7 @@ from .model import PyPiProject
 TEMPORARY_DIR_PREFIX_PATTERN = 'swh.loader.pypi.'
 
 
-class PyPiLoader(SWHStatelessLoader):
+class PyPiLoader(SWHLoader):
     CONFIG_BASE_FILENAME = 'loader/pypi'
     ADDITIONAL_CONFIG = {
         'temp_directory': ('str', '/tmp/swh.loader.pypi/'),
@@ -86,43 +86,29 @@ class PyPiLoader(SWHStatelessLoader):
         self.project = PyPiProject(self.pypi_client, self.project_name,
                                    self.origin_metadata_url)
 
-    def get_contents(self):
-        return self._contents
-
-    def get_directories(self):
-        return self._directories
-
-    def get_revisions(self):
-        return self._revisions
-
-    def get_releases(self):
-        return self._releases
-
-    def get_snapshot(self):
-        return self._snapshot
-
     def fetch_data(self):
-        """(override) Compute pypi data:
-
-        - 1. Retrieve project information
-        - 2. Fetch the releases and uncompress them
-        - 3. Collection object information (contents, directories,
-          revisions, releases, snapshot)
+        """(override) This will fetch and prepare the needed releases.
 
         """
-        pypi_releases = self.project.releases()
+        self.pypi_releases = self.project.releases()
 
-        _contents = []
-        _directories = []
-        _revisions = []
-        _releases = []
+    def store_data(self):
+        """(override) This collects the necessary objects information and send
+           them to storage.
+
+        """
         _snapshot = {
             'branches': {}
         }
 
         _last_rev = None
 
-        for version, _release in pypi_releases:
+        _contents = []
+        _directories = []
+        _revisions = []
+        _releases = []
+
+        for version, _release in self.pypi_releases:
             info = self.project.info(version)
             author = self.project.author(version)
             logging.debug('author: %s' % author)
@@ -159,6 +145,7 @@ class PyPiLoader(SWHStatelessLoader):
             _revision['id'] = identifier_to_bytes(
                 revision_identifier(_revision))
             _revisions.append(_revision)
+
             _last_rev = _revision
 
             _release = {
@@ -182,13 +169,8 @@ class PyPiLoader(SWHStatelessLoader):
         _snapshot['id'] = identifier_to_bytes(
             snapshot_identifier(_snapshot))
 
-        logging.debug('contents: %s' % len(_contents))
-        logging.debug('directories: %s' % len(_directories))
-        logging.debug('revisions: %s' % len(_revisions))
-        logging.debug('releases: %s' % len(_releases))
-
-        self._contents = _contents
-        self._directories = _directories
-        self._revisions = _revisions
-        self._releases = _releases
-        self._snapshot = _snapshot
+        self.maybe_load_contents(_contents)
+        self.maybe_load_directories(_directories)
+        self.maybe_load_revisions(_revisions)
+        self.maybe_load_releases(_releases)
+        self.maybe_load_snapshot(_snapshot)

[Phabricator Migration user]

Some references in the commit message have been migrated:

• T421 is now swh-loader-core#421 (closed)