New upstream version 0.0.20

PKG-INFO

 Metadata-Version: 2.1
 Name: swh.deposit
-Version: 0.0.19
+Version: 0.0.20
 Summary: Software Heritage Deposit Server
 Home-page: https://forge.softwareheritage.org/source/swh-deposit/
 Author: Software Heritage developers

debian/control

@@ -25,7 +25,6 @@ Package: python3-swh.deposit
 Architecture: all
 Depends: python3-swh.core (>= 0.0.14~),
          python3-swh.loader.tar (>= 0.0.26~),
-         python3-swh.scheduler (>= 0.0.17~),
          ${misc:Depends}, ${python3:Depends}
 Description: Software Heritage Deposit Server
 

requirements-swh.txt

 swh.core >= 0.0.14
 swh.loader.tar >= 0.0.27
-swh.scheduler >= 0.0.17

requirements.txt

 vcversioner
 
-retrying
 click
 Django
 djangorestframework

setup.py

@@ -24,6 +24,7 @@ setup(
     install_requires=parse_requirements(),
     extras_require={
         'injection': ['swh.loader.core >= 0.0.19',
+                      'swh.scheduler >= 0.0.17',
                       'requests'],
     },
     setup_requires=['vcversioner'],

swh.deposit.egg-info/PKG-INFO

 Metadata-Version: 2.1
 Name: swh.deposit
-Version: 0.0.19
+Version: 0.0.20
 Summary: Software Heritage Deposit Server
 Home-page: https://forge.softwareheritage.org/source/swh-deposit/
 Author: Software Heritage developers

swh.deposit.egg-info/SOURCES.txt

@@ -73,13 +73,12 @@ swh/deposit/fixtures/__init__.py
 swh/deposit/fixtures/deposit_data.yaml
 swh/deposit/injection/__init__.py
 swh/deposit/injection/loader.py
+swh/deposit/injection/scheduler.py
 swh/deposit/injection/tasks.py
 swh/deposit/migrations/0001_initial.py
 swh/deposit/migrations/0002_depositrequest_archive.py
 swh/deposit/migrations/0003_temporaryarchive.py
 swh/deposit/migrations/__init__.py
-swh/deposit/scheduler/__init__.py
-swh/deposit/scheduler/cli.py
 swh/deposit/service/__init__.py
 swh/deposit/service/clean_temporary_directory.py
 swh/deposit/settings/__init__.py
@@ -104,6 +103,7 @@ swh/deposit/tests/api/test_deposit_atom.py
 swh/deposit/tests/api/test_deposit_binary.py
 swh/deposit/tests/api/test_deposit_delete.py
 swh/deposit/tests/api/test_deposit_multipart.py
+swh/deposit/tests/api/test_deposit_read.py
 swh/deposit/tests/api/test_deposit_status.py
 swh/deposit/tests/api/test_deposit_update.py
 swh/deposit/tests/api/test_deposit_update_status.py

swh.deposit.egg-info/requires.txt

@@ -2,12 +2,11 @@ Django
 click
 djangorestframework
 djangorestframework-xml
-retrying
 swh.core>=0.0.14
 swh.loader.tar>=0.0.27
-swh.scheduler>=0.0.17
 vcversioner
 
 [injection]
 requests
 swh.loader.core>=0.0.19
+swh.scheduler>=0.0.17

swh/deposit/api/deposit_update.py

@@ -4,12 +4,10 @@
 # See top-level LICENSE file for more information
 
 from rest_framework import status
-from rest_framework.parsers import JSONParser
 
 from .common import SWHPostDepositAPI, SWHPutDepositAPI, SWHDeleteDepositAPI
 from ..config import CONT_FILE_IRI, EDIT_SE_IRI, EM_IRI
-from ..errors import make_error_response, make_error_dict, BAD_REQUEST
-from ..models import Deposit, DEPOSIT_STATUS_DETAIL
+from ..errors import make_error_response, BAD_REQUEST
 from ..parsers import SWHFileUploadParser, SWHAtomEntryParser
 from ..parsers import SWHMultiPartParser
 
@@ -155,42 +153,3 @@ class SWHUpdateMetadataDeposit(SWHPostDepositAPI, SWHPutDepositAPI,
         #protocoloperations_deleteconteiner
         """
         return self._delete_deposit(collection_name, deposit_id)
-
-
-class SWHUpdateStatusDeposit(SWHPutDepositAPI):
-    """Deposit request class to update the deposit's status.
-
-    HTTP verbs supported: PUT
-
-    """
-    parser_classes = (JSONParser, )
-
-    def restrict_access(self, req, deposit=None):
-        """Remove restriction modification to 'partial' deposit.
-           Update is possible regardless of the existing status.
-
-        """
-        return None
-
-    def process_put(self, req, headers, collection_name, deposit_id):
-        """Update the deposit's status
-
-        Returns:
-            204 No content
-
-        """
-        status = req.data.get('status')
-        if not status:
-            msg = 'The status key is mandatory with possible values %s' % list(
-                DEPOSIT_STATUS_DETAIL.keys())
-            return make_error_dict(BAD_REQUEST, msg)
-
-        if status not in DEPOSIT_STATUS_DETAIL:
-            msg = 'Possible status in %s' % list(DEPOSIT_STATUS_DETAIL.keys())
-            return make_error_dict(BAD_REQUEST, msg)
-
-        deposit = Deposit.objects.get(pk=deposit_id)
-        deposit.status = status
-        deposit.save()
-
-        return {}

swh/deposit/api/private/deposit_read.py

@@ -3,6 +3,7 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+import json
 import os
 import shutil
 import tempfile
@@ -10,9 +11,11 @@ import tempfile
 from rest_framework import status
 
 from swh.loader.tar import tarball
+from swh.model import hashutil, identifiers
 
 from ..common import SWHGetDepositAPI, SWHPrivateAPIView
 from ...models import Deposit, DepositRequest, TemporaryArchive
+from ...models import previous_revision_id
 
 
 def aggregate_tarballs(extraction_dir, archive_paths):
@@ -142,3 +145,91 @@ class SWHDepositReadArchives(SWHGetDepositAPI, SWHPrivateAPIView):
         self.cleanup(directory_to_cleanup)
 
         return status.HTTP_200_OK, stream, 'application/octet-stream'
+
+
+class SWHDepositReadMetadata(SWHGetDepositAPI, SWHPrivateAPIView):
+    """Class in charge of aggregating metadata on a deposit.
+
+    """
+    def _aggregate_metadata(self, deposit, metadata_requests):
+        """Retrieve and aggregates metadata information.
+
+        """
+        metadata = {}
+        for req in metadata_requests:
+            metadata.update(req.metadata)
+
+        return metadata
+
+    def aggregate(self, deposit, requests):
+        """Aggregate multiple data on deposit into one unified data dictionary.
+
+        Args:
+            deposit (Deposit): Deposit concerned by the data aggregation.
+            requests ([DepositRequest]): List of associated requests which
+                                         need aggregation.
+
+        Returns:
+            Dictionary of data representing the deposit to inject in swh.
+
+        """
+        data = {}
+        metadata_requests = []
+
+        # Retrieve tarballs/metadata information
+        metadata = self._aggregate_metadata(deposit, metadata_requests)
+
+        # Read information metadata
+        data['origin'] = {
+            'type': deposit.collection.name,
+            'url': deposit.external_id,
+        }
+
+        # revision
+
+        fullname = deposit.client.get_full_name()
+        author_committer = {
+            'name': deposit.client.last_name,
+            'fullname': fullname,
+            'email': deposit.client.email,
+        }
+
+        revision_type = 'tar'
+        revision_msg = '%s: Deposit %s in collection %s' % (
+            fullname, deposit.id, deposit.collection.name)
+        complete_date = identifiers.normalize_timestamp(deposit.complete_date)
+
+        data['revision'] = {
+            'synthetic': True,
+            'date': complete_date,
+            'committer_date': complete_date,
+            'author': author_committer,
+            'committer': author_committer,
+            'type': revision_type,
+            'message': revision_msg,
+            'metadata': metadata,
+        }
+
+        parent_revision = previous_revision_id(deposit.swh_id)
+        if parent_revision:
+            data['revision'] = {
+                'parents': [hashutil.hash_to_bytes(parent_revision)]
+            }
+
+        data['occurrence'] = {
+            'branch': 'master'
+        }
+
+        return data
+
+    def process_get(self, req, collection_name, deposit_id):
+        deposit = Deposit.objects.get(pk=deposit_id)
+        requests = DepositRequest.objects.filter(
+            deposit=deposit, type=self.deposit_request_types['metadata'])
+
+        data = self.aggregate(deposit, requests)
+        d = {}
+        if data:
+            d = json.dumps(data)
+
+        return status.HTTP_200_OK, d, 'application/json'

swh/deposit/api/urls.py

@@ -22,7 +22,7 @@ from django.conf.urls import url
 
 from ..config import EDIT_SE_IRI, EM_IRI, CONT_FILE_IRI
 from ..config import SD_IRI, COL_IRI, STATE_IRI, PRIVATE_GET_RAW_CONTENT
-from ..config import PRIVATE_PUT_DEPOSIT
+from ..config import PRIVATE_PUT_DEPOSIT, PRIVATE_GET_DEPOSIT_METADATA
 from .deposit import SWHDeposit
 from .deposit_status import SWHDepositStatus
 from .deposit_update import SWHUpdateMetadataDeposit
@@ -30,6 +30,7 @@ from .deposit_update import SWHUpdateArchiveDeposit
 from .deposit_content import SWHDepositContent
 from .service_document import SWHServiceDocument
 from .private.deposit_read import SWHDepositReadArchives
+from .private.deposit_read import SWHDepositReadMetadata
 from .private.deposit_update_status import SWHUpdateStatusDeposit
 
 
@@ -77,7 +78,14 @@ urlpatterns = [
     url(r'^(?P<collection_name>[^/]+)/(?P<deposit_id>[^/]+)/raw/$',
         SWHDepositReadArchives.as_view(),
         name=PRIVATE_GET_RAW_CONTENT),
+    # Update deposit's status
+    # -> PUT
     url(r'^(?P<collection_name>[^/]+)/(?P<deposit_id>[^/]+)/update/$',
         SWHUpdateStatusDeposit.as_view(),
         name=PRIVATE_PUT_DEPOSIT),
+    # Retrieve metadata information on a specific deposit
+    # -> GET
+    url(r'^(?P<collection_name>[^/]+)/(?P<deposit_id>[^/]+)/meta/$',
+        SWHDepositReadMetadata.as_view(),
+        name=PRIVATE_GET_DEPOSIT_METADATA),
 ]

swh/deposit/config.py

@@ -17,6 +17,7 @@ COL_IRI = 'upload'
 STATE_IRI = 'status'
 PRIVATE_GET_RAW_CONTENT = 'private-download'
 PRIVATE_PUT_DEPOSIT = 'private-update'
+PRIVATE_GET_DEPOSIT_METADATA = 'private-read'
 
 ARCHIVE_KEY = 'archive'
 METADATA_KEY = 'metadata'

swh/deposit/injection/loader.py

@@ -3,22 +3,22 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+import datetime
 import os
 import requests
 import tempfile
 
-
 from swh.model import hashutil
 from swh.loader.tar import loader
 from swh.loader.core.loader import SWHLoader
 
 
-def retrieve_archive_to(archive_url, archive_path):
+def retrieve_archive_to(archive_update_url, archive_path, log=None):
     """Retrieve the archive from the deposit to a local directory.
 
     Args:
 
-        archive_url (str): The full deposit archive(s)'s raw content
+        archive_update_url (str): The full deposit archive(s)'s raw content
                            to retrieve locally
 
         archive_path (str): the local archive's path where to store
@@ -29,30 +29,61 @@ def retrieve_archive_to(archive_url, archive_path):
         Or None if any problem arose.
 
     """
-    r = requests.get(archive_url, stream=True)
+    r = requests.get(archive_update_url, stream=True)
     if r.ok:
         with open(archive_path, 'wb') as f:
             for chunk in r.iter_content():
                 f.write(chunk)
 
         return archive_path
-    return None
+
+    msg = 'Problem when retrieving deposit archive at %s' % (
+        archive_update_url, )
+    if log:
+        log.error(msg)
+
+    raise ValueError(msg)
+
+
+def retrieve_metadata(metadata_url, log=None):
+    """Retrieve the metadata information on a given deposit.
+
+    Args:
+
+        metadata_url (str): The full deposit metadata url to retrieve
+        locally
+
+    Returns:
+        The dictionary of metadata for that deposit or None if any
+        problem arose.
+
+    """
+    r = requests.get(metadata_url)
+    if r.ok:
+        data = r.json()
+
+        return data
+
+    msg = 'Problem when retrieving metadata at %s' % metadata_url
+    if log:
+        log.error(msg)
+
+    raise ValueError(msg)
 
 
-def update_deposit_status(archive_url, status, revision_id=None):
+def update_deposit_status(update_status_url, status, revision_id=None):
     """Update the deposit's status.
 
     Args:
-        archive_url (str): the full deposit's archive
+        update_status_url (str): the full deposit's archive
         status (str): The status to update the deposit with
         revision_id (str/None): the revision's identifier to update to
 
     """
-    update_url = archive_url.replace('/raw/', '/update/')
     payload = {'status': status}
     if revision_id:
         payload['revision_id'] = revision_id
-    requests.put(update_url, json=payload)
+    requests.put(update_status_url, json=payload)
 
 
 class DepositLoader(loader.TarLoader):
@@ -71,36 +102,36 @@ class DepositLoader(loader.TarLoader):
     - update the deposit's status accordingly
 
     """
-    def load(self, *, deposit_archive_url, origin, visit_date, revision):
-        occurrence = {'branch': 'master'}
-        SWHLoader.load(self,
-                       deposit_archive_url=deposit_archive_url,
-                       origin=origin,
-                       visit_date=visit_date,
-                       revision=revision,
-                       occurrences=[occurrence])
-
-    def prepare(self, *, deposit_archive_url, origin, visit_date, revision,
-                occurrences):
+    def load(self, *, archive_url, deposit_meta_url, deposit_update_url):
+        SWHLoader.load(
+            self,
+            archive_url=archive_url,
+            deposit_meta_url=deposit_meta_url,
+            deposit_update_url=deposit_update_url)
+
+    def prepare(self, *, archive_url, deposit_meta_url, deposit_update_url):
         """Prepare the injection by first retrieving the deposit's raw archive
            content.
 
         """
-        self.archive_url = deposit_archive_url
+        self.deposit_update_url = deposit_update_url
         temporary_directory = tempfile.TemporaryDirectory()
         self.temporary_directory = temporary_directory
         archive_path = os.path.join(temporary_directory.name, 'archive.zip')
-        archive = retrieve_archive_to(deposit_archive_url, archive_path)
+        archive = retrieve_archive_to(archive_url, archive_path, log=self.log)
 
-        if not archive:
-            raise ValueError('Failure to retrieve archive')
+        metadata = retrieve_metadata(deposit_meta_url, log=self.log)
+        origin = metadata['origin']
+        visit_date = datetime.datetime.now(tz=datetime.timezone.utc)
+        revision = metadata['revision']
+        occurrence = metadata['occurrence']
 
-        update_deposit_status(self.archive_url, 'injecting')
+        update_deposit_status(deposit_update_url, 'injecting')
         super().prepare(tar_path=archive,
                         origin=origin,
                         visit_date=visit_date,
                         revision=revision,
-                        occurrences=occurrences)
+                        occurrences=[occurrence])
 
     def post_load(self, success=True):
         """Updating the deposit's status according to its loading status.
@@ -112,7 +143,8 @@ class DepositLoader(loader.TarLoader):
         """
         try:
             if not success:
-                update_deposit_status(self.archive_url, status='failure')
+                update_deposit_status(self.deposit_update_url,
+                                      status='failure')
                 return
             # first retrieve the new revision
             occs = list(self.storage.occurrence_get(self.origin_id))
@@ -121,7 +153,7 @@ class DepositLoader(loader.TarLoader):
                 revision_id = hashutil.hash_to_hex(occ['target'])
                 # then update the deposit's status to success with its
                 # revision-id
-                update_deposit_status(self.archive_url,
+                update_deposit_status(self.deposit_update_url,
                                       status='success',
                                       revision_id=revision_id)
         except:

swh/deposit/scheduler/cli.py → swh/deposit/injection/scheduler.py

@@ -3,7 +3,8 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
-"""Module in charge of scheduling deposit injection one-shot task to swh.
+"""Module in charge of sending deposit injection as celery task or
+scheduled one-shot tasks.
 
 """
 
@@ -11,26 +12,11 @@ import click
 import logging
 
 from abc import ABCMeta, abstractmethod
+from celery import group
 
+from swh.core import utils
 from swh.core.config import SWHConfig
 from swh.deposit.config import setup_django_for
-from swh.model import hashutil, identifiers
-
-
-def previous_revision_id(swh_id):
-    """Compute the parent's revision id (if any) from the swh_id.
-
-    Args:
-        swh_id (id): SWH Identifier from a previous deposit.
-
-    Returns:
-        None if no parent revision is detected.
-        The revision id's hash if any.
-
-    """
-    if swh_id:
-        return swh_id.split('-')[2]
-    return None
 
 
 class SWHScheduling(SWHConfig, metaclass=ABCMeta):
@@ -50,83 +36,11 @@ class SWHScheduling(SWHConfig, metaclass=ABCMeta):
                 additional_configs=[self.ADDITIONAL_CONFIG])
         self.log = logging.getLogger('swh.deposit.scheduling')
 
-    def _aggregate_metadata(self, deposit, metadata_requests):
-        """Retrieve and aggregates metadata information.
-
-        """
-        metadata = {}
-        for req in metadata_requests:
-            metadata.update(req.metadata)
-
-        return metadata
-
-    def aggregate(self, deposit, deposit_archive_url, requests):
-        """Aggregate multiple data on deposit into one unified data dictionary.
-
-        Args:
-            deposit (Deposit): Deposit concerned by the data aggregation.
-            deposit_archive_url (str): Url to retrieve a tarball from
-                                       the deposit instance
-            requests ([DepositRequest]): List of associated requests which
-                                         need aggregation.
-
-        Returns:
-            Dictionary of data representing the deposit to inject in swh.
-
-        """
-        data = {}
-        metadata_requests = []
-
-        # Retrieve tarballs/metadata information
-        metadata = self._aggregate_metadata(deposit, metadata_requests)
-
-        data['deposit_archive_url'] = deposit_archive_url
-
-        # Read information metadata
-        data['origin'] = {
-            'type': deposit.collection.name,
-            'url': deposit.external_id,
-        }
-
-        # revision
-
-        fullname = deposit.client.get_full_name()
-        author_committer = {
-            'name': deposit.client.last_name,
-            'fullname': fullname,
-            'email': deposit.client.email,
-        }
-
-        revision_type = 'tar'
-        revision_msg = '%s: Deposit %s in collection %s' % (
-            fullname, deposit.id, deposit.collection.name)
-        complete_date = identifiers.normalize_timestamp(deposit.complete_date)
-
-        data['revision'] = {
-            'synthetic': True,
-            'date': complete_date,
-            'committer_date': complete_date,
-            'author': author_committer,
-            'committer': author_committer,
-            'type': revision_type,
-            'message': revision_msg,
-            'metadata': metadata,
-        }
-
-        parent_revision = previous_revision_id(deposit.swh_id)
-        if parent_revision:
-            data['revision'] = {
-                'parents': [hashutil.hash_to_bytes(parent_revision)]
-            }
-
-        return data
-
     @abstractmethod
-    def schedule(self, deposit, data):
+    def schedule(self, deposits):
         """Schedule the new deposit injection.
 
         Args:
-            deposit (Deposit): Deposit concerned by the data aggregation.
             data (dict): Deposit aggregated data
 
         Returns:
@@ -153,32 +67,30 @@ class SWHCeleryScheduling(SWHScheduling):
             self.config.update(**config)
         self.dry_run = self.config['dry_run']
 
-    def schedule(self, deposit_data):
+    def _convert(self, deposits):
+        """Convert tuple to celery task signature.
+
+        """
+        task = self.task
+        for archive_url, deposit_meta_url, deposit_update_url in deposits:
+            yield task.s(archive_url=archive_url,
+                         deposit_meta_url=deposit_meta_url,
+                         deposit_update_url=deposit_update_url)
+
+    def schedule(self, deposits):
         """Schedule the new deposit injection directly through celery.
 
         Args:
-            deposit_data (dict): Deposit aggregated information.
+            depositdata (dict): Deposit aggregated information.
 
         Returns:
             None
 
         """
-        deposit_archive_url = deposit_data['deposit_archive_url']
-        origin = deposit_data['origin']
-        visit_date = None  # default to Now
-        revision = deposit_data['revision']
-
-        self.log.debug('args: %s %s %s %s' % (
-            deposit_archive_url, origin, visit_date, revision))
-
         if self.dry_run:
             return
 
-        return self.task.delay(
-            deposit_archive_url=deposit_archive_url,
-            origin=origin,
-            visit_date=visit_date,
-            revision=revision)
+        return group(self._convert(deposits)).delay()
 
 
 class SWHScheduling(SWHScheduling):
@@ -195,40 +107,66 @@ class SWHScheduling(SWHScheduling):
         self.dry_run = self.config['dry_run']
         self.scheduler = SchedulerBackend(**self.config)
 
-    def schedule(self, deposit_data):
+    def _convert(self, deposits):
+        """Convert tuple to one-shot scheduling tasks.
+
+        """
+        import datetime
+        for archive_url, deposit_meta_url, deposit_update_url in deposits:
+            yield {
+                'policy': 'oneshot',
+                'type': 'swh-deposit-archive-ingestion',
+                'next_run': datetime.datetime.now(tz=datetime.timezone.utc),
+                'arguments': {
+                    'args': [],
+                    'kwargs': {
+                        'archive_url': archive_url,
+                        'deposit_meta_url': deposit_meta_url,
+                        'deposit_update_url': deposit_update_url,
+                    },
+                }
+            }
+
+    def schedule(self, deposits):
         """Schedule the new deposit injection through swh.scheduler's api.
 
         Args:
-            deposit_data (dict): Deposit aggregated information.
+            deposits (dict): Deposit aggregated information.
 
         """
-        deposit_archive_url = deposit_data['deposit_archive_url']
-        origin = deposit_data['origin']
-        visit_date = None  # default to Now
-        revision = deposit_data['revision']
-
-        self.log.debug('args: %s %s %s %s' % (
-            deposit_archive_url, origin, visit_date, revision))
-
         if self.dry_run:
             return
 
-        import datetime
-        task = {
-            'policy': 'oneshot',
-            'type': 'swh-deposit-archive-ingestion',
-            'next_run': datetime.datetime.now(tz=datetime.timezone.utc),
-            'arguments': {
-                'args': [],
-                'kwargs': {
-                    'deposit_archive_url': deposit_archive_url,
-                    'origin': origin,
-                    'visit_date': visit_date,
-                    'revision': revision
-                },
-            }
-        }
-        self.scheduler.create_tasks([task])
+        self.scheduler.create_tasks(self._convert(deposits))
+
+
+def get_deposit_ready():
+    """Retrieve deposit ready to be task executed.
+
+    """
+    from swh.deposit.models import Deposit
+    yield from Deposit.objects.filter(status='ready')
+
+
+def prepare_task_arguments(server):
+    """Convert deposit to argument for task to be executed.
+
+    """
+    from swh.deposit.config import PRIVATE_GET_RAW_CONTENT
+    from swh.deposit.config import PRIVATE_GET_DEPOSIT_METADATA
+    from swh.deposit.config import PRIVATE_PUT_DEPOSIT
+    from django.core.urlresolvers import reverse
+
+    for deposit in get_deposit_ready():
+        args = [deposit.collection.name, deposit.id]
+        archive_url = '%s%s' % (server, reverse(
+            PRIVATE_GET_RAW_CONTENT, args=args))
+        deposit_meta_url = '%s%s' % (server, reverse(
+            PRIVATE_GET_DEPOSIT_METADATA, args=args))
+        deposit_update_url = '%s%s' % (server, reverse(
+            PRIVATE_PUT_DEPOSIT, args=args))
+
+        yield archive_url, deposit_meta_url, deposit_update_url
 
 
 @click.command(
@@ -239,13 +177,13 @@ class SWHScheduling(SWHScheduling):
               help='Scheduling method')
 @click.option('--server', default='http://127.0.0.1:5006',
               help='Deposit server')
+@click.option('--batch-size', default=1000, type=click.INT,
+              help='Task batch size')
 @click.option('--dry-run/--no-dry-run', is_flag=True, default=False,
               help='Dry run')
-def main(platform, scheduling_method, server, dry_run):
+def main(platform, scheduling_method, server, batch_size, dry_run):
     setup_django_for(platform)
 
-    from swh.deposit.models import Deposit, DepositRequest, DepositRequestType
-
     override_config = {}
     if dry_run:
         override_config['dry_run'] = dry_run
@@ -258,27 +196,8 @@ def main(platform, scheduling_method, server, dry_run):
         raise ValueError(
             'Only `celery` or `swh-scheduler` values are accepted')
 
-    from swh.deposit.config import PRIVATE_GET_RAW_CONTENT
-    from django.core.urlresolvers import reverse
-
-    _request_types = DepositRequestType.objects.all()
-    deposit_request_types = {
-        type.name: type for type in _request_types
-    }
-
-    deposits = Deposit.objects.filter(status='ready')
-    for deposit in deposits:
-        deposit_archive_url = '%s%s' % (server, reverse(
-            PRIVATE_GET_RAW_CONTENT,
-            args=[deposit.collection.name, deposit.id]))
-
-        requests = DepositRequest.objects.filter(
-            deposit=deposit, type=deposit_request_types['metadata'])
-
-        deposit_data = scheduling.aggregate(
-            deposit, deposit_archive_url, requests)
-
-        scheduling.schedule(deposit_data)
+    for deposits in utils.grouper(prepare_task_arguments(server), batch_size):
+        scheduling.schedule(deposits)
 
 
 if __name__ == '__main__':

swh/deposit/injection/tasks.py

@@ -20,8 +20,7 @@ class LoadDepositArchive(Task):
     """
     task_queue = 'swh_deposit_archive'
 
-    def run_task(self, *, deposit_archive_url, origin, visit_date,
-                 revision):
+    def run_task(self, *, archive_url, deposit_meta_url, deposit_update_url):
         """Import a deposit tarball into swh.
 
         Args: see :func:`DepositLoader.load`.
@@ -29,7 +28,6 @@ class LoadDepositArchive(Task):
         """
         loader = DepositLoader()
         loader.log = self.log
-        loader.load(deposit_archive_url=deposit_archive_url,
-                    origin=origin,
-                    visit_date=visit_date,
-                    revision=revision)
+        loader.load(archive_url=archive_url,
+                    deposit_meta_url=deposit_meta_url,
+                    deposit_update_url=deposit_update_url)

swh/deposit/models.py

@@ -89,6 +89,22 @@ def format_swh_id(collection_name, revision_id):
     return 'swh-%s-%s' % (collection_name, revision_id)
 
 
+def previous_revision_id(swh_id):
+    """Compute the parent's revision id (if any) from the swh_id.
+
+    Args:
+        swh_id (id): SWH Identifier from a previous deposit.
+
+    Returns:
+        None if no parent revision is detected.
+        The revision id's hash if any.
+
+    """
+    if swh_id:
+        return swh_id.split('-')[2]
+    return None
+
+
 class Deposit(models.Model):
     """Deposit reception table
 

swh/deposit/tests/api/test_deposit_read.py

+# Copyright (C) 2017  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import json
+
+from django.core.urlresolvers import reverse
+from rest_framework import status
+from rest_framework.test import APITestCase
+
+from swh.deposit.config import PRIVATE_GET_DEPOSIT_METADATA
+
+from ..common import BasicTestCase, WithAuthTestCase, CommonCreationRoutine
+
+
+class DepositReadMetadataTest(APITestCase, WithAuthTestCase, BasicTestCase,
+                              CommonCreationRoutine):
+    """Deposit access to read metadata information on deposit.
+
+    """
+    def test_access_to_an_existing_deposit_returns_metadata(self):
+        deposit_id = self.create_deposit_partial()
+
+        url = reverse(PRIVATE_GET_DEPOSIT_METADATA,
+                      args=[self.collection.name, deposit_id])
+
+        response = self.client.get(url)
+
+        self.assertEqual(response.status_code,
+                         status.HTTP_200_OK)
+        self.assertEquals(response._headers['content-type'][1],
+                          'application/json')
+        data = json.loads(response.content.decode('utf-8'))
+
+        expected_meta = {
+            'origin': {
+                'url': 'some-external-id',
+                'type': 'hal'
+            },
+            'revision': {
+                'synthetic': True,
+                'committer_date': None,
+                'message': ': Deposit %s in collection hal' % deposit_id,
+                'author': {
+                    'fullname': '', 'email': '', 'name': ''
+                },
+                'committer': {
+                    'fullname': '', 'email': '', 'name': ''
+                },
+                'date': None,
+                'metadata': {},
+                'type': 'tar'
+            },
+            'occurrence': {
+                'branch': 'master'
+            }
+        }
+
+        self.assertEquals(data, expected_meta)
+
+    def test_access_to_unexisting_collection_returns_404_response(self):
+        """Read unknown deposit should return a 404 response
+
+        """
+        unknown_id = '999'
+        url = reverse(PRIVATE_GET_DEPOSIT_METADATA,
+                      args=[self.collection.name, unknown_id])
+
+        response = self.client.get(url)
+        self.assertEqual(response.status_code,
+                         status.HTTP_404_NOT_FOUND)

version.txt

-v0.0.19-0-g662215d
\ No newline at end of file
+v0.0.20-0-g0df83b9
\ No newline at end of file