loader: Use metadata date as author/committer date if provided

Related T1637

swh/deposit/api/private/deposit_read.py

-# Copyright (C) 2017-2018 The Software Heritage developers
+# Copyright (C) 2017-2019 The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -9,6 +9,7 @@ import shutil
 import tempfile
 
 from contextlib import contextmanager
+from dateutil import parser
 from django.http import FileResponse
 from rest_framework import status
 
@@ -136,6 +137,45 @@ class SWHDepositReadMetadata(SWHGetDepositAPI, SWHPrivateAPIView,
                 if client_domain in metadata[field]:
                     return metadata[field]
 
+    def _prepare_date(self, date):
+        """Prepare date fields as normalized swh date
+
+        """
+        if isinstance(date, list):
+            date = date[0]
+        if isinstance(date, str):
+            date = parser.parse(date)
+
+        return identifiers.normalize_timestamp(date)
+
+    def _compute_date(self, deposit, metadata):
+        """Compute the date to use as a tuple of author date, committer date.
+        Each of those date are swh normalized immediately.
+
+        Args:
+            deposit (Deposit): Deposit model representation
+            metadata (Dict): Metadata dict representation
+
+        Returns:
+            Tuple of author date, committer date. Those dates are
+            swh normalized.
+
+        """
+        commit_date = metadata.get('codemeta:datePublished')
+        author_date = metadata.get('codemeta:dateCreated')
+
+        if author_date and commit_date:
+            t = (author_date, commit_date)
+        elif commit_date:
+            t = (commit_date, commit_date)
+        elif author_date:
+            t = (author_date, author_date)
+        else:
+            date = deposit.complete_date
+            t = (date, date)
+        return (
+            self._prepare_date(t[0]), self._prepare_date(t[1]))
+
     def metadata_read(self, deposit):
         """Read and aggregate multiple data on deposit into one unified data
            dictionary.
@@ -169,12 +209,13 @@ class SWHDepositReadMetadata(SWHGetDepositAPI, SWHPrivateAPIView,
         revision_type = 'tar'
         revision_msg = '%s: Deposit %s in collection %s' % (
             fullname, deposit.id, deposit.collection.name)
-        complete_date = identifiers.normalize_timestamp(deposit.complete_date)
+
+        author_date, commit_date = self._compute_date(deposit, metadata)
 
         data['revision'] = {
             'synthetic': True,
-            'date': complete_date,
-            'committer_date': complete_date,
+            'date': author_date,
+            'committer_date': commit_date,
             'author': author_committer,
             'committer': author_committer,
             'type': revision_type,

swh/deposit/tests/common.py

@@ -346,6 +346,7 @@ class CommonCreationRoutine(TestCase):
         <entry xmlns="http://www.w3.org/2005/Atom">
             <author>another one</author>
             <author>no one</author>
+            <codemeta:dateCreated>2017-10-07T15:17:08Z</codemeta:dateCreated>
         </entry>"""
 
         self.atom_entry_data2 = b"""<?xml version="1.0"?>

swh/deposit/tests/loader/test_loader.py

@@ -59,12 +59,10 @@ class DepositLoaderScenarioTest(APITestCase, WithAuthTestCase,
         # create the extraction dir used by the loader
         os.makedirs(TEST_LOADER_CONFIG['extraction_dir'], exist_ok=True)
 
-        # 1. create a deposit with archive and metadata
-        self.deposit_id = self.create_simple_binary_deposit()
-        # 2. Sets a basic client which accesses the test data
+        # Sets a basic client which accesses the test data
         loader_client = SWHDepositTestClient(self.client,
                                              config=CLIENT_TEST_CONFIG)
-        # 3. setup loader with that client
+        # Setup loader with that client
         self.loader = loader.DepositLoader(client=loader_client)
 
         self.storage = self.loader.storage
@@ -77,7 +75,11 @@ class DepositLoaderScenarioTest(APITestCase, WithAuthTestCase,
         """Load a deposit which is ready
 
         """
-        args = [self.collection.name, self.deposit_id]
+        # create a deposit with archive and metadata
+        deposit_id = self.create_simple_binary_deposit()
+        self.update_binary_deposit(deposit_id, status_partial=False)
+
+        args = [self.collection.name, deposit_id]
 
         archive_url = reverse(PRIVATE_GET_RAW_CONTENT, args=args)
         deposit_meta_url = reverse(PRIVATE_GET_DEPOSIT_METADATA, args=args)
@@ -100,9 +102,9 @@ class DepositLoaderScenarioTest(APITestCase, WithAuthTestCase,
         """Load a deposit with metadata, test metadata integrity
 
         """
-        self.deposit_metadata_id = self.add_metadata_to_deposit(
-            self.deposit_id)
-        args = [self.collection.name, self.deposit_metadata_id]
+        deposit_id = self.create_simple_binary_deposit()
+        self.add_metadata_to_deposit(deposit_id, status_partial=False)
+        args = [self.collection.name, deposit_id]
 
         archive_url = reverse(PRIVATE_GET_RAW_CONTENT, args=args)
         deposit_meta_url = reverse(PRIVATE_GET_DEPOSIT_METADATA, args=args)
@@ -157,7 +159,7 @@ class DepositLoaderScenarioTest(APITestCase, WithAuthTestCase,
         self.assertOriginMetadataContains('deposit', origin_url,
                                           expected_origin_metadata)
 
-        deposit = Deposit.objects.get(pk=self.deposit_id)
+        deposit = Deposit.objects.get(pk=deposit_id)
 
         self.assertRegex(deposit.swh_id, r'^swh:1:dir:.*')
         self.assertEqual(deposit.swh_id_context, '%s;origin=%s' % (