Generate snapshots with a PyPI-specific namespace convention

Summary:
After the conversation in D424, we've decided to move towards a
'releases/<version>' namespace for snapshots generated by the PyPI loader.

Default version pointers now use a branch alias instead of an explicit pointer.

Test Plan: Updated integration tests

Reviewers: zack

Differential Revision: https://forge.softwareheritage.org/D429

swh/loader/pypi/loader.py

@@ -242,19 +242,22 @@ class PyPILoader(SWHLoader):
             default_release = self.project.default_release()
             if len(artifacts) == 1:
                 # Only one artifact for this release, generate a single branch
-                branch_name = 'refs/tags/%s' % release
+                branch_name = 'releases/%s' % release
                 filename, sha256 = artifacts[0]
                 target = self.target_from_artifact(filename, sha256)
                 branches[branch_name.encode('utf-8')] = target
                 if release == default_release:
-                    branches[b'HEAD'] = target
+                    branches[b'HEAD'] = {
+                        'target_type': 'alias',
+                        'target': branch_name.encode('utf-8'),
+                    }
                 if not target:
                     self._visit_status = 'partial'
             else:
                 # Several artifacts for this release, generate a separate
                 # pointer for each of them
                 for filename, sha256 in artifacts:
-                    branch_name = 'refs/tags/%s/%s' % (release, filename)
+                    branch_name = 'releases/%s/%s' % (release, filename)
                     target = self.target_from_artifact(filename, sha256)
                     branches[branch_name.encode('utf-8')] = target
                     if not target:

swh/loader/pypi/tests/test_loader.py

@@ -95,12 +95,24 @@ class BaseLoaderITest(TestCase):
         snap_id = hashutil.hash_to_hex(snap['id'])
         self.assertEqual(snap_id, expected_snapshot)
 
+        def decode_target(target):
+            if not target:
+                return target
+            target_type = target['target_type']
+
+            if target_type == 'alias':
+                decoded_target = target['target'].decode('utf-8')
+            else:
+                decoded_target = hashutil.hash_to_hex(target['target'])
+
+            return {
+                'target': decoded_target,
+                'target_type': target_type
+            }
+
         branches = {
-            branch.decode('utf-8'): {
-                'target': hashutil.hash_to_hex(t['target']),
-                'target_type': t['target_type'],
-            } if t else t
-            for branch, t in snap['branches'].items()
+            branch.decode('utf-8'): decode_target(target)
+            for branch, target in snap['branches'].items()
         }
         self.assertEqual(expected_branches, branches)
 
@@ -189,21 +201,21 @@ class LoaderITest(BaseLoaderITest):
         self.assertRevisionsOk(expected_revisions)
 
         expected_branches = {
-            'refs/tags/1.1.0': {
+            'releases/1.1.0': {
                 'target': '4c99891f93b81450385777235a37b5e966dd1571',
                 'target_type': 'revision',
             },
-            'refs/tags/1.2.0': {
+            'releases/1.2.0': {
                 'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
                 'target_type': 'revision',
             },
             'HEAD': {
-                'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
-                'target_type': 'revision',
+                'target': 'releases/1.2.0',
+                'target_type': 'alias',
             },
         }
 
-        self.assertSnapshotOk('5c99339508de5eea67eecd3e0ba1efa8e804b96e',
+        self.assertSnapshotOk('ba6e158ada75d0b3cfb209ffdf6daa4ed34a227a',
                               expected_branches)
 
         self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
@@ -218,24 +230,22 @@ class PyPILoaderWithSnapshot(TestPyPILoader):
     def _last_snapshot(self):
         """Return last visited snapshot"""
         return {
-            'id': b'\\\x993\x95\x08\xde^\xeag\xee\xcd>\x0b\xa1\xef\xa8\xe8\x04\xb9n',  # noqa
+            'id': b'\xban\x15\x8a\xdau\xd0\xb3\xcf\xb2\t\xff\xdfm\xaaN\xd3J"z',  # noqa
             'branches': {
-                b'refs/tags/1.1.0': {
+                b'releases/1.1.0': {
                     'target': b'L\x99\x89\x1f\x93\xb8\x14P'
                     b'8Ww#Z7\xb5\xe9f\xdd\x15q',
                     'target_type': 'revision'
                 },
-                b'refs/tags/1.2.0': {
+                b'releases/1.2.0': {
                     'target': b'\xe4E\xdaM\xa2+1\xbf'
                     b'\xeb\xb6\xff\xc48=\xbf\x83'
                     b'\x9a\x07M!',
                     'target_type': 'revision'
                 },
                 b'HEAD': {
-                    'target': b'\xe4E\xdaM\xa2+1\xbf'
-                    b'\xeb\xb6\xff\xc48=\xbf\x83'
-                    b'\x9a\x07M!',
-                    'target_type': 'revision'
+                    'target': b'releases/1.2.0',
+                    'target_type': 'alias'
                 },
             },
         }
@@ -286,19 +296,19 @@ class LoaderNoNewChangesSinceLastVisitITest(BaseLoaderITest):
         self.assertDirectoriesOk([])
         self.assertRevisionsOk(expected_revisions={})
 
-        expected_snapshot_id = '5c99339508de5eea67eecd3e0ba1efa8e804b96e'
+        expected_snapshot_id = 'ba6e158ada75d0b3cfb209ffdf6daa4ed34a227a'
         expected_branches = {
-            'refs/tags/1.1.0': {
+            'releases/1.1.0': {
                 'target': '4c99891f93b81450385777235a37b5e966dd1571',
                 'target_type': 'revision',
             },
-            'refs/tags/1.2.0': {
+            'releases/1.2.0': {
                 'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
                 'target_type': 'revision',
             },
             'HEAD': {
-                'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
-                'target_type': 'revision',
+                'target': 'releases/1.2.0',
+                'target_type': 'alias',
             },
         }
         self.assertSnapshotOk(expected_snapshot_id, expected_branches)
@@ -378,23 +388,23 @@ class LoaderNewChangesSinceLastVisitITest(BaseLoaderITest):
         for rev, dir_id in old_revisions.items():
             expected_revisions[rev] = dir_id
 
-        expected_snapshot_id = 'c1500c9fae5c275f5a772105c36884dd01bc26e7'
+        expected_snapshot_id = '07322209e51618410b5e43ca4af7e04fe5113c9d'
         expected_branches = {
-            'refs/tags/1.1.0': {
+            'releases/1.1.0': {
                 'target': '4c99891f93b81450385777235a37b5e966dd1571',
                 'target_type': 'revision',
             },
-            'refs/tags/1.2.0': {
+            'releases/1.2.0': {
                 'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
                 'target_type': 'revision',
             },
-            'refs/tags/1.3.0': {
+            'releases/1.3.0': {
                 'target': 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d',
                 'target_type': 'revision',
             },
             'HEAD': {
-                'target': 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d',
-                'target_type': 'revision',
+                'target': 'releases/1.3.0',
+                'target_type': 'alias',
             },
         }
 
@@ -415,23 +425,23 @@ class PyPILoaderWithSnapshot2(TestPyPILoader):
     def _last_snapshot(self):
         """Return last visited snapshot"""
         return {
-            'id': b"\xc1P\x0c\x9f\xae\\'_Zw!\x05\xc3h\x84\xdd\x01\xbc&\xe7",  # noqa
+            'id': b'\x072"\t\xe5\x16\x18A\x0b^C\xcaJ\xf7\xe0O\xe5\x11<\x9d',  # noqa
             'branches': {
-                b'refs/tags/1.1.0': {
+                b'releases/1.1.0': {
                     'target': b'L\x99\x89\x1f\x93\xb8\x14P8Ww#Z7\xb5\xe9f\xdd\x15q',  # noqa
                     'target_type': 'revision'
                 },
-                b'refs/tags/1.2.0': {
+                b'releases/1.2.0': {
                     'target': b'\xe4E\xdaM\xa2+1\xbf\xeb\xb6\xff\xc48=\xbf\x83\x9a\x07M!',  # noqa
                     'target_type': 'revision'
                 },
-                b'refs/tags/1.3.0': {
+                b'releases/1.3.0': {
                     'target': b'\xfbF\xe4\x96\x05\xb0\xbb\xe6\x9f\x8cS\xd3\x15\xe8\x93p\xe7\xc6\xcb]',  # noqa
                     'target_type': 'revision'
                 },
                 b'HEAD': {
-                    'target': b'\xfbF\xe4\x96\x05\xb0\xbb\xe6\x9f\x8cS\xd3\x15\xe8\x93p\xe7\xc6\xcb]',  # noqa
-                    'target_type': 'revision'
+                    'target': b'releases/1.3.0',  # noqa
+                    'target_type': 'alias'
                 },
             }
         }
@@ -518,23 +528,23 @@ class LoaderChangesOldReleaseArtifactRemovedSinceLastVisit(BaseLoaderITest):
         }
         self.assertRevisionsOk(expected_revisions)
 
-        expected_snapshot_id = 'ddc940677eda1ca270c8b7e8d7796f8f98428522'
+        expected_snapshot_id = 'bb0b0c29040678eadb6dae9e43e496cc860123e4'
         expected_branches = {
-            'refs/tags/1.2.0': {
+            'releases/1.2.0': {
                 'target': 'e445da4da22b31bfebb6ffc4383dbf839a074d21',
                 'target_type': 'revision',
             },
-            'refs/tags/1.3.0': {
+            'releases/1.3.0': {
                 'target': 'fb46e49605b0bbe69f8c53d315e89370e7c6cb5d',
                 'target_type': 'revision',
             },
-            'refs/tags/1.4.0': {
+            'releases/1.4.0': {
                 'target': '5e91875f096ac48c98d74acf307439a3490f2827',
                 'target_type': 'revision',
             },
             'HEAD': {
-                'target': '5e91875f096ac48c98d74acf307439a3490f2827',
-                'target_type': 'revision',
+                'target': 'releases/1.4.0',
+                'target_type': 'alias',
             },
         }
         self.assertSnapshotOk(expected_snapshot_id, expected_branches)

[Phabricator Migration user]

Some references in the commit message have been migrated:

• D424 is now !10 (closed)
• D429 is now !15 (closed)