setup: extract db and http related parts to dedicated optional 'extras'

Note that for now, the main requirements.txt still contains all the
dependencies for backward compatibility reasons.
This will have to be fixed once all packages that depends on swh.core
have been updated.

closes T1678.

requirements-db.txt

+# requirements for swh.core.db
+psycopg2

requirements-http.txt

+# requirements for swh.core.api
+aiohttp
+arrow
+decorator
+Flask
+msgpack > 0.5
+python-dateutil
+requests

requirements.txt

+PyYAML
+systemd-python
+
+# these deps below are now handled in dedicated 'extras' and should be removed
+# from this main requirement file ASAP
 arrow
 aiohttp
 msgpack > 0.5
 psycopg2
 python-dateutil
-vcversioner
-PyYAML
 requests
 Flask
-systemd-python
 decorator

setup.py

@@ -17,22 +17,23 @@ with open(path.join(here, 'README.md'), encoding='utf-8') as f:
     long_description = f.read()
 
 
-def parse_requirements(name=None):
-    if name:
-        reqf = 'requirements-%s.txt' % name
-    else:
-        reqf = 'requirements.txt'
-
+def parse_requirements(*names):
     requirements = []
-    if not os.path.exists(reqf):
-        return requirements
+    for name in names:
+        if name:
+            reqf = 'requirements-%s.txt' % name
+        else:
+            reqf = 'requirements.txt'
+
+        if not os.path.exists(reqf):
+            return requirements
 
-    with open(reqf) as f:
-        for line in f.readlines():
-            line = line.strip()
-            if not line or line.startswith('#'):
-                continue
-            requirements.append(line)
+        with open(reqf) as f:
+            for line in f.readlines():
+                line = line.strip()
+                if not line or line.startswith('#'):
+                    continue
+                requirements.append(line)
     return requirements
 
 
@@ -46,9 +47,13 @@ setup(
     url='https://forge.softwareheritage.org/diffusion/DCORE/',
     packages=find_packages(),
     scripts=[],
-    install_requires=parse_requirements() + parse_requirements('swh'),
+    install_requires=parse_requirements(None, 'swh'),
     setup_requires=['vcversioner'],
-    extras_require={'testing': parse_requirements('test')},
+    extras_require={
+        'testing': parse_requirements('test', 'db', 'http'),
+        'db': parse_requirements('db'),
+        'http': parse_requirements('http'),
+    },
     vcversioner={},
     include_package_data=True,
     entry_points='''

[Phabricator Migration user]

Some references in the commit message have been migrated:

• T1678 is now #1678 (closed)