Add swh-datasets

docs/devel/api-reference.rst

@@ -18,6 +18,7 @@ by components:
    swh.auth <swh-auth/index>
    swh.core <swh-core/index>
    swh.counters <swh-counters/index>
+   swh.datasets <swh-datasets/index>
    swh.deposit <swh-deposit/index>
    swh.export <swh-export/index>
    swh.fuse <swh-fuse/index>

docs/devel/index.rst

@@ -107,6 +107,9 @@ of the corresponding Python module.
     service providing efficient estimates of the number of objects in the SWH archive,
     using Redis's Hyperloglog
 
+:ref:`swh.datasets <swh-datasets>`
+    datasets derived from periodic data dumps created by swh.export
+
 :ref:`swh.deposit <swh-deposit>`
     push-based deposit of software artifacts to the archive
 

requirements-swh-dev.txt

@@ -7,10 +7,11 @@
 ../swh-auth[django,starlette]
 ../swh-core[http,db,logging]
 ../swh-counters
+../swh-datasets[luigi]
 ../swh-deposit[server]
 ../swh-export[luigi]
 ../swh-fuse
-../swh-graph[luigi]
+../swh-graph[export,luigi]
 ../swh-graphql
 ../swh-icinga-plugins
 ../swh-indexer

requirements-swh.txt

@@ -3,6 +3,7 @@ swh.alter
 swh.auth[django,starlette]
 swh.core[db,http,logging]
 swh.counters
+swh.datasets[luigi]
 swh.deposit[server]
 swh.export[luigi]
 swh.fuse