LabelMapBuilder: mmap order file, use less RAM

java/src/main/java/org/softwareheritage/graph/maps/LabelMapBuilder.java

@@ -37,14 +37,15 @@ public class LabelMapBuilder {
     String debugPath;
     String tmpDir;
     ImmutableGraph graph;
+    long numNodes;
+    long numArcs;
 
-    Object2LongFunction<byte[]> swhIdMph;
-    long[][] orderMap;
+    NodeIdMap nodeIdMap;
     Object2LongFunction<byte[]> filenameMph;
     long numFilenames;
     int totalLabelWidth;
 
-    public LabelMapBuilder(String graphPath, String debugPath, String outputGraphPath, String tmpDir) {
+    public LabelMapBuilder(String graphPath, String debugPath, String outputGraphPath, String tmpDir) throws IOException {
         this.graphPath = graphPath;
         if (outputGraphPath == null) {
             this.outputGraphPath = graphPath;
@@ -53,6 +54,20 @@ public class LabelMapBuilder {
         }
         this.debugPath = debugPath;
         this.tmpDir = tmpDir;
+
+        // Load the graph in offline mode to retrieve the number of nodes/edges,
+        // then immediately destroy it. XXX: not even needed?
+        // ImmutableGraph graphOffline = BVGraph.loadMapped(graphPath);
+
+        graph = BVGraph.loadMapped(graphPath);
+        numArcs = graph.numArcs();
+        numNodes = graph.numNodes();
+
+        nodeIdMap = new NodeIdMap(graphPath, numNodes);
+
+        filenameMph = NodeIdMap.loadMph(graphPath + "-labels.mph");
+        numFilenames = getMPHSize(filenameMph);
+        totalLabelWidth = DirEntry.labelWidth(numFilenames);
     }
 
     private static JSAPResult parse_args(String[] args) {
@@ -89,16 +104,19 @@ public class LabelMapBuilder {
         LabelMapBuilder builder = new LabelMapBuilder(graphPath, debugPath, outputGraphPath, tmpDir);
 
         logger.info("Loading graph and MPH functions...");
-        builder.loadGraph();
-
-        builder.computeLabelMapSort();
-        // builder.computeLabelMapBsort();
+        builder.computeLabelMap();
     }
 
     static long getMPHSize(Object2LongFunction<byte[]> mph) {
         return (mph instanceof Size64) ? ((Size64) mph).size64() : mph.size();
     }
 
+    void computeLabelMap() throws IOException, InterruptedException {
+        this.loadGraph();
+        // this.computeLabelMapSort();
+        this.computeLabelMapBsort();
+    }
+
     void computeLabelMapSort() throws IOException {
         // Pass the intermediate representation to sort(1) so that we see the labels in the order they will
         // appear in the label file.
@@ -165,16 +183,6 @@ public class LabelMapBuilder {
     }
 
     void loadGraph() throws IOException {
-        graph = BVGraph.loadMapped(graphPath);
-
-        swhIdMph = NodeIdMap.loadMph(graphPath + ".mph");
-
-        orderMap = LongBigArrays.newBigArray(getMPHSize(swhIdMph));
-        BinIO.loadLongs(graphPath + ".order", orderMap);
-
-        filenameMph = NodeIdMap.loadMph(graphPath + "-labels.mph");
-        numFilenames = getMPHSize(filenameMph);
-        totalLabelWidth = DirEntry.labelWidth(numFilenames);
     }
 
     void hashLabelStream(FastBufferedInputStream input, EdgeLabelLineWriter writer) throws IOException {
@@ -182,7 +190,7 @@ public class LabelMapBuilder {
         // "<src node id> <dst node id> <label ids>\n"
         ProgressLogger plInter = new ProgressLogger(logger, 10, TimeUnit.SECONDS);
         plInter.itemsName = "edges";
-        plInter.expectedUpdates = graph.numArcs();
+        plInter.expectedUpdates = this.numArcs;
         plInter.start("Hashing the label stream");
 
         var charset = StandardCharsets.US_ASCII;
@@ -254,13 +262,10 @@ public class LabelMapBuilder {
 
             // System.err.format("DEBUG: read %s %s %s %d\n", ss, ts, ls, permission);
 
-            long s = swhIdMph.getLong(ss);
-            long t = swhIdMph.getLong(ts);
+            long srcNode = nodeIdMap.getNodeId(ss);
+            long dstNode = nodeIdMap.getNodeId(ts);
             long filenameId = filenameMph.getLong(ls);
 
-            long srcNode = BigArrays.get(orderMap, s);
-            long dstNode = BigArrays.get(orderMap, t);
-
             writer.writeLine(srcNode, dstNode, filenameId, permission);
             plInter.lightUpdate();
         }
@@ -271,7 +276,7 @@ public class LabelMapBuilder {
         // Get the sorted output and write the labels and label offsets
         ProgressLogger plLabels = new ProgressLogger(logger, 10, TimeUnit.SECONDS);
         plLabels.itemsName = "edges";
-        plLabels.expectedUpdates = graph.numArcs();
+        plLabels.expectedUpdates = this.numArcs;
         plLabels.start("Writing the labels to the label file.");
 
         FileWriter debugFile = null;

java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java

@@ -118,6 +118,21 @@ public class NodeIdMap {
         return res;
     }
 
+    /**
+     * Converts byte-form SWHID to corresponding long node id.
+     * Low-level function, does not check if the SWHID is valid.
+     *
+     * @param swhid node represented as bytes
+     * @return corresponding node as a long id
+     */
+    public long getNodeId(byte[] swhid) {
+        // 1. Hash the SWHID with the MPH to get its original ID
+        long origNodeId = mph.getLong(swhid);
+
+        // 2. Use the order permutation to get the position in the permuted graph
+        return this.orderMap.getLong(origNodeId);
+    }
+
     /**
      * Converts SWHID to corresponding long node id.
      *
@@ -128,13 +143,10 @@ public class NodeIdMap {
      * @see SWHID
      */
     public long getNodeId(SWHID swhid, boolean checkExists) {
-        // 1. Hash the SWHID with the MPH to get its original ID
-        long origNodeId = mph.getLong(swhid.toString().getBytes(StandardCharsets.US_ASCII));
-
-        // 2. Use the order permutation to get the position in the permuted graph
-        long nodeId = this.orderMap.getLong(origNodeId);
+        // Convert the SWHID to bytes and call getNodeId()
+        long nodeId = getNodeId(swhid.toString().getBytes(StandardCharsets.US_ASCII));
 
-        // 3. Check that the position effectively corresponds to a real node using the reverse map.
+        // Check that the position effectively corresponds to a real node using the reverse map.
         // This is necessary because the MPH makes no guarantees on whether the input SWHID is valid.
         if (!checkExists || getSWHID(nodeId).equals(swhid)) {
             return nodeId;