Skip to content
Snippets Groups Projects
Commit 4f978074 authored by vlorentz's avatar vlorentz Committed by vlorentz
Browse files

PopularContents: Make a single copy of the graph per thread

Copying it 1k or 10k for each thread is wasteful, even though they are
lightweight copies.

This brings the runtime down from 133 to ~100 hours.
parent 482f91e2
No related branches found
No related tags found
No related merge requests found
...@@ -37,6 +37,10 @@ import org.slf4j.LoggerFactory; ...@@ -37,6 +37,10 @@ import org.slf4j.LoggerFactory;
public class PopularContents { public class PopularContents {
private SwhBidirectionalGraph graph; private SwhBidirectionalGraph graph;
/*
* A copy of the graph for each thread to reuse between calls to processChunk
*/
private ThreadLocal<SwhBidirectionalGraph> threadGraph;
private int NUM_THREADS = 96; private int NUM_THREADS = 96;
final static Logger logger = LoggerFactory.getLogger(PopularContents.class); final static Logger logger = LoggerFactory.getLogger(PopularContents.class);
...@@ -52,6 +56,7 @@ public class PopularContents { ...@@ -52,6 +56,7 @@ public class PopularContents {
long popularityThreshold = Long.parseLong(args[2]); long popularityThreshold = Long.parseLong(args[2]);
PopularContents popular_contents = new PopularContents(); PopularContents popular_contents = new PopularContents();
popular_contents.threadGraph = new ThreadLocal<SwhBidirectionalGraph>();
popular_contents.loadGraph(graphPath); popular_contents.loadGraph(graphPath);
...@@ -70,7 +75,7 @@ public class PopularContents { ...@@ -70,7 +75,7 @@ public class PopularContents {
System.out.format("SWHID,length,filename,occurrences\n"); System.out.format("SWHID,length,filename,occurrences\n");
long totalNodes = graph.numNodes(); long totalNodes = graph.numNodes();
long numChunks = NUM_THREADS * 10000; long numChunks = NUM_THREADS * 1000;
ProgressLogger pl = new ProgressLogger(logger); ProgressLogger pl = new ProgressLogger(logger);
pl.itemsName = "nodes"; pl.itemsName = "nodes";
...@@ -94,9 +99,13 @@ public class PopularContents { ...@@ -94,9 +99,13 @@ public class PopularContents {
private void processChunk(long numChunks, long chunkId, int maxResults, long popularityThreshold, private void processChunk(long numChunks, long chunkId, int maxResults, long popularityThreshold,
ProgressLogger pl) { ProgressLogger pl) {
if (threadGraph.get() == null) {
threadGraph.set(this.graph.copy());
}
SwhBidirectionalGraph graph = threadGraph.get();
long totalNodes = graph.numNodes(); long totalNodes = graph.numNodes();
HashMap<Long, Long> names = new HashMap<>(); HashMap<Long, Long> names = new HashMap<>();
SwhUnidirectionalGraph backwardGraph = graph.getBackwardGraph().copy(); SwhUnidirectionalGraph backwardGraph = graph.getBackwardGraph();
long chunkSize = totalNodes / numChunks; long chunkSize = totalNodes / numChunks;
long chunkStart = chunkSize * chunkId; long chunkStart = chunkSize * chunkId;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment