This project has retired. For details please refer to its
Attic page.
TestOutOfCore xref
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.giraph;
20
21 import org.apache.giraph.conf.GiraphConfiguration;
22 import org.apache.giraph.conf.GiraphConstants;
23 import org.apache.giraph.examples.GeneratedVertexReader;
24 import org.apache.giraph.examples.SimplePageRankComputation;
25 import org.apache.giraph.examples.SimplePageRankComputation.SimplePageRankVertexInputFormat;
26 import org.apache.giraph.examples.SimplePageRankComputation.SimplePageRankVertexOutputFormat;
27
28 import org.apache.giraph.job.GiraphJob;
29 import org.apache.giraph.ooc.OutOfCoreIOScheduler;
30 import org.apache.giraph.ooc.persistence.InMemoryDataAccessor;
31 import org.junit.Before;
32 import org.junit.Test;
33
34 import java.io.IOException;
35
36 import static org.junit.Assert.assertEquals;
37 import static org.junit.Assert.assertTrue;
38
39
40
41
42 public class TestOutOfCore extends BspCase {
43 private final static int NUM_PARTITIONS = 400;
44 private final static int NUM_PARTITIONS_IN_MEMORY = 8;
45 private GiraphConfiguration conf;
46
47 public TestOutOfCore() {
48 super(TestOutOfCore.class.getName());
49 }
50
51 @Before
52 public void prepareTest() {
53 conf = new GiraphConfiguration();
54 conf.setComputationClass(SimplePageRankComputation.class);
55 conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
56 conf.setVertexOutputFormatClass(SimplePageRankVertexOutputFormat.class);
57 conf.setWorkerContextClass(
58 SimplePageRankComputation.SimplePageRankWorkerContext.class);
59 conf.setMasterComputeClass(
60 SimplePageRankComputation.SimplePageRankMasterCompute.class);
61 GiraphConstants.USER_PARTITION_COUNT.set(conf, NUM_PARTITIONS);
62 GiraphConstants.USE_OUT_OF_CORE_GRAPH.set(conf, true);
63 GiraphConstants.MAX_PARTITIONS_IN_MEMORY.set(conf, NUM_PARTITIONS_IN_MEMORY);
64 OutOfCoreIOScheduler.OOC_WAIT_INTERVAL.set(conf, 10);
65 GiraphConstants.NUM_COMPUTE_THREADS.set(conf, 8);
66 GiraphConstants.NUM_INPUT_THREADS.set(conf, 8);
67 GiraphConstants.NUM_OUTPUT_THREADS.set(conf, 8);
68 }
69
70 @Test
71 public void testOutOfCoreInMemoryAccessor()
72 throws IOException, InterruptedException, ClassNotFoundException {
73 GiraphConstants.OUT_OF_CORE_DATA_ACCESSOR.set(conf, InMemoryDataAccessor.class);
74 GiraphConstants.NUM_OUT_OF_CORE_THREADS.set(conf, 8);
75 runTest();
76 }
77
78 @Test
79 public void testOutOfCoreLocalDiskAccessor()
80 throws IOException, InterruptedException, ClassNotFoundException {
81 GiraphConstants.PARTITIONS_DIRECTORY.set(conf, "disk0,disk1,disk2");
82 runTest();
83 }
84
85
86
87
88
89
90
91
92 private void runTest()
93 throws IOException, InterruptedException, ClassNotFoundException {
94 GiraphJob job = prepareJob(getCallingMethodName(), conf,
95 getTempPath(getCallingMethodName()));
96
97 GeneratedVertexReader.READER_VERTICES.set(conf, 200);
98 assertTrue(job.run(false));
99 if (!runningInDistributedMode()) {
100 double maxPageRank =
101 SimplePageRankComputation.SimplePageRankWorkerContext.getFinalMax();
102 double minPageRank =
103 SimplePageRankComputation.SimplePageRankWorkerContext.getFinalMin();
104 long numVertices =
105 SimplePageRankComputation.SimplePageRankWorkerContext.getFinalSum();
106 System.out.println(getCallingMethodName() + ": maxPageRank=" +
107 maxPageRank + " minPageRank=" +
108 minPageRank + " numVertices=" + numVertices);
109 assertEquals(13591.5, maxPageRank, 0.01);
110 assertEquals(9.375e-5, minPageRank, 0.000000001);
111 assertEquals(8 * 200L, numVertices);
112 }
113 }
114 }