This project has retired. For details please refer to its Attic page.
TestOutOfCore xref
View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.giraph;
20  
21  import org.apache.giraph.conf.GiraphConfiguration;
22  import org.apache.giraph.conf.GiraphConstants;
23  import org.apache.giraph.examples.GeneratedVertexReader;
24  import org.apache.giraph.examples.SimplePageRankComputation;
25  import org.apache.giraph.examples.SimplePageRankComputation.SimplePageRankVertexInputFormat;
26  import org.apache.giraph.examples.SimplePageRankComputation.SimplePageRankVertexOutputFormat;
27  
28  import org.apache.giraph.job.GiraphJob;
29  import org.apache.giraph.ooc.OutOfCoreIOScheduler;
30  import org.apache.giraph.ooc.persistence.InMemoryDataAccessor;
31  import org.junit.Before;
32  import org.junit.Test;
33  
34  import java.io.IOException;
35  
36  import static org.junit.Assert.assertEquals;
37  import static org.junit.Assert.assertTrue;
38  
39  /**
40   * Unit test for out-of-core mechanism
41   */
42  public class TestOutOfCore extends BspCase {
43    private final static int NUM_PARTITIONS = 400;
44    private final static int NUM_PARTITIONS_IN_MEMORY = 8;
45    private GiraphConfiguration conf;
46  
47    public TestOutOfCore() {
48        super(TestOutOfCore.class.getName());
49    }
50  
51    @Before
52    public void prepareTest() {
53      conf = new GiraphConfiguration();
54      conf.setComputationClass(SimplePageRankComputation.class);
55      conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
56      conf.setVertexOutputFormatClass(SimplePageRankVertexOutputFormat.class);
57      conf.setWorkerContextClass(
58          SimplePageRankComputation.SimplePageRankWorkerContext.class);
59      conf.setMasterComputeClass(
60          SimplePageRankComputation.SimplePageRankMasterCompute.class);
61      GiraphConstants.USER_PARTITION_COUNT.set(conf, NUM_PARTITIONS);
62      GiraphConstants.USE_OUT_OF_CORE_GRAPH.set(conf, true);
63      GiraphConstants.MAX_PARTITIONS_IN_MEMORY.set(conf, NUM_PARTITIONS_IN_MEMORY);
64      OutOfCoreIOScheduler.OOC_WAIT_INTERVAL.set(conf, 10);
65      GiraphConstants.NUM_COMPUTE_THREADS.set(conf, 8);
66      GiraphConstants.NUM_INPUT_THREADS.set(conf, 8);
67      GiraphConstants.NUM_OUTPUT_THREADS.set(conf, 8);
68    }
69  
70    @Test
71    public void testOutOfCoreInMemoryAccessor()
72        throws IOException, InterruptedException, ClassNotFoundException {
73      GiraphConstants.OUT_OF_CORE_DATA_ACCESSOR.set(conf, InMemoryDataAccessor.class);
74      GiraphConstants.NUM_OUT_OF_CORE_THREADS.set(conf, 8);
75      runTest();
76    }
77  
78    @Test
79    public void testOutOfCoreLocalDiskAccessor()
80      throws IOException, InterruptedException, ClassNotFoundException {
81      GiraphConstants.PARTITIONS_DIRECTORY.set(conf, "disk0,disk1,disk2");
82      runTest();
83    }
84  
85    /**
86     * Run a job with fixed out-of-core policy and verify the result
87     *
88     * @throws IOException
89     * @throws InterruptedException
90     * @throws ClassNotFoundException
91     */
92    private void runTest()
93        throws IOException, InterruptedException, ClassNotFoundException {
94      GiraphJob job = prepareJob(getCallingMethodName(), conf,
95          getTempPath(getCallingMethodName()));
96      // Overwrite the number of vertices set in BspCase
97      GeneratedVertexReader.READER_VERTICES.set(conf, 200);
98      assertTrue(job.run(false));
99      if (!runningInDistributedMode()) {
100       double maxPageRank =
101           SimplePageRankComputation.SimplePageRankWorkerContext.getFinalMax();
102       double minPageRank =
103           SimplePageRankComputation.SimplePageRankWorkerContext.getFinalMin();
104       long numVertices =
105           SimplePageRankComputation.SimplePageRankWorkerContext.getFinalSum();
106       System.out.println(getCallingMethodName() + ": maxPageRank=" +
107           maxPageRank + " minPageRank=" +
108           minPageRank + " numVertices=" + numVertices);
109       assertEquals(13591.5, maxPageRank, 0.01);
110       assertEquals(9.375e-5, minPageRank, 0.000000001);
111       assertEquals(8 * 200L, numVertices);
112     }
113   }
114 }