1/*2 * Licensed to the Apache Software Foundation (ASF) under one3 * or more contributor license agreements. See the NOTICE file4 * distributed with this work for additional information5 * regarding copyright ownership. The ASF licenses this file6 * to you under the Apache License, Version 2.0 (the7 * "License"); you may not use this file except in compliance8 * with the License. You may obtain a copy of the License at9 *10 * http://www.apache.org/licenses/LICENSE-2.011 *12 * Unless required by applicable law or agreed to in writing, software13 * distributed under the License is distributed on an "AS IS" BASIS,14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.15 * See the License for the specific language governing permissions and16 * limitations under the License.17 */1819package org.apache.giraph.io.formats.multi;
2021import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
22import org.apache.giraph.io.EdgeInputFormat;
23import org.apache.giraph.io.EdgeReader;
24import org.apache.giraph.io.internal.WrappedEdgeReader;
25import org.apache.hadoop.conf.Configuration;
26import org.apache.hadoop.io.Writable;
27import org.apache.hadoop.io.WritableComparable;
28import org.apache.hadoop.mapreduce.InputSplit;
29import org.apache.hadoop.mapreduce.JobContext;
30import org.apache.hadoop.mapreduce.TaskAttemptContext;
3132import java.io.DataInput;
33import java.io.DataOutput;
34import java.io.IOException;
35import java.util.List;
3637/**38 * Edge input format which wraps several edge input formats.39 * Provides the way to read data from multiple sources,40 * using several different input formats.41 *42 * @param <I> Vertex id43 * @param <E> Edge data44 */45publicclass MultiEdgeInputFormat<I extends WritableComparable,
46 E extends Writable> extends EdgeInputFormat<I, E> {
47/** Edge input formats */48private List<EdgeInputFormat<I, E>> edgeInputFormats;
4950 @Override publicvoid checkInputSpecs(Configuration conf) {
51for (EdgeInputFormat edgeInputFormat : edgeInputFormats) {
52 edgeInputFormat.checkInputSpecs(conf);
53 }
54 }
5556 @Override
57publicvoid setConf(
58 ImmutableClassesGiraphConfiguration<I, Writable, E> conf) {
59super.setConf(conf);
60 edgeInputFormats =
61 EdgeInputFormatDescription.createEdgeInputFormats(getConf());
62if (edgeInputFormats.isEmpty()) {
63thrownew IllegalStateException("setConf: Using MultiEdgeInputFormat " +
64"without specifying edge inputs");
65 }
66 }
6768 @Override
69public EdgeReader<I, E> createEdgeReader(InputSplit inputSplit,
70 TaskAttemptContext context) throws IOException {
71if (inputSplit instanceof InputSplitWithInputFormatIndex) {
72// When multithreaded input is used we need to make sure other threads73// don't change context's configuration while we use it74synchronized (context) {
75InputSplitWithInputFormatIndex split =
76 (InputSplitWithInputFormatIndex) inputSplit;
77 EdgeInputFormat<I, E> edgeInputFormat =
78 edgeInputFormats.get(split.getInputFormatIndex());
79 EdgeReader<I, E> edgeReader =
80 edgeInputFormat.createEdgeReader(split.getSplit(), context);
81returnnew WrappedEdgeReader<I, E>(
82 edgeReader, edgeInputFormat.getConf()) {
83 @Override
84publicvoid initialize(InputSplit inputSplit,
85 TaskAttemptContext context) throws IOException,
86 InterruptedException {
87// When multithreaded input is used we need to make sure other88// threads don't change context's configuration while we use it89synchronized (context) {
90super.initialize(inputSplit, context);
91 }
92 }
93 };
94 }
95 } else {
96thrownew IllegalStateException("createEdgeReader: Got InputSplit which" +
97" was not created by this class: " + inputSplit.getClass().getName());
98 }
99 }
100101 @Override
102public List<InputSplit> getSplits(JobContext context,
103int minSplitCountHint) throws IOException, InterruptedException {
104// When multithreaded input is used we need to make sure other threads don't105// change context's configuration while we use it106synchronized (context) {
107return MultiInputUtils.getSplits(
108 context, minSplitCountHint, edgeInputFormats);
109 }
110 }
111112 @Override
113publicvoid writeInputSplit(InputSplit inputSplit,
114 DataOutput dataOutput) throws IOException {
115 MultiInputUtils.writeInputSplit(inputSplit, dataOutput, edgeInputFormats);
116 }
117118 @Override
119public InputSplit readInputSplit(
120 DataInput dataInput) throws IOException, ClassNotFoundException {
121return MultiInputUtils.readInputSplit(dataInput, edgeInputFormats);
122 }
123 }