1/*2 * Licensed to the Apache Software Foundation (ASF) under one3 * or more contributor license agreements. See the NOTICE file4 * distributed with this work for additional information5 * regarding copyright ownership. The ASF licenses this file6 * to you under the Apache License, Version 2.0 (the7 * "License"); you may not use this file except in compliance8 * with the License. You may obtain a copy of the License at9 *10 * http://www.apache.org/licenses/LICENSE-2.011 *12 * Unless required by applicable law or agreed to in writing, software13 * distributed under the License is distributed on an "AS IS" BASIS,14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.15 * See the License for the specific language governing permissions and16 * limitations under the License.17 */1819package org.apache.giraph.io.formats.multi;
2021import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
22import org.apache.giraph.io.MappingInputFormat;
23import org.apache.giraph.io.MappingReader;
24import org.apache.giraph.io.internal.WrappedMappingReader;
25import org.apache.hadoop.conf.Configuration;
26import org.apache.hadoop.io.Writable;
27import org.apache.hadoop.io.WritableComparable;
28import org.apache.hadoop.mapreduce.InputSplit;
29import org.apache.hadoop.mapreduce.JobContext;
30import org.apache.hadoop.mapreduce.TaskAttemptContext;
3132import java.io.DataInput;
33import java.io.DataOutput;
34import java.io.IOException;
35import java.util.List;
3637/**38 * Mapping input format which wraps several mapping input formats.39 * Provides the way to read data from multiple sources,40 * using several different input formats.41 *42 * @param <I> Vertex id43 * @param <V> Vertex value44 * @param <E> Edge data45 * @param <B> Mapping target46 */47publicclass MultiMappingInputFormat<I extends WritableComparable,
48 V extends Writable, E extends Writable, B extends Writable>
49extends MappingInputFormat<I, V, E, B> {
5051/** Mapping input formats */52private List<MappingInputFormat<I, V, E, B>> mappingInputFormats;
5354/**55 * Default constructor.56 */57publicMultiMappingInputFormat() {
58 }
5960 @Override
61publicvoid setConf(
62 ImmutableClassesGiraphConfiguration<I, V, E> conf) {
63super.setConf(conf);
64 mappingInputFormats =
65 MappingInputFormatDescription.createMappingInputFormats(getConf());
66if (mappingInputFormats.isEmpty()) {
67thrownew IllegalStateException("setConf: Using MultiVertexInputFormat " +
68"without specifying vertex inputs");
69 }
70 }
7172 @Override
73publicMappingReader createMappingReader(
74 InputSplit inputSplit, TaskAttemptContext context
75 ) throws IOException {
76if (inputSplit instanceof InputSplitWithInputFormatIndex) {
77// When multithreaded input is used we need to make sure other threads78// don't change context's configuration while we use it79synchronized (context) {
80InputSplitWithInputFormatIndex split =
81 (InputSplitWithInputFormatIndex) inputSplit;
82 MappingInputFormat<I, V, E, B> mappingInputFormat =
83 mappingInputFormats.get(split.getInputFormatIndex());
84 MappingReader<I, V, E, B> mappingReader =
85 mappingInputFormat.createMappingReader(split.getSplit(), context);
86returnnew WrappedMappingReader<I, V, E, B>(
87 mappingReader, mappingInputFormat.getConf()) {
88 @Override
89publicvoid initialize(InputSplit inputSplit,
90 TaskAttemptContext context) throws IOException,
91 InterruptedException {
92// When multithreaded input is used we need to make sure other93// threads don't change context's configuration while we use it94synchronized (context) {
95super.initialize(inputSplit, context);
96 }
97 }
98 };
99 }
100 } else {
101thrownew IllegalStateException("createVertexReader: Got InputSplit " +
102"which was not created by this class: " +
103 inputSplit.getClass().getName());
104 }
105 }
106107 @Override
108publicvoid checkInputSpecs(Configuration conf) {
109for (MappingInputFormat mappingInputFormat : mappingInputFormats) {
110 mappingInputFormat.checkInputSpecs(conf);
111 }
112 }
113114 @Override
115public List<InputSplit> getSplits(
116 JobContext context, int minSplitCountHint
117 ) throws IOException, InterruptedException {
118synchronized (context) {
119return MultiInputUtils.getSplits(
120 context, minSplitCountHint, mappingInputFormats);
121 }
122 }
123124 @Override
125publicvoid writeInputSplit(InputSplit inputSplit, DataOutput dataOutput)
126throws IOException {
127 MultiInputUtils.writeInputSplit(
128 inputSplit, dataOutput, mappingInputFormats);
129 }
130131 @Override
132public InputSplit readInputSplit(
133 DataInput dataInput) throws IOException, ClassNotFoundException {
134return MultiInputUtils.readInputSplit(dataInput, mappingInputFormats);
135 }
136 }