View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.giraph.writable.kryo;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.Arrays;
25  import java.util.Collections;
26  import java.util.Iterator;
27  import java.util.List;
28  import java.util.Random;
29  
30  import org.apache.giraph.conf.GiraphConfiguration;
31  import org.apache.giraph.conf.GiraphConfigurationSettable;
32  import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
33  import org.apache.giraph.types.ops.collections.array.WLongArrayList;
34  import org.apache.giraph.utils.WritableUtils;
35  import org.apache.giraph.writable.kryo.markers.NonKryoWritable;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.io.IntWritable;
38  import org.junit.Assert;
39  import org.junit.Ignore;
40  import org.junit.Test;
41  
42  import com.google.common.collect.ImmutableList;
43  import com.google.common.collect.Iterables;
44  import com.google.common.collect.Iterators;
45  
46  import it.unimi.dsi.fastutil.chars.Char2ObjectMap;
47  import it.unimi.dsi.fastutil.chars.Char2ObjectOpenHashMap;
48  import it.unimi.dsi.fastutil.floats.FloatArrayList;
49  import it.unimi.dsi.fastutil.ints.Int2BooleanMap;
50  import it.unimi.dsi.fastutil.ints.Int2BooleanOpenHashMap;
51  import it.unimi.dsi.fastutil.longs.LongArrayList;
52  import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
53  
54  
55  
56  /**
57   * Tests some subtle cases of kryo serialization.
58   */
59  public class KryoWritableWrapperTest {
60    public static <T> T kryoSerDeser(T t) throws IOException {
61      KryoWritableWrapper<T> wrapped = new KryoWritableWrapper<>(t);
62      KryoWritableWrapper<T> deser = new KryoWritableWrapper<>();
63      WritableUtils.copyInto(wrapped, deser, true);
64      return deser.get();
65    }
66  
67    @Test
68    public void testArraysAsList() throws IOException {
69      List res = kryoSerDeser(Arrays.asList(1, 2, 3));
70  
71      assertEquals(3, res.size());
72      assertEquals(1, res.get(0));
73      assertEquals(2, res.get(1));
74      assertEquals(3, res.get(2));
75    }
76  
77  
78    @Test
79    public void testArraysAsListMultiRef() throws IOException {
80      List list = Arrays.asList(1, 2, 3);
81      Object obj = new Object();
82      List wanted = Arrays.asList(list, list, obj, obj, null);
83      wanted.set(4, wanted);
84      List res = kryoSerDeser(wanted);
85  
86      assertTrue(res.get(0) == res.get(1));
87      assertTrue(res.get(2) == res.get(3));
88      // TODO see if this can be supported, though this is a rare case:
89      // assertTrue(res == res.get(4));
90    }
91  
92    @Test
93    public void testCollectionsNCopiesList() throws IOException {
94      List res = kryoSerDeser(Collections.nCopies(3, 42));
95  
96      assertEquals(3, res.size());
97      assertEquals(42, res.get(0));
98      assertEquals(42, res.get(1));
99      assertEquals(42, res.get(2));
100   }
101 
102   @Test
103   public void testCollectionsNCopiesObjectList() throws IOException {
104     String testObject = "Hello World!";
105     List<String> res = kryoSerDeser(Collections.nCopies(3, testObject));
106 
107     assertEquals(3, res.size());
108     assertEquals(testObject, res.get(0));
109     assertEquals(testObject, res.get(1));
110     assertEquals(testObject, res.get(2));
111   }
112 
113   @Test
114   public void testUnmodifiableIterator() throws IOException {
115     Iterator<Integer> in = Iterables.concat(
116         Arrays.asList(0, 1),
117         Arrays.asList(2, 3),
118         Arrays.asList(4, 5)).iterator();
119 
120     in.next();
121     in.next();
122     in.next();
123     Iterator res = kryoSerDeser(in);
124 
125     int cnt = 3;
126     for(; res.hasNext(); cnt++) {
127       assertEquals(cnt, res.next());
128     }
129     assertEquals(6, cnt);
130   }
131 
132   @Test
133   public void testIteratorsConcat() throws IOException {
134     Iterator<Integer> in = Iterators.concat(
135         Arrays.asList(0, 1).iterator(),
136         Arrays.asList(2, 3).iterator(),
137         Arrays.asList(4, 5).iterator());
138 
139     in.next();
140     in.next();
141     in.next();
142 
143     Iterator res = kryoSerDeser(in);
144 
145     int cnt = 3;
146     for(; res.hasNext(); cnt++) {
147       assertEquals(cnt, res.next());
148     }
149     assertEquals(6, cnt);
150 
151   }
152 
153   @Test
154   public void testImmutableList() throws IOException {
155     {
156       List res = kryoSerDeser(ImmutableList.of(1, 2));
157       assertEquals(2, res.size());
158       assertEquals(1, res.get(0));
159       assertEquals(2, res.get(1));
160     }
161 
162     {
163       List list = ImmutableList.of(1, 2, 3);
164       Object obj = new Object();
165       List wanted = ImmutableList.of(list, list, obj, obj);
166       List res = kryoSerDeser(wanted);
167 
168       assertTrue(res.get(0) == res.get(1));
169       assertTrue(res.get(2) == res.get(3));
170     }
171   }
172 
173   @Test
174   public void testFastutilSet() throws ClassNotFoundException, IOException {
175     LongOpenHashSet set = new LongOpenHashSet();
176     set.add(6);
177     LongOpenHashSet deser = kryoSerDeser(set);
178     deser.add(5);
179     set.add(5);
180     Assert.assertEquals(set, deser);
181   }
182 
183   @Test
184   public void testFastutilLongList() throws ClassNotFoundException, IOException {
185     LongArrayList list = new LongArrayList();
186     list.add(6);
187     LongArrayList deser = kryoSerDeser(list);
188     deser.add(5);
189     list.add(5);
190     Assert.assertEquals(list, deser);
191   }
192 
193   @Test
194   public void testWFastutilLongList() throws ClassNotFoundException, IOException {
195     WLongArrayList list = new WLongArrayList();
196     list.add(6);
197     WLongArrayList deser = kryoSerDeser(list);
198     deser.add(5);
199     list.add(5);
200     Assert.assertEquals(list, deser);
201   }
202 
203 
204   @Test
205   public void testFastutilFloatList() throws ClassNotFoundException, IOException {
206     FloatArrayList list = new FloatArrayList();
207     list.add(6L);
208     FloatArrayList deser = kryoSerDeser(list);
209     deser.add(5L);
210     list.add(5L);
211     Assert.assertEquals(list, deser);
212   }
213 
214   @Test
215   public void testFastutilMap() throws ClassNotFoundException, IOException {
216     Int2BooleanMap list = new Int2BooleanOpenHashMap();
217     list.put(6, true);
218     Int2BooleanMap deser = kryoSerDeser(list);
219     deser.put(5, false);
220     list.put(5, false);
221     Assert.assertEquals(list, deser);
222   }
223 
224   @Test
225   public void testFastutil2ObjMap() throws ClassNotFoundException, IOException {
226     Char2ObjectMap<IntWritable> list = new Char2ObjectOpenHashMap<>();
227     list.put('a', new IntWritable(6));
228     list.put('q', new IntWritable(7));
229     list.put('w', new IntWritable(8));
230     list.put('e', new IntWritable(9));
231     list.put('r', new IntWritable(7));
232     list.put('c', null);
233     Char2ObjectMap<IntWritable> deser = kryoSerDeser(list);
234     deser.put('b', null);
235     list.put('b', null);
236 
237     Assert.assertEquals(list, deser);
238   }
239 
240   @Test
241   @Ignore("Long test used for profiling compiling speed")
242   public void testLongFastutilListProfile() throws ClassNotFoundException, IOException {
243     int n = 100;
244     int rounds = 2000000;
245     LongArrayList list = new LongArrayList(n);
246     for (int i = 0; i < n; i++) {
247       list.add(i);
248     }
249 
250     for (int round = 0; round < rounds; round ++) {
251       LongArrayList deser = kryoSerDeser(list);
252       deser.add(round);
253       list.add(round);
254       Assert.assertEquals(list.size(), deser.size());
255       Assert.assertArrayEquals(list.elements(), deser.elements());
256 
257       list.popLong();
258     }
259   }
260 
261   @Test(expected=RuntimeException.class)
262   public void testRandom() throws ClassNotFoundException, IOException {
263     kryoSerDeser(new Random()).nextBoolean();
264   }
265 
266   private static class TestConf implements GiraphConfigurationSettable {
267     @Override
268     public void setConf(ImmutableClassesGiraphConfiguration configuration) {
269     }
270   }
271 
272   @Test(expected=RuntimeException.class)
273   public void testConfiguration() throws ClassNotFoundException, IOException {
274     kryoSerDeser(new Configuration());
275   }
276 
277   @Test(expected=RuntimeException.class)
278   public void testConfigurable() throws ClassNotFoundException, IOException {
279     kryoSerDeser(new TestConf());
280   }
281 
282   @Test(expected=RuntimeException.class)
283   public void testVertexReceiver() throws ClassNotFoundException, IOException {
284     kryoSerDeser(new NonKryoWritable() {
285     });
286   }
287 
288 
289   @Test
290   public void testBlacklistedClasses() throws ClassNotFoundException, IOException {
291     Assert.assertEquals(kryoSerDeser(Random.class), Random.class);
292     Assert.assertEquals(kryoSerDeser(TestConf.class), TestConf.class);
293     Assert.assertEquals(kryoSerDeser(GiraphConfiguration.class), GiraphConfiguration.class);
294   }
295 
296   @Test(expected=RuntimeException.class)
297   public void testRecursive() throws ClassNotFoundException, IOException {
298     kryoSerDeser(new KryoWritableWrapper<>(new Object())).get().hashCode();
299   }
300 
301   @Test
302   public void testNull() throws ClassNotFoundException, IOException {
303     Assert.assertNull(kryoSerDeser(null));
304   }
305 }