This project has retired. For details please refer to its Attic page.
JythonJob xref
View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.giraph.jython;
19  
20  import com.google.common.base.MoreObjects;
21  import org.apache.giraph.combiner.MessageCombiner;
22  
23  import com.google.common.collect.Lists;
24  import com.google.common.collect.Maps;
25  
26  import java.util.List;
27  import java.util.Map;
28  
29  // CHECKSTYLE: stop ParameterNameCheck
30  // CHECKSTYLE: stop MemberNameCheck
31  // CHECKSTYLE: stop MethodNameCheck
32  
33  /**
34   * Holder of Jython job information.
35   */
36  public class JythonJob {
37    /**
38     * Base class for input information
39     */
40    public static class InputBase {
41      /** Name of Hive table */
42      private String table;
43      /** Filter for partitions to read */
44      private String partition_filter;
45  
46      public String getPartition_filter() {
47        return partition_filter;
48      }
49  
50      public void setPartition_filter(String partition_filter) {
51        this.partition_filter = partition_filter;
52      }
53  
54      public String getTable() {
55        return table;
56      }
57  
58      public void setTable(String table) {
59        this.table = table;
60      }
61  
62      @Override public String toString() {
63        return MoreObjects.toStringHelper(this)
64            .add("table", table)
65            .add("partition_filter", partition_filter)
66            .toString();
67      }
68    }
69  
70    /**
71     * Info about vertex input
72     */
73    public static class VertexInput extends InputBase {
74      /** Name of vertex ID column */
75      private String id_column;
76      /** Name of vertex value column */
77      private String value_column;
78  
79      public String getId_column() {
80        return id_column;
81      }
82  
83      public void setId_column(String id_column) {
84        this.id_column = id_column;
85      }
86  
87      public String getValue_column() {
88        return value_column;
89      }
90  
91      public void setValue_column(String value_column) {
92        this.value_column = value_column;
93      }
94  
95      @Override public String toString() {
96        return MoreObjects.toStringHelper(this)
97            .add("table", getTable())
98            .add("partition_filter", getPartition_filter())
99            .add("id_column", id_column)
100           .add("value_column", value_column)
101           .toString();
102     }
103   }
104 
105   /**
106    * Info about edge input
107    */
108   public static class EdgeInput extends InputBase {
109     /** Name for source ID column */
110     private String source_id_column;
111     /** Name for target ID column */
112     private String target_id_column;
113     /** Name of edge value column */
114     private String value_column;
115 
116     public String getValue_column() {
117       return value_column;
118     }
119 
120     public void setValue_column(String value_column) {
121       this.value_column = value_column;
122     }
123 
124     public String getSource_id_column() {
125       return source_id_column;
126     }
127 
128     public void setSource_id_column(String source_id_column) {
129       this.source_id_column = source_id_column;
130     }
131 
132     public String getTarget_id_column() {
133       return target_id_column;
134     }
135 
136     public void setTarget_id_column(String target_id_column) {
137       this.target_id_column = target_id_column;
138     }
139 
140     @Override public String toString() {
141       return MoreObjects.toStringHelper(this)
142           .add("table", getTable())
143           .add("partition_filter", getPartition_filter())
144           .add("source_id_column", source_id_column)
145           .add("target_id_column", target_id_column)
146           .add("edge_value_column", value_column)
147           .toString();
148     }
149   }
150 
151   /**
152    * Info about vertex output
153    */
154   public static class VertexOutput {
155     /** Name of hive table */
156     private String table;
157     /** Partition data */
158     private final Map<String, String> partition = Maps.newHashMap();
159     /** Name of Vertex ID column */
160     private String id_column;
161     /** Name of Vertex value column */
162     private String value_column;
163 
164     public String getId_column() {
165       return id_column;
166     }
167 
168     public void setId_column(String id_column) {
169       this.id_column = id_column;
170     }
171 
172     public Map<String, String> getPartition() {
173       return partition;
174     }
175 
176     public String getTable() {
177       return table;
178     }
179 
180     public void setTable(String table) {
181       this.table = table;
182     }
183 
184     public String getValue_column() {
185       return value_column;
186     }
187 
188     public void setValue_column(String value_column) {
189       this.value_column = value_column;
190     }
191 
192     @Override public String toString() {
193       return MoreObjects.toStringHelper(this)
194           .add("table", table)
195           .add("partition", partition)
196           .add("id_column", id_column)
197           .add("value_column", value_column)
198           .toString();
199     }
200   }
201 
202   /**
203    * Holds a Java or Jython type.
204    */
205   public static class TypeHolder {
206     /**
207      * The Java or Jython class. Should be one of the following:
208      *
209      * 1) Name (string) of Jython class
210      * 2) Java Writable class
211      * 3) Name (string) of Java Writable class
212      * 4) Jython class directly. This is a bit problematic, it is preferred to
213      *    use the string name of the Jython class instead.
214      */
215     private Object type;
216 
217     public Object getType() {
218       return type;
219     }
220 
221     public void setType(Object type) {
222       this.type = type;
223     }
224   }
225 
226   /**
227    * A type along with its Hive input/output class.
228    */
229   public static class TypeWithHive extends TypeHolder {
230     /** Hive Reader */
231     private Object hive_reader;
232     /** Hive Writer */
233     private Object hive_writer;
234     /** Hive Reader and Writer in one */
235     private Object hive_io;
236 
237     public Object getHive_io() {
238       return hive_io;
239     }
240 
241     public void setHive_io(Object hive_io) {
242       this.hive_io = hive_io;
243     }
244 
245     public Object getHive_reader() {
246       return hive_reader;
247     }
248 
249     public void setHive_reader(Object hive_reader) {
250       this.hive_reader = hive_reader;
251     }
252 
253     public Object getHive_writer() {
254       return hive_writer;
255     }
256 
257     public void setHive_writer(Object hive_writer) {
258       this.hive_writer = hive_writer;
259     }
260   }
261 
262   /** Name of job */
263   private String name;
264   /** Hive database */
265   private String hive_database = "digraph";
266   /** Number of workers */
267   private int workers;
268   /** MapReduce pool */
269   private String pool;
270   /** Vertex ID */
271   private final TypeWithHive vertex_id = new TypeWithHive();
272   /** Vertex value */
273   private final TypeWithHive vertex_value = new TypeWithHive();
274   /** Edge value */
275   private final TypeWithHive edge_value = new TypeWithHive();
276   /** Incoming message value */
277   private final TypeHolder incoming_message_value = new TypeHolder();
278   /** Outgoing message value */
279   private final TypeHolder outgoing_message_value = new TypeHolder();
280   /** Message value type - used to set both in/out message types in one call. */
281   private final TypeHolder message_value = new TypeHolder();
282   /** Computation class */
283   private String computation_name;
284   /** MessageCombiner class */
285   private Class<? extends MessageCombiner> messageCombiner;
286   /** Java options */
287   private final List<String> java_options = Lists.newArrayList();
288   /** Giraph options */
289   private final Map<String, Object> giraph_options = Maps.newHashMap();
290   /** Vertex inputs */
291   private final List<VertexInput> vertex_inputs = Lists.newArrayList();
292   /** Edge inputs */
293   private final List<EdgeInput> edge_inputs = Lists.newArrayList();
294   /** Output */
295   private final VertexOutput vertex_output = new VertexOutput();
296 
297   /////// Read only info for jython scripts ////////
298   /** User running the job */
299   private String user;
300 
301   /**
302    * Constructor
303    */
304   public JythonJob() {
305     user = System.getProperty("user.name");
306     workers = 5;
307   }
308 
309   public String getUser() {
310     return user;
311   }
312 
313   public TypeWithHive getVertex_value() {
314     return vertex_value;
315   }
316 
317   public TypeWithHive getVertex_id() {
318     return vertex_id;
319   }
320 
321   public TypeWithHive getEdge_value() {
322     return edge_value;
323   }
324 
325   public TypeHolder getIncoming_message_value() {
326     return incoming_message_value;
327   }
328 
329   public TypeHolder getOutgoing_message_value() {
330     return outgoing_message_value;
331   }
332 
333   public TypeHolder getMessage_value() {
334     return message_value;
335   }
336 
337   public List<String> getJava_options() {
338     return java_options;
339   }
340 
341   public Map<String, Object> getGiraph_options() {
342     return giraph_options;
343   }
344 
345   public Class<? extends MessageCombiner> getMessageCombiner() {
346     return messageCombiner;
347   }
348 
349   public void setMessageCombiner(
350       Class<? extends MessageCombiner> messageCombiner) {
351     this.messageCombiner = messageCombiner;
352   }
353 
354   public String getComputation_name() {
355     return computation_name;
356   }
357 
358   public void setComputation_name(String computation_name) {
359     this.computation_name = computation_name;
360   }
361 
362   public String getHive_database() {
363     return hive_database;
364   }
365 
366   public void setHive_database(String hive_database) {
367     this.hive_database = hive_database;
368   }
369 
370   public List<EdgeInput> getEdge_inputs() {
371     return edge_inputs;
372   }
373 
374   public String getName() {
375     return name;
376   }
377 
378   public void setName(String name) {
379     this.name = name;
380   }
381 
382   public String getPool() {
383     return pool;
384   }
385 
386   public void setPool(String pool) {
387     this.pool = pool;
388   }
389 
390   public List<VertexInput> getVertex_inputs() {
391     return vertex_inputs;
392   }
393 
394   public VertexOutput getVertex_output() {
395     return vertex_output;
396   }
397 
398   public int getWorkers() {
399     return workers;
400   }
401 
402   public void setWorkers(int workers) {
403     this.workers = workers;
404   }
405 }
406 
407 // CHECKSTYLE: resume ParameterNameCheck
408 // CHECKSTYLE: resume MemberNameCheck
409 // CHECKSTYLE: resume MethodNameCheck