View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.giraph.conf;
19  
20  import org.apache.giraph.aggregators.AggregatorWriter;
21  import org.apache.giraph.aggregators.TextAggregatorWriter;
22  import org.apache.giraph.bsp.BspOutputFormat;
23  import org.apache.giraph.bsp.checkpoints.CheckpointSupportedChecker;
24  import org.apache.giraph.bsp.checkpoints.DefaultCheckpointSupportedChecker;
25  import org.apache.giraph.combiner.MessageCombiner;
26  import org.apache.giraph.comm.messages.InMemoryMessageStoreFactory;
27  import org.apache.giraph.comm.messages.MessageEncodeAndStoreType;
28  import org.apache.giraph.comm.messages.MessageStoreFactory;
29  import org.apache.giraph.edge.ByteArrayEdges;
30  import org.apache.giraph.edge.DefaultCreateSourceVertexCallback;
31  import org.apache.giraph.edge.CreateSourceVertexCallback;
32  import org.apache.giraph.edge.EdgeStoreFactory;
33  import org.apache.giraph.edge.InMemoryEdgeStoreFactory;
34  import org.apache.giraph.edge.OutEdges;
35  import org.apache.giraph.factories.ComputationFactory;
36  import org.apache.giraph.factories.DefaultComputationFactory;
37  import org.apache.giraph.factories.DefaultEdgeValueFactory;
38  import org.apache.giraph.factories.DefaultInputOutEdgesFactory;
39  import org.apache.giraph.factories.DefaultMessageValueFactory;
40  import org.apache.giraph.factories.DefaultOutEdgesFactory;
41  import org.apache.giraph.factories.DefaultVertexIdFactory;
42  import org.apache.giraph.factories.DefaultVertexValueFactory;
43  import org.apache.giraph.factories.EdgeValueFactory;
44  import org.apache.giraph.factories.MessageValueFactory;
45  import org.apache.giraph.factories.OutEdgesFactory;
46  import org.apache.giraph.factories.VertexIdFactory;
47  import org.apache.giraph.factories.VertexValueFactory;
48  import org.apache.giraph.graph.Computation;
49  import org.apache.giraph.graph.DefaultVertex;
50  import org.apache.giraph.graph.DefaultVertexResolver;
51  import org.apache.giraph.graph.DefaultVertexValueCombiner;
52  import org.apache.giraph.graph.Language;
53  import org.apache.giraph.graph.MapperObserver;
54  import org.apache.giraph.graph.Vertex;
55  import org.apache.giraph.graph.VertexResolver;
56  import org.apache.giraph.graph.VertexValueCombiner;
57  import org.apache.giraph.io.EdgeInputFormat;
58  import org.apache.giraph.io.EdgeOutputFormat;
59  import org.apache.giraph.io.MappingInputFormat;
60  import org.apache.giraph.io.VertexInputFormat;
61  import org.apache.giraph.io.VertexOutputFormat;
62  import org.apache.giraph.io.filters.DefaultEdgeInputFilter;
63  import org.apache.giraph.io.filters.DefaultVertexInputFilter;
64  import org.apache.giraph.io.filters.EdgeInputFilter;
65  import org.apache.giraph.io.filters.VertexInputFilter;
66  import org.apache.giraph.job.DefaultGiraphJobRetryChecker;
67  import org.apache.giraph.job.DefaultJobObserver;
68  import org.apache.giraph.job.DefaultJobProgressTrackerService;
69  import org.apache.giraph.job.GiraphJobObserver;
70  import org.apache.giraph.job.GiraphJobRetryChecker;
71  import org.apache.giraph.job.HaltApplicationUtils;
72  import org.apache.giraph.job.JobProgressTrackerService;
73  import org.apache.giraph.mapping.MappingStore;
74  import org.apache.giraph.mapping.MappingStoreOps;
75  import org.apache.giraph.mapping.translate.TranslateEdge;
76  import org.apache.giraph.master.DefaultMasterCompute;
77  import org.apache.giraph.master.MasterCompute;
78  import org.apache.giraph.master.MasterObserver;
79  import org.apache.giraph.ooc.persistence.OutOfCoreDataAccessor;
80  import org.apache.giraph.ooc.persistence.LocalDiskDataAccessor;
81  import org.apache.giraph.ooc.policy.MemoryEstimatorOracle;
82  import org.apache.giraph.ooc.policy.OutOfCoreOracle;
83  import org.apache.giraph.partition.GraphPartitionerFactory;
84  import org.apache.giraph.partition.HashPartitionerFactory;
85  import org.apache.giraph.partition.Partition;
86  import org.apache.giraph.partition.SimplePartition;
87  import org.apache.giraph.utils.GcObserver;
88  import org.apache.giraph.worker.DefaultWorkerContext;
89  import org.apache.giraph.worker.WorkerContext;
90  import org.apache.giraph.worker.WorkerObserver;
91  import org.apache.hadoop.io.Writable;
92  import org.apache.hadoop.io.WritableComparable;
93  import org.apache.hadoop.mapreduce.OutputFormat;
94  
95  import static java.util.concurrent.TimeUnit.HOURS;
96  import static java.util.concurrent.TimeUnit.MINUTES;
97  import static java.util.concurrent.TimeUnit.SECONDS;
98  
99  /**
100  * Constants used all over Giraph for configuration.
101  */
102 // CHECKSTYLE: stop InterfaceIsTypeCheck
103 public interface GiraphConstants {
104   /** 1KB in bytes */
105   int ONE_KB = 1024;
106   /** 1MB in bytes */
107   int ONE_MB = 1024 * 1024;
108 
109   /** Mapping related information */
110   ClassConfOption<MappingStore> MAPPING_STORE_CLASS =
111       ClassConfOption.create("giraph.mappingStoreClass", null,
112           MappingStore.class, "MappingStore Class");
113 
114   /** Class to use for performing read operations on mapping store */
115   ClassConfOption<MappingStoreOps> MAPPING_STORE_OPS_CLASS =
116       ClassConfOption.create("giraph.mappingStoreOpsClass", null,
117           MappingStoreOps.class, "MappingStoreOps class");
118 
119   /** Upper value of LongByteMappingStore */
120   IntConfOption LB_MAPPINGSTORE_UPPER =
121       new IntConfOption("giraph.lbMappingStoreUpper", -1,
122           "'upper' value used by lbmappingstore");
123   /** Lower value of LongByteMappingStore */
124   IntConfOption LB_MAPPINGSTORE_LOWER =
125       new IntConfOption("giraph.lbMappingStoreLower", -1,
126           "'lower' value used by lbMappingstore");
127   /** Class used to conduct expensive edge translation during vertex input */
128   ClassConfOption EDGE_TRANSLATION_CLASS =
129       ClassConfOption.create("giraph.edgeTranslationClass", null,
130           TranslateEdge.class, "Class used to conduct expensive edge " +
131               "translation during vertex input phase");
132 
133   /** Computation class - required */
134   ClassConfOption<Computation> COMPUTATION_CLASS =
135       ClassConfOption.create("giraph.computationClass", null,
136           Computation.class, "Computation class - required");
137   /** Computation factory class - optional */
138   ClassConfOption<ComputationFactory> COMPUTATION_FACTORY_CLASS =
139       ClassConfOption.create("giraph.computation.factory.class",
140           DefaultComputationFactory.class, ComputationFactory.class,
141           "Computation factory class - optional");
142 
143   /** TypesHolder, used if Computation not set - optional */
144   ClassConfOption<TypesHolder> TYPES_HOLDER_CLASS =
145       ClassConfOption.create("giraph.typesHolder", null,
146           TypesHolder.class,
147           "TypesHolder, used if Computation not set - optional");
148 
149   /** Edge Store Factory */
150   ClassConfOption<EdgeStoreFactory> EDGE_STORE_FACTORY_CLASS =
151       ClassConfOption.create("giraph.edgeStoreFactoryClass",
152           InMemoryEdgeStoreFactory.class,
153           EdgeStoreFactory.class,
154           "Edge Store Factory class to use for creating edgeStore");
155 
156   /** Message Store Factory */
157   ClassConfOption<MessageStoreFactory> MESSAGE_STORE_FACTORY_CLASS =
158       ClassConfOption.create("giraph.messageStoreFactoryClass",
159           InMemoryMessageStoreFactory.class,
160           MessageStoreFactory.class,
161           "Message Store Factory Class that is to be used");
162 
163   /** Language user's graph types are implemented in */
164   PerGraphTypeEnumConfOption<Language> GRAPH_TYPE_LANGUAGES =
165       PerGraphTypeEnumConfOption.create("giraph.types.language",
166           Language.class, Language.JAVA,
167           "Language user graph types (IVEMM) are implemented in");
168 
169   /** Whether user graph types need wrappers */
170   PerGraphTypeBooleanConfOption GRAPH_TYPES_NEEDS_WRAPPERS =
171       new PerGraphTypeBooleanConfOption("giraph.jython.type.wrappers",
172           false, "Whether user graph types (IVEMM) need Jython wrappers");
173 
174   /** Vertex id factory class - optional */
175   ClassConfOption<VertexIdFactory> VERTEX_ID_FACTORY_CLASS =
176       ClassConfOption.create("giraph.vertexIdFactoryClass",
177           DefaultVertexIdFactory.class, VertexIdFactory.class,
178           "Vertex ID factory class - optional");
179   /** Vertex value factory class - optional */
180   ClassConfOption<VertexValueFactory> VERTEX_VALUE_FACTORY_CLASS =
181       ClassConfOption.create("giraph.vertexValueFactoryClass",
182           DefaultVertexValueFactory.class, VertexValueFactory.class,
183           "Vertex value factory class - optional");
184   /** Edge value factory class - optional */
185   ClassConfOption<EdgeValueFactory> EDGE_VALUE_FACTORY_CLASS =
186       ClassConfOption.create("giraph.edgeValueFactoryClass",
187           DefaultEdgeValueFactory.class, EdgeValueFactory.class,
188           "Edge value factory class - optional");
189   /** Outgoing message value factory class - optional */
190   ClassConfOption<MessageValueFactory>
191   OUTGOING_MESSAGE_VALUE_FACTORY_CLASS =
192       ClassConfOption.create("giraph.outgoingMessageValueFactoryClass",
193           DefaultMessageValueFactory.class, MessageValueFactory.class,
194           "Outgoing message value factory class - optional");
195 
196   /** Vertex edges class - optional */
197   ClassConfOption<OutEdges> VERTEX_EDGES_CLASS =
198       ClassConfOption.create("giraph.outEdgesClass", ByteArrayEdges.class,
199           OutEdges.class, "Vertex edges class - optional");
200   /** Vertex edges class to be used during edge input only - optional */
201   ClassConfOption<OutEdges> INPUT_VERTEX_EDGES_CLASS =
202       ClassConfOption.create("giraph.inputOutEdgesClass",
203           ByteArrayEdges.class, OutEdges.class,
204           "Vertex edges class to be used during edge input only - optional");
205   /** OutEdges factory class - optional */
206   ClassConfOption<OutEdgesFactory> VERTEX_EDGES_FACTORY_CLASS =
207       ClassConfOption.create("giraph.outEdgesFactoryClass",
208         DefaultOutEdgesFactory.class, OutEdgesFactory.class,
209           "OutEdges factory class - optional");
210   /** OutEdges for input factory class - optional */
211   ClassConfOption<OutEdgesFactory> INPUT_VERTEX_EDGES_FACTORY_CLASS =
212       ClassConfOption.create("giraph.inputOutEdgesFactoryClass",
213         DefaultInputOutEdgesFactory.class, OutEdgesFactory.class,
214           "OutEdges for input factory class - optional");
215 
216   /** Class for Master - optional */
217   ClassConfOption<MasterCompute> MASTER_COMPUTE_CLASS =
218       ClassConfOption.create("giraph.masterComputeClass",
219           DefaultMasterCompute.class, MasterCompute.class,
220           "Class for Master - optional");
221   /** Classes for Master Observer - optional */
222   ClassConfOption<MasterObserver> MASTER_OBSERVER_CLASSES =
223       ClassConfOption.create("giraph.master.observers",
224           null, MasterObserver.class, "Classes for Master Observer - optional");
225   /** Classes for Worker Observer - optional */
226   ClassConfOption<WorkerObserver> WORKER_OBSERVER_CLASSES =
227       ClassConfOption.create("giraph.worker.observers", null,
228           WorkerObserver.class, "Classes for Worker Observer - optional");
229   /** Classes for Mapper Observer - optional */
230   ClassConfOption<MapperObserver> MAPPER_OBSERVER_CLASSES =
231       ClassConfOption.create("giraph.mapper.observers", null,
232           MapperObserver.class, "Classes for Mapper Observer - optional");
233   /** Classes for GC Observer - optional */
234   ClassConfOption<GcObserver> GC_OBSERVER_CLASSES =
235       ClassConfOption.create("giraph.gc.observers", null,
236           GcObserver.class, "Classes for GC oObserver - optional");
237   /** Message combiner class - optional */
238   ClassConfOption<MessageCombiner> MESSAGE_COMBINER_CLASS =
239       ClassConfOption.create("giraph.messageCombinerClass", null,
240           MessageCombiner.class, "Message combiner class - optional");
241   /** Vertex resolver class - optional */
242   ClassConfOption<VertexResolver> VERTEX_RESOLVER_CLASS =
243       ClassConfOption.create("giraph.vertexResolverClass",
244           DefaultVertexResolver.class, VertexResolver.class,
245           "Vertex resolver class - optional");
246   /** Vertex value combiner class - optional */
247   ClassConfOption<VertexValueCombiner> VERTEX_VALUE_COMBINER_CLASS =
248       ClassConfOption.create("giraph.vertexValueCombinerClass",
249           DefaultVertexValueCombiner.class, VertexValueCombiner.class,
250           "Vertex value combiner class - optional");
251 
252   /** Which language computation is implemented in */
253   EnumConfOption<Language> COMPUTATION_LANGUAGE =
254       EnumConfOption.create("giraph.computation.language",
255           Language.class, Language.JAVA,
256           "Which language computation is implemented in");
257 
258   /**
259    * Option of whether to create vertexes that were not existent before but
260    * received messages
261    */
262   BooleanConfOption RESOLVER_CREATE_VERTEX_ON_MSGS =
263       new BooleanConfOption("giraph.vertex.resolver.create.on.msgs", true,
264           "Option of whether to create vertexes that were not existent " +
265           "before but received messages");
266   /** Graph partitioner factory class - optional */
267   ClassConfOption<GraphPartitionerFactory> GRAPH_PARTITIONER_FACTORY_CLASS =
268       ClassConfOption.create("giraph.graphPartitionerFactoryClass",
269           HashPartitionerFactory.class, GraphPartitionerFactory.class,
270           "Graph partitioner factory class - optional");
271 
272   /** Observer class to watch over job status - optional */
273   ClassConfOption<GiraphJobObserver> JOB_OBSERVER_CLASS =
274       ClassConfOption.create("giraph.jobObserverClass",
275           DefaultJobObserver.class, GiraphJobObserver.class,
276           "Observer class to watch over job status - optional");
277 
278   /** Observer class to watch over job status - optional */
279   ClassConfOption<GiraphJobRetryChecker> JOB_RETRY_CHECKER_CLASS =
280       ClassConfOption.create("giraph.jobRetryCheckerClass",
281           DefaultGiraphJobRetryChecker.class, GiraphJobRetryChecker.class,
282           "Class which decides whether a failed job should be retried - " +
283               "optional");
284 
285   /**
286    * Maximum allowed time for job to run after getting all resources before it
287    * will be killed, in milliseconds (-1 if it has no limit)
288    */
289   LongConfOption MAX_ALLOWED_JOB_TIME_MS =
290       new LongConfOption("giraph.maxAllowedJobTimeMilliseconds", -1,
291           "Maximum allowed time for job to run after getting all resources " +
292               "before it will be killed, in milliseconds " +
293               "(-1 if it has no limit)");
294 
295   // At least one of the input format classes is required.
296   /** VertexInputFormat class */
297   ClassConfOption<VertexInputFormat> VERTEX_INPUT_FORMAT_CLASS =
298       ClassConfOption.create("giraph.vertexInputFormatClass", null,
299           VertexInputFormat.class, "VertexInputFormat class (at least " +
300           "one of the input format classes is required)");
301   /** EdgeInputFormat class */
302   ClassConfOption<EdgeInputFormat> EDGE_INPUT_FORMAT_CLASS =
303       ClassConfOption.create("giraph.edgeInputFormatClass", null,
304           EdgeInputFormat.class, "EdgeInputFormat class");
305   /** MappingInputFormat class */
306   ClassConfOption<MappingInputFormat> MAPPING_INPUT_FORMAT_CLASS =
307       ClassConfOption.create("giraph.mappingInputFormatClass", null,
308           MappingInputFormat.class, "MappingInputFormat class");
309 
310   /** EdgeInputFilter class */
311   ClassConfOption<EdgeInputFilter> EDGE_INPUT_FILTER_CLASS =
312       ClassConfOption.create("giraph.edgeInputFilterClass",
313           DefaultEdgeInputFilter.class, EdgeInputFilter.class,
314           "EdgeInputFilter class");
315   /** VertexInputFilter class */
316   ClassConfOption<VertexInputFilter> VERTEX_INPUT_FILTER_CLASS =
317       ClassConfOption.create("giraph.vertexInputFilterClass",
318           DefaultVertexInputFilter.class, VertexInputFilter.class,
319           "VertexInputFilter class");
320   /** Vertex class */
321   ClassConfOption<Vertex> VERTEX_CLASS =
322       ClassConfOption.create("giraph.vertexClass",
323           DefaultVertex.class, Vertex.class,
324           "Vertex class");
325   /** VertexOutputFormat class */
326   ClassConfOption<VertexOutputFormat> VERTEX_OUTPUT_FORMAT_CLASS =
327       ClassConfOption.create("giraph.vertexOutputFormatClass", null,
328           VertexOutputFormat.class, "VertexOutputFormat class");
329   /** EdgeOutputFormat sub-directory */
330   StrConfOption VERTEX_OUTPUT_FORMAT_SUBDIR =
331     new StrConfOption("giraph.vertex.output.subdir", "",
332                       "VertexOutputFormat sub-directory");
333   /** EdgeOutputFormat class */
334   ClassConfOption<EdgeOutputFormat> EDGE_OUTPUT_FORMAT_CLASS =
335       ClassConfOption.create("giraph.edgeOutputFormatClass", null,
336           EdgeOutputFormat.class, "EdgeOutputFormat class");
337   /** EdgeOutputFormat sub-directory */
338   StrConfOption EDGE_OUTPUT_FORMAT_SUBDIR =
339     new StrConfOption("giraph.edge.output.subdir", "",
340                       "EdgeOutputFormat sub-directory");
341 
342   /** GiraphTextOuputFormat Separator */
343   StrConfOption GIRAPH_TEXT_OUTPUT_FORMAT_SEPARATOR =
344     new StrConfOption("giraph.textoutputformat.separator", "\t",
345                       "GiraphTextOuputFormat Separator");
346   /** Reverse values in the output */
347   BooleanConfOption GIRAPH_TEXT_OUTPUT_FORMAT_REVERSE =
348       new BooleanConfOption("giraph.textoutputformat.reverse", false,
349                             "Reverse values in the output");
350 
351   /**
352    * If you use this option, instead of having saving vertices in the end of
353    * application, saveVertex will be called right after each vertex.compute()
354    * is called.
355    * NOTE: This feature doesn't work well with checkpointing - if you restart
356    * from a checkpoint you won't have any output from previous supersteps.
357    */
358   BooleanConfOption DO_OUTPUT_DURING_COMPUTATION =
359       new BooleanConfOption("giraph.doOutputDuringComputation", false,
360           "If you use this option, instead of having saving vertices in the " +
361           "end of application, saveVertex will be called right after each " +
362           "vertex.compute() is called." +
363           "NOTE: This feature doesn't work well with checkpointing - if you " +
364           "restart from a checkpoint you won't have any ouptut from previous " +
365           "supresteps.");
366   /**
367    * Vertex output format thread-safe - if your VertexOutputFormat allows
368    * several vertexWriters to be created and written to in parallel,
369    * you should set this to true.
370    */
371   BooleanConfOption VERTEX_OUTPUT_FORMAT_THREAD_SAFE =
372       new BooleanConfOption("giraph.vertexOutputFormatThreadSafe", false,
373           "Vertex output format thread-safe - if your VertexOutputFormat " +
374           "allows several vertexWriters to be created and written to in " +
375           "parallel, you should set this to true.");
376   /** Number of threads for writing output in the end of the application */
377   IntConfOption NUM_OUTPUT_THREADS =
378       new IntConfOption("giraph.numOutputThreads", 1,
379           "Number of threads for writing output in the end of the application");
380 
381   /** conf key for comma-separated list of jars to export to YARN workers */
382   StrConfOption GIRAPH_YARN_LIBJARS =
383     new StrConfOption("giraph.yarn.libjars", "",
384         "conf key for comma-separated list of jars to export to YARN workers");
385   /** Name of the XML file that will export our Configuration to YARN workers */
386   String GIRAPH_YARN_CONF_FILE = "giraph-conf.xml";
387   /** Giraph default heap size for all tasks when running on YARN profile */
388   int GIRAPH_YARN_TASK_HEAP_MB_DEFAULT = 1024;
389   /** Name of Giraph property for user-configurable heap memory per worker */
390   IntConfOption GIRAPH_YARN_TASK_HEAP_MB = new IntConfOption(
391     "giraph.yarn.task.heap.mb", GIRAPH_YARN_TASK_HEAP_MB_DEFAULT,
392     "Name of Giraph property for user-configurable heap memory per worker");
393   /** Default priority level in YARN for our task containers */
394   int GIRAPH_YARN_PRIORITY = 10;
395   /** Is this a pure YARN job (i.e. no MapReduce layer managing Giraph tasks) */
396   BooleanConfOption IS_PURE_YARN_JOB =
397     new BooleanConfOption("giraph.pure.yarn.job", false,
398         "Is this a pure YARN job (i.e. no MapReduce layer managing Giraph " +
399         "tasks)");
400 
401   /** Vertex index class */
402   ClassConfOption<WritableComparable> VERTEX_ID_CLASS =
403       ClassConfOption.create("giraph.vertexIdClass", null,
404           WritableComparable.class, "Vertex index class");
405   /** Vertex value class */
406   ClassConfOption<Writable> VERTEX_VALUE_CLASS =
407       ClassConfOption.create("giraph.vertexValueClass", null, Writable.class,
408           "Vertex value class");
409   /** Edge value class */
410   ClassConfOption<Writable> EDGE_VALUE_CLASS =
411       ClassConfOption.create("giraph.edgeValueClass", null, Writable.class,
412           "Edge value class");
413   /** Outgoing message value class */
414   ClassConfOption<Writable> OUTGOING_MESSAGE_VALUE_CLASS =
415       ClassConfOption.create("giraph.outgoingMessageValueClass", null,
416           Writable.class, "Outgoing message value class");
417   /** Worker context class */
418   ClassConfOption<WorkerContext> WORKER_CONTEXT_CLASS =
419       ClassConfOption.create("giraph.workerContextClass",
420           DefaultWorkerContext.class, WorkerContext.class,
421           "Worker contextclass");
422   /** AggregatorWriter class - optional */
423   ClassConfOption<AggregatorWriter> AGGREGATOR_WRITER_CLASS =
424       ClassConfOption.create("giraph.aggregatorWriterClass",
425           TextAggregatorWriter.class, AggregatorWriter.class,
426           "AggregatorWriter class - optional");
427 
428   /** Partition class - optional */
429   ClassConfOption<Partition> PARTITION_CLASS =
430       ClassConfOption.create("giraph.partitionClass", SimplePartition.class,
431           Partition.class, "Partition class - optional");
432 
433   /**
434    * Minimum number of simultaneous workers before this job can run (int)
435    */
436   String MIN_WORKERS = "giraph.minWorkers";
437   /**
438    * Maximum number of simultaneous worker tasks started by this job (int).
439    */
440   String MAX_WORKERS = "giraph.maxWorkers";
441 
442   /**
443    * Separate the workers and the master tasks.  This is required
444    * to support dynamic recovery. (boolean)
445    */
446   BooleanConfOption SPLIT_MASTER_WORKER =
447       new BooleanConfOption("giraph.SplitMasterWorker", true,
448           "Separate the workers and the master tasks.  This is required to " +
449           "support dynamic recovery. (boolean)");
450 
451   /** Indicates whether this job is run in an internal unit test */
452   BooleanConfOption LOCAL_TEST_MODE =
453       new BooleanConfOption("giraph.localTestMode", false,
454           "Indicates whether this job is run in an internal unit test");
455 
456   /** Override the Hadoop log level and set the desired log level. */
457   StrConfOption LOG_LEVEL = new StrConfOption("giraph.logLevel", "info",
458       "Override the Hadoop log level and set the desired log level.");
459 
460   /** Use thread level debugging? */
461   BooleanConfOption LOG_THREAD_LAYOUT =
462       new BooleanConfOption("giraph.logThreadLayout", false,
463           "Use thread level debugging?");
464 
465   /** Configuration key to enable jmap printing */
466   BooleanConfOption JMAP_ENABLE =
467       new BooleanConfOption("giraph.jmap.histo.enable", false,
468           "Configuration key to enable jmap printing");
469 
470   /** Configuration key for msec to sleep between calls */
471   IntConfOption JMAP_SLEEP_MILLIS =
472       new IntConfOption("giraph.jmap.histo.msec", SECONDS.toMillis(30),
473           "Configuration key for msec to sleep between calls");
474 
475   /** Configuration key for how many lines to print */
476   IntConfOption JMAP_PRINT_LINES =
477       new IntConfOption("giraph.jmap.histo.print_lines", 30,
478           "Configuration key for how many lines to print");
479 
480   /**
481    * Configuration key for printing live objects only
482    * This option will trigger Full GC for every jmap dump
483    * and so can significantly hinder performance.
484    */
485   BooleanConfOption JMAP_LIVE_ONLY =
486       new BooleanConfOption("giraph.jmap.histo.live", false,
487           "Only print live objects in jmap?");
488 
489   /**
490    * Option used by ReactiveJMapHistoDumper to check for an imminent
491    * OOM in worker or master process
492    */
493   IntConfOption MIN_FREE_MBS_ON_HEAP =
494       new IntConfOption("giraph.heap.minFreeMb", 128, "Option used by " +
495           "worker and master observers to check for imminent OOM exception");
496   /**
497    * Option can be used to enable reactively dumping jmap histo when
498    * OOM is imminent
499    */
500   BooleanConfOption REACTIVE_JMAP_ENABLE =
501       new BooleanConfOption("giraph.heap.enableReactiveJmapDumping", false,
502           "Option to enable dumping jmap histogram reactively based on " +
503               "free memory on heap");
504 
505   /**
506    * Minimum percent of the maximum number of workers that have responded
507    * in order to continue progressing. (float)
508    */
509   FloatConfOption MIN_PERCENT_RESPONDED =
510       new FloatConfOption("giraph.minPercentResponded", 100.0f,
511           "Minimum percent of the maximum number of workers that have " +
512           "responded in order to continue progressing. (float)");
513 
514   /** Enable the Metrics system */
515   BooleanConfOption METRICS_ENABLE =
516       new BooleanConfOption("giraph.metrics.enable", false,
517           "Enable the Metrics system");
518 
519   /** Directory in HDFS to write master metrics to, instead of stderr */
520   StrConfOption METRICS_DIRECTORY =
521       new StrConfOption("giraph.metrics.directory", "",
522           "Directory in HDFS to write master metrics to, instead of stderr");
523 
524   /**
525    *  ZooKeeper comma-separated list (if not set,
526    *  will start up ZooKeeper locally). Consider that after locally-starting
527    *  zookeeper, this parameter will updated the configuration with the corrent
528    *  configuration value.
529    */
530   StrConfOption ZOOKEEPER_LIST =
531       new StrConfOption("giraph.zkList", "",
532           "ZooKeeper comma-separated list (if not set, will start up " +
533           "ZooKeeper locally). Consider that after locally-starting " +
534           "zookeeper, this parameter will updated the configuration with " +
535           "the corrent configuration value.");
536 
537   /**
538    * Zookeeper List will always hold a value during the computation while
539    * this option provides information regarding whether the zookeeper was
540    * internally started or externally provided.
541    */
542   BooleanConfOption ZOOKEEPER_IS_EXTERNAL =
543     new BooleanConfOption("giraph.zkIsExternal", true,
544                           "Zookeeper List will always hold a value during " +
545                           "the computation while this option provides " +
546                           "information regarding whether the zookeeper was " +
547                           "internally started or externally provided.");
548 
549   /** ZooKeeper session millisecond timeout */
550   IntConfOption ZOOKEEPER_SESSION_TIMEOUT =
551       new IntConfOption("giraph.zkSessionMsecTimeout", MINUTES.toMillis(1),
552           "ZooKeeper session millisecond timeout");
553 
554   /** Polling interval to check for the ZooKeeper server data */
555   IntConfOption ZOOKEEPER_SERVERLIST_POLL_MSECS =
556       new IntConfOption("giraph.zkServerlistPollMsecs", SECONDS.toMillis(3),
557           "Polling interval to check for the ZooKeeper server data");
558 
559   /** ZooKeeper port to use */
560   IntConfOption ZOOKEEPER_SERVER_PORT =
561       new IntConfOption("giraph.zkServerPort", 22181, "ZooKeeper port to use");
562 
563   /** Local ZooKeeper directory to use */
564   String ZOOKEEPER_DIR = "giraph.zkDir";
565 
566   /** Max attempts for handling ZooKeeper connection loss */
567   IntConfOption ZOOKEEPER_OPS_MAX_ATTEMPTS =
568       new IntConfOption("giraph.zkOpsMaxAttempts", 3,
569           "Max attempts for handling ZooKeeper connection loss");
570 
571   /**
572    * Msecs to wait before retrying a failed ZooKeeper op due to connection loss.
573    */
574   IntConfOption ZOOKEEPER_OPS_RETRY_WAIT_MSECS =
575       new IntConfOption("giraph.zkOpsRetryWaitMsecs", SECONDS.toMillis(5),
576           "Msecs to wait before retrying a failed ZooKeeper op due to " +
577           "connection loss.");
578 
579   /** TCP backlog (defaults to number of workers) */
580   IntConfOption TCP_BACKLOG = new IntConfOption("giraph.tcpBacklog", 1,
581       "TCP backlog (defaults to number of workers)");
582 
583   /** Use netty pooled memory buffer allocator */
584   BooleanConfOption NETTY_USE_POOLED_ALLOCATOR = new BooleanConfOption(
585       "giraph.useNettyPooledAllocator", false, "Should netty use pooled " +
586       "memory allocator?");
587 
588   /** Use direct memory buffers in netty */
589   BooleanConfOption NETTY_USE_DIRECT_MEMORY = new BooleanConfOption(
590       "giraph.useNettyDirectMemory", false, "Should netty use direct " +
591       "memory buffers");
592 
593   /** How big to make the encoder buffer? */
594   IntConfOption NETTY_REQUEST_ENCODER_BUFFER_SIZE =
595       new IntConfOption("giraph.nettyRequestEncoderBufferSize", 32 * ONE_KB,
596           "How big to make the encoder buffer?");
597 
598   /** Netty client threads */
599   IntConfOption NETTY_CLIENT_THREADS =
600       new IntConfOption("giraph.nettyClientThreads", 4, "Netty client threads");
601 
602   /** Netty server threads */
603   IntConfOption NETTY_SERVER_THREADS =
604       new IntConfOption("giraph.nettyServerThreads", 16,
605           "Netty server threads");
606 
607   /** Use the execution handler in netty on the client? */
608   BooleanConfOption NETTY_CLIENT_USE_EXECUTION_HANDLER =
609       new BooleanConfOption("giraph.nettyClientUseExecutionHandler", true,
610           "Use the execution handler in netty on the client?");
611 
612   /** Netty client execution threads (execution handler) */
613   IntConfOption NETTY_CLIENT_EXECUTION_THREADS =
614       new IntConfOption("giraph.nettyClientExecutionThreads", 8,
615           "Netty client execution threads (execution handler)");
616 
617   /** Where to place the netty client execution handle? */
618   StrConfOption NETTY_CLIENT_EXECUTION_AFTER_HANDLER =
619       new StrConfOption("giraph.nettyClientExecutionAfterHandler",
620           "request-encoder",
621           "Where to place the netty client execution handle?");
622 
623   /** Use the execution handler in netty on the server? */
624   BooleanConfOption NETTY_SERVER_USE_EXECUTION_HANDLER =
625       new BooleanConfOption("giraph.nettyServerUseExecutionHandler", true,
626           "Use the execution handler in netty on the server?");
627 
628   /** Netty server execution threads (execution handler) */
629   IntConfOption NETTY_SERVER_EXECUTION_THREADS =
630       new IntConfOption("giraph.nettyServerExecutionThreads", 8,
631           "Netty server execution threads (execution handler)");
632 
633   /** Where to place the netty server execution handle? */
634   StrConfOption NETTY_SERVER_EXECUTION_AFTER_HANDLER =
635       new StrConfOption("giraph.nettyServerExecutionAfterHandler",
636           "requestFrameDecoder",
637           "Where to place the netty server execution handle?");
638 
639   /** Netty simulate a first request closed */
640   BooleanConfOption NETTY_SIMULATE_FIRST_REQUEST_CLOSED =
641       new BooleanConfOption("giraph.nettySimulateFirstRequestClosed", false,
642           "Netty simulate a first request closed");
643 
644   /** Netty simulate a first response failed */
645   BooleanConfOption NETTY_SIMULATE_FIRST_RESPONSE_FAILED =
646       new BooleanConfOption("giraph.nettySimulateFirstResponseFailed", false,
647           "Netty simulate a first response failed");
648 
649   /** Netty - set which compression to use */
650   StrConfOption NETTY_COMPRESSION_ALGORITHM =
651       new StrConfOption("giraph.nettyCompressionAlgorithm", "",
652           "Which compression algorithm to use in netty");
653 
654   /**
655    * Whether netty should pro-actively read requests and feed them to its
656    * processing pipeline
657    */
658   BooleanConfOption NETTY_AUTO_READ =
659       new BooleanConfOption("giraph.nettyAutoRead", true,
660           "Whether netty should pro-actively read requests and feed them to " +
661               "its processing pipeline");
662 
663   /** Max resolve address attempts */
664   IntConfOption MAX_RESOLVE_ADDRESS_ATTEMPTS =
665       new IntConfOption("giraph.maxResolveAddressAttempts", 5,
666           "Max resolve address attempts");
667 
668   /** Msecs to wait between waiting for all requests to finish */
669   IntConfOption WAITING_REQUEST_MSECS =
670       new IntConfOption("giraph.waitingRequestMsecs", SECONDS.toMillis(15),
671           "Msecs to wait between waiting for all requests to finish");
672 
673   /** Millseconds to wait for an event before continuing */
674   IntConfOption EVENT_WAIT_MSECS =
675       new IntConfOption("giraph.eventWaitMsecs", SECONDS.toMillis(30),
676           "Millseconds to wait for an event before continuing");
677 
678   /**
679    * Maximum milliseconds to wait before giving up trying to get the minimum
680    * number of workers before a superstep (int).
681    */
682   IntConfOption MAX_MASTER_SUPERSTEP_WAIT_MSECS =
683       new IntConfOption("giraph.maxMasterSuperstepWaitMsecs",
684           MINUTES.toMillis(10),
685           "Maximum milliseconds to wait before giving up trying to get the " +
686           "minimum number of workers before a superstep (int).");
687 
688   /** Milliseconds for a request to complete (or else resend) */
689   IntConfOption MAX_REQUEST_MILLISECONDS =
690       new IntConfOption("giraph.maxRequestMilliseconds", MINUTES.toMillis(10),
691           "Milliseconds for a request to complete (or else resend)");
692 
693   /** Netty max connection failures */
694   IntConfOption NETTY_MAX_CONNECTION_FAILURES =
695       new IntConfOption("giraph.nettyMaxConnectionFailures", 1000,
696           "Netty max connection failures");
697 
698   /** How long to wait before trying to reconnect failed connections */
699   IntConfOption WAIT_TIME_BETWEEN_CONNECTION_RETRIES_MS =
700       new IntConfOption("giraph.waitTimeBetweenConnectionRetriesMs", 500,
701           "");
702 
703   /** Initial port to start using for the IPC communication */
704   IntConfOption IPC_INITIAL_PORT =
705       new IntConfOption("giraph.ipcInitialPort", 30000,
706           "Initial port to start using for the IPC communication");
707 
708   /** Maximum bind attempts for different IPC ports */
709   IntConfOption MAX_IPC_PORT_BIND_ATTEMPTS =
710       new IntConfOption("giraph.maxIpcPortBindAttempts", 20,
711           "Maximum bind attempts for different IPC ports");
712   /**
713    * Fail first IPC port binding attempt, simulate binding failure
714    * on real grid testing
715    */
716   BooleanConfOption FAIL_FIRST_IPC_PORT_BIND_ATTEMPT =
717       new BooleanConfOption("giraph.failFirstIpcPortBindAttempt", false,
718           "Fail first IPC port binding attempt, simulate binding failure " +
719           "on real grid testing");
720 
721   /** Client send buffer size */
722   IntConfOption CLIENT_SEND_BUFFER_SIZE =
723       new IntConfOption("giraph.clientSendBufferSize", 512 * ONE_KB,
724           "Client send buffer size");
725 
726   /** Client receive buffer size */
727   IntConfOption CLIENT_RECEIVE_BUFFER_SIZE =
728       new IntConfOption("giraph.clientReceiveBufferSize", 32 * ONE_KB,
729           "Client receive buffer size");
730 
731   /** Server send buffer size */
732   IntConfOption SERVER_SEND_BUFFER_SIZE =
733       new IntConfOption("giraph.serverSendBufferSize", 32 * ONE_KB,
734           "Server send buffer size");
735 
736   /** Server receive buffer size */
737   IntConfOption SERVER_RECEIVE_BUFFER_SIZE =
738       new IntConfOption("giraph.serverReceiveBufferSize", 512 * ONE_KB,
739           "Server receive buffer size");
740 
741   /** Maximum size of messages (in bytes) per peer before flush */
742   IntConfOption MAX_MSG_REQUEST_SIZE =
743       new IntConfOption("giraph.msgRequestSize", 512 * ONE_KB,
744           "Maximum size of messages (in bytes) per peer before flush");
745 
746   /**
747    * How much bigger than the average per partition size to make initial per
748    * partition buffers.
749    * If this value is A, message request size is M,
750    * and a worker has P partitions, than its initial partition buffer size
751    * will be (M / P) * (1 + A).
752    */
753   FloatConfOption ADDITIONAL_MSG_REQUEST_SIZE =
754       new FloatConfOption("giraph.additionalMsgRequestSize", 0.2f,
755           "How much bigger than the average per partition size to make " +
756           "initial per partition buffers. If this value is A, message " +
757           "request size is M, and a worker has P partitions, than its " +
758           "initial partition buffer size will be (M / P) * (1 + A).");
759 
760 
761   /** Warn if msg request size exceeds default size by this factor */
762   FloatConfOption REQUEST_SIZE_WARNING_THRESHOLD = new FloatConfOption(
763       "giraph.msgRequestWarningThreshold", 2.0f,
764       "If request sizes are bigger than the buffer size by this factor " +
765       "warnings are printed to the log and to the command line");
766 
767   /** Maximum size of vertices (in bytes) per peer before flush */
768   IntConfOption MAX_VERTEX_REQUEST_SIZE =
769       new IntConfOption("giraph.vertexRequestSize", 512 * ONE_KB,
770           "Maximum size of vertices (in bytes) per peer before flush");
771 
772   /**
773    * Additional size (expressed as a ratio) of each per-partition buffer on
774    * top of the average size for vertices.
775    */
776   FloatConfOption ADDITIONAL_VERTEX_REQUEST_SIZE =
777       new FloatConfOption("giraph.additionalVertexRequestSize", 0.2f,
778           "Additional size (expressed as a ratio) of each per-partition " +
779               "buffer on top of the average size.");
780 
781   /** Maximum size of edges (in bytes) per peer before flush */
782   IntConfOption MAX_EDGE_REQUEST_SIZE =
783       new IntConfOption("giraph.edgeRequestSize", 512 * ONE_KB,
784           "Maximum size of edges (in bytes) per peer before flush");
785 
786   /**
787    * Additional size (expressed as a ratio) of each per-partition buffer on
788    * top of the average size for edges.
789    */
790   FloatConfOption ADDITIONAL_EDGE_REQUEST_SIZE =
791       new FloatConfOption("giraph.additionalEdgeRequestSize", 0.2f,
792           "Additional size (expressed as a ratio) of each per-partition " +
793           "buffer on top of the average size.");
794 
795   /** Maximum number of mutations per partition before flush */
796   IntConfOption MAX_MUTATIONS_PER_REQUEST =
797       new IntConfOption("giraph.maxMutationsPerRequest", 100,
798           "Maximum number of mutations per partition before flush");
799 
800   /**
801    * Use message size encoding (typically better for complex objects,
802    * not meant for primitive wrapped messages)
803    */
804   BooleanConfOption USE_MESSAGE_SIZE_ENCODING =
805       new BooleanConfOption("giraph.useMessageSizeEncoding", false,
806           "Use message size encoding (typically better for complex objects, " +
807           "not meant for primitive wrapped messages)");
808 
809   /** Number of channels used per server */
810   IntConfOption CHANNELS_PER_SERVER =
811       new IntConfOption("giraph.channelsPerServer", 1,
812           "Number of channels used per server");
813 
814   /** Number of flush threads per peer */
815   String MSG_NUM_FLUSH_THREADS = "giraph.msgNumFlushThreads";
816 
817   /** Number of threads for vertex computation */
818   IntConfOption NUM_COMPUTE_THREADS =
819       new IntConfOption("giraph.numComputeThreads", 1,
820           "Number of threads for vertex computation");
821 
822   /** Number of threads for input split loading */
823   IntConfOption NUM_INPUT_THREADS =
824       new IntConfOption("giraph.numInputThreads", 1,
825           "Number of threads for input split loading");
826 
827   /** Minimum stragglers of the superstep before printing them out */
828   IntConfOption PARTITION_LONG_TAIL_MIN_PRINT =
829       new IntConfOption("giraph.partitionLongTailMinPrint", 1,
830           "Minimum stragglers of the superstep before printing them out");
831 
832   /** Use superstep counters? (boolean) */
833   BooleanConfOption USE_SUPERSTEP_COUNTERS =
834       new BooleanConfOption("giraph.useSuperstepCounters", true,
835           "Use superstep counters? (boolean)");
836 
837   /**
838    * Input split sample percent - Used only for sampling and testing, rather
839    * than an actual job.  The idea is that to test, you might only want a
840    * fraction of the actual input splits from your VertexInputFormat to
841    * load (values should be [0, 100]).
842    */
843   FloatConfOption INPUT_SPLIT_SAMPLE_PERCENT =
844       new FloatConfOption("giraph.inputSplitSamplePercent", 100f,
845           "Input split sample percent - Used only for sampling and testing, " +
846           "rather than an actual job.  The idea is that to test, you might " +
847           "only want a fraction of the actual input splits from your " +
848           "VertexInputFormat to load (values should be [0, 100]).");
849 
850   /**
851    * To limit outlier vertex input splits from producing too many vertices or
852    * to help with testing, the number of vertices loaded from an input split
853    * can be limited.  By default, everything is loaded.
854    */
855   LongConfOption INPUT_SPLIT_MAX_VERTICES =
856       new LongConfOption("giraph.InputSplitMaxVertices", -1,
857           "To limit outlier vertex input splits from producing too many " +
858               "vertices or to help with testing, the number of vertices " +
859               "loaded from an input split can be limited. By default, " +
860               "everything is loaded.");
861 
862   /**
863    * To limit outlier vertex input splits from producing too many vertices or
864    * to help with testing, the number of edges loaded from an input split
865    * can be limited.  By default, everything is loaded.
866    */
867   LongConfOption INPUT_SPLIT_MAX_EDGES =
868       new LongConfOption("giraph.InputSplitMaxEdges", -1,
869           "To limit outlier vertex input splits from producing too many " +
870               "vertices or to help with testing, the number of edges loaded " +
871               "from an input split can be limited. By default, everything is " +
872               "loaded.");
873 
874   /**
875    * To minimize network usage when reading input splits,
876    * each worker can prioritize splits that reside on its host.
877    * This, however, comes at the cost of increased load on ZooKeeper.
878    * Hence, users with a lot of splits and input threads (or with
879    * configurations that can't exploit locality) may want to disable it.
880    */
881   BooleanConfOption USE_INPUT_SPLIT_LOCALITY =
882       new BooleanConfOption("giraph.useInputSplitLocality", true,
883           "To minimize network usage when reading input splits, each worker " +
884           "can prioritize splits that reside on its host. " +
885           "This, however, comes at the cost of increased load on ZooKeeper. " +
886           "Hence, users with a lot of splits and input threads (or with " +
887           "configurations that can't exploit locality) may want to disable " +
888           "it.");
889 
890   /** Multiplier for the current workers squared */
891   FloatConfOption PARTITION_COUNT_MULTIPLIER =
892       new FloatConfOption("giraph.masterPartitionCountMultiplier", 1.0f,
893           "Multiplier for the current workers squared");
894 
895   /** Minimum number of partitions to have per compute thread */
896   IntConfOption MIN_PARTITIONS_PER_COMPUTE_THREAD =
897       new IntConfOption("giraph.minPartitionsPerComputeThread", 1,
898           "Minimum number of partitions to have per compute thread");
899 
900   /** Overrides default partition count calculation if not -1 */
901   IntConfOption USER_PARTITION_COUNT =
902       new IntConfOption("giraph.userPartitionCount", -1,
903           "Overrides default partition count calculation if not -1");
904 
905   /** Vertex key space size for
906    * {@link org.apache.giraph.partition.WorkerGraphPartitionerImpl}
907    */
908   String PARTITION_VERTEX_KEY_SPACE_SIZE = "giraph.vertexKeySpaceSize";
909 
910   /**
911    *  How often to checkpoint (i.e. 0, means no checkpoint,
912    *  1 means every superstep, 2 is every two supersteps, etc.).
913    */
914   IntConfOption CHECKPOINT_FREQUENCY =
915       new IntConfOption("giraph.checkpointFrequency", 0,
916           "How often to checkpoint (i.e. 0, means no checkpoint, 1 means " +
917           "every superstep, 2 is every two supersteps, etc.).");
918 
919   /**
920    * Delete checkpoints after a successful job run?
921    */
922   BooleanConfOption CLEANUP_CHECKPOINTS_AFTER_SUCCESS =
923       new BooleanConfOption("giraph.cleanupCheckpointsAfterSuccess", true,
924           "Delete checkpoints after a successful job run?");
925 
926   /**
927    * An application can be restarted manually by selecting a superstep.  The
928    * corresponding checkpoint must exist for this to work.  The user should
929    * set a long value.  Default is start from scratch.
930    */
931   String RESTART_SUPERSTEP = "giraph.restartSuperstep";
932 
933   /**
934    * If application is restarted manually we need to specify job ID
935    * to restart from.
936    */
937   StrConfOption RESTART_JOB_ID = new StrConfOption("giraph.restart.jobId",
938       null, "Which job ID should I try to restart?");
939 
940   /**
941    * Base ZNode for Giraph's state in the ZooKeeper cluster.  Must be a root
942    * znode on the cluster beginning with "/"
943    */
944   String BASE_ZNODE_KEY = "giraph.zkBaseZNode";
945 
946   /**
947    * If ZOOKEEPER_LIST is not set, then use this directory to manage
948    * ZooKeeper
949    */
950   StrConfOption ZOOKEEPER_MANAGER_DIRECTORY =
951       new StrConfOption("giraph.zkManagerDirectory",
952           "_bsp/_defaultZkManagerDir",
953           "If ZOOKEEPER_LIST is not set, then use this directory to manage " +
954           "ZooKeeper");
955 
956   /** Number of ZooKeeper client connection attempts before giving up. */
957   IntConfOption ZOOKEEPER_CONNECTION_ATTEMPTS =
958       new IntConfOption("giraph.zkConnectionAttempts", 10,
959           "Number of ZooKeeper client connection attempts before giving up.");
960 
961   /** This directory has/stores the available checkpoint files in HDFS. */
962   StrConfOption CHECKPOINT_DIRECTORY =
963       new StrConfOption("giraph.checkpointDirectory", "_bsp/_checkpoints/",
964           "This directory has/stores the available checkpoint files in HDFS.");
965 
966   /**
967    * Comma-separated list of directories in the local filesystem for
968    * out-of-core partitions.
969    */
970   StrConfOption PARTITIONS_DIRECTORY =
971       new StrConfOption("giraph.partitionsDirectory", "_bsp/_partitions",
972           "Comma-separated list of directories in the local filesystem for " +
973           "out-of-core partitions.");
974 
975   /**
976    * Number of IO threads used in out-of-core mechanism. If local disk is used
977    * for spilling data to and reading data from, this number should be equal to
978    * the number of available disks on each machine. In such case, one should
979    * use giraph.partitionsDirectory to specify directories mounted on different
980    * disks.
981    */
982   IntConfOption NUM_OUT_OF_CORE_THREADS =
983       new IntConfOption("giraph.numOutOfCoreThreads", 1, "Number of IO " +
984           "threads used in out-of-core mechanism. If using local disk to " +
985           "spill data, this should be equal to the number of available " +
986           "disks. In such case, use giraph.partitionsDirectory to specify " +
987           "mount points on different disks.");
988 
989   /** Enable out-of-core graph. */
990   BooleanConfOption USE_OUT_OF_CORE_GRAPH =
991       new BooleanConfOption("giraph.useOutOfCoreGraph", false,
992           "Enable out-of-core graph.");
993 
994   /** Data accessor resource/object */
995   ClassConfOption<OutOfCoreDataAccessor> OUT_OF_CORE_DATA_ACCESSOR =
996       ClassConfOption.create("giraph.outOfCoreDataAccessor",
997           LocalDiskDataAccessor.class, OutOfCoreDataAccessor.class,
998           "Data accessor used in out-of-core computation (local-disk, " +
999               "in-memory, HDFS, etc.)");
1000 
1001   /**
1002    * Out-of-core oracle that is to be used for adaptive out-of-core engine. If
1003    * the `MAX_PARTITIONS_IN_MEMORY` is already set, this will be over-written
1004    * to be `FixedPartitionsOracle`.
1005    */
1006   ClassConfOption<OutOfCoreOracle> OUT_OF_CORE_ORACLE =
1007       ClassConfOption.create("giraph.outOfCoreOracle",
1008           MemoryEstimatorOracle.class, OutOfCoreOracle.class,
1009           "Out-of-core oracle that is to be used for adaptive out-of-core " +
1010               "engine");
1011 
1012   /** Maximum number of partitions to hold in memory for each worker. */
1013   IntConfOption MAX_PARTITIONS_IN_MEMORY =
1014       new IntConfOption("giraph.maxPartitionsInMemory", 0,
1015           "Maximum number of partitions to hold in memory for each worker. By" +
1016               " default it is set to 0 (for adaptive out-of-core mechanism");
1017 
1018   /** Directory to write YourKit snapshots to */
1019   String YOURKIT_OUTPUT_DIR = "giraph.yourkit.outputDir";
1020   /** Default directory to write YourKit snapshots to */
1021   String YOURKIT_OUTPUT_DIR_DEFAULT = "/tmp/giraph/%JOB_ID%/%TASK_ID%";
1022 
1023   /** Keep the zookeeper output for debugging? Default is to remove it. */
1024   BooleanConfOption KEEP_ZOOKEEPER_DATA =
1025       new BooleanConfOption("giraph.keepZooKeeperData", false,
1026           "Keep the zookeeper output for debugging? Default is to remove it.");
1027   /** Default ZooKeeper snap count. */
1028   int DEFAULT_ZOOKEEPER_SNAP_COUNT = 50000;
1029   /** Default ZooKeeper tick time. */
1030   int DEFAULT_ZOOKEEPER_TICK_TIME = 6000;
1031   /** Default ZooKeeper maximum client connections. */
1032   int DEFAULT_ZOOKEEPER_MAX_CLIENT_CNXNS = 10000;
1033   /** Number of snapshots to be retained after purge */
1034   int ZOOKEEPER_SNAP_RETAIN_COUNT = 3;
1035   /** Zookeeper purge interval in hours */
1036   int ZOOKEEPER_PURGE_INTERVAL = 1;
1037   /** ZooKeeper minimum session timeout */
1038   IntConfOption ZOOKEEPER_MIN_SESSION_TIMEOUT =
1039       new IntConfOption("giraph.zKMinSessionTimeout", MINUTES.toMillis(10),
1040           "ZooKeeper minimum session timeout");
1041   /** ZooKeeper maximum session timeout */
1042   IntConfOption ZOOKEEPER_MAX_SESSION_TIMEOUT =
1043       new IntConfOption("giraph.zkMaxSessionTimeout", MINUTES.toMillis(15),
1044           "ZooKeeper maximum session timeout");
1045 
1046   /** ZooKeeper force sync */
1047   BooleanConfOption ZOOKEEPER_FORCE_SYNC =
1048       new BooleanConfOption("giraph.zKForceSync", false,
1049           "ZooKeeper force sync");
1050 
1051   /** ZooKeeper skip ACLs */
1052   BooleanConfOption ZOOKEEPER_SKIP_ACL =
1053       new BooleanConfOption("giraph.ZkSkipAcl", true, "ZooKeeper skip ACLs");
1054 
1055   /**
1056    * Whether to use SASL with DIGEST and Hadoop Job Tokens to authenticate
1057    * and authorize Netty BSP Clients to Servers.
1058    */
1059   BooleanConfOption AUTHENTICATE =
1060       new BooleanConfOption("giraph.authenticate", false,
1061           "Whether to use SASL with DIGEST and Hadoop Job Tokens to " +
1062           "authenticate and authorize Netty BSP Clients to Servers.");
1063 
1064   /** Use unsafe serialization? */
1065   BooleanConfOption USE_UNSAFE_SERIALIZATION =
1066       new BooleanConfOption("giraph.useUnsafeSerialization", true,
1067           "Use unsafe serialization?");
1068 
1069   /**
1070    * Use BigDataIO for messages? If there are super-vertices in the
1071    * graph which receive a lot of messages (total serialized size of messages
1072    * goes beyond the maximum size of a byte array), setting this option to true
1073    * will remove that limit. The maximum memory available for a single vertex
1074    * will be limited to the maximum heap size available.
1075    */
1076   BooleanConfOption USE_BIG_DATA_IO_FOR_MESSAGES =
1077       new BooleanConfOption("giraph.useBigDataIOForMessages", false,
1078           "Use BigDataIO for messages?");
1079 
1080   /**
1081    * Maximum number of attempts a master/worker will retry before killing
1082    * the job.  This directly maps to the number of map task attempts in
1083    * Hadoop.
1084    */
1085   IntConfOption MAX_TASK_ATTEMPTS =
1086       new IntConfOption("mapred.map.max.attempts", -1,
1087           "Maximum number of attempts a master/worker will retry before " +
1088           "killing the job.  This directly maps to the number of map task " +
1089           "attempts in Hadoop.");
1090 
1091   /** Interface to use for hostname resolution */
1092   StrConfOption DNS_INTERFACE =
1093       new StrConfOption("giraph.dns.interface", "default",
1094           "Interface to use for hostname resolution");
1095   /** Server for hostname resolution */
1096   StrConfOption DNS_NAMESERVER =
1097       new StrConfOption("giraph.dns.nameserver", "default",
1098           "Server for hostname resolution");
1099 
1100   /**
1101    * The application will halt after this many supersteps is completed.  For
1102    * instance, if it is set to 3, the application will run at most 0, 1,
1103    * and 2 supersteps and then go into the shutdown superstep.
1104    */
1105   IntConfOption MAX_NUMBER_OF_SUPERSTEPS =
1106       new IntConfOption("giraph.maxNumberOfSupersteps", 1,
1107           "The application will halt after this many supersteps is " +
1108           "completed. For instance, if it is set to 3, the application will " +
1109           "run at most 0, 1, and 2 supersteps and then go into the shutdown " +
1110           "superstep.");
1111 
1112   /**
1113    * The application will not mutate the graph topology (the edges). It is used
1114    * to optimise out-of-core graph, by not writing back edges every time.
1115    */
1116   BooleanConfOption STATIC_GRAPH =
1117       new BooleanConfOption("giraph.isStaticGraph", false,
1118           "The application will not mutate the graph topology (the edges). " +
1119           "It is used to optimise out-of-core graph, by not writing back " +
1120           "edges every time.");
1121 
1122   /**
1123    * This option will tell which message encode &amp; store enum to use when
1124    * combining is not enabled
1125    */
1126   EnumConfOption<MessageEncodeAndStoreType> MESSAGE_ENCODE_AND_STORE_TYPE =
1127       EnumConfOption.create("giraph.messageEncodeAndStoreType",
1128           MessageEncodeAndStoreType.class,
1129           MessageEncodeAndStoreType.BYTEARRAY_PER_PARTITION,
1130           "Select the message_encode_and_store_type to use");
1131 
1132   /**
1133    * This option can be used to specify if a source vertex present in edge
1134    * input but not in vertex input can be created
1135    */
1136   BooleanConfOption CREATE_EDGE_SOURCE_VERTICES =
1137       new BooleanConfOption("giraph.createEdgeSourceVertices", true,
1138           "Create a source vertex if present in edge input but not " +
1139           "necessarily in vertex input");
1140 
1141   /**
1142    * Defines a call back that can be used to make decisions on
1143    * whether the vertex should be created or not in the runtime.
1144    */
1145   ClassConfOption<CreateSourceVertexCallback>
1146       CREATE_EDGE_SOURCE_VERTICES_CALLBACK =
1147       ClassConfOption.create("giraph.createEdgeSourceVerticesCallback",
1148           DefaultCreateSourceVertexCallback.class,
1149           CreateSourceVertexCallback.class,
1150           "Decide whether we should create a source vertex when id is " +
1151               "present in the edge input but not in vertex input");
1152 
1153   /**
1154    * This counter group will contain one counter whose name is the ZooKeeper
1155    * server:port which this job is using
1156    */
1157   String ZOOKEEPER_SERVER_PORT_COUNTER_GROUP = "Zookeeper server:port";
1158 
1159   /**
1160    * This counter group will contain one counter whose name is the ZooKeeper
1161    * node path which should be created to trigger computation halt
1162    */
1163   String ZOOKEEPER_HALT_NODE_COUNTER_GROUP = "Zookeeper halt node";
1164 
1165   /**
1166    * This counter group will contain one counter whose name is the ZooKeeper
1167    * node path which contains all data about this job
1168    */
1169   String ZOOKEEPER_BASE_PATH_COUNTER_GROUP = "Zookeeper base path";
1170 
1171   /**
1172    * Which class to use to write instructions on how to halt the application
1173    */
1174   ClassConfOption<HaltApplicationUtils.HaltInstructionsWriter>
1175   HALT_INSTRUCTIONS_WRITER_CLASS = ClassConfOption.create(
1176       "giraph.haltInstructionsWriter",
1177       HaltApplicationUtils.DefaultHaltInstructionsWriter.class,
1178       HaltApplicationUtils.HaltInstructionsWriter.class,
1179       "Class used to write instructions on how to halt the application");
1180 
1181   /**
1182    * Maximum timeout (in milliseconds) for waiting for all tasks
1183    * to complete after the job is done.  Defaults to 15 minutes.
1184    */
1185   IntConfOption WAIT_TASK_DONE_TIMEOUT_MS =
1186       new IntConfOption("giraph.waitTaskDoneTimeoutMs", MINUTES.toMillis(15),
1187           "Maximum timeout (in ms) for waiting for all all tasks to " +
1188               "complete");
1189 
1190   /** Whether to track job progress on client or not */
1191   BooleanConfOption TRACK_JOB_PROGRESS_ON_CLIENT =
1192       new BooleanConfOption("giraph.trackJobProgressOnClient", false,
1193           "Whether to track job progress on client or not");
1194 
1195   /** Class to use to track job progress on client */
1196   ClassConfOption<JobProgressTrackerService> JOB_PROGRESS_TRACKER_CLASS =
1197       ClassConfOption.create("giraph.jobProgressTrackerClass",
1198           DefaultJobProgressTrackerService.class,
1199           JobProgressTrackerService.class,
1200           "Class to use to track job progress on client");
1201 
1202   /**
1203    * Minimum number of vertices to compute before adding to worker progress.
1204    */
1205   LongConfOption VERTICES_TO_UPDATE_PROGRESS =
1206           new LongConfOption("giraph.VerticesToUpdateProgress", 100000,
1207                   "Minimum number of vertices to compute before " +
1208                           "updating worker progress");
1209 
1210 
1211   /** Number of retries for creating the HDFS files */
1212   IntConfOption HDFS_FILE_CREATION_RETRIES =
1213       new IntConfOption("giraph.hdfs.file.creation.retries", 10,
1214           "Retries to create an HDFS file before failing");
1215 
1216   /** Number of milliseconds to wait before retrying HDFS file creation */
1217   IntConfOption HDFS_FILE_CREATION_RETRY_WAIT_MS =
1218       new IntConfOption("giraph.hdfs.file.creation.retry.wait.ms", 30_000,
1219           "Milliseconds to wait prior to retrying creation of an HDFS file");
1220 
1221   /** Number of threads for writing and reading checkpoints */
1222   IntConfOption NUM_CHECKPOINT_IO_THREADS =
1223       new IntConfOption("giraph.checkpoint.io.threads", 8,
1224           "Number of threads for writing and reading checkpoints");
1225 
1226   /**
1227    * Compression algorithm to be used for checkpointing.
1228    * Defined by extension for hadoop compatibility reasons.
1229    */
1230   StrConfOption CHECKPOINT_COMPRESSION_CODEC =
1231       new StrConfOption("giraph.checkpoint.compression.codec",
1232           ".deflate",
1233           "Defines compression algorithm we will be using for " +
1234               "storing checkpoint. Available options include but " +
1235               "not restricted to: .deflate, .gz, .bz2, .lzo");
1236 
1237   /**
1238    * Defines if and when checkpointing is supported by this job.
1239    * By default checkpointing is always supported unless output during the
1240    * computation is enabled.
1241    */
1242   ClassConfOption<CheckpointSupportedChecker> CHECKPOINT_SUPPORTED_CHECKER =
1243       ClassConfOption.create("giraph.checkpoint.supported.checker",
1244           DefaultCheckpointSupportedChecker.class,
1245           CheckpointSupportedChecker.class,
1246           "This is the way to specify if checkpointing is " +
1247               "supported by the job");
1248 
1249 
1250   /** Number of threads to use in async message store, 0 means
1251    * we should not use async message processing */
1252   IntConfOption ASYNC_MESSAGE_STORE_THREADS_COUNT =
1253       new IntConfOption("giraph.async.message.store.threads", 0,
1254           "Number of threads to be used in async message store.");
1255 
1256   /** Output format class for hadoop to use (for committing) */
1257   ClassConfOption<OutputFormat> HADOOP_OUTPUT_FORMAT_CLASS =
1258       ClassConfOption.create("giraph.hadoopOutputFormatClass",
1259           BspOutputFormat.class, OutputFormat.class,
1260           "Output format class for hadoop to use (for committing)");
1261 
1262   /**
1263    * For worker to worker communication we can use IPs or host names, by
1264    * default prefer IPs.
1265    */
1266   BooleanConfOption PREFER_IP_ADDRESSES =
1267       new BooleanConfOption("giraph.preferIP", false,
1268       "Prefer IP addresses instead of host names");
1269 
1270   /**
1271    * Timeout for "waitForever", when we need to wait for zookeeper.
1272    * Since we should never really have to wait forever.
1273    * We should only wait some reasonable but large amount of time.
1274    */
1275   LongConfOption WAIT_ZOOKEEPER_TIMEOUT_MSEC =
1276       new LongConfOption("giraph.waitZookeeperTimeoutMsec",
1277           MINUTES.toMillis(15),
1278           "How long should we stay in waitForever loops in various " +
1279               "places that require network connection");
1280 
1281   /**
1282    * Timeout for "waitForever", when we need to wait for other workers
1283    * to complete their job.
1284    * Since we should never really have to wait forever.
1285    * We should only wait some reasonable but large amount of time.
1286    */
1287   LongConfOption WAIT_FOR_OTHER_WORKERS_TIMEOUT_MSEC =
1288       new LongConfOption("giraph.waitForOtherWorkersMsec",
1289           HOURS.toMillis(48),
1290           "How long should workers wait to finish superstep");
1291 
1292   /** Number of supersteps job will run for */
1293   IntConfOption SUPERSTEP_COUNT = new IntConfOption("giraph.numSupersteps", -1,
1294       "Number of supersteps job will run for");
1295 
1296   /** Whether to disable GiraphClassResolver which is an efficient
1297    * implementation of kryo class resolver. By default this resolver is used by
1298    * KryoSimpleWritable and KryoSimpleWrapper, and can be disabled with this
1299    * option */
1300   BooleanConfOption DISABLE_GIRAPH_CLASS_RESOLVER =
1301           new BooleanConfOption("giraph.disableGiraphClassResolver", false,
1302             "Disables GiraphClassResolver, which is a custom implementation " +
1303             "of kryo class resolver that avoids writing class names to the " +
1304             "underlying stream for faster serialization.");
1305 }
1306 // CHECKSTYLE: resume InterfaceIsTypeCheck