View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.giraph.counters;
20  
21  import org.apache.hadoop.mapreduce.Mapper.Context;
22  
23  import java.util.Arrays;
24  import java.util.Iterator;
25  
26  /**
27   * Hadoop Counters in group "Giraph Stats". General statistics about job.
28   */
29  public class GiraphStats extends HadoopCountersBase {
30    /** Counter group name for the Giraph statistics */
31    public static final String GROUP_NAME = "Giraph Stats";
32    /** superstep counter name */
33    public static final String SUPERSTEP_NAME = "Superstep";
34    /** aggregate vertices counter name */
35    public static final String VERTICES_NAME = "Aggregate vertices";
36    /** aggregate finished vertices counter name */
37    public static final String FINISHED_VERTICES_NAME =
38        "Aggregate finished vertices";
39    /** aggregate edges counter name */
40    public static final String EDGES_NAME = "Aggregate edges";
41    /** sent messages counter name */
42    public static final String SENT_MESSAGES_NAME = "Sent messages";
43    /** sent message bytes counter name */
44    public static final String SENT_MESSAGE_BYTES_NAME = "Sent message bytes";
45    /** aggregate sent messages counter name */
46    public static final String AGGREGATE_SENT_MESSAGES_NAME
47      = "Aggregate sent messages";
48    /** aggregate sent messages bytes counter name */
49    public static final String AGGREGATE_SENT_MESSAGE_BYTES_NAME
50      = "Aggregate sent message bytes";
51    /** workers counter name */
52    public static final String CURRENT_WORKERS_NAME = "Current workers";
53    /** current master partition task counter name */
54    public static final String CURRENT_MASTER_PARTITION_TASK_NAME =
55        "Current master task partition";
56    /** last checkpointed superstep counter name */
57    public static final String LAST_CHECKPOINTED_SUPERSTEP_NAME =
58        "Last checkpointed superstep";
59    /** aggregate bytes loaded from local disks in out-of-core */
60    public static final String OOC_BYTES_LOADED_NAME =
61        "Aggregate bytes loaded from local disks (out-of-core)";
62    /** aggregate bytes stored to local disks in out-of-core */
63    public static final String OOC_BYTES_STORED_NAME =
64        "Aggregate bytes stored to local disks (out-of-core)";
65    /** lowest percentage of graph in memory throughout the execution */
66    public static final String LOWEST_GRAPH_PERCENTAGE_IN_MEMORY_NAME =
67        "Lowest percentage of graph in memory so far (out-of-core)";
68  
69    /** Singleton instance for everyone to use */
70    private static GiraphStats INSTANCE;
71  
72    /** Superstep counter */
73    private static final int SUPERSTEP = 0;
74    /** Vertex counter */
75    private static final int VERTICES = 1;
76    /** Finished vertex counter */
77    private static final int FINISHED_VERTICES = 2;
78    /** Edge counter */
79    private static final int EDGES = 3;
80    /** Sent messages counter */
81    private static final int SENT_MESSAGES = 4;
82    /** Workers on this superstep */
83    private static final int CURRENT_WORKERS = 5;
84    /** Current master task partition */
85    private static final int CURRENT_MASTER_TASK_PARTITION = 6;
86    /** Last checkpointed superstep */
87    private static final int LAST_CHECKPOINTED_SUPERSTEP = 7;
88    /** Sent message bytes counter */
89    private static final int SENT_MESSAGE_BYTES = 8;
90    /** Aggregate sent messages counter */
91    private static final int AGG_SENT_MESSAGES = 9;
92    /** Aggregate sent message bytes counter */
93    private static final int AGG_SENT_MESSAGE_BYTES = 10;
94    /** Aggregate OOC loaded bytes counter */
95    private static final int OOC_BYTES_LOADED = 11;
96    /** Aggregate OOC stored bytes counter */
97    private static final int OOC_BYTES_STORED = 12;
98    /** Lowest percentage of graph in memory over time */
99    private static final int LOWEST_GRAPH_PERCENTAGE_IN_MEMORY = 13;
100   /** Number of counters in this class */
101   private static final int NUM_COUNTERS = 14;
102 
103   /** All the counters stored */
104   private final GiraphHadoopCounter[] counters;
105 
106   /**
107    * Create with Hadoop Context.
108    *
109    * @param context Hadoop Context to use.
110    */
111   private GiraphStats(Context context) {
112     super(context, GROUP_NAME);
113     counters = new GiraphHadoopCounter[NUM_COUNTERS];
114     counters[SUPERSTEP] = getCounter(SUPERSTEP_NAME);
115     counters[VERTICES] = getCounter(VERTICES_NAME);
116     counters[FINISHED_VERTICES] = getCounter(FINISHED_VERTICES_NAME);
117     counters[EDGES] = getCounter(EDGES_NAME);
118     counters[SENT_MESSAGES] = getCounter(SENT_MESSAGES_NAME);
119     counters[SENT_MESSAGE_BYTES] = getCounter(SENT_MESSAGE_BYTES_NAME);
120     counters[CURRENT_WORKERS] = getCounter(CURRENT_WORKERS_NAME);
121     counters[CURRENT_MASTER_TASK_PARTITION] =
122         getCounter(CURRENT_MASTER_PARTITION_TASK_NAME);
123     counters[LAST_CHECKPOINTED_SUPERSTEP] =
124         getCounter(LAST_CHECKPOINTED_SUPERSTEP_NAME);
125     counters[AGG_SENT_MESSAGES] =
126         getCounter(AGGREGATE_SENT_MESSAGES_NAME);
127     counters[AGG_SENT_MESSAGE_BYTES] =
128         getCounter(AGGREGATE_SENT_MESSAGE_BYTES_NAME);
129     counters[OOC_BYTES_LOADED] = getCounter(OOC_BYTES_LOADED_NAME);
130     counters[OOC_BYTES_STORED] = getCounter(OOC_BYTES_STORED_NAME);
131     counters[LOWEST_GRAPH_PERCENTAGE_IN_MEMORY] =
132         getCounter(LOWEST_GRAPH_PERCENTAGE_IN_MEMORY_NAME);
133     counters[LOWEST_GRAPH_PERCENTAGE_IN_MEMORY].setValue(100);
134   }
135 
136   /**
137    * Initialize with Hadoop Context.
138    *
139    * @param context Hadoop Context to use.
140    */
141   public static void init(Context context) {
142     INSTANCE = new GiraphStats(context);
143   }
144 
145   /**
146    * Get singleton instance.
147    *
148    * @return GiraphStats singleton
149    */
150   public static GiraphStats getInstance() {
151     return INSTANCE;
152   }
153 
154   /**
155    * Get SuperstepCounter counter
156    *
157    * @return SuperstepCounter counter
158    */
159   public GiraphHadoopCounter getSuperstepCounter() {
160     return counters[SUPERSTEP];
161   }
162 
163   /**
164    * Get Vertices counter
165    *
166    * @return Vertices counter
167    */
168   public GiraphHadoopCounter getVertices() {
169     return counters[VERTICES];
170   }
171 
172   /**
173    * Get FinishedVertexes counter
174    *
175    * @return FinishedVertexes counter
176    */
177   public GiraphHadoopCounter getFinishedVertexes() {
178     return counters[FINISHED_VERTICES];
179   }
180 
181   /**
182    * Get Edges counter
183    *
184    * @return Edges counter
185    */
186   public GiraphHadoopCounter getEdges() {
187     return counters[EDGES];
188   }
189 
190   /**
191    * Get SentMessages counter
192    *
193    * @return SentMessages counter
194    */
195   public GiraphHadoopCounter getSentMessages() {
196     return counters[SENT_MESSAGES];
197   }
198 
199   /**
200    * Get SentMessageBytes counter
201    *
202    * @return SentMessageBytes counter
203    */
204   public GiraphHadoopCounter getSentMessageBytes() {
205     return counters[SENT_MESSAGE_BYTES];
206   }
207 
208   /**
209    * Get AggregateSentMessages counter
210    *
211    * @return AggregateSentMessages counter
212    */
213   public GiraphHadoopCounter getAggregateSentMessages() {
214     return counters[AGG_SENT_MESSAGES];
215   }
216 
217   /**
218    * Get AggregateSentMessageBytes counter
219    *
220    * @return AggregateSentMessageBytes counter
221    */
222   public GiraphHadoopCounter getAggregateSentMessageBytes() {
223     return counters[AGG_SENT_MESSAGE_BYTES];
224   }
225 
226   /**
227    * Get CurrentWorkers counter
228    *
229    * @return CurrentWorkers counter
230    */
231   public GiraphHadoopCounter getCurrentWorkers() {
232     return counters[CURRENT_WORKERS];
233   }
234 
235   /**
236    * Get CurrentMasterTaskPartition counter
237    *
238    * @return CurrentMasterTaskPartition counter
239    */
240   public GiraphHadoopCounter getCurrentMasterTaskPartition() {
241     return counters[CURRENT_MASTER_TASK_PARTITION];
242   }
243 
244   /**
245    * Get LastCheckpointedSuperstep counter
246    *
247    * @return LastCheckpointedSuperstep counter
248    */
249   public GiraphHadoopCounter getLastCheckpointedSuperstep() {
250     return counters[LAST_CHECKPOINTED_SUPERSTEP];
251   }
252 
253   /**
254    * Get OOCBytesLoaded counter
255    *
256    * @return OOCBytesLoaded counter
257    */
258   public GiraphHadoopCounter getAggregateOOCBytesLoaded() {
259     return counters[OOC_BYTES_LOADED];
260   }
261 
262   /**
263    * Get OOCBytesStored counter
264    *
265    * @return OOCBytesStored counter
266    */
267   public GiraphHadoopCounter getAggregateOOCBytesStored() {
268     return counters[OOC_BYTES_STORED];
269   }
270 
271   public GiraphHadoopCounter getLowestGraphPercentageInMemory() {
272     return counters[LOWEST_GRAPH_PERCENTAGE_IN_MEMORY];
273   }
274 
275   @Override
276   public Iterator<GiraphHadoopCounter> iterator() {
277     return Arrays.asList(counters).iterator();
278   }
279 }