View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.giraph.block_app.library.internal;
19  
20  import java.util.Iterator;
21  
22  import org.apache.giraph.block_app.framework.api.BlockWorkerReceiveApi;
23  import org.apache.giraph.block_app.framework.api.BlockWorkerSendApi;
24  import org.apache.giraph.block_app.framework.block.Block;
25  import org.apache.giraph.block_app.framework.piece.Piece;
26  import org.apache.giraph.block_app.framework.piece.delegate.FilteringPiece;
27  import org.apache.giraph.block_app.framework.piece.interfaces.VertexReceiver;
28  import org.apache.giraph.block_app.framework.piece.interfaces.VertexSender;
29  import org.apache.giraph.block_app.library.striping.StripingUtils;
30  import org.apache.giraph.combiner.MessageCombiner;
31  import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
32  import org.apache.giraph.function.Function;
33  import org.apache.giraph.function.Predicate;
34  import org.apache.giraph.function.primitive.Int2ObjFunction;
35  import org.apache.giraph.function.vertex.ConsumerWithVertex;
36  import org.apache.giraph.function.vertex.SupplierFromVertex;
37  import org.apache.giraph.graph.Vertex;
38  import org.apache.hadoop.io.Writable;
39  import org.apache.hadoop.io.WritableComparable;
40  
41  import com.google.common.base.Preconditions;
42  import com.google.common.collect.Iterators;
43  
44  /**
45   * Piece that sends a message provided through messageProducer to given set of
46   * neighbors, uses a message combiner and passes them to messagesConsumer.
47   *
48   * @param <I> Vertex id type
49   * @param <V> Vertex value type
50   * @param <E> Edge value type
51   * @param <M> Message type
52   */
53  public class SendMessageWithCombinerPiece<I extends WritableComparable,
54      V extends Writable, E extends Writable, M extends Writable>
55      extends Piece<I, V, E, M, Object> {
56    private final String name;
57    private final MessageCombiner<? super I, M> messageCombiner;
58    private final SupplierFromVertex<I, V, E, M> messageSupplier;
59    private final SupplierFromVertex<I, V, E, Iterator<I>> targetsSupplier;
60    private final ConsumerWithVertex<I, V, E, M> messagesConsumer;
61  
62    public SendMessageWithCombinerPiece(String name,
63        MessageCombiner<? super I, M> messageCombiner,
64        SupplierFromVertex<I, V, E, M> messageSupplier,
65        SupplierFromVertex<I, V, E, Iterator<I>> targetsSupplier,
66        ConsumerWithVertex<I, V, E, M> messagesConsumer) {
67      Preconditions.checkNotNull(messageCombiner);
68      this.name = name;
69      this.messageCombiner = messageCombiner;
70      this.messageSupplier = messageSupplier;
71      this.targetsSupplier = targetsSupplier;
72      this.messagesConsumer = messagesConsumer;
73    }
74  
75    /**
76     * Stripe message sending computation across multiple stripes, in
77     * each stripe only part of the vertices will receive messages.
78     *
79     * @param stripes Number of stripes
80     * @param stripeSupplier Stripe supplier function, if IDs are Longs, you can
81     *                       use StripingUtils::fastHashStripingPredicate
82     * @return Resulting block
83     */
84    public Block stripeByReceiver(
85        int stripes,
86        Int2ObjFunction<Int2ObjFunction<Predicate<I>>> stripeSupplier) {
87      return StripingUtils.generateStripedBlock(
88          stripes,
89          new Function<Predicate<I>, Block>() {
90            @Override
91            public Block apply(final Predicate<I> stripePredicate) {
92              return FilteringPiece.createReceiveFiltering(
93                  new SupplierFromVertex<I, V, E, Boolean>() {
94                    @Override
95                    public Boolean get(Vertex<I, V, E> vertex) {
96                      return stripePredicate.apply(vertex.getId());
97                    }
98                  },
99                  new SendMessageWithCombinerPiece<>(
100                   name,
101                   messageCombiner,
102                   messageSupplier,
103                   new SupplierFromVertex<I, V, E, Iterator<I>>() {
104                     @Override
105                     public Iterator<I> get(Vertex<I, V, E> vertex) {
106                       return Iterators.filter(
107                           targetsSupplier.get(vertex),
108                           new com.google.common.base.Predicate<I>() {
109                             @Override
110                             public boolean apply(I targetId) {
111                               return stripePredicate.apply(targetId);
112                             }
113                           });
114                     }
115                   },
116                   messagesConsumer));
117           }
118         },
119         stripeSupplier);
120   }
121 
122   @Override
123   public VertexSender<I, V, E> getVertexSender(
124       final BlockWorkerSendApi<I, V, E, M> workerApi,
125       Object executionStage) {
126     return new InnerVertexSender() {
127       @Override
128       public void vertexSend(Vertex<I, V, E> vertex) {
129         Iterator<I> targets = targetsSupplier.get(vertex);
130         M message = messageSupplier.get(vertex);
131         if (message != null && targets != null && targets.hasNext()) {
132           workerApi.sendMessageToMultipleEdges(targets, message);
133         }
134       }
135     };
136   }
137 
138   @Override
139   public VertexReceiver<I, V, E, M> getVertexReceiver(
140       BlockWorkerReceiveApi<I> workerApi,
141       Object executionStage) {
142     return new InnerVertexReceiver() {
143       @Override
144       public void vertexReceive(Vertex<I, V, E> vertex, Iterable<M> messages) {
145         Iterator<M> iter = messages.iterator();
146         M combinedMessage = null;
147         if (iter.hasNext()) {
148           combinedMessage = iter.next();
149           // When message combiner is used, there is never more then one message
150           Preconditions.checkArgument(!iter.hasNext());
151         }
152         messagesConsumer.apply(vertex, combinedMessage);
153       }
154     };
155   }
156 
157   @Override
158   public MessageCombiner<? super I, M> getMessageCombiner(
159       ImmutableClassesGiraphConfiguration conf) {
160     return messageCombiner;
161   }
162 
163   @Override
164   public String toString() {
165     return name;
166   }
167 }