Zoltan2
Zoltan2_AlltoAll.hpp
Go to the documentation of this file.
00001 // @HEADER
00002 //
00003 // ***********************************************************************
00004 //
00005 //   Zoltan2: A package of combinatorial algorithms for scientific computing
00006 //                  Copyright 2012 Sandia Corporation
00007 //
00008 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00009 // the U.S. Government retains certain rights in this software.
00010 //
00011 // Redistribution and use in source and binary forms, with or without
00012 // modification, are permitted provided that the following conditions are
00013 // met:
00014 //
00015 // 1. Redistributions of source code must retain the above copyright
00016 // notice, this list of conditions and the following disclaimer.
00017 //
00018 // 2. Redistributions in binary form must reproduce the above copyright
00019 // notice, this list of conditions and the following disclaimer in the
00020 // documentation and/or other materials provided with the distribution.
00021 //
00022 // 3. Neither the name of the Corporation nor the names of the
00023 // contributors may be used to endorse or promote products derived from
00024 // this software without specific prior written permission.
00025 //
00026 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00027 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00028 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00029 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00030 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00031 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00032 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00033 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00034 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00035 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00036 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00037 //
00038 // Questions? Contact Karen Devine      (kddevin@sandia.gov)
00039 //                    Erik Boman        (egboman@sandia.gov)
00040 //                    Siva Rajamanickam (srajama@sandia.gov)
00041 //
00042 // ***********************************************************************
00043 //
00044 // @HEADER
00045 
00050 #ifndef _ZOLTAN2_ALLTOALL_HPP_
00051 #define _ZOLTAN2_ALLTOALL_HPP_
00052 
00053 #include <Zoltan2_Standards.hpp>
00054 #include <Zoltan2_Environment.hpp>
00055 
00056 #include <vector>
00057 #include <climits>
00058 
00059 namespace Zoltan2
00060 {
00061 
00062 extern void AlltoAllCount(const Comm<int> &comm, const Environment &env,
00063   const ArrayView<const int> &sendCount, const ArrayView<int> &recvCount);
00064 
00092 template <typename T>
00093 void AlltoAllv(const Comm<int> &comm,
00094               const Environment &env,
00095               const ArrayView<const T> &sendBuf,
00096               const ArrayView<const int> &sendCount,
00097               ArrayRCP<T> &recvBuf,      // output, allocated here
00098               const ArrayView<int> &recvCount   // output
00099 )
00100 {
00101   int nprocs = comm.getSize();
00102   int rank = comm.getRank();
00103 
00104   try{
00105     Zoltan2::AlltoAllCount(comm, env, sendCount, recvCount);
00106   }
00107   Z2_FORWARD_EXCEPTIONS;
00108 
00109   // Allocate the receive buffer.
00110   size_t totalrecv = 0;
00111   int maxMsg = 0;
00112   int nrecvranks = 0;
00113   for(int i = 0; i < nprocs; i++) {
00114     if (recvCount[i] > 0) {
00115       totalrecv += recvCount[i];
00116       nrecvranks++;
00117       if (recvCount[i] > maxMsg) maxMsg = recvCount[i];
00118     }
00119   }
00120 
00121 
00122   T *rbuf = NULL;
00123   if (totalrecv) rbuf = new T[totalrecv];
00124 
00125   if (nprocs > 1) {
00126 
00127     RCP<CommRequest<int> > *requests = new RCP<CommRequest<int> > [nrecvranks];
00128 
00129     // Error checking for memory and message size.
00130     int OK[2] = {1,1};
00131                 // OK[0] -- true/false indicating whether each message size
00132                 //          fits in an int (for MPI).
00133                 // OK[1] -- true/false indicating whether memory allocs are OK
00134     int gOK[2]; // For global reduce of OK.
00135 
00136     if (size_t(maxMsg) * sizeof(T) > INT_MAX && nprocs > 1) OK[0] = false;
00137     if (totalrecv && !rbuf) OK[1] = 0;
00138     if (!requests) OK[1] = 0;
00139 
00140     // Post receives
00141 
00142     size_t offset = 0;
00143     size_t myrecvoffset = 0;
00144     size_t mysendoffset = 0;
00145 
00146     if (OK[0] && OK[1]) {
00147       int rcnt = 0;
00148       for (int i = 0; i < nprocs; i++) {
00149         if (i != rank && recvCount[i]) {
00150           try {
00151             requests[rcnt++] = Teuchos::ireceive<int,T>(comm,
00152                              Teuchos::arcp(&rbuf[offset],0,recvCount[i],false),
00153                              i);
00154           }
00155           Z2_THROW_OUTSIDE_ERROR(env);
00156         }
00157         else if (i == rank) {
00158           myrecvoffset = offset;
00159         }
00160         offset += recvCount[i];
00161       }
00162     }
00163 
00164     // Use barrier for error checking
00165     Teuchos::reduceAll<int>(comm, Teuchos::REDUCE_MIN, 2, OK, gOK);
00166     if (!gOK[0] || !gOK[1]) {
00167       delete [] rbuf;
00168       delete [] requests;
00169       if (!gOK[0])
00170         throw std::runtime_error("Max single message length exceeded");
00171       else
00172         throw std::bad_alloc();
00173     }
00174 
00175     // Send data; can use readySend since receives are posted.
00176     offset = 0;
00177     for (int i = 0; i < nprocs; i++) {
00178       if (i != rank && sendCount[i]) {
00179         try {
00180           Teuchos::readySend<int,T>(comm,
00181                             Teuchos::arrayView(&sendBuf[offset],sendCount[i]),
00182                             i);
00183         }
00184         Z2_THROW_OUTSIDE_ERROR(env);
00185       }
00186       else if (i == rank) {
00187         mysendoffset = offset;
00188       }
00189       offset += sendCount[i];
00190     }
00191 
00192     // Copy local data
00193     for (int j = 0; j < sendCount[rank]; j++)
00194       rbuf[myrecvoffset++] = sendBuf[mysendoffset++];
00195 
00196     // Wait for messages to return.
00197     try {
00198       Teuchos::waitAll<int>(comm, Teuchos::arrayView(requests, nrecvranks));
00199     }
00200     Z2_THROW_OUTSIDE_ERROR(env);
00201 
00202     delete [] requests;
00203   }
00204   else { // nprocs == 1; no communication needed
00205 
00206     if (totalrecv && !rbuf)
00207       throw std::bad_alloc();
00208 
00209     for (int j = 0; j < sendCount[0]; j++)
00210       rbuf[j] = sendBuf[j];
00211   }
00212 
00213   if (totalrecv)
00214     recvBuf = ArrayRCP<T>(rbuf, 0, totalrecv, true);
00215   else
00216     recvBuf = Teuchos::null;
00217 }
00218 
00219 /* \brief Specialization for std::string.
00220 
00221     For string of char. Number of chars in a string limited to SCHAR_MAX.
00222     Send as chars: 1 char for length of string, then chars in string,
00223      1 char for length of next string, and so on.
00224     \todo error checking
00225  */
00226 template <>
00227 void AlltoAllv(const Comm<int> &comm,
00228               const Environment &env,
00229               const ArrayView<const std::string> &sendBuf,
00230               const ArrayView<const int> &sendCount,
00231               ArrayRCP<std::string> &recvBuf,
00232               const ArrayView<int> &recvCount);
00233 
00234 #ifdef HAVE_ZOLTAN2_LONG_LONG
00235 
00236 /* \brief Specialization for unsigned long long
00237  */
00238 template <>
00239 void AlltoAllv(const Comm<int> &comm,
00240               const Environment &env,
00241               const ArrayView<const unsigned long long> &sendBuf,
00242               const ArrayView<const int> &sendCount,
00243               ArrayRCP<unsigned long long> &recvBuf,
00244               const ArrayView<int> &recvCount);
00245 #endif
00246 
00247 /* \brief Specialization for unsigned short
00248  */
00249 template <>
00250 void AlltoAllv(const Comm<int> &comm,
00251               const Environment &env,
00252               const ArrayView<const unsigned short> &sendBuf,
00253               const ArrayView<const int> &sendCount,
00254               ArrayRCP<unsigned short> &recvBuf,
00255               const ArrayView<int> &recvCount);
00256 
00257 /* \brief For data type unsigned char (no Teuchos::DirectSerializationTraits)
00258  */
00259 template <>
00260 void AlltoAllv(const Comm<int> &comm,
00261               const Environment &env,
00262               const ArrayView<const unsigned char> &sendBuf,
00263               const ArrayView<const int> &sendCount,
00264               ArrayRCP<unsigned char> &recvBuf,
00265               const ArrayView<int> &recvCount);
00266 
00267 }                   // namespace Z2
00268 #endif