Zoltan2 Version of the Day
Zoltan2_AlltoAll.hpp
Go to the documentation of this file.
00001 // @HEADER
00002 //
00003 // ***********************************************************************
00004 //
00005 //   Zoltan2: A package of combinatorial algorithms for scientific computing
00006 //                  Copyright 2012 Sandia Corporation
00007 //
00008 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00009 // the U.S. Government retains certain rights in this software.
00010 //
00011 // Redistribution and use in source and binary forms, with or without
00012 // modification, are permitted provided that the following conditions are
00013 // met:
00014 //
00015 // 1. Redistributions of source code must retain the above copyright
00016 // notice, this list of conditions and the following disclaimer.
00017 //
00018 // 2. Redistributions in binary form must reproduce the above copyright
00019 // notice, this list of conditions and the following disclaimer in the
00020 // documentation and/or other materials provided with the distribution.
00021 //
00022 // 3. Neither the name of the Corporation nor the names of the
00023 // contributors may be used to endorse or promote products derived from
00024 // this software without specific prior written permission.
00025 //
00026 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00027 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00028 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00029 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00030 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00031 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00032 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00033 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00034 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00035 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00036 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00037 //
00038 // Questions? Contact Karen Devine      (kddevin@sandia.gov)
00039 //                    Erik Boman        (egboman@sandia.gov)
00040 //                    Siva Rajamanickam (srajama@sandia.gov)
00041 //
00042 // ***********************************************************************
00043 //
00044 // @HEADER
00045 
00050 #ifndef _ZOLTAN2_ALLTOALL_HPP_
00051 #define _ZOLTAN2_ALLTOALL_HPP_
00052 
00053 #include <Zoltan2_Standards.hpp>
00054 #include <Zoltan2_Environment.hpp>
00055 
00056 #include <vector>
00057 #include <climits>
00058 
00059 namespace Zoltan2
00060 {
00061 
00062 extern void AlltoAllCount(const Comm<int> &comm, const Environment &env,
00063   const ArrayView<const int> &sendCount, const ArrayView<int> &recvCount);
00064 
00092 template <typename T>
00093 void AlltoAllv(const Comm<int> &comm,
00094               const Environment &env,
00095               const ArrayView<const T> &sendBuf,
00096               const ArrayView<const int> &sendCount,
00097               ArrayRCP<T> &recvBuf,      // output, allocated here
00098               const ArrayView<int> &recvCount   // output
00099 )
00100 {
00101   int nprocs = comm.getSize();
00102   int rank = comm.getRank();
00103 
00104   try{
00105     Zoltan2::AlltoAllCount(comm, env, sendCount, recvCount);
00106   }
00107   Z2_FORWARD_EXCEPTIONS;
00108 
00109   // Allocate the receive buffer.
00110   size_t totalrecv = 0;
00111   int maxMsg = 0;
00112   int nrecvranks = 0;
00113   for(int i = 0; i < nprocs; i++) {
00114     if (recvCount[i] > 0) {
00115       totalrecv += recvCount[i];
00116       nrecvranks++;
00117       if (recvCount[i] > maxMsg) maxMsg = recvCount[i];
00118     }
00119   }
00120 
00121 
00122   T *rbuf = new T[totalrecv];
00123 
00124   if (nprocs > 1) {
00125 
00126     RCP<CommRequest<int> > *requests = new RCP<CommRequest<int> > [nrecvranks];
00127 
00128     // Error checking for memory and message size.
00129     int OK[2] = {1,1};
00130                 // OK[0] -- true/false indicating whether each message size
00131                 //          fits in an int (for MPI).
00132                 // OK[1] -- true/false indicating whether memory allocs are OK
00133     int gOK[2]; // For global reduce of OK.
00134 
00135     if (size_t(maxMsg) * sizeof(T) > INT_MAX && nprocs > 1) OK[0] = false;
00136     if (totalrecv && !rbuf) OK[1] = 0;
00137     if (!requests) OK[1] = 0;
00138 
00139     // Post receives
00140 
00141     size_t offset = 0;
00142     size_t myrecvoffset = 0;
00143     size_t mysendoffset = 0;
00144 
00145     if (OK[0] && OK[1]) {
00146       int rcnt = 0;
00147       for (int i = 0; i < nprocs; i++) {
00148         if (i != rank && recvCount[i]) {
00149           try {
00150             requests[rcnt++] = Teuchos::ireceive<int,T>(comm,
00151                              Teuchos::arcp(&rbuf[offset],0,recvCount[i],false),
00152                              i);
00153           }
00154           Z2_THROW_OUTSIDE_ERROR(env);
00155         }
00156         else if (i == rank) {
00157           myrecvoffset = offset;
00158         }
00159         offset += recvCount[i];
00160       }
00161     }
00162 
00163     // Use barrier for error checking
00164     Teuchos::reduceAll<int>(comm, Teuchos::REDUCE_MIN, 2, OK, gOK);
00165     if (!gOK[0] || !gOK[1]) {
00166       delete [] rbuf;
00167       delete [] requests;
00168       if (!gOK[0])
00169         throw std::runtime_error("Max single message length exceeded");
00170       else
00171         throw std::bad_alloc();
00172     }
00173 
00174     // Send data; can use readySend since receives are posted.
00175     offset = 0;
00176     for (int i = 0; i < nprocs; i++) {
00177       if (i != rank && sendCount[i]) {
00178         try {
00179           Teuchos::readySend<int,T>(comm,
00180                             Teuchos::arrayView(&sendBuf[offset],sendCount[i]),
00181                             i);
00182         }
00183         Z2_THROW_OUTSIDE_ERROR(env);
00184       }
00185       else if (i == rank) {
00186         mysendoffset = offset;
00187       }
00188       offset += sendCount[i];
00189     }
00190 
00191     // Copy local data
00192     for (int j = 0; j < sendCount[rank]; j++)
00193       rbuf[myrecvoffset++] = sendBuf[mysendoffset++];
00194 
00195     // Wait for messages to return.
00196     try {
00197       Teuchos::waitAll<int>(comm, Teuchos::arrayView(requests, nrecvranks));
00198     }
00199     Z2_THROW_OUTSIDE_ERROR(env);
00200 
00201     delete [] requests;
00202   }
00203   else { // nprocs == 1; no communication needed
00204 
00205     if (totalrecv && !rbuf)
00206       throw std::bad_alloc();
00207 
00208     for (int j = 0; j < sendCount[0]; j++)
00209       rbuf[j] = sendBuf[j];
00210   }
00211 
00212   if (totalrecv)
00213     recvBuf = ArrayRCP<T>(rbuf, 0, totalrecv, true);
00214   else
00215     recvBuf = Teuchos::null;
00216 }
00217 
00218 /* \brief Specialization for std::string.
00219 
00220     For string of char. Number of chars in a string limited to SCHAR_MAX.
00221     Send as chars: 1 char for length of string, then chars in string,
00222      1 char for length of next string, and so on.
00223     \todo error checking
00224  */
00225 template <>
00226 void AlltoAllv(const Comm<int> &comm,
00227               const Environment &env,
00228               const ArrayView<const string> &sendBuf,
00229               const ArrayView<const int> &sendCount,
00230               ArrayRCP<string> &recvBuf,
00231               const ArrayView<int> &recvCount);
00232 
00233 #ifdef HAVE_ZOLTAN2_LONG_LONG
00234 
00235 /* \brief Specialization for unsigned long long
00236  */
00237 template <>
00238 void AlltoAllv(const Comm<int> &comm,
00239               const Environment &env,
00240               const ArrayView<const unsigned long long> &sendBuf,
00241               const ArrayView<const int> &sendCount,
00242               ArrayRCP<unsigned long long> &recvBuf,
00243               const ArrayView<int> &recvCount);
00244 #endif
00245 
00246 /* \brief Specialization for unsigned short
00247  */
00248 template <>
00249 void AlltoAllv(const Comm<int> &comm,
00250               const Environment &env,
00251               const ArrayView<const unsigned short> &sendBuf,
00252               const ArrayView<const int> &sendCount,
00253               ArrayRCP<unsigned short> &recvBuf,
00254               const ArrayView<int> &recvCount);
00255 
00256 /* \brief For data type unsigned char (no Teuchos::DirectSerializationTraits)
00257  */
00258 template <>
00259 void AlltoAllv(const Comm<int> &comm,
00260               const Environment &env,
00261               const ArrayView<const unsigned char> &sendBuf,
00262               const ArrayView<const int> &sendCount,
00263               ArrayRCP<unsigned char> &recvBuf,
00264               const ArrayView<int> &recvCount);
00265 
00266 }                   // namespace Z2
00267 #endif