Zoltan2 Version of the Day
Zoltan2_AlltoAll.cpp
Go to the documentation of this file.
00001 // @HEADER
00002 //
00003 // ***********************************************************************
00004 //
00005 //   Zoltan2: A package of combinatorial algorithms for scientific computing
00006 //                  Copyright 2012 Sandia Corporation
00007 //
00008 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00009 // the U.S. Government retains certain rights in this software.
00010 //
00011 // Redistribution and use in source and binary forms, with or without
00012 // modification, are permitted provided that the following conditions are
00013 // met:
00014 //
00015 // 1. Redistributions of source code must retain the above copyright
00016 // notice, this list of conditions and the following disclaimer.
00017 //
00018 // 2. Redistributions in binary form must reproduce the above copyright
00019 // notice, this list of conditions and the following disclaimer in the
00020 // documentation and/or other materials provided with the distribution.
00021 //
00022 // 3. Neither the name of the Corporation nor the names of the
00023 // contributors may be used to endorse or promote products derived from
00024 // this software without specific prior written permission.
00025 //
00026 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00027 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00028 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00029 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00030 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00031 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00032 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00033 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00034 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00035 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00036 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00037 //
00038 // Questions? Contact Karen Devine      (kddevin@sandia.gov)
00039 //                    Erik Boman        (egboman@sandia.gov)
00040 //                    Siva Rajamanickam (srajama@sandia.gov)
00041 //
00042 // ***********************************************************************
00043 //
00044 // @HEADER
00045 
00051 #include <Zoltan2_AlltoAll.hpp>
00052 #include <Zoltan2_Standards.hpp>
00053 #include <Zoltan2_Environment.hpp>
00054 
00055 #include <vector>
00056 #include <climits>
00057 
00058 namespace Zoltan2
00059 {
00060 
00072 void AlltoAllCount(
00073   const Comm<int> &comm,      // Communicator to use for AlltoAll
00074   const Environment &env,     // Needed only for error handling
00075   const ArrayView<const int> &sendCount,   // Input:  # of data items to
00076                                            //         send to each process
00077   const ArrayView<int> &recvCount          // Output: # of data itmes to
00078                                            //         receive from each process
00079 )
00080 {
00081   int nprocs = comm.getSize();
00082   int rank = comm.getRank();
00083 
00084   recvCount[rank] = sendCount[rank];
00085 
00086   if (nprocs > 1) {
00087 
00088     // Post receives
00089     RCP<CommRequest<int> > *requests = new RCP<CommRequest<int> > [nprocs];
00090     for (int cnt = 0, i = 0; i < nprocs; i++) {
00091       if (i != rank) {
00092         try {
00093           requests[cnt++] = Teuchos::ireceive<int,int>(comm,
00094                                                      rcp(&(recvCount[i]),false),
00095                                                      i);
00096         }
00097         Z2_THROW_OUTSIDE_ERROR(env);
00098       }
00099     }
00100 
00101     Teuchos::barrier<int>(comm);
00102 
00103     // Send data; can use readySend since receives are posted.
00104     for (int i = 0; i < nprocs; i++) {
00105       if (i != rank) {
00106         try {
00107           Teuchos::readySend<int,int>(comm, sendCount[i], i);
00108         }
00109         Z2_THROW_OUTSIDE_ERROR(env);
00110       }
00111     }
00112 
00113     // Wait for messages to return.
00114     try {
00115       Teuchos::waitAll<int>(comm, arrayView(requests, nprocs-1));
00116     }
00117     Z2_THROW_OUTSIDE_ERROR(env);
00118 
00119     delete [] requests;
00120   }
00121 }
00122 
00123 /* \brief Specialization for std::string.
00124 
00125     For string of char. Number of chars in a string limited to SCHAR_MAX.
00126     Send as chars: 1 char for length of string, then chars in string,
00127      1 char for length of next string, and so on.
00128     \todo error checking
00129  */
00130 template <>
00131 void AlltoAllv(const Comm<int> &comm,
00132                const Environment &env,
00133                const ArrayView<const string> &sendBuf,
00134                const ArrayView<const int> &sendCount,
00135                ArrayRCP<string> &recvBuf,
00136                const ArrayView<int> &recvCount
00137 )
00138 {
00139   int nprocs = comm.getSize();
00140   int *newCount = new int [nprocs];
00141   memset(newCount, 0, sizeof(int) * nprocs);
00142   ArrayView<const int> newSendCount(newCount, nprocs);
00143 
00144   size_t numStrings = sendBuf.size();
00145   size_t numChars = 0;
00146   bool fail=false;
00147 
00148   for (int p=0, i=0; !fail && p < nprocs; p++){
00149     for (int c=0; !fail && c < sendCount[p]; c++, i++){
00150       size_t nchars = sendBuf[i].size();
00151       if (nchars > SCHAR_MAX)
00152         fail = true;
00153       else
00154         newCount[p] += nchars;
00155     }
00156     newCount[p] += sendCount[p];
00157     numChars += newCount[p];
00158   }
00159 
00160   if (fail)
00161     throw std::runtime_error("id string length exceeds SCHAR_MAX");
00162 
00163   char *sbuf = NULL;
00164   if (numChars > 0)
00165     sbuf = new char [numChars];
00166   char *sbufptr = sbuf;
00167 
00168   ArrayView<const char> newSendBuf(sbuf, numChars);
00169 
00170   for (size_t i=0; i < numStrings; i++){
00171     size_t nchars = sendBuf[i].size();
00172     *sbufptr++ = static_cast<char>(nchars);
00173     for (size_t j=0; j < nchars; j++)
00174       *sbufptr++ = sendBuf[i][j];
00175   }
00176 
00177   ArrayRCP<char> newRecvBuf;
00178   Array<int> newRecvCount(nprocs, 0);
00179 
00180   AlltoAllv<char>(comm, env, newSendBuf, newSendCount,
00181                   newRecvBuf, newRecvCount());
00182 
00183   delete [] sbuf;
00184   delete [] newCount;
00185 
00186   char *inBuf = newRecvBuf.getRawPtr();
00187 
00188   int numNewStrings = 0;
00189   char *buf = inBuf;
00190   char *endChar = inBuf + newRecvBuf.size();
00191   while (buf < endChar){
00192     int slen = static_cast<int>(*buf++);
00193     buf += slen;
00194     numNewStrings++;
00195   }
00196 
00197   // Data to return
00198   string *newStrings = new string [numNewStrings];
00199 
00200   buf = inBuf;
00201   int next = 0;
00202 
00203   for (int p=0; p < nprocs; p++){
00204     int nchars = newRecvCount[p];
00205     endChar = buf + nchars;
00206     while (buf < endChar){
00207       int slen = *buf++;
00208       string nextString;
00209       for (int i=0; i < slen; i++)
00210         nextString.push_back(*buf++);
00211       newStrings[next++] = nextString;
00212       recvCount[p]++;
00213     }
00214   }
00215 
00216   recvBuf = arcp<string>(newStrings, 0, numNewStrings, true);
00217 }
00218 
00219 
00220 
00221 #ifdef HAVE_ZOLTAN2_LONG_LONG
00222 
00223 /* \brief Specialization for unsigned long long
00224  */
00225 template <>
00226 void AlltoAllv(const Comm<int> &comm,
00227               const Environment &env,
00228               const ArrayView<const unsigned long long> &sendBuf,
00229               const ArrayView<const int> &sendCount,
00230               ArrayRCP<unsigned long long> &recvBuf,  // output, allocated here
00231               const ArrayView<int> &recvCount   // output
00232 )
00233 {
00234   const long long *sbuf =
00235     reinterpret_cast<const long long *>(sendBuf.getRawPtr());
00236   ArrayView<const long long> newSendBuf(sbuf, sendBuf.size());
00237   ArrayRCP<long long> newRecvBuf;
00238 
00239   AlltoAllv<long long>(comm, env, newSendBuf, sendCount,
00240     newRecvBuf, recvCount);
00241 
00242   recvBuf = arcp_reinterpret_cast<unsigned long long>(newRecvBuf);
00243 }
00244 
00245 #endif
00246 
00247 /* \brief Specialization for unsigned short
00248  */
00249 template <>
00250 void AlltoAllv(const Comm<int> &comm,
00251               const Environment &env,
00252               const ArrayView<const unsigned short> &sendBuf,
00253               const ArrayView<const int> &sendCount,
00254               ArrayRCP<unsigned short> &recvBuf,  // output, allocated here
00255               const ArrayView<int> &recvCount   // output
00256 )
00257 {
00258   const short *sbuf = reinterpret_cast<const short *>(sendBuf.getRawPtr());
00259   ArrayView<const short> newSendBuf(sbuf, sendBuf.size());
00260   ArrayRCP<short> newRecvBuf;
00261 
00262   AlltoAllv<short>(comm, env, newSendBuf, sendCount,
00263     newRecvBuf, recvCount);
00264 
00265   recvBuf = arcp_reinterpret_cast<unsigned short>(newRecvBuf);
00266 }
00267 
00268 
00269 /* \brief For data type unsigned char (no Teuchos::DirectSerializationTraits)
00270  */
00271 template <>
00272 void AlltoAllv(const Comm<int> &comm,
00273               const Environment &env,
00274               const ArrayView<const unsigned char> &sendBuf,
00275               const ArrayView<const int> &sendCount,
00276               ArrayRCP<unsigned char> &recvBuf,      // output, allocated here
00277               const ArrayView<int> &recvCount   // output
00278 )
00279 {
00280   const char *sbuf = reinterpret_cast<const char *>(sendBuf.getRawPtr());
00281   ArrayView<const char> newSendBuf(sbuf, sendBuf.size());
00282   ArrayRCP<char> newRecvBuf;
00283 
00284   AlltoAllv<char>(comm, env, newSendBuf, sendCount,
00285     newRecvBuf, recvCount);
00286 
00287   recvBuf = arcp_reinterpret_cast<unsigned char>(newRecvBuf);
00288 }
00289 
00290 }                   // namespace Z2