Teuchos - Trilinos Tools Package Version of the Day
Teuchos_MPIComm.cpp
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 //
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ***********************************************************************
00040 // @HEADER
00041 
00042 // Make deprecated warnings go away for compling this source file.   All of this
00043 // code is deprecated so there is little point in flagging it as such!
00044 #define TEUCHOS_DEPRECATED
00045 
00046 #include "Teuchos_MPIComm.hpp"
00047 #include "Teuchos_ErrorPolling.hpp"
00048 
00049 
00050 using namespace Teuchos;
00051 
00052 namespace Teuchos
00053 {
00054 const int MPIComm::INT = 1;
00055 const int MPIComm::FLOAT = 2;
00056 const int MPIComm::DOUBLE = 3;
00057 const int MPIComm::DOUBLE_INT = 4;
00058 const int MPIComm::CHAR = 5;
00059 
00060 const int MPIComm::SUM = 6;
00061 const int MPIComm::MIN = 7;
00062 const int MPIComm::MAX = 8;
00063 const int MPIComm::MINLOC = 8;
00064 const int MPIComm::MAXLOC = 10;
00065 const int MPIComm::PROD = 11;
00066 }
00067 
00068 
00069 MPIComm::MPIComm()
00070   :
00071 #ifdef HAVE_MPI
00072   comm_(MPI_COMM_WORLD),
00073 #endif
00074   nProc_(0), myRank_(0)
00075 {
00076   init();
00077 }
00078 
00079 #ifdef HAVE_MPI
00080 MPIComm::MPIComm(MPI_Comm comm)
00081   : comm_(comm), nProc_(0), myRank_(0)
00082 {
00083   init();
00084 }
00085 #endif
00086 
00087 int MPIComm::mpiIsRunning() const
00088 {
00089   int mpiStarted = 0;
00090 #ifdef HAVE_MPI
00091   MPI_Initialized(&mpiStarted);
00092 #endif
00093   return mpiStarted;
00094 }
00095 
00096 void MPIComm::init()
00097 {
00098 #ifdef HAVE_MPI
00099 
00100   if (mpiIsRunning())
00101     {
00102       errCheck(MPI_Comm_rank(comm_, &myRank_), "Comm_rank");
00103       errCheck(MPI_Comm_size(comm_, &nProc_), "Comm_size");
00104     }
00105   else
00106     {
00107       nProc_ = 1;
00108       myRank_ = 0;
00109     }
00110   
00111 #else
00112   nProc_ = 1;
00113   myRank_ = 0;
00114 #endif
00115 }
00116 
00117 #ifdef USE_MPI_GROUPS /* we're ignoring groups for now */
00118 
00119 MPIComm::MPIComm(const MPIComm& parent, const MPIGroup& group)
00120   :
00121 #ifdef HAVE_MPI
00122   comm_(MPI_COMM_WORLD), 
00123 #endif
00124   nProc_(0), myRank_(0)
00125 {
00126 #ifdef HAVE_MPI
00127   if (group.getNProc()==0)
00128     {
00129       rank_ = -1;
00130       nProc_ = 0;
00131     }
00132   else if (parent.containsMe())
00133     {
00134       MPI_Comm parentComm = parent.comm_;
00135       MPI_Group newGroup = group.group_;
00136       
00137       errCheck(MPI_Comm_create(parentComm, newGroup, &comm_), 
00138                "Comm_create");
00139       
00140       if (group.containsProc(parent.getRank()))
00141         {
00142           errCheck(MPI_Comm_rank(comm_, &rank_), "Comm_rank");
00143           
00144           errCheck(MPI_Comm_size(comm_, &nProc_), "Comm_size");
00145         }
00146       else
00147         {
00148           rank_ = -1;
00149           nProc_ = -1;
00150           return;
00151         }
00152     }
00153   else
00154     {
00155       rank_ = -1;
00156       nProc_ = -1;
00157     }
00158 #endif
00159 }
00160 
00161 #endif /* USE_MPI_GROUPS */
00162 
00163 MPIComm& MPIComm::world()
00164 {
00165   static MPIComm w = MPIComm();
00166   return w;
00167 }
00168 
00169 
00170 MPIComm& MPIComm::self()
00171 {
00172 #ifdef HAVE_MPI
00173   static MPIComm w = MPIComm(MPI_COMM_SELF);
00174 #else
00175   static MPIComm w = MPIComm();
00176 #endif
00177   return w;
00178 }
00179 
00180 
00181 void MPIComm::synchronize() const 
00182 {
00183 #ifdef HAVE_MPI
00184   //mutex_.lock();
00185   {
00186     if (mpiIsRunning())
00187       {
00188         /* test whether errors have been detected on another proc before
00189          * doing the collective operation. */
00190         TEUCHOS_POLL_FOR_FAILURES(*this);
00191         /* if we're to this point, all processors are OK */
00192         
00193         errCheck(::MPI_Barrier(comm_), "Barrier");
00194       }
00195   }
00196   //mutex_.unlock();
00197 #endif
00198 }
00199 
00200 void MPIComm::allToAll(void* sendBuf, int sendCount, int sendType,
00201                        void* recvBuf, int recvCount, int recvType) const
00202 {
00203 #ifdef HAVE_MPI
00204   //mutex_.lock();
00205   {
00206     MPI_Datatype mpiSendType = getDataType(sendType);
00207     MPI_Datatype mpiRecvType = getDataType(recvType);
00208 
00209 
00210     if (mpiIsRunning())
00211       {
00212         /* test whether errors have been detected on another proc before
00213          * doing the collective operation. */
00214         TEUCHOS_POLL_FOR_FAILURES(*this);
00215         /* if we're to this point, all processors are OK */
00216         
00217         errCheck(::MPI_Alltoall(sendBuf, sendCount, mpiSendType,
00218                                 recvBuf, recvCount, mpiRecvType,
00219                                 comm_), "Alltoall");
00220       }
00221   }
00222   //mutex_.unlock();
00223 #else
00224   (void)sendBuf;
00225   (void)sendCount;
00226   (void)sendType;
00227   (void)recvBuf;
00228   (void)recvCount;
00229   (void)recvType;
00230 #endif
00231 }
00232 
00233 void MPIComm::allToAllv(void* sendBuf, int* sendCount, 
00234                         int* sendDisplacements, int sendType,
00235                         void* recvBuf, int* recvCount, 
00236                         int* recvDisplacements, int recvType) const
00237 {
00238 #ifdef HAVE_MPI
00239   //mutex_.lock();
00240   {
00241     MPI_Datatype mpiSendType = getDataType(sendType);
00242     MPI_Datatype mpiRecvType = getDataType(recvType);
00243 
00244     if (mpiIsRunning())
00245       {
00246         /* test whether errors have been detected on another proc before
00247          * doing the collective operation. */
00248         TEUCHOS_POLL_FOR_FAILURES(*this);
00249         /* if we're to this point, all processors are OK */   
00250         
00251         errCheck(::MPI_Alltoallv(sendBuf, sendCount, sendDisplacements, mpiSendType,
00252                                  recvBuf, recvCount, recvDisplacements, mpiRecvType,
00253                                  comm_), "Alltoallv");
00254       }
00255   }
00256   //mutex_.unlock();
00257 #else
00258   (void)sendBuf;
00259   (void)sendCount;
00260   (void)sendDisplacements;
00261   (void)sendType;
00262   (void)recvBuf;
00263   (void)recvCount;
00264   (void)recvDisplacements;
00265   (void)recvType;
00266 #endif
00267 }
00268 
00269 void MPIComm::gather(void* sendBuf, int sendCount, int sendType,
00270                      void* recvBuf, int recvCount, int recvType,
00271                      int root) const
00272 {
00273 #ifdef HAVE_MPI
00274   //mutex_.lock();
00275   {
00276     MPI_Datatype mpiSendType = getDataType(sendType);
00277     MPI_Datatype mpiRecvType = getDataType(recvType);
00278 
00279 
00280     if (mpiIsRunning())
00281       {
00282         /* test whether errors have been detected on another proc before
00283          * doing the collective operation. */
00284         TEUCHOS_POLL_FOR_FAILURES(*this);
00285         /* if we're to this point, all processors are OK */
00286         
00287         errCheck(::MPI_Gather(sendBuf, sendCount, mpiSendType,
00288                               recvBuf, recvCount, mpiRecvType,
00289                               root, comm_), "Gather");
00290       }
00291   }
00292   //mutex_.unlock();
00293 #endif
00294 }
00295 
00296 void MPIComm::gatherv(void* sendBuf, int sendCount, int sendType,
00297                      void* recvBuf, int* recvCount, int* displacements, int recvType,
00298                      int root) const
00299 {
00300 #ifdef HAVE_MPI
00301   //mutex_.lock();
00302   {
00303     MPI_Datatype mpiSendType = getDataType(sendType);
00304     MPI_Datatype mpiRecvType = getDataType(recvType);
00305     
00306     if (mpiIsRunning())
00307       {
00308         /* test whether errors have been detected on another proc before
00309          * doing the collective operation. */
00310         TEUCHOS_POLL_FOR_FAILURES(*this);
00311         /* if we're to this point, all processors are OK */
00312         
00313         errCheck(::MPI_Gatherv(sendBuf, sendCount, mpiSendType,
00314                                recvBuf, recvCount, displacements, mpiRecvType,
00315                                root, comm_), "Gatherv");
00316       }
00317   }
00318   //mutex_.unlock();
00319 #endif
00320 }
00321 
00322 void MPIComm::allGather(void* sendBuf, int sendCount, int sendType,
00323                         void* recvBuf, int recvCount, 
00324                         int recvType) const
00325 {
00326 #ifdef HAVE_MPI
00327   //mutex_.lock();
00328   {
00329     MPI_Datatype mpiSendType = getDataType(sendType);
00330     MPI_Datatype mpiRecvType = getDataType(recvType);
00331     
00332     if (mpiIsRunning())
00333       {
00334         /* test whether errors have been detected on another proc before
00335          * doing the collective operation. */
00336         TEUCHOS_POLL_FOR_FAILURES(*this);
00337         /* if we're to this point, all processors are OK */
00338         
00339         errCheck(::MPI_Allgather(sendBuf, sendCount, mpiSendType,
00340                                  recvBuf, recvCount, 
00341                                  mpiRecvType, comm_), 
00342                  "AllGather");
00343       }
00344   }
00345   //mutex_.unlock();
00346 #endif
00347 }
00348 
00349 
00350 void MPIComm::allGatherv(void* sendBuf, int sendCount, int sendType,
00351                          void* recvBuf, int* recvCount, 
00352                          int* recvDisplacements,
00353                          int recvType) const
00354 {
00355 #ifdef HAVE_MPI
00356   //mutex_.lock();
00357   {
00358     MPI_Datatype mpiSendType = getDataType(sendType);
00359     MPI_Datatype mpiRecvType = getDataType(recvType);
00360     
00361     if (mpiIsRunning())
00362       {
00363         /* test whether errors have been detected on another proc before
00364          * doing the collective operation. */
00365         TEUCHOS_POLL_FOR_FAILURES(*this);
00366         /* if we're to this point, all processors are OK */
00367         
00368         errCheck(::MPI_Allgatherv(sendBuf, sendCount, mpiSendType,
00369                                   recvBuf, recvCount, recvDisplacements,
00370                                   mpiRecvType, 
00371                                   comm_), 
00372                  "AllGatherv");
00373       }
00374   }
00375   //mutex_.unlock();
00376 #endif
00377 }
00378 
00379 
00380 void MPIComm::bcast(void* msg, int length, int type, int src) const
00381 {
00382 #ifdef HAVE_MPI
00383   //mutex_.lock();
00384   {
00385     if (mpiIsRunning())
00386       {
00387         /* test whether errors have been detected on another proc before
00388          * doing the collective operation. */
00389         TEUCHOS_POLL_FOR_FAILURES(*this);
00390         /* if we're to this point, all processors are OK */
00391         
00392         MPI_Datatype mpiType = getDataType(type);
00393         errCheck(::MPI_Bcast(msg, length, mpiType, src, 
00394                              comm_), "Bcast");
00395       }
00396   }
00397   //mutex_.unlock();
00398 #endif
00399 }
00400 
00401 void MPIComm::allReduce(void* input, void* result, int inputCount, 
00402                         int type, int op) const
00403 {
00404 #ifdef HAVE_MPI
00405 
00406   //mutex_.lock();
00407   {
00408     MPI_Op mpiOp = getOp(op);
00409     MPI_Datatype mpiType = getDataType(type);
00410     
00411     if (mpiIsRunning())
00412       {
00413         errCheck(::MPI_Allreduce(input, result, inputCount, mpiType,
00414                                  mpiOp, comm_), 
00415                  "Allreduce");
00416       }
00417   }
00418   //mutex_.unlock();
00419 #endif
00420 }
00421 
00422 
00423 #ifdef HAVE_MPI
00424 
00425 MPI_Datatype MPIComm::getDataType(int type)
00426 {
00427   TEUCHOS_TEST_FOR_EXCEPTION(
00428     !(type == INT || type==FLOAT 
00429       || type==DOUBLE || type==CHAR),
00430     std::range_error,
00431     "invalid type " << type << " in MPIComm::getDataType");
00432   
00433   if(type == INT) return MPI_INT;
00434   if(type == FLOAT) return MPI_FLOAT;
00435   if(type == DOUBLE) return MPI_DOUBLE;
00436   
00437   return MPI_CHAR;
00438 }
00439 
00440 
00441 void MPIComm::errCheck(int errCode, const std::string& methodName)
00442 {
00443   TEUCHOS_TEST_FOR_EXCEPTION(errCode != 0, std::runtime_error,
00444                      "MPI function MPI_" << methodName 
00445                      << " returned error code=" << errCode);
00446 }
00447 
00448 MPI_Op MPIComm::getOp(int op)
00449 {
00450 
00451   TEUCHOS_TEST_FOR_EXCEPTION(
00452     !(op == SUM || op==MAX 
00453       || op==MIN || op==PROD),
00454     std::range_error,
00455     "invalid operator " 
00456     << op << " in MPIComm::getOp");
00457 
00458   if( op == SUM) return MPI_SUM;
00459   else if( op == MAX) return MPI_MAX;
00460   else if( op == MIN) return MPI_MIN;
00461   return MPI_PROD;
00462 }
00463 
00464 #endif
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines