Teuchos Package Browser (Single Doxygen Collection) Version of the Day
Teuchos_MPIComm.cpp
Go to the documentation of this file.
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 //
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ***********************************************************************
00040 // @HEADER
00041 
00042 #include "Teuchos_MPIComm.hpp"
00043 #include "Teuchos_ErrorPolling.hpp"
00044 
00045 
00046 using namespace Teuchos;
00047 
00048 namespace Teuchos
00049 {
00050 const int MPIComm::INT = 1;
00051 const int MPIComm::FLOAT = 2;
00052 const int MPIComm::DOUBLE = 3;
00053 const int MPIComm::DOUBLE_INT = 4;
00054 const int MPIComm::CHAR = 5;
00055 
00056 const int MPIComm::SUM = 6;
00057 const int MPIComm::MIN = 7;
00058 const int MPIComm::MAX = 8;
00059 const int MPIComm::MINLOC = 8;
00060 const int MPIComm::MAXLOC = 10;
00061 const int MPIComm::PROD = 11;
00062 }
00063 
00064 
00065 MPIComm::MPIComm()
00066   :
00067 #ifdef HAVE_MPI
00068   comm_(MPI_COMM_WORLD),
00069 #endif
00070   nProc_(0), myRank_(0)
00071 {
00072   init();
00073 }
00074 
00075 #ifdef HAVE_MPI
00076 MPIComm::MPIComm(MPI_Comm comm)
00077   : comm_(comm), nProc_(0), myRank_(0)
00078 {
00079   init();
00080 }
00081 #endif
00082 
00083 int MPIComm::mpiIsRunning() const
00084 {
00085   int mpiStarted = 0;
00086 #ifdef HAVE_MPI
00087   MPI_Initialized(&mpiStarted);
00088 #endif
00089   return mpiStarted;
00090 }
00091 
00092 void MPIComm::init()
00093 {
00094 #ifdef HAVE_MPI
00095 
00096   if (mpiIsRunning())
00097     {
00098       errCheck(MPI_Comm_rank(comm_, &myRank_), "Comm_rank");
00099       errCheck(MPI_Comm_size(comm_, &nProc_), "Comm_size");
00100     }
00101   else
00102     {
00103       nProc_ = 1;
00104       myRank_ = 0;
00105     }
00106   
00107 #else
00108   nProc_ = 1;
00109   myRank_ = 0;
00110 #endif
00111 }
00112 
00113 #ifdef USE_MPI_GROUPS /* we're ignoring groups for now */
00114 
00115 MPIComm::MPIComm(const MPIComm& parent, const MPIGroup& group)
00116   :
00117 #ifdef HAVE_MPI
00118   comm_(MPI_COMM_WORLD), 
00119 #endif
00120   nProc_(0), myRank_(0)
00121 {
00122 #ifdef HAVE_MPI
00123   if (group.getNProc()==0)
00124     {
00125       rank_ = -1;
00126       nProc_ = 0;
00127     }
00128   else if (parent.containsMe())
00129     {
00130       MPI_Comm parentComm = parent.comm_;
00131       MPI_Group newGroup = group.group_;
00132       
00133       errCheck(MPI_Comm_create(parentComm, newGroup, &comm_), 
00134                "Comm_create");
00135       
00136       if (group.containsProc(parent.getRank()))
00137         {
00138           errCheck(MPI_Comm_rank(comm_, &rank_), "Comm_rank");
00139           
00140           errCheck(MPI_Comm_size(comm_, &nProc_), "Comm_size");
00141         }
00142       else
00143         {
00144           rank_ = -1;
00145           nProc_ = -1;
00146           return;
00147         }
00148     }
00149   else
00150     {
00151       rank_ = -1;
00152       nProc_ = -1;
00153     }
00154 #endif
00155 }
00156 
00157 #endif /* USE_MPI_GROUPS */
00158 
00159 MPIComm& MPIComm::world()
00160 {
00161   static MPIComm w = MPIComm();
00162   return w;
00163 }
00164 
00165 
00166 MPIComm& MPIComm::self()
00167 {
00168 #ifdef HAVE_MPI
00169   static MPIComm w = MPIComm(MPI_COMM_SELF);
00170 #else
00171   static MPIComm w = MPIComm();
00172 #endif
00173   return w;
00174 }
00175 
00176 
00177 void MPIComm::synchronize() const 
00178 {
00179 #ifdef HAVE_MPI
00180   //mutex_.lock();
00181   {
00182     if (mpiIsRunning())
00183       {
00184         /* test whether errors have been detected on another proc before
00185          * doing the collective operation. */
00186         TEUCHOS_POLL_FOR_FAILURES(*this);
00187         /* if we're to this point, all processors are OK */
00188         
00189         errCheck(::MPI_Barrier(comm_), "Barrier");
00190       }
00191   }
00192   //mutex_.unlock();
00193 #endif
00194 }
00195 
00196 void MPIComm::allToAll(void* sendBuf, int sendCount, int sendType,
00197                        void* recvBuf, int recvCount, int recvType) const
00198 {
00199 #ifdef HAVE_MPI
00200   //mutex_.lock();
00201   {
00202     MPI_Datatype mpiSendType = getDataType(sendType);
00203     MPI_Datatype mpiRecvType = getDataType(recvType);
00204 
00205 
00206     if (mpiIsRunning())
00207       {
00208         /* test whether errors have been detected on another proc before
00209          * doing the collective operation. */
00210         TEUCHOS_POLL_FOR_FAILURES(*this);
00211         /* if we're to this point, all processors are OK */
00212         
00213         errCheck(::MPI_Alltoall(sendBuf, sendCount, mpiSendType,
00214                                 recvBuf, recvCount, mpiRecvType,
00215                                 comm_), "Alltoall");
00216       }
00217   }
00218   //mutex_.unlock();
00219 #else
00220   (void)sendBuf;
00221   (void)sendCount;
00222   (void)sendType;
00223   (void)recvBuf;
00224   (void)recvCount;
00225   (void)recvType;
00226 #endif
00227 }
00228 
00229 void MPIComm::allToAllv(void* sendBuf, int* sendCount, 
00230                         int* sendDisplacements, int sendType,
00231                         void* recvBuf, int* recvCount, 
00232                         int* recvDisplacements, int recvType) const
00233 {
00234 #ifdef HAVE_MPI
00235   //mutex_.lock();
00236   {
00237     MPI_Datatype mpiSendType = getDataType(sendType);
00238     MPI_Datatype mpiRecvType = getDataType(recvType);
00239 
00240     if (mpiIsRunning())
00241       {
00242         /* test whether errors have been detected on another proc before
00243          * doing the collective operation. */
00244         TEUCHOS_POLL_FOR_FAILURES(*this);
00245         /* if we're to this point, all processors are OK */   
00246         
00247         errCheck(::MPI_Alltoallv(sendBuf, sendCount, sendDisplacements, mpiSendType,
00248                                  recvBuf, recvCount, recvDisplacements, mpiRecvType,
00249                                  comm_), "Alltoallv");
00250       }
00251   }
00252   //mutex_.unlock();
00253 #else
00254   (void)sendBuf;
00255   (void)sendCount;
00256   (void)sendDisplacements;
00257   (void)sendType;
00258   (void)recvBuf;
00259   (void)recvCount;
00260   (void)recvDisplacements;
00261   (void)recvType;
00262 #endif
00263 }
00264 
00265 void MPIComm::gather(void* sendBuf, int sendCount, int sendType,
00266                      void* recvBuf, int recvCount, int recvType,
00267                      int root) const
00268 {
00269 #ifdef HAVE_MPI
00270   //mutex_.lock();
00271   {
00272     MPI_Datatype mpiSendType = getDataType(sendType);
00273     MPI_Datatype mpiRecvType = getDataType(recvType);
00274 
00275 
00276     if (mpiIsRunning())
00277       {
00278         /* test whether errors have been detected on another proc before
00279          * doing the collective operation. */
00280         TEUCHOS_POLL_FOR_FAILURES(*this);
00281         /* if we're to this point, all processors are OK */
00282         
00283         errCheck(::MPI_Gather(sendBuf, sendCount, mpiSendType,
00284                               recvBuf, recvCount, mpiRecvType,
00285                               root, comm_), "Gather");
00286       }
00287   }
00288   //mutex_.unlock();
00289 #endif
00290 }
00291 
00292 void MPIComm::gatherv(void* sendBuf, int sendCount, int sendType,
00293                      void* recvBuf, int* recvCount, int* displacements, int recvType,
00294                      int root) const
00295 {
00296 #ifdef HAVE_MPI
00297   //mutex_.lock();
00298   {
00299     MPI_Datatype mpiSendType = getDataType(sendType);
00300     MPI_Datatype mpiRecvType = getDataType(recvType);
00301     
00302     if (mpiIsRunning())
00303       {
00304         /* test whether errors have been detected on another proc before
00305          * doing the collective operation. */
00306         TEUCHOS_POLL_FOR_FAILURES(*this);
00307         /* if we're to this point, all processors are OK */
00308         
00309         errCheck(::MPI_Gatherv(sendBuf, sendCount, mpiSendType,
00310                                recvBuf, recvCount, displacements, mpiRecvType,
00311                                root, comm_), "Gatherv");
00312       }
00313   }
00314   //mutex_.unlock();
00315 #endif
00316 }
00317 
00318 void MPIComm::allGather(void* sendBuf, int sendCount, int sendType,
00319                         void* recvBuf, int recvCount, 
00320                         int recvType) const
00321 {
00322 #ifdef HAVE_MPI
00323   //mutex_.lock();
00324   {
00325     MPI_Datatype mpiSendType = getDataType(sendType);
00326     MPI_Datatype mpiRecvType = getDataType(recvType);
00327     
00328     if (mpiIsRunning())
00329       {
00330         /* test whether errors have been detected on another proc before
00331          * doing the collective operation. */
00332         TEUCHOS_POLL_FOR_FAILURES(*this);
00333         /* if we're to this point, all processors are OK */
00334         
00335         errCheck(::MPI_Allgather(sendBuf, sendCount, mpiSendType,
00336                                  recvBuf, recvCount, 
00337                                  mpiRecvType, comm_), 
00338                  "AllGather");
00339       }
00340   }
00341   //mutex_.unlock();
00342 #endif
00343 }
00344 
00345 
00346 void MPIComm::allGatherv(void* sendBuf, int sendCount, int sendType,
00347                          void* recvBuf, int* recvCount, 
00348                          int* recvDisplacements,
00349                          int recvType) const
00350 {
00351 #ifdef HAVE_MPI
00352   //mutex_.lock();
00353   {
00354     MPI_Datatype mpiSendType = getDataType(sendType);
00355     MPI_Datatype mpiRecvType = getDataType(recvType);
00356     
00357     if (mpiIsRunning())
00358       {
00359         /* test whether errors have been detected on another proc before
00360          * doing the collective operation. */
00361         TEUCHOS_POLL_FOR_FAILURES(*this);
00362         /* if we're to this point, all processors are OK */
00363         
00364         errCheck(::MPI_Allgatherv(sendBuf, sendCount, mpiSendType,
00365                                   recvBuf, recvCount, recvDisplacements,
00366                                   mpiRecvType, 
00367                                   comm_), 
00368                  "AllGatherv");
00369       }
00370   }
00371   //mutex_.unlock();
00372 #endif
00373 }
00374 
00375 
00376 void MPIComm::bcast(void* msg, int length, int type, int src) const
00377 {
00378 #ifdef HAVE_MPI
00379   //mutex_.lock();
00380   {
00381     if (mpiIsRunning())
00382       {
00383         /* test whether errors have been detected on another proc before
00384          * doing the collective operation. */
00385         TEUCHOS_POLL_FOR_FAILURES(*this);
00386         /* if we're to this point, all processors are OK */
00387         
00388         MPI_Datatype mpiType = getDataType(type);
00389         errCheck(::MPI_Bcast(msg, length, mpiType, src, 
00390                              comm_), "Bcast");
00391       }
00392   }
00393   //mutex_.unlock();
00394 #endif
00395 }
00396 
00397 void MPIComm::allReduce(void* input, void* result, int inputCount, 
00398                         int type, int op) const
00399 {
00400 #ifdef HAVE_MPI
00401 
00402   //mutex_.lock();
00403   {
00404     MPI_Op mpiOp = getOp(op);
00405     MPI_Datatype mpiType = getDataType(type);
00406     
00407     if (mpiIsRunning())
00408       {
00409         errCheck(::MPI_Allreduce(input, result, inputCount, mpiType,
00410                                  mpiOp, comm_), 
00411                  "Allreduce");
00412       }
00413   }
00414   //mutex_.unlock();
00415 #endif
00416 }
00417 
00418 
00419 #ifdef HAVE_MPI
00420 
00421 MPI_Datatype MPIComm::getDataType(int type)
00422 {
00423   TEST_FOR_EXCEPTION(
00424     !(type == INT || type==FLOAT 
00425       || type==DOUBLE || type==CHAR),
00426     std::range_error,
00427     "invalid type " << type << " in MPIComm::getDataType");
00428   
00429   if(type == INT) return MPI_INT;
00430   if(type == FLOAT) return MPI_FLOAT;
00431   if(type == DOUBLE) return MPI_DOUBLE;
00432   
00433   return MPI_CHAR;
00434 }
00435 
00436 
00437 void MPIComm::errCheck(int errCode, const std::string& methodName)
00438 {
00439   TEST_FOR_EXCEPTION(errCode != 0, std::runtime_error,
00440                      "MPI function MPI_" << methodName 
00441                      << " returned error code=" << errCode);
00442 }
00443 
00444 MPI_Op MPIComm::getOp(int op)
00445 {
00446 
00447   TEST_FOR_EXCEPTION(
00448     !(op == SUM || op==MAX 
00449       || op==MIN || op==PROD),
00450     std::range_error,
00451     "invalid operator " 
00452     << op << " in MPIComm::getOp");
00453 
00454   if( op == SUM) return MPI_SUM;
00455   else if( op == MAX) return MPI_MAX;
00456   else if( op == MIN) return MPI_MIN;
00457   return MPI_PROD;
00458 }
00459 
00460 #endif
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines