Teuchos - Trilinos Tools Package Version of the Day
Teuchos_DefaultMpiComm.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 //
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ***********************************************************************
00040 // @HEADER
00041 
00042 #ifndef TEUCHOS_MPI_COMM_HPP
00043 #define TEUCHOS_MPI_COMM_HPP
00044 
00045 
00046 #include "Teuchos_Comm.hpp"
00047 #include "Teuchos_CommUtilities.hpp"
00048 #include "Teuchos_OrdinalTraits.hpp"
00049 #include "Teuchos_OpaqueWrapper.hpp"
00050 #include "Teuchos_MpiReductionOpSetter.hpp"
00051 #include "Teuchos_SerializationTraitsHelpers.hpp"
00052 #include "Teuchos_Workspace.hpp"
00053 #include "Teuchos_TypeNameTraits.hpp"
00054 #include "Teuchos_as.hpp"
00055 #include "Teuchos_Assert.hpp"
00056 #include "mpi.h"
00057 
00058 
00059 // This must be defined globally for the whole program!
00060 //#define TEUCHOS_MPI_COMM_DUMP
00061 
00062 
00063 #ifdef TEUCHOS_MPI_COMM_DUMP
00064 #  include "Teuchos_VerboseObject.hpp"
00065 #endif
00066 
00067 
00068 namespace Teuchos {
00069 
00070 
00071 #ifdef TEUCHOS_MPI_COMM_DUMP
00072 template<typename Ordinal, typename T>
00073 void dumpBuffer(
00074   const std::string &funcName, const std::string &buffName
00075   ,const Ordinal bytes, const T buff[]
00076   )
00077 {
00078   Teuchos::RCP<Teuchos::FancyOStream>
00079     out = Teuchos::VerboseObjectBase::getDefaultOStream();
00080   Teuchos::OSTab tab(out);
00081   *out
00082     << "\n" << funcName << "::" << buffName << ":\n";
00083   tab.incrTab();
00084   for( Ordinal i = 0; i < bytes; ++i ) {
00085     *out << buffName << "[" << i << "] = '" << buff[i] << "'\n";
00086   }
00087   *out << "\n";
00088 }
00089 #endif // TEUCHOS_MPI_COMM_DUMP
00090 
00091 
00093 class MpiCommRequest : public CommRequest {
00094 public:
00096   MpiCommRequest( MPI_Request rawMpiRequest )
00097     :rawMpiRequest_(rawMpiRequest)
00098     {}
00100   MPI_Request releaseRawMpiRequest()
00101     {
00102       MPI_Request tmp_rawMpiRequest = rawMpiRequest_;
00103       rawMpiRequest_ = MPI_REQUEST_NULL;
00104       return tmp_rawMpiRequest;
00105     }
00106 private:
00107   MPI_Request rawMpiRequest_;
00108   MpiCommRequest(); // Not defined
00109 };
00110 
00111 
00116 inline
00117 const RCP<MpiCommRequest>
00118 mpiCommRequest( MPI_Request rawMpiRequest  )
00119 {
00120   return Teuchos::rcp(new MpiCommRequest(rawMpiRequest));
00121 }
00122 
00123 
00134 template<typename Ordinal>
00135 class MpiComm : public Comm<Ordinal> {
00136 public:
00137 
00139 
00140 
00147   MpiComm(
00148     const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00149     );
00150 
00167   MpiComm(const MpiComm<Ordinal>& other);
00168 
00170   RCP<const OpaqueWrapper<MPI_Comm> > getRawMpiComm() const
00171   {return rawMpiComm_;}
00172 
00174 
00176 
00177 
00179   virtual int getRank() const;
00181   virtual int getSize() const;
00183   virtual void barrier() const;
00185   virtual void broadcast(
00186     const int rootRank, const Ordinal bytes, char buffer[]
00187     ) const;
00189   virtual void gatherAll(
00190     const Ordinal sendBytes, const char sendBuffer[]
00191     ,const Ordinal recvBytes, char recvBuffer[]
00192     ) const;
00194   virtual void reduceAll(
00195     const ValueTypeReductionOp<Ordinal,char> &reductOp
00196     ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00197     ) const;
00199   virtual void reduceAllAndScatter(
00200     const ValueTypeReductionOp<Ordinal,char> &reductOp
00201     ,const Ordinal sendBytes, const char sendBuffer[]
00202     ,const Ordinal recvCounts[], char myGlobalReducts[]
00203     ) const;
00205   virtual void scan(
00206     const ValueTypeReductionOp<Ordinal,char> &reductOp
00207     ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00208     ) const;
00210   virtual void send(
00211     const Ordinal bytes, const char sendBuffer[], const int destRank
00212     ) const;
00214   virtual int receive(
00215     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00216     ) const;
00218   virtual void readySend(
00219     const ArrayView<const char> &sendBuffer,
00220     const int destRank
00221     ) const;
00223   virtual RCP<CommRequest> isend(
00224     const ArrayView<const char> &sendBuffer,
00225     const int destRank
00226     ) const;
00228   virtual RCP<CommRequest> ireceive(
00229     const ArrayView<char> &Buffer,
00230     const int sourceRank
00231     ) const;
00233   virtual void waitAll(
00234     const ArrayView<RCP<CommRequest> > &requests
00235     ) const;
00237   virtual void wait(
00238     const Ptr<RCP<CommRequest> > &request
00239     ) const;
00241   virtual RCP< Comm<Ordinal> > duplicate() const;
00243   virtual RCP< Comm<Ordinal> > split(const int color, const int key) const;
00245   virtual RCP< Comm<Ordinal> > createSubcommunicator(
00246     const ArrayView<const int>& ranks) const;
00248 
00250 
00251 
00253   std::string description() const;
00254 
00256 
00257   // These should be private but the PGI compiler requires them be public
00258 
00259   static int const minTag_ = 26000; // These came from Teuchos::MpiComm???
00260   static int const maxTag_ = 26099; // ""
00261 
00262 private:
00263 
00264   // Set internal data members once the rawMpiComm_ data member is valid.
00265   void setupMembersFromComm();
00266   static int tagCounter_;
00267 
00268   RCP<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;
00269   int rank_;
00270   int size_;
00271   int tag_;
00272 
00273   void assertRank(const int rank, const std::string &rankName) const;
00274 
00275   // Not defined and not to be called!
00276   MpiComm();
00277 
00278 #ifdef TEUCHOS_MPI_COMM_DUMP
00279 public:
00280   static bool show_dump;
00281 #endif // TEUCHOS_MPI_COMM_DUMP
00282   
00283 };
00284 
00285 
00299 template<typename Ordinal>
00300 RCP<MpiComm<Ordinal> >
00301 createMpiComm(
00302   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00303   );
00304 
00305 
00306 // ////////////////////////
00307 // Implementations
00308 
00309 
00310 // Static members
00311 
00312 
00313 template<typename Ordinal>
00314 int MpiComm<Ordinal>::tagCounter_ = MpiComm<Ordinal>::minTag_;
00315 
00316 
00317 // Constructors
00318 
00319 
00320 template<typename Ordinal>
00321 MpiComm<Ordinal>::MpiComm(
00322   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00323   )
00324 {
00325   TEUCHOS_TEST_FOR_EXCEPT( rawMpiComm.get()==NULL );
00326   TEUCHOS_TEST_FOR_EXCEPT( *rawMpiComm == MPI_COMM_NULL );
00327   rawMpiComm_ = rawMpiComm;
00328   setupMembersFromComm();
00329 }
00330 
00331 
00332 template<typename Ordinal>
00333 MpiComm<Ordinal>::MpiComm(const MpiComm<Ordinal>& other)
00334 {
00335   TEUCHOS_TEST_FOR_EXCEPT(other.getRawMpiComm().get() == NULL);
00336   TEUCHOS_TEST_FOR_EXCEPT(*other.getRawMpiComm() == MPI_COMM_NULL);
00337   MPI_Comm newComm;
00338   MPI_Comm_dup(*other.getRawMpiComm(), &newComm);
00339   rawMpiComm_ = opaqueWrapper(newComm);
00340   setupMembersFromComm();
00341 }
00342 
00343 
00344 template<typename Ordinal>
00345 void MpiComm<Ordinal>::setupMembersFromComm()
00346 {
00347   MPI_Comm_size(*rawMpiComm_, &size_);
00348   MPI_Comm_rank(*rawMpiComm_, &rank_);
00349   if(tagCounter_ > maxTag_)
00350     tagCounter_ = minTag_;
00351   tag_ = tagCounter_++;
00352 }
00353 
00354 
00355 // Overridden from Comm
00356 
00357   
00358 template<typename Ordinal>
00359 int MpiComm<Ordinal>::getRank() const
00360 {
00361   return rank_;
00362 }
00363 
00364   
00365 template<typename Ordinal>
00366 int MpiComm<Ordinal>::getSize() const
00367 {
00368   return size_;
00369 }
00370 
00371   
00372 template<typename Ordinal>
00373 void MpiComm<Ordinal>::barrier() const
00374 {
00375   TEUCHOS_COMM_TIME_MONITOR(
00376     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::barrier()"
00377     );
00378   MPI_Barrier(*rawMpiComm_);
00379 }
00380 
00381   
00382 template<typename Ordinal>
00383 void MpiComm<Ordinal>::broadcast(
00384   const int rootRank, const Ordinal bytes, char buffer[]
00385   ) const
00386 {
00387   TEUCHOS_COMM_TIME_MONITOR(
00388     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::broadcast(...)"
00389     );
00390   MPI_Bcast(buffer,bytes,MPI_CHAR,rootRank,*rawMpiComm_);
00391 }
00392 
00393   
00394 template<typename Ordinal>
00395 void MpiComm<Ordinal>::gatherAll(
00396   const Ordinal sendBytes, const char sendBuffer[],
00397   const Ordinal recvBytes, char recvBuffer[]
00398   ) const
00399 {
00400   TEUCHOS_COMM_TIME_MONITOR(
00401     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::gatherAll(...)"
00402     );
00403   TEUCHOS_ASSERT_EQUALITY((sendBytes*size_), recvBytes );
00404   MPI_Allgather(
00405     const_cast<char *>(sendBuffer), sendBytes, MPI_CHAR,
00406     recvBuffer, sendBytes, MPI_CHAR,
00407     *rawMpiComm_
00408     );
00409   // NOTE: 'sendBytes' is being sent above for the MPI arg recvcount (which is
00410   // very confusing in the MPI documentation) for MPI_Allgether(...).
00411 }
00412 
00413   
00414 template<typename Ordinal>
00415 void MpiComm<Ordinal>::reduceAll(
00416   const ValueTypeReductionOp<Ordinal,char> &reductOp
00417   ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00418   ) const
00419 {
00420   TEUCHOS_COMM_TIME_MONITOR(
00421     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAll(...)"
00422     );
00423   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00424   MPI_Datatype char_block;
00425   MPI_Type_contiguous(bytes, MPI_CHAR, &char_block);
00426   MPI_Type_commit(&char_block);
00427   MPI_Allreduce(
00428     const_cast<char*>(sendBuffer),globalReducts,1,char_block,op.mpi_op()
00429     ,*rawMpiComm_
00430     );
00431 }
00432 
00433 
00434 template<typename Ordinal>
00435 void MpiComm<Ordinal>::reduceAllAndScatter(
00436   const ValueTypeReductionOp<Ordinal,char> &reductOp
00437   ,const Ordinal sendBytes, const char sendBuffer[]
00438   ,const Ordinal recvCounts[], char myGlobalReducts[]
00439   ) const
00440 {
00441 
00442   (void)sendBytes; // Ignore if not in debug mode
00443 
00444   TEUCHOS_COMM_TIME_MONITOR(
00445     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAllAndScatter(...)"
00446     );
00447 
00448 #ifdef TEUCHOS_DEBUG
00449   Ordinal sumRecvBytes = 0;
00450   for( Ordinal i = 0; i < size_; ++i ) {
00451     sumRecvBytes += recvCounts[i];
00452   }
00453   TEUCHOS_TEST_FOR_EXCEPT(!(sumRecvBytes==sendBytes));
00454 #endif // TEUCHOS_DEBUG
00455 
00456 #ifdef TEUCHOS_MPI_COMM_DUMP
00457   if(show_dump) {
00458     dumpBuffer<Ordinal,char>(
00459       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00460       "sendBuffer", sendBytes, sendBuffer );
00461     dumpBuffer<Ordinal,Ordinal>(
00462       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00463       "recvCounts", as<Ordinal>(size_), recvCounts );
00464     dumpBuffer<Ordinal,char>(
00465       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00466       "myGlobalReducts", as<char>(recvCounts[rank_]), myGlobalReducts );
00467   }
00468 #endif // TEUCHOS_MPI_COMM_DUMP
00469 
00470   // Create a new recvCount[] if Ordinal!=int
00471   WorkspaceStore* wss = get_default_workspace_store().get();
00472   const bool Ordinal_is_int = typeid(int)==typeid(Ordinal);
00473   Workspace<int> ws_int_recvCounts(wss,Ordinal_is_int?0:size_);
00474   const int *int_recvCounts = 0;
00475   if(Ordinal_is_int) {
00476     int_recvCounts = reinterpret_cast<const int*>(recvCounts);
00477     // Note: We must do an reinterpet cast since this must
00478     // compile even if it is not executed.  I could implement
00479     // code that would not need to do this using template
00480     // conditionals but I don't want to bother.
00481   }
00482   else {
00483     std::copy(recvCounts, recvCounts+size_, &ws_int_recvCounts[0]);
00484     int_recvCounts = &ws_int_recvCounts[0];
00485   }
00486 
00487   // Perform the operation
00488   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp, false)));
00489   MPI_Reduce_scatter(
00490     const_cast<char*>(sendBuffer), myGlobalReducts,
00491     const_cast<int*>(int_recvCounts),
00492     MPI_CHAR,
00493     op.mpi_op(),
00494     *rawMpiComm_
00495     );
00496 
00497 }
00498 
00499 
00500 template<typename Ordinal>
00501 void MpiComm<Ordinal>::scan(
00502   const ValueTypeReductionOp<Ordinal,char> &reductOp
00503   ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00504   ) const
00505 {
00506   TEUCHOS_COMM_TIME_MONITOR(
00507     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::scan(...)"
00508     );
00509   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00510   MPI_Scan(
00511     const_cast<char*>(sendBuffer),scanReducts,bytes,MPI_CHAR,op.mpi_op()
00512     ,*rawMpiComm_
00513     );
00514 }
00515 
00516 
00517 template<typename Ordinal>
00518 void MpiComm<Ordinal>::send(
00519   const Ordinal bytes, const char sendBuffer[], const int destRank
00520   ) const
00521 {
00522   TEUCHOS_COMM_TIME_MONITOR(
00523     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::send(...)"
00524     );
00525 #ifdef TEUCHOS_DEBUG
00526   TEUCHOS_TEST_FOR_EXCEPTION(
00527     ! ( 0 <= destRank && destRank < size_ ), std::logic_error
00528     ,"Error, destRank = " << destRank << " is not < 0 or is not"
00529     " in the range [0,"<<size_-1<<"]!"
00530     );
00531 #endif // TEUCHOS_DEBUG
00532 #ifdef TEUCHOS_MPI_COMM_DUMP
00533   if(show_dump) {
00534     dumpBuffer<Ordinal,char>(
00535       "Teuchos::MpiComm<Ordinal>::send(...)"
00536       ,"sendBuffer", bytes, sendBuffer
00537       );
00538   }
00539 #endif // TEUCHOS_MPI_COMM_DUMP
00540   MPI_Send(
00541     const_cast<char*>(sendBuffer),bytes,MPI_CHAR,destRank,tag_,*rawMpiComm_
00542     );
00543   // ToDo: What about error handling???
00544 }
00545 
00546 
00547 template<typename Ordinal>
00548 void MpiComm<Ordinal>::readySend(
00549   const ArrayView<const char> &sendBuffer,
00550   const int destRank
00551   ) const
00552 {
00553   TEUCHOS_COMM_TIME_MONITOR(
00554     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::readySend(...)"
00555     );
00556 #ifdef TEUCHOS_DEBUG
00557   TEUCHOS_TEST_FOR_EXCEPTION(
00558     ! ( 0 <= destRank && destRank < size_ ), std::logic_error
00559     ,"Error, destRank = " << destRank << " is not < 0 or is not"
00560     " in the range [0,"<<size_-1<<"]!"
00561     );
00562 #endif // TEUCHOS_DEBUG
00563 #ifdef TEUCHOS_MPI_COMM_DUMP
00564   if(show_dump) {
00565     dumpBuffer<Ordinal,char>(
00566       "Teuchos::MpiComm<Ordinal>::readySend(...)"
00567       ,"sendBuffer", bytes, sendBuffer
00568       );
00569   }
00570 #endif // TEUCHOS_MPI_COMM_DUMP
00571   MPI_Rsend(
00572     const_cast<char*>(sendBuffer.getRawPtr()),sendBuffer.size(),MPI_CHAR,destRank,tag_,*rawMpiComm_
00573     );
00574   // ToDo: What about error handling???
00575 }
00576 
00577 
00578 template<typename Ordinal>
00579 int MpiComm<Ordinal>::receive(
00580     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00581   ) const
00582 {
00583   TEUCHOS_COMM_TIME_MONITOR(
00584     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::receive(...)"
00585     );
00586 #ifdef TEUCHOS_DEBUG
00587   TEUCHOS_TEST_FOR_EXCEPTION(
00588     sourceRank >=0 && !(sourceRank < size_), std::logic_error
00589     ,"Error, sourceRank = " << sourceRank << " is not < 0 or is not"
00590     " in the range [0,"<<(size_-1)<<"]!"
00591     );
00592 #endif // TEUCHOS_DEBUG
00593   MPI_Status status;
00594   MPI_Recv(
00595     recvBuffer,bytes,MPI_CHAR
00596     ,sourceRank >= 0 ? sourceRank : MPI_ANY_SOURCE
00597     ,tag_,*rawMpiComm_
00598     ,&status
00599     );
00600 #ifdef TEUCHOS_MPI_COMM_DUMP
00601   if(show_dump) {
00602     dumpBuffer<Ordinal,char>(
00603       "Teuchos::MpiComm<Ordinal>::receive(...)"
00604       ,"recvBuffer", bytes, recvBuffer
00605       );
00606   }
00607 #endif // TEUCHOS_MPI_COMM_DUMP
00608   return status.MPI_SOURCE;
00609   // ToDo: What about error handling???
00610 }
00611 
00612 
00613 template<typename Ordinal>
00614 RCP<CommRequest> MpiComm<Ordinal>::isend(
00615   const ArrayView<const char> &sendBuffer,
00616   const int destRank
00617   ) const
00618 {
00619   TEUCHOS_COMM_TIME_MONITOR(
00620     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::isend(...)"
00621     );
00622 #ifdef TEUCHOS_DEBUG
00623   assertRank(destRank, "destRank");
00624 #endif // TEUCHOS_DEBUG
00625   MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
00626   MPI_Isend(
00627     const_cast<char*>(sendBuffer.getRawPtr()), sendBuffer.size(), MPI_CHAR, destRank,
00628     tag_, *rawMpiComm_, &rawMpiRequest );
00629   return mpiCommRequest(rawMpiRequest);
00630   // ToDo: What about MPI error handling???
00631 }
00632 
00633 
00634 template<typename Ordinal>
00635 RCP<CommRequest> MpiComm<Ordinal>::ireceive(
00636   const ArrayView<char> &recvBuffer,
00637   const int sourceRank
00638   ) const
00639 {
00640   TEUCHOS_COMM_TIME_MONITOR(
00641     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::ireceive(...)"
00642     );
00643 #ifdef TEUCHOS_DEBUG
00644   assertRank(sourceRank, "sourceRank");
00645 #endif // TEUCHOS_DEBUG
00646   MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
00647   MPI_Irecv(
00648     const_cast<char*>(recvBuffer.getRawPtr()), recvBuffer.size(), MPI_CHAR, sourceRank,
00649     tag_, *rawMpiComm_, &rawMpiRequest );
00650   return mpiCommRequest(rawMpiRequest);
00651   // ToDo: What about MPI error handling???
00652 }
00653 
00654 
00655 template<typename Ordinal>
00656 void MpiComm<Ordinal>::waitAll(
00657   const ArrayView<RCP<CommRequest> > &requests
00658   ) const
00659 {
00660   TEUCHOS_COMM_TIME_MONITOR(
00661     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::waitAll(...)"
00662     );
00663   const int count = requests.size();
00664 #ifdef TEUCHOS_DEBUG
00665   TEUCHOS_TEST_FOR_EXCEPT( requests.size() == 0 );
00666 #endif
00667   
00668   Array<MPI_Request> rawMpiRequests(count, MPI_REQUEST_NULL);
00669   for (int i = 0; i < count; ++i) {
00670     RCP<CommRequest> &request = requests[i];
00671     if (!is_null(request)) {
00672       const RCP<MpiCommRequest> mpiCommRequest =
00673         rcp_dynamic_cast<MpiCommRequest>(request);
00674       rawMpiRequests[i] = mpiCommRequest->releaseRawMpiRequest();
00675     }
00676     // else already null
00677     request = null;
00678   }
00679 
00680   Array<MPI_Status> rawMpiStatuses(count);
00681   MPI_Waitall( count, rawMpiRequests.getRawPtr(), rawMpiStatuses.getRawPtr() );
00682   // ToDo: We really should check the status?
00683 
00684 }
00685 
00686 
00687 template<typename Ordinal>
00688 void MpiComm<Ordinal>::wait(
00689   const Ptr<RCP<CommRequest> > &request
00690   ) const
00691 {
00692   TEUCHOS_COMM_TIME_MONITOR(
00693     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::wait(...)"
00694     );
00695   if (is_null(*request)) {
00696     return; // Nothing to wait on ...
00697   }
00698   const RCP<MpiCommRequest> mpiCommRequest =
00699     rcp_dynamic_cast<MpiCommRequest>(*request);
00700   MPI_Request rawMpiRequest = mpiCommRequest->releaseRawMpiRequest();
00701   MPI_Status status;
00702   MPI_Wait( &rawMpiRequest, &status );
00703   // ToDo: We really should check the status?
00704   *request = null;
00705 }
00706 
00707 
00708 template<typename Ordinal>
00709 RCP< Comm<Ordinal> >
00710 MpiComm<Ordinal>::duplicate() const
00711 {
00712   return rcp(new MpiComm<Ordinal>(*this));
00713 }
00714 
00715 
00716 template<typename Ordinal>
00717 RCP< Comm<Ordinal> >
00718 MpiComm<Ordinal>::split(const int color, const int key) const
00719 {
00720   MPI_Comm newComm;
00721   int splitReturn = MPI_Comm_split(
00722     *rawMpiComm_,
00723     color < 0 ? MPI_UNDEFINED : color,
00724     key,
00725     &newComm);
00726   TEUCHOS_TEST_FOR_EXCEPTION(
00727     splitReturn != MPI_SUCCESS,
00728     std::logic_error,
00729     "Failed to create communicator with color " << color <<
00730     "and key " << key << ".");
00731   if (newComm == MPI_COMM_NULL) {
00732     return RCP< Comm<Ordinal> >();
00733   } else {
00734     return rcp(new MpiComm<Ordinal>(opaqueWrapper(newComm)));
00735   }
00736 }
00737 
00738 
00739 template<typename Ordinal>
00740 RCP< Comm<Ordinal> >
00741 MpiComm<Ordinal>::createSubcommunicator(const ArrayView<const int> &ranks) const
00742 {
00743   int mpiReturn;
00744 
00745   // Get the group that this communicator is in.
00746   MPI_Group thisGroup;
00747   mpiReturn = MPI_Comm_group(*rawMpiComm_, &thisGroup);
00748   TEUCHOS_TEST_FOR_EXCEPTION(mpiReturn != MPI_SUCCESS, std::logic_error,
00749                      "Failed to obtain group.");
00750   // Create a new group with the specified members.
00751   MPI_Group newGroup;
00752   mpiReturn = MPI_Group_incl(
00753     thisGroup, ranks.size(), const_cast<int *>(&ranks[0]), &newGroup);
00754   TEUCHOS_TEST_FOR_EXCEPTION(mpiReturn != MPI_SUCCESS, std::logic_error,
00755                      "Failed to create subgroup.");
00756   // Create a new communicator from the new group.
00757   MPI_Comm newComm;
00758   mpiReturn = MPI_Comm_create(*rawMpiComm_, newGroup, &newComm);
00759   TEUCHOS_TEST_FOR_EXCEPTION(mpiReturn != MPI_SUCCESS, std::logic_error,
00760                      "Failed to create subcommunicator.");
00761   if (newComm == MPI_COMM_NULL) {
00762     return RCP< Comm<Ordinal> >();
00763   } else {
00764     return rcp(new MpiComm<Ordinal>(opaqueWrapper(newComm)));
00765   }
00766 }
00767 
00768 
00769 // Overridden from Describable
00770 
00771 
00772 template<typename Ordinal>
00773 std::string MpiComm<Ordinal>::description() const
00774 {
00775   std::ostringstream oss;
00776   oss
00777     << typeName(*this)
00778     << "{"
00779     << "size="<<size_
00780     << ",rank="<<rank_
00781     << ",rawMpiComm="<<static_cast<MPI_Comm>(*rawMpiComm_)
00782     <<"}";
00783   return oss.str();
00784 }
00785 
00786 
00787 #ifdef TEUCHOS_MPI_COMM_DUMP
00788 template<typename Ordinal>
00789 bool MpiComm<Ordinal>::show_dump = false;
00790 #endif
00791 
00792 
00793 // private
00794 
00795 
00796 template<typename Ordinal>
00797 void MpiComm<Ordinal>::assertRank(const int rank, const std::string &rankName) const
00798 {
00799   TEUCHOS_TEST_FOR_EXCEPTION(
00800     ! ( 0 <= rank && rank < size_ ), std::logic_error
00801     ,"Error, "<<rankName<<" = " << rank << " is not < 0 or is not"
00802     " in the range [0,"<<size_-1<<"]!"
00803     );
00804 }
00805 
00806 
00807 } // namespace Teuchos
00808 
00809 
00810 template<typename Ordinal>
00811 Teuchos::RCP<Teuchos::MpiComm<Ordinal> >
00812 Teuchos::createMpiComm(
00813   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00814   )
00815 {
00816   if( rawMpiComm.get()!=NULL && *rawMpiComm != MPI_COMM_NULL )
00817     return rcp(new MpiComm<Ordinal>(rawMpiComm));
00818   return Teuchos::null;
00819 }
00820 
00821 
00822 #endif // TEUCHOS_MPI_COMM_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines