Teuchos - Trilinos Tools Package Version of the Day
Teuchos_DefaultMpiComm.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 //
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ***********************************************************************
00040 // @HEADER
00041 
00042 #ifndef TEUCHOS_MPI_COMM_HPP
00043 #define TEUCHOS_MPI_COMM_HPP
00044 
00045 
00046 #include "Teuchos_Comm.hpp"
00047 #include "Teuchos_CommUtilities.hpp"
00048 #include "Teuchos_OrdinalTraits.hpp"
00049 #include "Teuchos_OpaqueWrapper.hpp"
00050 #include "Teuchos_MpiReductionOpSetter.hpp"
00051 #include "Teuchos_SerializationTraitsHelpers.hpp"
00052 #include "Teuchos_Workspace.hpp"
00053 #include "Teuchos_TypeNameTraits.hpp"
00054 #include "Teuchos_as.hpp"
00055 #include "Teuchos_Assert.hpp"
00056 #include "mpi.h"
00057 
00058 
00059 // This must be defined globally for the whole program!
00060 //#define TEUCHOS_MPI_COMM_DUMP
00061 
00062 
00063 #ifdef TEUCHOS_MPI_COMM_DUMP
00064 #  include "Teuchos_VerboseObject.hpp"
00065 #endif
00066 
00067 
00068 namespace Teuchos {
00069 
00070 
00071 #ifdef TEUCHOS_MPI_COMM_DUMP
00072 template<typename Ordinal, typename T>
00073 void dumpBuffer(
00074   const std::string &funcName, const std::string &buffName
00075   ,const Ordinal bytes, const T buff[]
00076   )
00077 {
00078   Teuchos::RCP<Teuchos::FancyOStream>
00079     out = Teuchos::VerboseObjectBase::getDefaultOStream();
00080   Teuchos::OSTab tab(out);
00081   *out
00082     << "\n" << funcName << "::" << buffName << ":\n";
00083   tab.incrTab();
00084   for( Ordinal i = 0; i < bytes; ++i ) {
00085     *out << buffName << "[" << i << "] = '" << buff[i] << "'\n";
00086   }
00087   *out << "\n";
00088 }
00089 #endif // TEUCHOS_MPI_COMM_DUMP
00090 
00091 
00093 class MpiCommRequest : public CommRequest {
00094 public:
00096   MpiCommRequest( MPI_Request rawMpiRequest )
00097     :rawMpiRequest_(rawMpiRequest)
00098     {}
00100   MPI_Request releaseRawMpiRequest()
00101     {
00102       MPI_Request tmp_rawMpiRequest = rawMpiRequest_;
00103       rawMpiRequest_ = MPI_REQUEST_NULL;
00104       return tmp_rawMpiRequest;
00105     }
00106 private:
00107   MPI_Request rawMpiRequest_;
00108   MpiCommRequest(); // Not defined
00109 };
00110 
00111 
00116 inline
00117 const RCP<MpiCommRequest>
00118 mpiCommRequest( MPI_Request rawMpiRequest  )
00119 {
00120   return Teuchos::rcp(new MpiCommRequest(rawMpiRequest));
00121 }
00122 
00123 
00134 template<typename Ordinal>
00135 class MpiComm : public Comm<Ordinal> {
00136 public:
00137 
00139 
00140 
00147   MpiComm(
00148     const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00149     );
00150 
00152   RCP<const OpaqueWrapper<MPI_Comm> > getRawMpiComm() const
00153   {return rawMpiComm_;}
00154 
00156 
00158 
00159 
00161   virtual int getRank() const;
00163   virtual int getSize() const;
00165   virtual void barrier() const;
00167   virtual void broadcast(
00168     const int rootRank, const Ordinal bytes, char buffer[]
00169     ) const;
00171   virtual void gatherAll(
00172     const Ordinal sendBytes, const char sendBuffer[]
00173     ,const Ordinal recvBytes, char recvBuffer[]
00174     ) const;
00176   virtual void reduceAll(
00177     const ValueTypeReductionOp<Ordinal,char> &reductOp
00178     ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00179     ) const;
00181   virtual void reduceAllAndScatter(
00182     const ValueTypeReductionOp<Ordinal,char> &reductOp
00183     ,const Ordinal sendBytes, const char sendBuffer[]
00184     ,const Ordinal recvCounts[], char myGlobalReducts[]
00185     ) const;
00187   virtual void scan(
00188     const ValueTypeReductionOp<Ordinal,char> &reductOp
00189     ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00190     ) const;
00192   virtual void send(
00193     const Ordinal bytes, const char sendBuffer[], const int destRank
00194     ) const;
00196   virtual int receive(
00197     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00198     ) const;
00200   virtual void readySend(
00201     const ArrayView<const char> &sendBuffer,
00202     const int destRank
00203     ) const;
00205   virtual RCP<CommRequest> isend(
00206     const ArrayView<const char> &sendBuffer,
00207     const int destRank
00208     ) const;
00210   virtual RCP<CommRequest> ireceive(
00211     const ArrayView<char> &Buffer,
00212     const int sourceRank
00213     ) const;
00215   virtual void waitAll(
00216     const ArrayView<RCP<CommRequest> > &requests
00217     ) const;
00219   virtual void wait(
00220     const Ptr<RCP<CommRequest> > &request
00221     ) const;
00222 
00224 
00226 
00227 
00229   std::string description() const;
00230 
00232 
00233   // These should be private but the PGI compiler requires them be public
00234 
00235   static int const minTag_ = 26000; // These came from Teuchos::MpiComm???
00236   static int const maxTag_ = 26099; // ""
00237 
00238 private:
00239 
00240   static int tagCounter_;
00241 
00242   RCP<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;
00243   int rank_;
00244   int size_;
00245   int tag_;
00246 
00247   void assertRank(const int rank, const std::string &rankName) const;
00248 
00249   // Not defined and not to be called!
00250   MpiComm();
00251 
00252 #ifdef TEUCHOS_MPI_COMM_DUMP
00253 public:
00254   static bool show_dump;
00255 #endif // TEUCHOS_MPI_COMM_DUMP
00256   
00257 };
00258 
00259 
00273 template<typename Ordinal>
00274 RCP<MpiComm<Ordinal> >
00275 createMpiComm(
00276   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00277   );
00278 
00279 
00280 // ////////////////////////
00281 // Implementations
00282 
00283 
00284 // Static members
00285 
00286 
00287 template<typename Ordinal>
00288 int MpiComm<Ordinal>::tagCounter_ = MpiComm<Ordinal>::minTag_;
00289 
00290 
00291 // Constructors
00292 
00293 
00294 template<typename Ordinal>
00295 MpiComm<Ordinal>::MpiComm(
00296   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00297   )
00298 {
00299   TEST_FOR_EXCEPT( rawMpiComm.get()==NULL );
00300   TEST_FOR_EXCEPT( *rawMpiComm == MPI_COMM_NULL );
00301   rawMpiComm_ = rawMpiComm;
00302   MPI_Comm_size(*rawMpiComm_,&size_);
00303   MPI_Comm_rank(*rawMpiComm_,&rank_);
00304   if(tagCounter_ > maxTag_)
00305     tagCounter_ = minTag_;
00306   tag_ = tagCounter_++;
00307 }
00308 
00309 
00310 // Overridden from Comm
00311 
00312   
00313 template<typename Ordinal>
00314 int MpiComm<Ordinal>::getRank() const
00315 {
00316   return rank_;
00317 }
00318 
00319   
00320 template<typename Ordinal>
00321 int MpiComm<Ordinal>::getSize() const
00322 {
00323   return size_;
00324 }
00325 
00326   
00327 template<typename Ordinal>
00328 void MpiComm<Ordinal>::barrier() const
00329 {
00330   TEUCHOS_COMM_TIME_MONITOR(
00331     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::barrier()"
00332     );
00333   MPI_Barrier(*rawMpiComm_);
00334 }
00335 
00336   
00337 template<typename Ordinal>
00338 void MpiComm<Ordinal>::broadcast(
00339   const int rootRank, const Ordinal bytes, char buffer[]
00340   ) const
00341 {
00342   TEUCHOS_COMM_TIME_MONITOR(
00343     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::broadcast(...)"
00344     );
00345   MPI_Bcast(buffer,bytes,MPI_CHAR,rootRank,*rawMpiComm_);
00346 }
00347 
00348   
00349 template<typename Ordinal>
00350 void MpiComm<Ordinal>::gatherAll(
00351   const Ordinal sendBytes, const char sendBuffer[],
00352   const Ordinal recvBytes, char recvBuffer[]
00353   ) const
00354 {
00355   TEUCHOS_COMM_TIME_MONITOR(
00356     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::gatherAll(...)"
00357     );
00358   TEUCHOS_ASSERT_EQUALITY((sendBytes*size_), recvBytes );
00359   MPI_Allgather(
00360     const_cast<char *>(sendBuffer), sendBytes, MPI_CHAR,
00361     recvBuffer, sendBytes, MPI_CHAR,
00362     *rawMpiComm_
00363     );
00364   // NOTE: 'sendBytes' is being sent above for the MPI arg recvcount (which is
00365   // very confusing in the MPI documentation) for MPI_Allgether(...).
00366 }
00367 
00368   
00369 template<typename Ordinal>
00370 void MpiComm<Ordinal>::reduceAll(
00371   const ValueTypeReductionOp<Ordinal,char> &reductOp
00372   ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00373   ) const
00374 {
00375   TEUCHOS_COMM_TIME_MONITOR(
00376     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAll(...)"
00377     );
00378   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00379   MPI_Allreduce(
00380     const_cast<char*>(sendBuffer),globalReducts,bytes,MPI_CHAR,op.mpi_op()
00381     ,*rawMpiComm_
00382     );
00383 }
00384 
00385 
00386 template<typename Ordinal>
00387 void MpiComm<Ordinal>::reduceAllAndScatter(
00388   const ValueTypeReductionOp<Ordinal,char> &reductOp
00389   ,const Ordinal sendBytes, const char sendBuffer[]
00390   ,const Ordinal recvCounts[], char myGlobalReducts[]
00391   ) const
00392 {
00393 
00394   (void)sendBytes; // Ignore if not in debug mode
00395 
00396   TEUCHOS_COMM_TIME_MONITOR(
00397     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAllAndScatter(...)"
00398     );
00399 
00400 #ifdef TEUCHOS_DEBUG
00401   Ordinal sumRecvBytes = 0;
00402   for( Ordinal i = 0; i < size_; ++i ) {
00403     sumRecvBytes += recvCounts[i];
00404   }
00405   TEST_FOR_EXCEPT(!(sumRecvBytes==sendBytes));
00406 #endif // TEUCHOS_DEBUG
00407 
00408 #ifdef TEUCHOS_MPI_COMM_DUMP
00409   if(show_dump) {
00410     dumpBuffer<Ordinal,char>(
00411       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00412       "sendBuffer", sendBytes, sendBuffer );
00413     dumpBuffer<Ordinal,Ordinal>(
00414       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00415       "recvCounts", as<Ordinal>(size_), recvCounts );
00416     dumpBuffer<Ordinal,char>(
00417       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00418       "myGlobalReducts", as<char>(recvCounts[rank_]), myGlobalReducts );
00419   }
00420 #endif // TEUCHOS_MPI_COMM_DUMP
00421 
00422   // Create a new recvCount[] if Ordinal!=int
00423   WorkspaceStore* wss = get_default_workspace_store().get();
00424   const bool Ordinal_is_int = typeid(int)==typeid(Ordinal);
00425   Workspace<int> ws_int_recvCounts(wss,Ordinal_is_int?0:size_);
00426   const int *int_recvCounts = 0;
00427   if(Ordinal_is_int) {
00428     int_recvCounts = reinterpret_cast<const int*>(recvCounts);
00429     // Note: We must do an reinterpet cast since this must
00430     // compile even if it is not executed.  I could implement
00431     // code that would not need to do this using template
00432     // conditionals but I don't want to bother.
00433   }
00434   else {
00435     std::copy(recvCounts, recvCounts+size_, &ws_int_recvCounts[0]);
00436     int_recvCounts = &ws_int_recvCounts[0];
00437   }
00438 
00439   // Perform the operation
00440   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp, false)));
00441   MPI_Reduce_scatter(
00442     const_cast<char*>(sendBuffer), myGlobalReducts,
00443     const_cast<int*>(int_recvCounts),
00444     MPI_CHAR,
00445     op.mpi_op(),
00446     *rawMpiComm_
00447     );
00448 
00449 }
00450 
00451 
00452 template<typename Ordinal>
00453 void MpiComm<Ordinal>::scan(
00454   const ValueTypeReductionOp<Ordinal,char> &reductOp
00455   ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00456   ) const
00457 {
00458   TEUCHOS_COMM_TIME_MONITOR(
00459     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::scan(...)"
00460     );
00461   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00462   MPI_Scan(
00463     const_cast<char*>(sendBuffer),scanReducts,bytes,MPI_CHAR,op.mpi_op()
00464     ,*rawMpiComm_
00465     );
00466 }
00467 
00468 
00469 template<typename Ordinal>
00470 void MpiComm<Ordinal>::send(
00471   const Ordinal bytes, const char sendBuffer[], const int destRank
00472   ) const
00473 {
00474   TEUCHOS_COMM_TIME_MONITOR(
00475     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::send(...)"
00476     );
00477 #ifdef TEUCHOS_DEBUG
00478   TEST_FOR_EXCEPTION(
00479     ! ( 0 <= destRank && destRank < size_ ), std::logic_error
00480     ,"Error, destRank = " << destRank << " is not < 0 or is not"
00481     " in the range [0,"<<size_-1<<"]!"
00482     );
00483 #endif // TEUCHOS_DEBUG
00484 #ifdef TEUCHOS_MPI_COMM_DUMP
00485   if(show_dump) {
00486     dumpBuffer<Ordinal,char>(
00487       "Teuchos::MpiComm<Ordinal>::send(...)"
00488       ,"sendBuffer", bytes, sendBuffer
00489       );
00490   }
00491 #endif // TEUCHOS_MPI_COMM_DUMP
00492   MPI_Send(
00493     const_cast<char*>(sendBuffer),bytes,MPI_CHAR,destRank,tag_,*rawMpiComm_
00494     );
00495   // ToDo: What about error handling???
00496 }
00497 
00498 
00499 template<typename Ordinal>
00500 void MpiComm<Ordinal>::readySend(
00501   const ArrayView<const char> &sendBuffer,
00502   const int destRank
00503   ) const
00504 {
00505   TEUCHOS_COMM_TIME_MONITOR(
00506     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::readySend(...)"
00507     );
00508 #ifdef TEUCHOS_DEBUG
00509   TEST_FOR_EXCEPTION(
00510     ! ( 0 <= destRank && destRank < size_ ), std::logic_error
00511     ,"Error, destRank = " << destRank << " is not < 0 or is not"
00512     " in the range [0,"<<size_-1<<"]!"
00513     );
00514 #endif // TEUCHOS_DEBUG
00515 #ifdef TEUCHOS_MPI_COMM_DUMP
00516   if(show_dump) {
00517     dumpBuffer<Ordinal,char>(
00518       "Teuchos::MpiComm<Ordinal>::readySend(...)"
00519       ,"sendBuffer", bytes, sendBuffer
00520       );
00521   }
00522 #endif // TEUCHOS_MPI_COMM_DUMP
00523   MPI_Rsend(
00524     const_cast<char*>(sendBuffer.getRawPtr()),sendBuffer.size(),MPI_CHAR,destRank,tag_,*rawMpiComm_
00525     );
00526   // ToDo: What about error handling???
00527 }
00528 
00529 
00530 template<typename Ordinal>
00531 int MpiComm<Ordinal>::receive(
00532     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00533   ) const
00534 {
00535   TEUCHOS_COMM_TIME_MONITOR(
00536     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::receive(...)"
00537     );
00538 #ifdef TEUCHOS_DEBUG
00539   TEST_FOR_EXCEPTION(
00540     sourceRank >=0 && !(sourceRank < size_), std::logic_error
00541     ,"Error, sourceRank = " << sourceRank << " is not < 0 or is not"
00542     " in the range [0,"<<(size_-1)<<"]!"
00543     );
00544 #endif // TEUCHOS_DEBUG
00545   MPI_Status status;
00546   MPI_Recv(
00547     recvBuffer,bytes,MPI_CHAR
00548     ,sourceRank >= 0 ? sourceRank : MPI_ANY_SOURCE
00549     ,tag_,*rawMpiComm_
00550     ,&status
00551     );
00552 #ifdef TEUCHOS_MPI_COMM_DUMP
00553   if(show_dump) {
00554     dumpBuffer<Ordinal,char>(
00555       "Teuchos::MpiComm<Ordinal>::receive(...)"
00556       ,"recvBuffer", bytes, recvBuffer
00557       );
00558   }
00559 #endif // TEUCHOS_MPI_COMM_DUMP
00560   return status.MPI_SOURCE;
00561   // ToDo: What about error handling???
00562 }
00563 
00564 
00565 template<typename Ordinal>
00566 RCP<CommRequest> MpiComm<Ordinal>::isend(
00567   const ArrayView<const char> &sendBuffer,
00568   const int destRank
00569   ) const
00570 {
00571   TEUCHOS_COMM_TIME_MONITOR(
00572     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::isend(...)"
00573     );
00574 #ifdef TEUCHOS_DEBUG
00575   assertRank(destRank, "destRank");
00576 #endif // TEUCHOS_DEBUG
00577   MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
00578   MPI_Isend(
00579     const_cast<char*>(sendBuffer.getRawPtr()), sendBuffer.size(), MPI_CHAR, destRank,
00580     tag_, *rawMpiComm_, &rawMpiRequest );
00581   return mpiCommRequest(rawMpiRequest);
00582   // ToDo: What about MPI error handling???
00583 }
00584 
00585 
00586 template<typename Ordinal>
00587 RCP<CommRequest> MpiComm<Ordinal>::ireceive(
00588   const ArrayView<char> &recvBuffer,
00589   const int sourceRank
00590   ) const
00591 {
00592   TEUCHOS_COMM_TIME_MONITOR(
00593     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::ireceive(...)"
00594     );
00595 #ifdef TEUCHOS_DEBUG
00596   assertRank(sourceRank, "sourceRank");
00597 #endif // TEUCHOS_DEBUG
00598   MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
00599   MPI_Irecv(
00600     const_cast<char*>(recvBuffer.getRawPtr()), recvBuffer.size(), MPI_CHAR, sourceRank,
00601     tag_, *rawMpiComm_, &rawMpiRequest );
00602   return mpiCommRequest(rawMpiRequest);
00603   // ToDo: What about MPI error handling???
00604 }
00605 
00606 
00607 template<typename Ordinal>
00608 void MpiComm<Ordinal>::waitAll(
00609   const ArrayView<RCP<CommRequest> > &requests
00610   ) const
00611 {
00612   TEUCHOS_COMM_TIME_MONITOR(
00613     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::waitAll(...)"
00614     );
00615   const int count = requests.size();
00616 #ifdef TEUCHOS_DEBUG
00617   TEST_FOR_EXCEPT( requests.size() == 0 );
00618 #endif
00619   
00620   Array<MPI_Request> rawMpiRequests(count, MPI_REQUEST_NULL);
00621   for (int i = 0; i < count; ++i) {
00622     RCP<CommRequest> &request = requests[i];
00623     if (!is_null(request)) {
00624       const RCP<MpiCommRequest> mpiCommRequest =
00625         rcp_dynamic_cast<MpiCommRequest>(request);
00626       rawMpiRequests[i] = mpiCommRequest->releaseRawMpiRequest();
00627     }
00628     // else already null
00629     request = null;
00630   }
00631 
00632   Array<MPI_Status> rawMpiStatuses(count);
00633   MPI_Waitall( count, rawMpiRequests.getRawPtr(), rawMpiStatuses.getRawPtr() );
00634   // ToDo: We really should check the status?
00635 
00636 }
00637 
00638 
00639 template<typename Ordinal>
00640 void MpiComm<Ordinal>::wait(
00641   const Ptr<RCP<CommRequest> > &request
00642   ) const
00643 {
00644   TEUCHOS_COMM_TIME_MONITOR(
00645     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::wait(...)"
00646     );
00647   if (is_null(*request)) {
00648     return; // Nothing to wait on ...
00649   }
00650   const RCP<MpiCommRequest> mpiCommRequest =
00651     rcp_dynamic_cast<MpiCommRequest>(*request);
00652   MPI_Request rawMpiRequest = mpiCommRequest->releaseRawMpiRequest();
00653   MPI_Status status;
00654   MPI_Wait( &rawMpiRequest, &status );
00655   // ToDo: We really should check the status?
00656   *request = null;
00657 }
00658 
00659 
00660 // Overridden from Describable
00661 
00662 
00663 template<typename Ordinal>
00664 std::string MpiComm<Ordinal>::description() const
00665 {
00666   std::ostringstream oss;
00667   oss
00668     << typeName(*this)
00669     << "{"
00670     << "size="<<size_
00671     << ",rank="<<rank_
00672     << ",rawMpiComm="<<static_cast<MPI_Comm>(*rawMpiComm_)
00673     <<"}";
00674   return oss.str();
00675 }
00676 
00677 
00678 #ifdef TEUCHOS_MPI_COMM_DUMP
00679 template<typename Ordinal>
00680 bool MpiComm<Ordinal>::show_dump = false;
00681 #endif
00682 
00683 
00684 // private
00685 
00686 
00687 template<typename Ordinal>
00688 void MpiComm<Ordinal>::assertRank(const int rank, const std::string &rankName) const
00689 {
00690   TEST_FOR_EXCEPTION(
00691     ! ( 0 <= rank && rank < size_ ), std::logic_error
00692     ,"Error, "<<rankName<<" = " << rank << " is not < 0 or is not"
00693     " in the range [0,"<<size_-1<<"]!"
00694     );
00695 }
00696 
00697 
00698 } // namespace Teuchos
00699 
00700 
00701 template<typename Ordinal>
00702 Teuchos::RCP<Teuchos::MpiComm<Ordinal> >
00703 Teuchos::createMpiComm(
00704   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00705   )
00706 {
00707   if( rawMpiComm.get()!=NULL && *rawMpiComm != MPI_COMM_NULL )
00708     return rcp(new MpiComm<Ordinal>(rawMpiComm));
00709   return Teuchos::null;
00710 }
00711 
00712 
00713 #endif // TEUCHOS_MPI_COMM_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines