Teuchos - Trilinos Tools Package Version of the Day
Teuchos_DefaultMpiComm.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 //
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ***********************************************************************
00040 // @HEADER
00041 
00042 #ifndef TEUCHOS_MPI_COMM_HPP
00043 #define TEUCHOS_MPI_COMM_HPP
00044 
00045 
00046 #include "Teuchos_Comm.hpp"
00047 #include "Teuchos_CommUtilities.hpp"
00048 #include "Teuchos_OrdinalTraits.hpp"
00049 #include "Teuchos_OpaqueWrapper.hpp"
00050 #include "Teuchos_MpiReductionOpSetter.hpp"
00051 #include "Teuchos_SerializationTraitsHelpers.hpp"
00052 #include "Teuchos_Workspace.hpp"
00053 #include "Teuchos_TypeNameTraits.hpp"
00054 #include "Teuchos_as.hpp"
00055 #include "Teuchos_Assert.hpp"
00056 #include "mpi.h"
00057 #include <iterator>
00058 
00059 // This must be defined globally for the whole program!
00060 //#define TEUCHOS_MPI_COMM_DUMP
00061 
00062 
00063 #ifdef TEUCHOS_MPI_COMM_DUMP
00064 #  include "Teuchos_VerboseObject.hpp"
00065 #endif
00066 
00067 
00068 namespace Teuchos {
00069 
00071 std::string
00072 mpiErrorCodeToString (const int err);
00073 
00074 namespace details {
00088   void safeCommFree (MPI_Comm* comm);
00089 
00094   int setCommErrhandler (MPI_Comm comm, MPI_Errhandler handler);
00095 
00096 } // namespace details
00097 
00098 #ifdef TEUCHOS_MPI_COMM_DUMP
00099 template<typename Ordinal, typename T>
00100 void dumpBuffer(
00101   const std::string &funcName, const std::string &buffName
00102   ,const Ordinal bytes, const T buff[]
00103   )
00104 {
00105   Teuchos::RCP<Teuchos::FancyOStream>
00106     out = Teuchos::VerboseObjectBase::getDefaultOStream();
00107   Teuchos::OSTab tab(out);
00108   *out
00109     << "\n" << funcName << "::" << buffName << ":\n";
00110   tab.incrTab();
00111   for( Ordinal i = 0; i < bytes; ++i ) {
00112     *out << buffName << "[" << i << "] = '" << buff[i] << "'\n";
00113   }
00114   *out << "\n";
00115 }
00116 #endif // TEUCHOS_MPI_COMM_DUMP
00117 
00118 
00130 template<class OrdinalType>
00131 class MpiCommStatus : public CommStatus<OrdinalType> {
00132 public:
00133   MpiCommStatus (MPI_Status status) : status_ (status) {}
00134 
00136   virtual ~MpiCommStatus() {}
00137 
00139   OrdinalType getSourceRank () { return status_.MPI_SOURCE; }
00140 
00142   OrdinalType getTag () { return status_.MPI_TAG; }
00143 
00145   OrdinalType getError () { return status_.MPI_ERROR; }
00146 
00147 private:
00149   MpiCommStatus ();
00150 
00152   MPI_Status status_;
00153 };
00154 
00158 template<class OrdinalType>
00159 inline RCP<MpiCommStatus<OrdinalType> >
00160 mpiCommStatus (MPI_Status rawMpiStatus)
00161 {
00162   return rcp (new MpiCommStatus<OrdinalType> (rawMpiStatus));
00163 }
00164 
00180 template<class OrdinalType>
00181 class MpiCommRequest : public CommRequest<OrdinalType> {
00182 public:
00184   MpiCommRequest (MPI_Request rawMpiRequest,
00185                   const ArrayView<char>::size_type numBytesInMessage) :
00186     rawMpiRequest_ (rawMpiRequest), numBytes_ (numBytesInMessage)
00187   {}
00188 
00195   MPI_Request releaseRawMpiRequest()
00196   {
00197     MPI_Request tmp_rawMpiRequest = rawMpiRequest_;
00198     rawMpiRequest_ = MPI_REQUEST_NULL;
00199     return tmp_rawMpiRequest;
00200   }
00201 
00203   bool isNull() const {
00204     return rawMpiRequest_ == MPI_REQUEST_NULL;
00205   }
00206 
00212   ArrayView<char>::size_type numBytes () const {
00213     return numBytes_;
00214   }
00215 
00221   RCP<CommStatus<OrdinalType> > wait () {
00222     MPI_Status rawMpiStatus;
00223     // Whether this function satisfies the strong exception guarantee
00224     // depends on whether MPI_Wait modifies its input request on error.
00225     const int err = MPI_Wait (&rawMpiRequest_, &rawMpiStatus);
00226     TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00227       "Teuchos::MpiCommStatus::wait: MPI_Wait() failed with error \""
00228       << mpiErrorCodeToString (err) << "\".");
00229     // MPI_Wait sets the MPI_Request to MPI_REQUEST_NULL on success.
00230     return mpiCommStatus<OrdinalType> (rawMpiStatus);
00231   }
00232 
00237   RCP<CommStatus<OrdinalType> > cancel () {
00238     if (rawMpiRequest_ == MPI_REQUEST_NULL) {
00239       return null;
00240     }
00241     else {
00242       int err = MPI_Cancel (&rawMpiRequest_);
00243       TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00244         "Teuchos::MpiCommStatus::cancel: MPI_Cancel failed with the following "
00245         "error: " << mpiErrorCodeToString (err));
00246 
00247       // Wait on the request.  If successful, MPI_Wait will set the
00248       // MPI_Request to MPI_REQUEST_NULL.  The returned status may
00249       // still be useful; for example, one may call MPI_Test_cancelled
00250       // to test an MPI_Status from a nonblocking send.
00251       MPI_Status status;
00252       err = MPI_Wait (&rawMpiRequest_, &status);
00253       TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00254         "Teuchos::MpiCommStatus::cancel: MPI_Wait failed with the following "
00255         "error: " << mpiErrorCodeToString (err));
00256       return mpiCommStatus<OrdinalType> (status);
00257     }
00258   }
00259 
00261   ~MpiCommRequest () {
00262     if (rawMpiRequest_ != MPI_REQUEST_NULL) {
00263       // We're in a destructor, so don't throw errors.  However, if
00264       // MPI_Cancel fails, it's probably a bad idea to call MPI_Wait.
00265       const int err = MPI_Cancel (&rawMpiRequest_);
00266       if (err == MPI_SUCCESS) {
00267   // The MPI_Cancel succeeded.  Now wait on the request.  Ignore
00268   // any reported error, since we can't do anything about those
00269   // in the destructor (other than kill the program).  If
00270   // successful, MPI_Wait will set the MPI_Request to
00271   // MPI_REQUEST_NULL.  We ignore the returned MPI_Status, since
00272   // if the user let the request fall out of scope, she must not
00273   // care about the status.
00274   //
00275   // mfh 21 Oct 2012: The MPI standard requires completing a
00276   // canceled request by calling a function like MPI_Wait,
00277   // MPI_Test, or MPI_Request_free.  MPI_Wait on a canceled
00278   // request behaves like a local operation (it does not
00279   // communicate or block waiting for communication).  One could
00280   // also call MPI_Request_free instead of MPI_Wait, but
00281   // MPI_Request_free is intended more for persistent requests
00282   // (created with functions like MPI_Recv_init).
00283   (void) MPI_Wait (&rawMpiRequest_, MPI_STATUS_IGNORE);
00284       } 
00285     }
00286   }
00287 
00288 private:
00290   MPI_Request rawMpiRequest_;
00292   ArrayView<char>::size_type numBytes_;
00293 
00294   MpiCommRequest(); // Not defined
00295 };
00296 
00305 template<class OrdinalType>
00306 inline RCP<MpiCommRequest<OrdinalType> >
00307 mpiCommRequest (MPI_Request rawMpiRequest,
00308                 const ArrayView<char>::size_type numBytes)
00309 {
00310   return rcp (new MpiCommRequest<OrdinalType> (rawMpiRequest, numBytes));
00311 }
00312 
00328 template<typename Ordinal>
00329 class MpiComm : public Comm<Ordinal> {
00330 public:
00331 
00333 
00334 
00355   explicit MpiComm (MPI_Comm rawMpiComm);
00356 
00371   MpiComm(
00372     const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00373     );
00374 
00391   MpiComm(const MpiComm<Ordinal>& other);
00392 
00394   RCP<const OpaqueWrapper<MPI_Comm> > getRawMpiComm() const
00395   {return rawMpiComm_;}
00396 
00415   void setErrorHandler (const RCP<const OpaqueWrapper<MPI_Errhandler> >& errHandler);
00416 
00418 
00420 
00421 
00423   virtual int getRank() const;
00425   virtual int getSize() const;
00427   virtual void barrier() const;
00429   virtual void broadcast(
00430     const int rootRank, const Ordinal bytes, char buffer[]
00431     ) const;
00433   virtual void gatherAll(
00434     const Ordinal sendBytes, const char sendBuffer[]
00435     ,const Ordinal recvBytes, char recvBuffer[]
00436     ) const;
00438   virtual void reduceAll(
00439     const ValueTypeReductionOp<Ordinal,char> &reductOp
00440     ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00441     ) const;
00443   virtual void reduceAllAndScatter(
00444     const ValueTypeReductionOp<Ordinal,char> &reductOp
00445     ,const Ordinal sendBytes, const char sendBuffer[]
00446     ,const Ordinal recvCounts[], char myGlobalReducts[]
00447     ) const;
00449         virtual void scan(
00450     const ValueTypeReductionOp<Ordinal,char> &reductOp
00451     ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00452     ) const;
00454   virtual void send(
00455     const Ordinal bytes, const char sendBuffer[], const int destRank
00456     ) const;
00458   virtual void ssend(
00459     const Ordinal bytes, const char sendBuffer[], const int destRank
00460     ) const;
00462   virtual int receive(
00463     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00464     ) const;
00466   virtual void readySend(
00467     const ArrayView<const char> &sendBuffer,
00468     const int destRank
00469     ) const;
00471   virtual RCP<CommRequest<Ordinal> > isend(
00472     const ArrayView<const char> &sendBuffer,
00473     const int destRank
00474     ) const;
00476   virtual RCP<CommRequest<Ordinal> > ireceive(
00477     const ArrayView<char> &Buffer,
00478     const int sourceRank
00479     ) const;
00481   virtual void waitAll(
00482     const ArrayView<RCP<CommRequest<Ordinal> > > &requests
00483     ) const;
00485   virtual void
00486   waitAll (const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
00487            const ArrayView<RCP<CommStatus<Ordinal> > >& statuses) const;
00489   virtual RCP<CommStatus<Ordinal> >
00490   wait (const Ptr<RCP<CommRequest<Ordinal> > >& request) const;
00492   virtual RCP< Comm<Ordinal> > duplicate() const;
00494   virtual RCP< Comm<Ordinal> > split(const int color, const int key) const;
00496   virtual RCP< Comm<Ordinal> > createSubcommunicator(
00497     const ArrayView<const int>& ranks) const;
00499 
00501 
00502 
00504   std::string description() const;
00505 
00507 
00508   // These should be private but the PGI compiler requires them be public
00509 
00510   static int const minTag_ = 26000; // These came from Teuchos::MpiComm???
00511   static int const maxTag_ = 26099; // ""
00512 
00518   int getTag () const { return tag_; }
00519 
00520 private:
00521 
00522   // Set internal data members once the rawMpiComm_ data member is valid.
00523   void setupMembersFromComm();
00524   static int tagCounter_;
00525 
00533   RCP<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;
00534 
00536   int rank_;
00537 
00539   int size_;
00540   
00548   int tag_;
00549 
00551   RCP<const OpaqueWrapper<MPI_Errhandler> > customErrorHandler_;
00552 
00553   void assertRank(const int rank, const std::string &rankName) const;
00554 
00555   // Not defined and not to be called!
00556   MpiComm();
00557 
00558 #ifdef TEUCHOS_MPI_COMM_DUMP
00559 public:
00560   static bool show_dump;
00561 #endif // TEUCHOS_MPI_COMM_DUMP
00562 
00563 };
00564 
00565 
00579 template<typename Ordinal>
00580 RCP<MpiComm<Ordinal> >
00581 createMpiComm(
00582   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00583   );
00584 
00585 
00586 // ////////////////////////
00587 // Implementations
00588 
00589 
00590 // Static members
00591 
00592 
00593 template<typename Ordinal>
00594 int MpiComm<Ordinal>::tagCounter_ = MpiComm<Ordinal>::minTag_;
00595 
00596 
00597 // Constructors
00598 
00599 
00600 template<typename Ordinal>
00601 MpiComm<Ordinal>::MpiComm(
00602   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00603   )
00604 {
00605   TEUCHOS_TEST_FOR_EXCEPTION(rawMpiComm.get()==NULL, std::invalid_argument,
00606     "Teuchos::MpiComm constructor: The input RCP is null.");
00607   TEUCHOS_TEST_FOR_EXCEPTION(*rawMpiComm == MPI_COMM_NULL,
00608     std::invalid_argument, "Teuchos::MpiComm constructor: The given MPI_Comm "
00609     "is MPI_COMM_NULL.");
00610   rawMpiComm_ = rawMpiComm;
00611 
00612   // FIXME (mfh 26 Mar 2012) The following is a bit wicked in that it
00613   // changes the behavior of existing applications that use MpiComm,
00614   // without warning.  I've chosen to do it because I can't figure out
00615   // any other way to help me debug MPI_Waitall failures on some (but
00616   // not all) of the testing platforms.  The problem is that MPI's
00617   // default error handler is MPI_ERRORS_ARE_FATAL, which immediately
00618   // aborts on error without returning an error code from the MPI
00619   // function.  Also, the testing platforms' MPI implementations'
00620   // diagnostics are not giving me useful information.  Thus, I'm
00621   // setting the default error handler to MPI_ERRORS_RETURN, so that
00622   // MPI_Waitall will return an error code.
00623   //
00624   // Note that all MpiComm methods check error codes returned by MPI
00625   // functions, and throw an exception if the code is not MPI_SUCCESS.
00626   // Thus, this change in behavior will only affect your program in
00627   // the following case: You call a function f() in the try block of a
00628   // try-catch, and expect f() to throw an exception (generally
00629   // std::runtime_error) in a particular case not related to MpiComm,
00630   // but MpiComm throws the exception instead.  It's probably a bad
00631   // idea for you to do this, because MpiComm might very well throw
00632   // exceptions for things like invalid arguments.
00633   const bool makeMpiErrorsReturn = true;
00634   if (makeMpiErrorsReturn) {
00635     RCP<const OpaqueWrapper<MPI_Errhandler> > errHandler =
00636       rcp (new OpaqueWrapper<MPI_Errhandler> (MPI_ERRORS_RETURN));
00637     setErrorHandler (errHandler);
00638   }
00639 
00640   setupMembersFromComm();
00641 }
00642 
00643 
00644 template<typename Ordinal>
00645 MpiComm<Ordinal>::
00646 MpiComm (MPI_Comm rawMpiComm)
00647 {
00648   TEUCHOS_TEST_FOR_EXCEPTION(rawMpiComm == MPI_COMM_NULL,
00649     std::invalid_argument, "Teuchos::MpiComm constructor: The given MPI_Comm "
00650     "is MPI_COMM_NULL.");
00651   // We don't supply a "free" function here, since this version of the
00652   // constructor makes the caller responsible for freeing rawMpiComm
00653   // after use if necessary.
00654   rawMpiComm_ = opaqueWrapper<MPI_Comm> (rawMpiComm);
00655 
00656   // FIXME (mfh 26 Mar 2012) The following is a bit wicked in that it
00657   // changes the behavior of existing applications that use MpiComm,
00658   // without warning.  I've chosen to do it because I can't figure out
00659   // any other way to help me debug MPI_Waitall failures on some (but
00660   // not all) of the testing platforms.  The problem is that MPI's
00661   // default error handler is MPI_ERRORS_ARE_FATAL, which immediately
00662   // aborts on error without returning an error code from the MPI
00663   // function.  Also, the testing platforms' MPI implementations'
00664   // diagnostics are not giving me useful information.  Thus, I'm
00665   // setting the default error handler to MPI_ERRORS_RETURN, so that
00666   // MPI_Waitall will return an error code.
00667   //
00668   // Note that all MpiComm methods check error codes returned by MPI
00669   // functions, and throw an exception if the code is not MPI_SUCCESS.
00670   // Thus, this change in behavior will only affect your program in
00671   // the following case: You call a function f() in the try block of a
00672   // try-catch, and expect f() to throw an exception (generally
00673   // std::runtime_error) in a particular case not related to MpiComm,
00674   // but MpiComm throws the exception instead.  It's probably a bad
00675   // idea for you to do this, because MpiComm might very well throw
00676   // exceptions for things like invalid arguments.
00677   const bool makeMpiErrorsReturn = true;
00678   if (makeMpiErrorsReturn) {
00679     RCP<const OpaqueWrapper<MPI_Errhandler> > errHandler =
00680       rcp (new OpaqueWrapper<MPI_Errhandler> (MPI_ERRORS_RETURN));
00681     setErrorHandler (errHandler);
00682   }
00683 
00684   setupMembersFromComm();
00685 }
00686 
00687 
00688 template<typename Ordinal>
00689 MpiComm<Ordinal>::MpiComm (const MpiComm<Ordinal>& other) : 
00690   rawMpiComm_ (opaqueWrapper<MPI_Comm> (MPI_COMM_NULL)) // <- This will be set below
00691 {
00692   // These are logic errors, since they violate MpiComm's invariants.
00693   RCP<const OpaqueWrapper<MPI_Comm> > origCommPtr = other.getRawMpiComm ();
00694   TEUCHOS_TEST_FOR_EXCEPTION(origCommPtr == null, std::logic_error, 
00695     "Teuchos::MpiComm copy constructor: "
00696     "The input's getRawMpiComm() method returns null.");
00697   MPI_Comm origComm = *origCommPtr;
00698   TEUCHOS_TEST_FOR_EXCEPTION(origComm == MPI_COMM_NULL, std::logic_error,
00699     "Teuchos::MpiComm copy constructor: "
00700     "The input's raw MPI_Comm is MPI_COMM_NULL.");
00701 
00702   // mfh 19 Oct 2012: Don't change the behavior of MpiComm's copy
00703   // constructor for now.  Later, we'll switch to the version that
00704   // calls MPI_Comm_dup.  For now, we just copy other's handle over.
00705   // Note that the new MpiComm's tag is still different than the input
00706   // MpiComm's tag.  See Bug 5740.
00707   if (true) {
00708     rawMpiComm_ = origCommPtr;
00709   } 
00710   else { // false (not run)
00711     MPI_Comm newComm;
00712     const int err = MPI_Comm_dup (origComm, &newComm);
00713     TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00714       "Teuchos::MpiComm copy constructor: MPI_Comm_dup failed with "
00715       "the following error: " << mpiErrorCodeToString (err));
00716     // No side effects until after everything has succeeded.
00717     rawMpiComm_ = opaqueWrapper (newComm, details::safeCommFree);
00718   }
00719 
00720   setupMembersFromComm();
00721 }
00722 
00723 
00724 template<typename Ordinal>
00725 void MpiComm<Ordinal>::setupMembersFromComm()
00726 {
00727   MPI_Comm_size(*rawMpiComm_, &size_);
00728   MPI_Comm_rank(*rawMpiComm_, &rank_);
00729   if(tagCounter_ > maxTag_)
00730     tagCounter_ = minTag_;
00731   tag_ = tagCounter_++;
00732 }
00733 
00734 
00735 template<typename Ordinal>
00736 void
00737 MpiComm<Ordinal>::
00738 setErrorHandler (const RCP<const OpaqueWrapper<MPI_Errhandler> >& errHandler)
00739 {
00740   if (! is_null (errHandler)) {
00741     const int err = details::setCommErrhandler (*getRawMpiComm (), *errHandler);
00742     TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00743       "Teuchos::MpiComm: Setting the MPI_Comm's error handler failed with "
00744       "error \"" << mpiErrorCodeToString (err) << "\".");
00745   }
00746   // Wait to set this until the end, in case setting the error handler
00747   // doesn't succeed.
00748   customErrorHandler_ = errHandler;
00749 }
00750 
00751 
00752 
00753 // Overridden from Comm
00754 
00755 
00756 template<typename Ordinal>
00757 int MpiComm<Ordinal>::getRank() const
00758 {
00759   return rank_;
00760 }
00761 
00762 
00763 template<typename Ordinal>
00764 int MpiComm<Ordinal>::getSize() const
00765 {
00766   return size_;
00767 }
00768 
00769 
00770 template<typename Ordinal>
00771 void MpiComm<Ordinal>::barrier() const
00772 {
00773   TEUCHOS_COMM_TIME_MONITOR(
00774     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::barrier()"
00775     );
00776   const int err = MPI_Barrier(*rawMpiComm_);
00777   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00778     "Teuchos::MpiComm::barrier: MPI_Barrier failed with error \""
00779     << mpiErrorCodeToString (err) << "\".");
00780 }
00781 
00782 
00783 template<typename Ordinal>
00784 void MpiComm<Ordinal>::broadcast(
00785   const int rootRank, const Ordinal bytes, char buffer[]
00786   ) const
00787 {
00788   TEUCHOS_COMM_TIME_MONITOR(
00789     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::broadcast(...)"
00790     );
00791   const int err = MPI_Bcast(buffer,bytes,MPI_CHAR,rootRank,*rawMpiComm_);
00792   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00793     "Teuchos::MpiComm::broadcast: MPI_Bcast failed with error \""
00794     << mpiErrorCodeToString (err) << "\".");
00795 }
00796 
00797 
00798 template<typename Ordinal>
00799 void MpiComm<Ordinal>::gatherAll(
00800   const Ordinal sendBytes, const char sendBuffer[],
00801   const Ordinal recvBytes, char recvBuffer[]
00802   ) const
00803 {
00804   TEUCHOS_COMM_TIME_MONITOR(
00805     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::gatherAll(...)"
00806     );
00807   TEUCHOS_ASSERT_EQUALITY((sendBytes*size_), recvBytes );
00808   const int err =
00809     MPI_Allgather (const_cast<char *>(sendBuffer), sendBytes, MPI_CHAR,
00810                    recvBuffer, sendBytes, MPI_CHAR, *rawMpiComm_);
00811   // NOTE: 'sendBytes' is being sent above for the MPI arg recvcount (which is
00812   // very confusing in the MPI documentation) for MPI_Allgether(...).
00813 
00814   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00815     "Teuchos::MpiComm::gatherAll: MPI_Allgather failed with error \""
00816     << mpiErrorCodeToString (err) << "\".");
00817 }
00818 
00819 
00820 template<typename Ordinal>
00821 void
00822 MpiComm<Ordinal>::
00823 reduceAll (const ValueTypeReductionOp<Ordinal,char> &reductOp,
00824            const Ordinal bytes,
00825            const char sendBuffer[],
00826            char globalReducts[]) const
00827 {
00828   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::reduceAll(...)" );
00829 
00830   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00831   MPI_Datatype char_block;
00832 
00833   // TODO (mfh 26 Mar 2012) Check returned error codes of the MPI
00834   // custom datatype functions.
00835   MPI_Type_contiguous(bytes, MPI_CHAR, &char_block);
00836   MPI_Type_commit(&char_block);
00837 
00838   const int err =
00839     MPI_Allreduce (const_cast<char*>(sendBuffer), globalReducts, 1, char_block,
00840                    op.mpi_op(), *rawMpiComm_);
00841   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00842     "Teuchos::MpiComm::reduceAll (custom op): MPI_Allreduce failed with error \""
00843     << mpiErrorCodeToString (err) << "\".");
00844 
00845   // TODO (mfh 26 Mar 2012) Check returned error codes of the MPI
00846   // custom datatype functions.
00847   MPI_Type_free(&char_block);
00848 }
00849 
00850 
00851 template<typename Ordinal>
00852 void MpiComm<Ordinal>::reduceAllAndScatter(
00853   const ValueTypeReductionOp<Ordinal,char> &reductOp
00854   ,const Ordinal sendBytes, const char sendBuffer[]
00855   ,const Ordinal recvCounts[], char myGlobalReducts[]
00856   ) const
00857 {
00858 
00859   (void)sendBytes; // Ignore if not in debug mode
00860 
00861   TEUCHOS_COMM_TIME_MONITOR(
00862     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAllAndScatter(...)"
00863     );
00864 
00865 #ifdef TEUCHOS_DEBUG
00866   Ordinal sumRecvBytes = 0;
00867   for( Ordinal i = 0; i < size_; ++i ) {
00868     sumRecvBytes += recvCounts[i];
00869   }
00870   TEUCHOS_TEST_FOR_EXCEPT(!(sumRecvBytes==sendBytes));
00871 #endif // TEUCHOS_DEBUG
00872 
00873 #ifdef TEUCHOS_MPI_COMM_DUMP
00874   if(show_dump) {
00875     dumpBuffer<Ordinal,char>(
00876       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00877       "sendBuffer", sendBytes, sendBuffer );
00878     dumpBuffer<Ordinal,Ordinal>(
00879       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00880       "recvCounts", as<Ordinal>(size_), recvCounts );
00881     dumpBuffer<Ordinal,char>(
00882       "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)",
00883       "myGlobalReducts", as<char>(recvCounts[rank_]), myGlobalReducts );
00884   }
00885 #endif // TEUCHOS_MPI_COMM_DUMP
00886 
00887   // Create a new recvCount[] if Ordinal!=int
00888   WorkspaceStore* wss = get_default_workspace_store().get();
00889   const bool Ordinal_is_int = typeid(int)==typeid(Ordinal);
00890   Workspace<int> ws_int_recvCounts(wss,Ordinal_is_int?0:size_);
00891   const int *int_recvCounts = 0;
00892   if(Ordinal_is_int) {
00893     int_recvCounts = reinterpret_cast<const int*>(recvCounts);
00894     // Note: We must do an reinterpet cast since this must
00895     // compile even if it is not executed.  I could implement
00896     // code that would not need to do this using template
00897     // conditionals but I don't want to bother.
00898   }
00899   else {
00900     std::copy(recvCounts, recvCounts+size_, &ws_int_recvCounts[0]);
00901     int_recvCounts = &ws_int_recvCounts[0];
00902   }
00903 
00904   // Perform the operation
00905   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp, false)));
00906 
00907   const int err = MPI_Reduce_scatter(
00908     const_cast<char*>(sendBuffer), myGlobalReducts,
00909     const_cast<int*>(int_recvCounts),
00910     MPI_CHAR,
00911     op.mpi_op(),
00912     *rawMpiComm_
00913     );
00914   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00915     "Teuchos::MpiComm::reduceAllAndScatter: MPI_Reduce_scatter failed with "
00916     "error \"" << mpiErrorCodeToString (err) << "\".");
00917 }
00918 
00919 
00920 template<typename Ordinal>
00921 void MpiComm<Ordinal>::scan(
00922   const ValueTypeReductionOp<Ordinal,char> &reductOp
00923   ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00924   ) const
00925 {
00926   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::scan(...)" );
00927 
00928   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00929   const int err =
00930     MPI_Scan (const_cast<char*>(sendBuffer), scanReducts, bytes, MPI_CHAR,
00931               op.mpi_op(), *rawMpiComm_);
00932   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00933     "Teuchos::MpiComm::scan: MPI_Scan() failed with error \""
00934     << mpiErrorCodeToString (err) << "\".");
00935 }
00936 
00937 
00938 template<typename Ordinal>
00939 void
00940 MpiComm<Ordinal>::send (const Ordinal bytes,
00941                         const char sendBuffer[],
00942                         const int destRank) const
00943 {
00944   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::send(...)" );
00945 
00946 #ifdef TEUCHOS_MPI_COMM_DUMP
00947   if(show_dump) {
00948     dumpBuffer<Ordinal,char>(
00949       "Teuchos::MpiComm<Ordinal>::send(...)"
00950       ,"sendBuffer", bytes, sendBuffer
00951       );
00952   }
00953 #endif // TEUCHOS_MPI_COMM_DUMP
00954 
00955   const int err = MPI_Send (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
00956                             destRank, tag_, *rawMpiComm_);
00957   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00958     "Teuchos::MpiComm::send: MPI_Send() failed with error \""
00959     << mpiErrorCodeToString (err) << "\".");
00960 }
00961 
00962 
00963 template<typename Ordinal>
00964 void
00965 MpiComm<Ordinal>::ssend (const Ordinal bytes,
00966                          const char sendBuffer[],
00967                          const int destRank) const
00968 {
00969   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::ssend(...)" );
00970 
00971 #ifdef TEUCHOS_MPI_COMM_DUMP
00972   if(show_dump) {
00973     dumpBuffer<Ordinal,char>(
00974       "Teuchos::MpiComm<Ordinal>::send(...)"
00975       ,"sendBuffer", bytes, sendBuffer
00976       );
00977   }
00978 #endif // TEUCHOS_MPI_COMM_DUMP
00979 
00980   const int err = MPI_Ssend (const_cast<char*>(sendBuffer), bytes, MPI_CHAR,
00981                              destRank, tag_, *rawMpiComm_);
00982   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
00983     "Teuchos::MpiComm::send: MPI_Ssend() failed with error \""
00984     << mpiErrorCodeToString (err) << "\".");
00985 }
00986 
00987 
00988 template<typename Ordinal>
00989 void MpiComm<Ordinal>::readySend(
00990   const ArrayView<const char> &sendBuffer,
00991   const int destRank
00992   ) const
00993 {
00994   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::readySend" );
00995 
00996 #ifdef TEUCHOS_MPI_COMM_DUMP
00997   if(show_dump) {
00998     dumpBuffer<Ordinal,char>(
00999       "Teuchos::MpiComm<Ordinal>::readySend(...)"
01000       ,"sendBuffer", bytes, sendBuffer
01001       );
01002   }
01003 #endif // TEUCHOS_MPI_COMM_DUMP
01004 
01005   const int err =
01006     MPI_Rsend (const_cast<char*>(sendBuffer.getRawPtr()), sendBuffer.size(),
01007                MPI_CHAR, destRank, tag_, *rawMpiComm_);
01008   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
01009     "Teuchos::MpiComm::readySend: MPI_Rsend() failed with error \""
01010     << mpiErrorCodeToString (err) << "\".");
01011 }
01012 
01013 
01014 template<typename Ordinal>
01015 int
01016 MpiComm<Ordinal>::receive (const int sourceRank,
01017                            const Ordinal bytes,
01018                            char recvBuffer[]) const
01019 {
01020   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::receive(...)" );
01021 
01022   // A negative source rank indicates MPI_ANY_SOURCE, namely that we
01023   // will take an incoming message from any process, as long as the
01024   // tag matches.
01025   const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
01026 
01027   MPI_Status status;
01028   const int err = MPI_Recv (recvBuffer, bytes, MPI_CHAR, theSrcRank, tag_,
01029                             *rawMpiComm_, &status);
01030   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
01031     "Teuchos::MpiComm::receive: MPI_Recv() failed with error \""
01032     << mpiErrorCodeToString (err) << "\".");
01033 
01034 #ifdef TEUCHOS_MPI_COMM_DUMP
01035   if (show_dump) {
01036     dumpBuffer<Ordinal,char> ("Teuchos::MpiComm<Ordinal>::receive(...)",
01037                               "recvBuffer", bytes, recvBuffer);
01038   }
01039 #endif // TEUCHOS_MPI_COMM_DUMP
01040 
01041   // Returning the source rank is useful in the MPI_ANY_SOURCE case.
01042   return status.MPI_SOURCE;
01043 }
01044 
01045 
01046 template<typename Ordinal>
01047 RCP<CommRequest<Ordinal> >
01048 MpiComm<Ordinal>::isend (const ArrayView<const char> &sendBuffer,
01049                          const int destRank) const
01050 {
01051   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::isend(...)" );
01052 
01053   MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
01054   const int err =
01055     MPI_Isend (const_cast<char*>(sendBuffer.getRawPtr()), sendBuffer.size(),
01056                MPI_CHAR, destRank, tag_, *rawMpiComm_, &rawMpiRequest);
01057   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
01058     "Teuchos::MpiComm::isend: MPI_Isend() failed with error \""
01059     << mpiErrorCodeToString (err) << "\".");
01060 
01061   return mpiCommRequest<Ordinal> (rawMpiRequest, sendBuffer.size());
01062 }
01063 
01064 
01065 template<typename Ordinal>
01066 RCP<CommRequest<Ordinal> >
01067 MpiComm<Ordinal>::ireceive (const ArrayView<char> &recvBuffer,
01068                             const int sourceRank) const
01069 {
01070   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::ireceive(...)" );
01071 
01072   // A negative source rank indicates MPI_ANY_SOURCE, namely that we
01073   // will take an incoming message from any process, as long as the
01074   // tag matches.
01075   const int theSrcRank = (sourceRank < 0) ? MPI_ANY_SOURCE : sourceRank;
01076 
01077   MPI_Request rawMpiRequest = MPI_REQUEST_NULL;
01078   const int err =
01079     MPI_Irecv (const_cast<char*>(recvBuffer.getRawPtr()), recvBuffer.size(),
01080                MPI_CHAR, theSrcRank, tag_, *rawMpiComm_, &rawMpiRequest);
01081   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error,
01082     "Teuchos::MpiComm::ireceive: MPI_Irecv() failed with error \""
01083     << mpiErrorCodeToString (err) << "\".");
01084 
01085   return mpiCommRequest<Ordinal> (rawMpiRequest, recvBuffer.size());
01086 }
01087 
01088 
01089 namespace {
01090   // Called by both MpiComm::waitAll() implementations.
01091   template<typename Ordinal>
01092   void
01093   waitAllImpl (const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
01094                const ArrayView<MPI_Status>& rawMpiStatuses)
01095   {
01096     typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
01097     const size_type count = requests.size();
01098     // waitAllImpl() is not meant to be called by users, so it's a bug
01099     // for the two views to have different lengths.
01100     TEUCHOS_TEST_FOR_EXCEPTION(rawMpiStatuses.size() != count,
01101       std::logic_error, "Teuchos::MpiComm's waitAllImpl: rawMpiStatus.size() = "
01102       << rawMpiStatuses.size() << " != requests.size() = " << requests.size()
01103       << ".  Please report this bug to the Tpetra developers.");
01104     if (count == 0) {
01105       return; // No requests on which to wait
01106     }
01107 
01108     // MpiComm wraps MPI and can't expose any MPI structs or opaque
01109     // objects.  Thus, we have to unpack requests into a separate array.
01110     // If that's too slow, then your code should just call into MPI
01111     // directly.
01112     //
01113     // Pull out the raw MPI requests from the wrapped requests.
01114     // MPI_Waitall should not fail if a request is MPI_REQUEST_NULL, but
01115     // we keep track just to inform the user.
01116     bool someNullRequests = false;
01117     Array<MPI_Request> rawMpiRequests (count, MPI_REQUEST_NULL);
01118     for (int i = 0; i < count; ++i) {
01119       RCP<CommRequest<Ordinal> > request = requests[i];
01120       if (! is_null (request)) {
01121         RCP<MpiCommRequest<Ordinal> > mpiRequest =
01122           rcp_dynamic_cast<MpiCommRequest<Ordinal> > (request);
01123         // releaseRawMpiRequest() sets the MpiCommRequest's raw
01124         // MPI_Request to MPI_REQUEST_NULL.  This makes waitAll() not
01125         // satisfy the strong exception guarantee.  That's OK because
01126         // MPI_Waitall() doesn't promise that it satisfies the strong
01127         // exception guarantee, and we would rather conservatively
01128         // invalidate the handles than leave dangling requests around
01129         // and risk users trying to wait on the same request twice.
01130         rawMpiRequests[i] = mpiRequest->releaseRawMpiRequest();
01131       }
01132       else { // Null requests map to MPI_REQUEST_NULL
01133         rawMpiRequests[i] = MPI_REQUEST_NULL;
01134         someNullRequests = true;
01135       }
01136     }
01137 
01138     // This is the part where we've finally peeled off the wrapper and
01139     // we can now interact with MPI directly.
01140     //
01141     // One option in the one-argument version of waitAll() is to ignore
01142     // the statuses completely.  MPI lets you pass in the named constant
01143     // MPI_STATUSES_IGNORE for the MPI_Status array output argument in
01144     // MPI_Waitall(), which would tell MPI not to bother with the
01145     // statuses.  However, we want the statuses because we can use them
01146     // for detailed error diagnostics in case something goes wrong.
01147     const int err = MPI_Waitall (count, rawMpiRequests.getRawPtr(),
01148                                  rawMpiStatuses.getRawPtr());
01149 
01150     // In MPI_Waitall(), an error indicates that one or more requests
01151     // failed.  In that case, there could be requests that completed
01152     // (their MPI_Status' error field is MPI_SUCCESS), and other
01153     // requests that have not completed yet but have not necessarily
01154     // failed (MPI_PENDING).  We make no attempt here to wait on the
01155     // pending requests.  It doesn't make sense for us to do so, because
01156     // in general Teuchos::Comm doesn't attempt to provide robust
01157     // recovery from failed messages.
01158     if (err != MPI_SUCCESS) {
01159       if (err == MPI_ERR_IN_STATUS) {
01160         //
01161         // When MPI_Waitall returns MPI_ERR_IN_STATUS (a standard error
01162         // class), it's telling us to check the error codes in the
01163         // returned statuses.  In that case, we do so and generate a
01164         // detailed exception message.
01165         //
01166         // Figure out which of the requests failed.
01167         Array<std::pair<size_type, int> > errorLocationsAndCodes;
01168         for (size_type k = 0; k < rawMpiStatuses.size(); ++k) {
01169           const int curErr = rawMpiStatuses[k].MPI_ERROR;
01170           if (curErr != MPI_SUCCESS) {
01171             errorLocationsAndCodes.push_back (std::make_pair (k, curErr));
01172           }
01173         }
01174         const size_type numErrs = errorLocationsAndCodes.size();
01175         if (numErrs > 0) {
01176           // There was at least one error.  Assemble a detailed
01177           // exception message reporting which requests failed,
01178           // their error codes, and their source
01179           std::ostringstream os;
01180           os << "Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \""
01181              << mpiErrorCodeToString (err) << "\".  Of the " << count
01182              << " total request" << (count != 1 ? "s" : "") << ", " << numErrs
01183              << " failed.  Here are the indices of the failed requests, and the "
01184             "error codes extracted from their returned MPI_Status objects:"
01185              << std::endl;
01186           for (size_type k = 0; k < numErrs; ++k) {
01187             const size_type errInd = errorLocationsAndCodes[k].first;
01188             os << "Request " << errInd << ": MPI_ERROR = "
01189                << mpiErrorCodeToString (rawMpiStatuses[errInd].MPI_ERROR)
01190                << std::endl;
01191           }
01192           if (someNullRequests) {
01193             os << "  On input to MPI_Waitall, there was at least one MPI_"
01194               "Request that was MPI_REQUEST_NULL.  MPI_Waitall should not "
01195               "normally fail in that case, but we thought we should let you know "
01196               "regardless.";
01197           }
01198           TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, os.str());
01199         }
01200         // If there were no actual errors in the returned statuses,
01201         // well, then I guess everything is OK.  Just keep going.
01202       }
01203       else {
01204         std::ostringstream os;
01205         os << "Teuchos::MpiComm::waitAll: MPI_Waitall() failed with error \""
01206            << mpiErrorCodeToString (err) << "\".";
01207         if (someNullRequests) {
01208           os << "  On input to MPI_Waitall, there was at least one MPI_Request "
01209             "that was MPI_REQUEST_NULL.  MPI_Waitall should not normally fail in "
01210             "that case, but we thought we should let you know regardless.";
01211         }
01212         TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, os.str());
01213       }
01214     }
01215 
01216 #ifdef HAVE_TEUCHOS_DEBUG
01217     if (false) // mfh 02 Apr 2012: This test fails in some cases (e.g., Belos BlockCG), with the MPI_Request reporting 8 bytes and the MPI_Status reporting 0 bytes.  The tests pass otherwise, so I'm disabling this check for now.
01218     {
01219       // In debug mode, test whether the requests' message lengths
01220       // matched the message lengths on completion.
01221       Array<size_type> nonmatchingIndices;
01222       Array<std::pair<size_type, size_type> > nonmatchingLengthPairs;
01223       for (size_type k = 0; k < count; ++k) {
01224         if (! is_null (requests[k])) {
01225           RCP<MpiCommRequest<Ordinal> > mpiRequest =
01226             rcp_dynamic_cast<MpiCommRequest<Ordinal> > (requests[k]);
01227 
01228           int statusCount = -1;
01229           (void) MPI_Get_count (&rawMpiStatuses[k], MPI_CHAR, &statusCount);
01230           if (mpiRequest->numBytes() != as<size_type> (statusCount)) {
01231             nonmatchingIndices.push_back (k);
01232             nonmatchingLengthPairs.push_back (std::make_pair (mpiRequest->numBytes(), Teuchos::as<size_type> (statusCount)));
01233           }
01234         }
01235       }
01236       const size_type numNonmatching = nonmatchingIndices.size();
01237       if (numNonmatching > 0) {
01238         std::ostringstream os;
01239         os << "Teuchos::MpiComm::waitAll(): " << numNonmatching << " message "
01240           "request" << (numNonmatching != 1 ? "s" : "") << " have a number of "
01241           "bytes which does not match the number of bytes in "
01242            << (numNonmatching != 1 ? "their" : "its") << " corresponding status"
01243            << (numNonmatching != 1 ? "es" : "") << "." << std::endl;
01244         os << "Here are the lengths that don't match (from MPI_Request, MPI_Status resp.): " << std::endl;
01245         for (typename Array<std::pair<size_type, size_type> >::const_iterator it = nonmatchingLengthPairs.begin(); it != nonmatchingLengthPairs.end(); ++it) {
01246           os << "(" << it->first << "," << it->second << ") ";
01247         }
01248         if (err == MPI_ERR_IN_STATUS) {
01249           os << std::endl << "This is that weird case where MPI_Waitall returned MPI_ERR_IN_STATUS, but all of the MPI_Statuses' error codes were MPI_SUCCESS.";
01250         }
01251         // This is a bug, so we throw std::logic_error.
01252         TEUCHOS_TEST_FOR_EXCEPTION(numNonmatching > 0, std::logic_error, os.str());
01253       }
01254     }
01255 #endif // HAVE_TEUCHOS_DEBUG
01256 
01257     // Invalidate the input array of requests by setting all entries to
01258     // null.
01259     std::fill (requests.begin(), requests.end(), null);
01260   }
01261 } // namespace (anonymous)
01262 
01263 
01264 
01265 template<typename Ordinal>
01266 void
01267 MpiComm<Ordinal>::
01268 waitAll (const ArrayView<RCP<CommRequest<Ordinal> > >& requests) const
01269 {
01270   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::waitAll(requests)" );
01271 
01272   Array<MPI_Status> rawMpiStatuses (requests.size());
01273   waitAllImpl<Ordinal> (requests, rawMpiStatuses());
01274 }
01275 
01276 
01277 template<typename Ordinal>
01278 void
01279 MpiComm<Ordinal>::
01280 waitAll (const ArrayView<RCP<CommRequest<Ordinal> > >& requests,
01281          const ArrayView<RCP<CommStatus<Ordinal> > >& statuses) const
01282 {
01283   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::waitAll(requests, statuses)" );
01284 
01285   typedef typename ArrayView<RCP<CommRequest<Ordinal> > >::size_type size_type;
01286   const size_type count = requests.size();
01287 
01288   TEUCHOS_TEST_FOR_EXCEPTION(count != statuses.size(),
01289     std::invalid_argument, "Teuchos::MpiComm::waitAll: requests.size() = "
01290     << count << " != statuses.size() = " << statuses.size() << ".");
01291 
01292   Array<MPI_Status> rawMpiStatuses (count);
01293   waitAllImpl<Ordinal> (requests, rawMpiStatuses());
01294 
01295   // Repackage the raw MPI_Status structs into the wrappers.
01296   for (size_type i = 0; i < count; ++i) {
01297     statuses[i] = mpiCommStatus<Ordinal> (rawMpiStatuses[i]);
01298   }
01299 }
01300 
01301 
01302 template<typename Ordinal>
01303 RCP<CommStatus<Ordinal> >
01304 MpiComm<Ordinal>::wait (const Ptr<RCP<CommRequest<Ordinal> > >& request) const
01305 {
01306   TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm::wait(...)" );
01307 
01308   if (is_null (*request)) {
01309     return null; // Nothing to wait on ...
01310   }
01311   else {
01312     RCP<CommStatus<Ordinal> > status = (*request)->wait ();
01313     // mfh 22 Oct 2012: The unit tests expect waiting on the
01314     // CommRequest to invalidate it by setting it to null.
01315     *request = null;
01316     return status;
01317   }
01318 }
01319 
01320 template<typename Ordinal>
01321 RCP< Comm<Ordinal> >
01322 MpiComm<Ordinal>::duplicate() const
01323 {
01324   MPI_Comm origRawComm = *rawMpiComm_;
01325   MPI_Comm newRawComm = MPI_COMM_NULL;
01326   const int err = MPI_Comm_dup (origRawComm, &newRawComm);
01327   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::runtime_error, "Teuchos"
01328     "::MpiComm::duplicate: MPI_Comm_dup failed with the following error: " 
01329     << mpiErrorCodeToString (err));
01330 
01331   // Wrap the raw communicator, and pass the (const) wrapped
01332   // communicator to MpiComm's constructor.  We created the raw comm,
01333   // so we have to supply a function that frees it after use.
01334   RCP<OpaqueWrapper<MPI_Comm> > wrapped = opaqueWrapper<MPI_Comm> (newRawComm, details::safeCommFree);
01335   RCP<MpiComm<Ordinal> > newComm (new MpiComm<Ordinal> (wrapped.getConst ()));
01336   return rcp_implicit_cast<Comm<Ordinal> > (newComm);
01337 }
01338 
01339 
01340 template<typename Ordinal>
01341 RCP< Comm<Ordinal> >
01342 MpiComm<Ordinal>::split(const int color, const int key) const
01343 {
01344   MPI_Comm newComm;
01345   const int splitReturn =
01346     MPI_Comm_split (*rawMpiComm_,
01347                     color < 0 ? MPI_UNDEFINED : color,
01348                     key,
01349                     &newComm);
01350   TEUCHOS_TEST_FOR_EXCEPTION(
01351     splitReturn != MPI_SUCCESS,
01352     std::logic_error,
01353     "Teuchos::MpiComm::split: Failed to create communicator with color "
01354     << color << "and key " << key << ".  MPI_Comm_split failed with error \""
01355     << mpiErrorCodeToString (splitReturn) << "\".");
01356   if (newComm == MPI_COMM_NULL) {
01357     return RCP< Comm<Ordinal> >();
01358   } else {
01359     return rcp(new MpiComm<Ordinal>(
01360                    rcp_implicit_cast<const OpaqueWrapper<MPI_Comm> >( 
01361                                      opaqueWrapper(newComm,MPI_Comm_free))));
01362   }
01363 }
01364 
01365 
01366 template<typename Ordinal>
01367 RCP< Comm<Ordinal> >
01368 MpiComm<Ordinal>::createSubcommunicator(const ArrayView<const int> &ranks) const
01369 {
01370   int err = MPI_SUCCESS; // For error codes returned by MPI functions
01371 
01372   // Get the group that this communicator is in.
01373   MPI_Group thisGroup;
01374   err = MPI_Comm_group(*rawMpiComm_, &thisGroup);
01375   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::logic_error,
01376     "Failed to obtain the current communicator's group.  "
01377     "MPI_Comm_group failed with error \""
01378     << mpiErrorCodeToString (err) << "\".");
01379 
01380   // Create a new group with the specified members.
01381   MPI_Group newGroup;
01382   // It's rude to cast away const, but MPI functions demand it.
01383   //
01384   // NOTE (mfh 14 Aug 2012) Please don't ask for &ranks[0] unless you
01385   // know that ranks.size() > 0.  That's why I'm using getRawPtr().
01386   err = MPI_Group_incl (thisGroup, ranks.size(),
01387                         const_cast<int*> (ranks.getRawPtr ()), &newGroup);
01388   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::logic_error,
01389     "Failed to create subgroup.  MPI_Group_incl failed with error \""
01390     << mpiErrorCodeToString (err) << "\".");
01391 
01392   // Create a new communicator from the new group.
01393   MPI_Comm newComm;
01394   try {
01395     err = MPI_Comm_create(*rawMpiComm_, newGroup, &newComm);
01396     TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::logic_error,
01397       "Failed to create subcommunicator.  MPI_Comm_create failed with error \""
01398       << mpiErrorCodeToString (err) << "\".");
01399   } catch (...) {
01400     // Attempt to free the new group before rethrowing.  If
01401     // successful, this will prevent a memory leak due to the "lost"
01402     // group that was allocated successfully above.  Since we're
01403     // throwing std::logic_error anyway, we can only promise
01404     // best-effort recovery; thus, we don't check the error code.
01405     (void) MPI_Group_free (&newGroup);
01406     (void) MPI_Group_free (&thisGroup);
01407     throw;
01408   }
01409 
01410   // We don't need the group any more, so free it.
01411   err = MPI_Group_free (&newGroup);
01412   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::logic_error,
01413     "Failed to free subgroup.  MPI_Group_free failed with error \""
01414     << mpiErrorCodeToString (err) << "\".");
01415   err = MPI_Group_free (&thisGroup);
01416   TEUCHOS_TEST_FOR_EXCEPTION(err != MPI_SUCCESS, std::logic_error,
01417     "Failed to free subgroup.  MPI_Group_free failed with error \""
01418     << mpiErrorCodeToString (err) << "\".");
01419 
01420   if (newComm == MPI_COMM_NULL) {
01421     return RCP< Comm<Ordinal> >();
01422   } else {
01423     return rcp(new MpiComm<Ordinal>(
01424                    rcp_implicit_cast<const OpaqueWrapper<MPI_Comm> >(
01425                                      opaqueWrapper(newComm,MPI_Comm_free))));
01426   }
01427 }
01428 
01429 
01430 // Overridden from Describable
01431 
01432 
01433 template<typename Ordinal>
01434 std::string MpiComm<Ordinal>::description() const
01435 {
01436   std::ostringstream oss;
01437   oss
01438     << typeName(*this)
01439     << "{"
01440     << "size="<<size_
01441     << ",rank="<<rank_
01442     << ",rawMpiComm="<<static_cast<MPI_Comm>(*rawMpiComm_)
01443     <<"}";
01444   return oss.str();
01445 }
01446 
01447 
01448 #ifdef TEUCHOS_MPI_COMM_DUMP
01449 template<typename Ordinal>
01450 bool MpiComm<Ordinal>::show_dump = false;
01451 #endif
01452 
01453 
01454 // private
01455 
01456 
01457 template<typename Ordinal>
01458 void MpiComm<Ordinal>::assertRank(const int rank, const std::string &rankName) const
01459 {
01460   TEUCHOS_TEST_FOR_EXCEPTION(
01461     ! ( 0 <= rank && rank < size_ ), std::logic_error
01462     ,"Error, "<<rankName<<" = " << rank << " is not < 0 or is not"
01463     " in the range [0,"<<size_-1<<"]!"
01464     );
01465 }
01466 
01467 
01468 } // namespace Teuchos
01469 
01470 
01471 template<typename Ordinal>
01472 Teuchos::RCP<Teuchos::MpiComm<Ordinal> >
01473 Teuchos::createMpiComm(
01474   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
01475   )
01476 {
01477   if( rawMpiComm.get()!=NULL && *rawMpiComm != MPI_COMM_NULL )
01478     return rcp(new MpiComm<Ordinal>(rawMpiComm));
01479   return Teuchos::null;
01480 }
01481 
01482 
01483 #endif // TEUCHOS_MPI_COMM_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines