Teuchos_DefaultMpiComm.hpp

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TEUCHOS_MPI_COMM_HPP
00030 #define TEUCHOS_MPI_COMM_HPP
00031 
00032 #include "Teuchos_Comm.hpp"
00033 #include "Teuchos_CommUtilities.hpp"
00034 #include "Teuchos_OrdinalTraits.hpp"
00035 #include "Teuchos_OpaqueWrapper.hpp"
00036 #include "Teuchos_MpiReductionOpSetter.hpp"
00037 #include "Teuchos_SerializationTraitsHelpers.hpp"
00038 #include "Teuchos_Workspace.hpp"
00039 #include "mpi.h"
00040 
00041 //#define TEUCHOS_MPI_COMM_DUMP
00042 
00043 #ifdef TEUCHOS_MPI_COMM_DUMP
00044 #  include "Teuchos_VerboseObject.hpp"
00045 #endif
00046 
00047 namespace Teuchos {
00048 
00049 #ifdef TEUCHOS_MPI_COMM_DUMP
00050 template<typename Ordinal>
00051 void dumpCharBuffer(
00052   const std::string &funcName, const std::string &buffName
00053   ,const Ordinal bytes, const char buff[]
00054   )
00055 {
00056   Teuchos::RefCountPtr<Teuchos::FancyOStream>
00057     out = Teuchos::VerboseObjectBase::getDefaultOStream();
00058   Teuchos::OSTab tab(out);
00059   *out
00060     << "\n" << funcName << "::" << buffName << ":";
00061   tab.incrTab();
00062   for( Ordinal i = 0; i < bytes; ++i ) {
00063     *out << buffName << "[" << i << "] = " << buff[i] << "\n";
00064   }
00065   *out << "\n";
00066 }
00067 #endif // TEUCHOS_MPI_COMM_DUMP
00068 
00079 template<typename Ordinal>
00080 class MpiComm : public Comm<Ordinal> {
00081 public:
00082 
00084 
00085 
00092   MpiComm(
00093     const RefCountPtr<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00094     );
00095 
00097   RefCountPtr<const OpaqueWrapper<MPI_Comm> > getRawMpiComm() const
00098   {return rawMpiComm_;}
00099 
00101 
00103 
00104 
00106   int getRank() const;
00108   int getSize() const;
00110   void barrier() const;
00112   void broadcast(
00113     const int rootRank, const Ordinal bytes, char buffer[]
00114     ) const;
00116   void gatherAll(
00117     const Ordinal sendBytes, const char sendBuffer[]
00118     ,const Ordinal recvBytes, char recvBuffer[]
00119     ) const;
00121   void reduceAll(
00122     const ValueTypeReductionOp<Ordinal,char> &reductOp
00123     ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00124     ) const;
00126   void reduceAllAndScatter(
00127     const ValueTypeReductionOp<Ordinal,char> &reductOp
00128     ,const Ordinal sendBytes, const char sendBuffer[]
00129     ,const Ordinal recvCounts[], const Ordinal blockSize, char myGlobalReducts[]
00130     ) const;
00132   void scan(
00133     const ValueTypeReductionOp<Ordinal,char> &reductOp
00134     ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00135     ) const;
00137   void send(
00138     const Ordinal bytes, const char sendBuffer[], const int destRank
00139     ) const;
00141   int receive(
00142     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00143     ) const;
00144 
00146 
00148 
00149 
00151   std::string description() const;
00152 
00154 
00155 private:
00156 
00157   static int const minTag_ = 26000; // These came from Teuchos::MpiComm???
00158   static int const maxTag_ = 26099; // ""
00159   static int tagCounter_;
00160 
00161   RefCountPtr<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;
00162   int                                         rank_;
00163   int                                         size_;
00164   int                                         tag_;
00165 
00166   // Not defined and not to be called!
00167   MpiComm();
00168 
00169 #ifdef TEUCHOS_MPI_COMM_DUMP
00170 public:
00171   static bool show_dump;
00172 #endif // TEUCHOS_MPI_COMM_DUMP
00173   
00174 };
00175 
00189 template<typename Ordinal>
00190 RefCountPtr<MpiComm<Ordinal> >
00191 createMpiComm(
00192   const RefCountPtr<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00193   );
00194 
00195 // ////////////////////////
00196 // Implementations
00197 
00198 // Static members
00199 
00200 template<typename Ordinal>
00201 int MpiComm<Ordinal>::tagCounter_ = MpiComm<Ordinal>::minTag_;
00202 
00203 // Constructors
00204 
00205 template<typename Ordinal>
00206 MpiComm<Ordinal>::MpiComm(
00207   const RefCountPtr<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00208   )
00209 {
00210   TEST_FOR_EXCEPT( rawMpiComm.get()==NULL );
00211   TEST_FOR_EXCEPT( *rawMpiComm == MPI_COMM_NULL );
00212   rawMpiComm_ = rawMpiComm;
00213   MPI_Comm_size(*rawMpiComm_,&size_);
00214   MPI_Comm_rank(*rawMpiComm_,&rank_);
00215   if(tagCounter_ > maxTag_)
00216     tagCounter_ = minTag_;
00217   tag_ = tagCounter_++;
00218 }
00219 
00220 // Overridden from Comm
00221   
00222 template<typename Ordinal>
00223 int MpiComm<Ordinal>::getRank() const
00224 {
00225   return rank_;
00226 }
00227   
00228 template<typename Ordinal>
00229 int MpiComm<Ordinal>::getSize() const
00230 {
00231   return size_;
00232 }
00233   
00234 template<typename Ordinal>
00235 void MpiComm<Ordinal>::barrier() const
00236 {
00237   TEUCHOS_COMM_TIME_MONITOR(
00238     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::barrier()"
00239     );
00240   MPI_Barrier(*rawMpiComm_);
00241 }
00242   
00243 template<typename Ordinal>
00244 void MpiComm<Ordinal>::broadcast(
00245   const int rootRank, const Ordinal bytes, char buffer[]
00246   ) const
00247 {
00248   TEUCHOS_COMM_TIME_MONITOR(
00249     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::broadcast(...)"
00250     );
00251   MPI_Bcast(buffer,bytes,MPI_CHAR,rootRank,*rawMpiComm_);
00252 }
00253   
00254 template<typename Ordinal>
00255 void MpiComm<Ordinal>::gatherAll(
00256     const Ordinal sendBytes, const char sendBuffer[]
00257     ,const Ordinal recvBytes, char recvBuffer[]
00258   ) const
00259 {
00260   TEUCHOS_COMM_TIME_MONITOR(
00261     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::gatherAll(...)"
00262     );
00263   TEST_FOR_EXCEPT(!(sendBytes*size_==recvBytes));
00264   MPI_Allgather(
00265     const_cast<char *>(sendBuffer),sendBytes,MPI_CHAR
00266     ,recvBuffer,sendBytes,MPI_CHAR
00267     ,*rawMpiComm_
00268     );
00269 }
00270   
00271 template<typename Ordinal>
00272 void MpiComm<Ordinal>::reduceAll(
00273   const ValueTypeReductionOp<Ordinal,char> &reductOp
00274   ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00275   ) const
00276 {
00277   TEUCHOS_COMM_TIME_MONITOR(
00278     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAll(...)"
00279     );
00280   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00281   MPI_Allreduce(
00282     const_cast<char*>(sendBuffer),globalReducts,bytes,MPI_CHAR,op.mpi_op()
00283     ,*rawMpiComm_
00284     );
00285 }
00286 
00287 template<typename Ordinal>
00288 void MpiComm<Ordinal>::reduceAllAndScatter(
00289   const ValueTypeReductionOp<Ordinal,char> &reductOp
00290   ,const Ordinal sendBytes, const char sendBuffer[]
00291   ,const Ordinal recvCounts[], const Ordinal blockSize, char myGlobalReducts[]
00292   ) const
00293 {
00294   TEUCHOS_COMM_TIME_MONITOR(
00295     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAllAndScatter(...)"
00296     );
00297 #ifdef TEUCHOS_DEBUG
00298   Ordinal sumRecvBytes = 0;
00299   for( Ordinal i = 0; i < size_; ++i )
00300     sumRecvBytes += recvCounts[i];
00301   sumRecvBytes *= blockSize;
00302   TEST_FOR_EXCEPT(!(sumRecvBytes==sendBytes));
00303 #endif // TEUCHOS_DEBUG
00304   WorkspaceStore* wss = get_default_workspace_store().get();
00305   // Create a of recvCount[] if Ordinal!=int
00306   const bool Ordinal_is_int = typeid(int)==typeid(Ordinal);
00307   Workspace<int> _recvCounts(wss,Ordinal_is_int?0:size_);
00308   const int *int_recvCounts = 0;
00309   if(Ordinal_is_int) {
00310     int_recvCounts = reinterpret_cast<const int*>(recvCounts);
00311     // Note: We must do an reinterpet cast since this must
00312     // compile even if it is not executed.  I could implement
00313     // code that would not need to do this using template
00314     // conditionals but I don't want to bother.
00315   }
00316   else {
00317     std::copy(recvCounts,recvCounts+size_,&_recvCounts[0]);
00318     int_recvCounts = &_recvCounts[0];
00319   }
00320   MPI_Datatype _chars_type;
00321   MPI_Type_contiguous(blockSize,MPI_CHAR,&_chars_type);
00322   RefCountPtr<const OpaqueWrapper<MPI_Datatype> >
00323     chars_type = opaqueWrapper(_chars_type,MPI_Type_free);
00324   // Perform the operation
00325   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00326   MPI_Reduce_scatter(
00327     const_cast<char*>(sendBuffer), myGlobalReducts
00328     ,const_cast<int*>(int_recvCounts)
00329     ,*chars_type
00330     ,op.mpi_op()
00331     ,*rawMpiComm_
00332     );
00333 }
00334   
00335 template<typename Ordinal>
00336 void MpiComm<Ordinal>::scan(
00337   const ValueTypeReductionOp<Ordinal,char> &reductOp
00338   ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00339   ) const
00340 {
00341   TEUCHOS_COMM_TIME_MONITOR(
00342     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::scan(...)"
00343     );
00344   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00345   MPI_Scan(
00346     const_cast<char*>(sendBuffer),scanReducts,bytes,MPI_CHAR,op.mpi_op()
00347     ,*rawMpiComm_
00348     );
00349 }
00350 
00351 template<typename Ordinal>
00352 void MpiComm<Ordinal>::send(
00353   const Ordinal bytes, const char sendBuffer[], const int destRank
00354   ) const
00355 {
00356   TEUCHOS_COMM_TIME_MONITOR(
00357     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::send(...)"
00358     );
00359 #ifdef TEUCHOS_DEBUG
00360   TEST_FOR_EXCEPTION(
00361     ! ( 0 <= destRank && destRank < size_ ), std::logic_error
00362     ,"Error, destRank = " << destRank << " is not < 0 or is not"
00363     " in the range [0,"<<size_-1<<"]!"
00364     );
00365 #endif // TEUCHOS_DEBUG
00366 #ifdef TEUCHOS_MPI_COMM_DUMP
00367   if(show_dump) {
00368     dumpCharBuffer<Ordinal>(
00369       "Teuchos::MpiComm<Ordinal>::send(...)"
00370       ,"sendBuffer", bytes, sendBuffer
00371       );
00372   }
00373 #endif // TEUCHOS_MPI_COMM_DUMP
00374   MPI_Send(
00375     const_cast<char*>(sendBuffer),bytes,MPI_CHAR,destRank,tag_,*rawMpiComm_
00376     );
00377   // ToDo: What about error handling???
00378 }
00379 
00380 template<typename Ordinal>
00381 int MpiComm<Ordinal>::receive(
00382     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00383   ) const
00384 {
00385   TEUCHOS_COMM_TIME_MONITOR(
00386     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::receive(...)"
00387     );
00388 #ifdef TEUCHOS_DEBUG
00389   TEST_FOR_EXCEPTION(
00390     sourceRank >=0 && !(sourceRank < size_), std::logic_error
00391     ,"Error, sourceRank = " << sourceRank << " is not < 0 or is not"
00392     " in the range [0,"<<(size_-1)<<"]!"
00393     );
00394 #endif // TEUCHOS_DEBUG
00395   MPI_Status status;
00396   MPI_Recv(
00397     recvBuffer,bytes,MPI_CHAR
00398     ,sourceRank >= 0 ? sourceRank : MPI_ANY_SOURCE
00399     ,tag_,*rawMpiComm_
00400     ,&status
00401     );
00402 #ifdef TEUCHOS_MPI_COMM_DUMP
00403   if(show_dump) {
00404     dumpCharBuffer<Ordinal>(
00405       "Teuchos::MpiComm<Ordinal>::receive(...)"
00406       ,"recvBuffer", bytes, recvBuffer
00407       );
00408   }
00409 #endif // TEUCHOS_MPI_COMM_DUMP
00410   return status.MPI_SOURCE;
00411   // ToDo: What about error handling???
00412 }
00413 
00414 // Overridden from Describable
00415 
00416 template<typename Ordinal>
00417 std::string MpiComm<Ordinal>::description() const
00418 {
00419   std::ostringstream oss;
00420   oss
00421     << "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">"
00422     << "{"
00423     << "size="<<size_
00424     << ",rawMpiComm="<<static_cast<MPI_Comm>(*rawMpiComm_)
00425     <<"}";
00426   return oss.str();
00427 }
00428 
00429 #ifdef TEUCHOS_MPI_COMM_DUMP
00430 template<typename Ordinal>
00431 bool MpiComm<Ordinal>::show_dump = false;
00432 #endif // TEUCHOS_MPI_COMM_DUMP
00433 
00434 } // namespace Teuchos
00435 
00436 template<typename Ordinal>
00437 Teuchos::RefCountPtr<Teuchos::MpiComm<Ordinal> >
00438 Teuchos::createMpiComm(
00439   const RefCountPtr<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00440   )
00441 {
00442   if( rawMpiComm.get()!=NULL && *rawMpiComm != MPI_COMM_NULL )
00443     return rcp(new MpiComm<Ordinal>(rawMpiComm));
00444   return Teuchos::null;
00445 }
00446 
00447 #endif // TEUCHOS_MPI_COMM_HPP

Generated on Thu Sep 18 12:30:29 2008 for Teuchos - Trilinos Tools Package by doxygen 1.3.9.1