Teuchos_DefaultMpiComm.hpp

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TEUCHOS_MPI_COMM_HPP
00030 #define TEUCHOS_MPI_COMM_HPP
00031 
00032 #include "Teuchos_Comm.hpp"
00033 #include "Teuchos_CommUtilities.hpp"
00034 #include "Teuchos_OrdinalTraits.hpp"
00035 #include "Teuchos_OpaqueWrapper.hpp"
00036 #include "Teuchos_MpiReductionOpSetter.hpp"
00037 #include "Teuchos_SerializationTraitsHelpers.hpp"
00038 #include "Teuchos_Workspace.hpp"
00039 #include "Teuchos_TypeNameTraits.hpp"
00040 #include "mpi.h"
00041 
00042 //#define TEUCHOS_MPI_COMM_DUMP
00043 
00044 #ifdef TEUCHOS_MPI_COMM_DUMP
00045 #  include "Teuchos_VerboseObject.hpp"
00046 #endif
00047 
00048 namespace Teuchos {
00049 
00050 #ifdef TEUCHOS_MPI_COMM_DUMP
00051 template<typename Ordinal>
00052 void dumpCharBuffer(
00053   const std::string &funcName, const std::string &buffName
00054   ,const Ordinal bytes, const char buff[]
00055   )
00056 {
00057   Teuchos::RCP<Teuchos::FancyOStream>
00058     out = Teuchos::VerboseObjectBase::getDefaultOStream();
00059   Teuchos::OSTab tab(out);
00060   *out
00061     << "\n" << funcName << "::" << buffName << ":";
00062   tab.incrTab();
00063   for( Ordinal i = 0; i < bytes; ++i ) {
00064     *out << buffName << "[" << i << "] = " << buff[i] << "\n";
00065   }
00066   *out << "\n";
00067 }
00068 #endif // TEUCHOS_MPI_COMM_DUMP
00069 
00080 template<typename Ordinal>
00081 class MpiComm : public Comm<Ordinal> {
00082 public:
00083 
00085 
00086 
00093   MpiComm(
00094     const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00095     );
00096 
00098   RCP<const OpaqueWrapper<MPI_Comm> > getRawMpiComm() const
00099   {return rawMpiComm_;}
00100 
00102 
00104 
00105 
00107   int getRank() const;
00109   int getSize() const;
00111   void barrier() const;
00113   void broadcast(
00114     const int rootRank, const Ordinal bytes, char buffer[]
00115     ) const;
00117   void gatherAll(
00118     const Ordinal sendBytes, const char sendBuffer[]
00119     ,const Ordinal recvBytes, char recvBuffer[]
00120     ) const;
00122   void reduceAll(
00123     const ValueTypeReductionOp<Ordinal,char> &reductOp
00124     ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00125     ) const;
00127   void reduceAllAndScatter(
00128     const ValueTypeReductionOp<Ordinal,char> &reductOp
00129     ,const Ordinal sendBytes, const char sendBuffer[]
00130     ,const Ordinal recvCounts[], const Ordinal blockSize, char myGlobalReducts[]
00131     ) const;
00133   void scan(
00134     const ValueTypeReductionOp<Ordinal,char> &reductOp
00135     ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00136     ) const;
00138   void send(
00139     const Ordinal bytes, const char sendBuffer[], const int destRank
00140     ) const;
00142   int receive(
00143     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00144     ) const;
00145 
00147 
00149 
00150 
00152   std::string description() const;
00153 
00155 
00156 private:
00157 
00158   static int const minTag_ = 26000; // These came from Teuchos::MpiComm???
00159   static int const maxTag_ = 26099; // ""
00160   static int tagCounter_;
00161 
00162   RCP<const OpaqueWrapper<MPI_Comm> > rawMpiComm_;
00163   int                                         rank_;
00164   int                                         size_;
00165   int                                         tag_;
00166 
00167   // Not defined and not to be called!
00168   MpiComm();
00169 
00170 #ifdef TEUCHOS_MPI_COMM_DUMP
00171 public:
00172   static bool show_dump;
00173 #endif // TEUCHOS_MPI_COMM_DUMP
00174   
00175 };
00176 
00190 template<typename Ordinal>
00191 RCP<MpiComm<Ordinal> >
00192 createMpiComm(
00193   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00194   );
00195 
00196 // ////////////////////////
00197 // Implementations
00198 
00199 // Static members
00200 
00201 template<typename Ordinal>
00202 int MpiComm<Ordinal>::tagCounter_ = MpiComm<Ordinal>::minTag_;
00203 
00204 // Constructors
00205 
00206 template<typename Ordinal>
00207 MpiComm<Ordinal>::MpiComm(
00208   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00209   )
00210 {
00211   TEST_FOR_EXCEPT( rawMpiComm.get()==NULL );
00212   TEST_FOR_EXCEPT( *rawMpiComm == MPI_COMM_NULL );
00213   rawMpiComm_ = rawMpiComm;
00214   MPI_Comm_size(*rawMpiComm_,&size_);
00215   MPI_Comm_rank(*rawMpiComm_,&rank_);
00216   if(tagCounter_ > maxTag_)
00217     tagCounter_ = minTag_;
00218   tag_ = tagCounter_++;
00219 }
00220 
00221 // Overridden from Comm
00222   
00223 template<typename Ordinal>
00224 int MpiComm<Ordinal>::getRank() const
00225 {
00226   return rank_;
00227 }
00228   
00229 template<typename Ordinal>
00230 int MpiComm<Ordinal>::getSize() const
00231 {
00232   return size_;
00233 }
00234   
00235 template<typename Ordinal>
00236 void MpiComm<Ordinal>::barrier() const
00237 {
00238   TEUCHOS_COMM_TIME_MONITOR(
00239     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::barrier()"
00240     );
00241   MPI_Barrier(*rawMpiComm_);
00242 }
00243   
00244 template<typename Ordinal>
00245 void MpiComm<Ordinal>::broadcast(
00246   const int rootRank, const Ordinal bytes, char buffer[]
00247   ) const
00248 {
00249   TEUCHOS_COMM_TIME_MONITOR(
00250     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::broadcast(...)"
00251     );
00252   MPI_Bcast(buffer,bytes,MPI_CHAR,rootRank,*rawMpiComm_);
00253 }
00254   
00255 template<typename Ordinal>
00256 void MpiComm<Ordinal>::gatherAll(
00257     const Ordinal sendBytes, const char sendBuffer[]
00258     ,const Ordinal recvBytes, char recvBuffer[]
00259   ) const
00260 {
00261   TEUCHOS_COMM_TIME_MONITOR(
00262     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::gatherAll(...)"
00263     );
00264   TEST_FOR_EXCEPT(!(sendBytes*size_==recvBytes));
00265   MPI_Allgather(
00266     const_cast<char *>(sendBuffer),sendBytes,MPI_CHAR
00267     ,recvBuffer,sendBytes,MPI_CHAR
00268     ,*rawMpiComm_
00269     );
00270 }
00271   
00272 template<typename Ordinal>
00273 void MpiComm<Ordinal>::reduceAll(
00274   const ValueTypeReductionOp<Ordinal,char> &reductOp
00275   ,const Ordinal bytes, const char sendBuffer[], char globalReducts[]
00276   ) const
00277 {
00278   TEUCHOS_COMM_TIME_MONITOR(
00279     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAll(...)"
00280     );
00281   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00282   MPI_Allreduce(
00283     const_cast<char*>(sendBuffer),globalReducts,bytes,MPI_CHAR,op.mpi_op()
00284     ,*rawMpiComm_
00285     );
00286 }
00287 
00288 template<typename Ordinal>
00289 void MpiComm<Ordinal>::reduceAllAndScatter(
00290   const ValueTypeReductionOp<Ordinal,char> &reductOp
00291   ,const Ordinal sendBytes, const char sendBuffer[]
00292   ,const Ordinal recvCounts[], const Ordinal blockSize, char myGlobalReducts[]
00293   ) const
00294 {
00295   TEUCHOS_COMM_TIME_MONITOR(
00296     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAllAndScatter(...)"
00297     );
00298 #ifdef TEUCHOS_DEBUG
00299   Ordinal sumRecvBytes = 0;
00300   for( Ordinal i = 0; i < size_; ++i )
00301     sumRecvBytes += recvCounts[i];
00302   sumRecvBytes *= blockSize;
00303   TEST_FOR_EXCEPT(!(sumRecvBytes==sendBytes));
00304 #endif // TEUCHOS_DEBUG
00305   WorkspaceStore* wss = get_default_workspace_store().get();
00306   // Create a of recvCount[] if Ordinal!=int
00307   const bool Ordinal_is_int = typeid(int)==typeid(Ordinal);
00308   Workspace<int> _recvCounts(wss,Ordinal_is_int?0:size_);
00309   const int *int_recvCounts = 0;
00310   if(Ordinal_is_int) {
00311     int_recvCounts = reinterpret_cast<const int*>(recvCounts);
00312     // Note: We must do an reinterpet cast since this must
00313     // compile even if it is not executed.  I could implement
00314     // code that would not need to do this using template
00315     // conditionals but I don't want to bother.
00316   }
00317   else {
00318     std::copy(recvCounts,recvCounts+size_,&_recvCounts[0]);
00319     int_recvCounts = &_recvCounts[0];
00320   }
00321   MPI_Datatype _chars_type;
00322   MPI_Type_contiguous(blockSize,MPI_CHAR,&_chars_type);
00323   RCP<const OpaqueWrapper<MPI_Datatype> >
00324     chars_type = opaqueWrapper(_chars_type,MPI_Type_free);
00325   // Perform the operation
00326   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00327   MPI_Reduce_scatter(
00328     const_cast<char*>(sendBuffer), myGlobalReducts
00329     ,const_cast<int*>(int_recvCounts)
00330     ,*chars_type
00331     ,op.mpi_op()
00332     ,*rawMpiComm_
00333     );
00334 }
00335   
00336 template<typename Ordinal>
00337 void MpiComm<Ordinal>::scan(
00338   const ValueTypeReductionOp<Ordinal,char> &reductOp
00339   ,const Ordinal bytes, const char sendBuffer[], char scanReducts[]
00340   ) const
00341 {
00342   TEUCHOS_COMM_TIME_MONITOR(
00343     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::scan(...)"
00344     );
00345   MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp,false)));
00346   MPI_Scan(
00347     const_cast<char*>(sendBuffer),scanReducts,bytes,MPI_CHAR,op.mpi_op()
00348     ,*rawMpiComm_
00349     );
00350 }
00351 
00352 template<typename Ordinal>
00353 void MpiComm<Ordinal>::send(
00354   const Ordinal bytes, const char sendBuffer[], const int destRank
00355   ) const
00356 {
00357   TEUCHOS_COMM_TIME_MONITOR(
00358     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::send(...)"
00359     );
00360 #ifdef TEUCHOS_DEBUG
00361   TEST_FOR_EXCEPTION(
00362     ! ( 0 <= destRank && destRank < size_ ), std::logic_error
00363     ,"Error, destRank = " << destRank << " is not < 0 or is not"
00364     " in the range [0,"<<size_-1<<"]!"
00365     );
00366 #endif // TEUCHOS_DEBUG
00367 #ifdef TEUCHOS_MPI_COMM_DUMP
00368   if(show_dump) {
00369     dumpCharBuffer<Ordinal>(
00370       "Teuchos::MpiComm<Ordinal>::send(...)"
00371       ,"sendBuffer", bytes, sendBuffer
00372       );
00373   }
00374 #endif // TEUCHOS_MPI_COMM_DUMP
00375   MPI_Send(
00376     const_cast<char*>(sendBuffer),bytes,MPI_CHAR,destRank,tag_,*rawMpiComm_
00377     );
00378   // ToDo: What about error handling???
00379 }
00380 
00381 template<typename Ordinal>
00382 int MpiComm<Ordinal>::receive(
00383     const int sourceRank, const Ordinal bytes, char recvBuffer[]
00384   ) const
00385 {
00386   TEUCHOS_COMM_TIME_MONITOR(
00387     "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::receive(...)"
00388     );
00389 #ifdef TEUCHOS_DEBUG
00390   TEST_FOR_EXCEPTION(
00391     sourceRank >=0 && !(sourceRank < size_), std::logic_error
00392     ,"Error, sourceRank = " << sourceRank << " is not < 0 or is not"
00393     " in the range [0,"<<(size_-1)<<"]!"
00394     );
00395 #endif // TEUCHOS_DEBUG
00396   MPI_Status status;
00397   MPI_Recv(
00398     recvBuffer,bytes,MPI_CHAR
00399     ,sourceRank >= 0 ? sourceRank : MPI_ANY_SOURCE
00400     ,tag_,*rawMpiComm_
00401     ,&status
00402     );
00403 #ifdef TEUCHOS_MPI_COMM_DUMP
00404   if(show_dump) {
00405     dumpCharBuffer<Ordinal>(
00406       "Teuchos::MpiComm<Ordinal>::receive(...)"
00407       ,"recvBuffer", bytes, recvBuffer
00408       );
00409   }
00410 #endif // TEUCHOS_MPI_COMM_DUMP
00411   return status.MPI_SOURCE;
00412   // ToDo: What about error handling???
00413 }
00414 
00415 // Overridden from Describable
00416 
00417 template<typename Ordinal>
00418 std::string MpiComm<Ordinal>::description() const
00419 {
00420   std::ostringstream oss;
00421   oss
00422     << typeName(*this)
00423     << "{"
00424     << "size="<<size_
00425     << ",rank="<<rank_
00426     << ",rawMpiComm="<<static_cast<MPI_Comm>(*rawMpiComm_)
00427     <<"}";
00428   return oss.str();
00429 }
00430 
00431 #ifdef TEUCHOS_MPI_COMM_DUMP
00432 template<typename Ordinal>
00433 bool MpiComm<Ordinal>::show_dump = false;
00434 #endif // TEUCHOS_MPI_COMM_DUMP
00435 
00436 } // namespace Teuchos
00437 
00438 template<typename Ordinal>
00439 Teuchos::RCP<Teuchos::MpiComm<Ordinal> >
00440 Teuchos::createMpiComm(
00441   const RCP<const OpaqueWrapper<MPI_Comm> > &rawMpiComm
00442   )
00443 {
00444   if( rawMpiComm.get()!=NULL && *rawMpiComm != MPI_COMM_NULL )
00445     return rcp(new MpiComm<Ordinal>(rawMpiComm));
00446   return Teuchos::null;
00447 }
00448 
00449 #endif // TEUCHOS_MPI_COMM_HPP

Generated on Tue Oct 20 12:45:25 2009 for Teuchos - Trilinos Tools Package by doxygen 1.4.7