Kokkos Node API and Local Linear Algebra Kernels Version of the Day
Tsqr_TeuchosMessenger.hpp
00001 //@HEADER
00002 // ************************************************************************
00003 // 
00004 //          Kokkos: Node API and Parallel Node Kernels
00005 //              Copyright (2008) Sandia Corporation
00006 // 
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 // 
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00038 // 
00039 // ************************************************************************
00040 //@HEADER
00041 
00042 #ifndef __TSQR_TeuchosMessenger_hpp
00043 #define __TSQR_TeuchosMessenger_hpp
00044 
00045 #include <Teuchos_CommHelpers.hpp>
00046 #include <Tsqr_MessengerBase.hpp>
00047 
00048 #include <algorithm>
00049 
00050 
00051 namespace TSQR { 
00052 
00063   template<class Datum>
00064   class TeuchosMessenger : public MessengerBase<Datum> {
00065   public:
00066     typedef Teuchos::RCP<const Teuchos::Comm<int> > comm_ptr;
00067 
00069     TeuchosMessenger (const comm_ptr& pComm) : pComm_ (pComm) {}
00070 
00072     virtual ~TeuchosMessenger() {}
00073 
00080     void 
00081     send (const Datum sendData[], 
00082     const int sendCount, 
00083     const int destProc, 
00084     const int tag) 
00085     {
00086       // NOTE (mfh 14 June 2010): Teuchos generates "tag" arguments to
00087       // MPI calls internally, so we ignore the tag here.  I don't use
00088       // tags for anything in TSQR, so it doesn't matter.
00089       Teuchos::send (*pComm_, sendCount, sendData, destProc);
00090     }
00091 
00098     void 
00099     recv (Datum recvData[], 
00100     const int recvCount, 
00101     const int srcProc, 
00102     const int tag) 
00103     {
00104       // NOTE (mfh 14 June 2010): Teuchos generates "tag" arguments to
00105       // MPI calls internally, so we ignore the tag here.  I don't use
00106       // tags for anything in TSQR, so it doesn't matter.
00107       Teuchos::receive (*pComm_, srcProc, recvCount, recvData);
00108     }
00109 
00128     void 
00129     swapData (const Datum sendData[], 
00130         Datum recvData[], 
00131         const int sendRecvCount, 
00132         const int destProc, 
00133         const int tag)
00134     {
00135       if (destProc == rank())
00136   {
00137     // If the sending and receiving processes are the same,
00138     // then all we need to do is copy the data.  Hopefully in
00139     // that case you aren't aliasing.  std::copy assumes that
00140     // the third argument does not point to an element in the
00141     // range of the first two arguments.
00142     std::copy (sendData, sendData+sendRecvCount, recvData);
00143   }
00144       else
00145   {
00146     using Teuchos::RCP;
00147     using Teuchos::ArrayRCP;
00148     using Teuchos::CommRequest;
00149 
00150     const int srcProc = Teuchos::rank (*pComm_);
00151 
00152     // If we can prove that sendData and recvData don't alias
00153     // one another, use an isend and an ireceive to exchange
00154     // them.  (Our test may not necessarily be safe in general,
00155     // since we only check whether the pointers are equal and
00156     // not whether the arrays overlap.  However, it is safe for
00157     // the specific case of TSQR.)
00158     //
00159     // Otherwise, if the arrays do alias one another, safely
00160     // perform a send and then a receive (or a receive and then
00161     // a send, depending on whether this MPI process is the
00162     // source or destination process).  
00163     //
00164     // (It would be nice if Teuchos had a sendRecv() routine, as
00165     // of summer 2010 when this code was written.  As it stands,
00166     // we have to do a send and then a receive.)
00167     if (sendData == recvData)
00168       {
00169         // The smaller-rank process sends first, and the
00170         // larger-rank process receives first.
00171         //
00172         // Teuchos::send() and Teuchos::recv() are blocking,
00173         // so we may safely write to recvBuf even if it
00174         // aliases sendBuf.
00175         if (srcProc < destProc)
00176     {
00177       Teuchos::send (*pComm_, sendRecvCount, sendData, destProc);
00178       Teuchos::receive (*pComm_, destProc, sendRecvCount, recvData);
00179     }
00180         else 
00181     {
00182       Teuchos::receive (*pComm_, destProc, sendRecvCount, recvData);
00183       Teuchos::send (*pComm_, sendRecvCount, sendData, destProc);
00184     }
00185       }
00186     else
00187       {
00188         ArrayRCP<const Datum> sendBuf (sendData, 0, sendRecvCount, false);
00189         ArrayRCP<Datum> recvBuf (recvData, 0, sendRecvCount, false);
00190 
00191         RCP<CommRequest> sendReq, recvReq;
00192         if (srcProc < destProc)
00193     {
00194       sendReq = Teuchos::isend (*pComm_, sendBuf, destProc);
00195       recvReq = Teuchos::ireceive (*pComm_, recvBuf, destProc);
00196     }
00197         else
00198     {
00199       recvReq = Teuchos::ireceive (*pComm_, recvBuf, destProc);
00200       sendReq = Teuchos::isend (*pComm_, sendBuf, destProc);
00201     }
00202         // Wait on both the send and the receive to complete.  The
00203         // two can happen independently, because sendBuf and recvBuf
00204         // are different.  (We assert no aliasing of buffers here,
00205         // and we've also checked above that destProc != rank().)
00206         Teuchos::waitAll (*pComm_, Teuchos::tuple (sendReq, recvReq));
00207       }
00208   }
00209     }
00210 
00212     Datum 
00213     globalSum (const Datum& inDatum) 
00214     {
00215       Datum outDatum;
00216       Teuchos::reduceAll (*pComm_, Teuchos::REDUCE_SUM, inDatum, 
00217         Teuchos::outArg(outDatum));
00218       return outDatum;
00219     }
00220 
00224     Datum 
00225     globalMin (const Datum& inDatum)
00226     {
00227       Datum outDatum;
00228       Teuchos::reduceAll (*pComm_, Teuchos::REDUCE_MIN, inDatum, 
00229         Teuchos::outArg(outDatum));
00230       return outDatum;
00231     }
00232 
00236     Datum 
00237     globalMax (const Datum& inDatum)
00238     {
00239       Datum outDatum;
00240       Teuchos::reduceAll (*pComm_, Teuchos::REDUCE_MAX, inDatum, 
00241         Teuchos::outArg(outDatum));
00242       return outDatum;
00243     }
00244 
00246     void
00247     globalVectorSum (const Datum inData[], 
00248          Datum outData[], 
00249          const int count) 
00250     {
00251       Teuchos::reduceAll (*pComm_, Teuchos::REDUCE_SUM, count, 
00252         inData, outData);
00253     }
00254 
00256     void
00257     broadcast (Datum data[], 
00258          const int count,
00259          const int root)
00260     {
00261       Teuchos::broadcast (*pComm_, root, count, data);
00262     }
00263 
00265     int rank () const { return Teuchos::rank (*pComm_); }
00266 
00268     int size () const { return Teuchos::size (*pComm_); }
00269 
00271     void barrier () const { Teuchos::barrier (*pComm_); }
00272 
00273   private:
00274 
00276     comm_ptr pComm_;
00277   };
00278 } // namespace TSQR
00279 
00280 #endif // __TSQR_TeuchosMessenger_hpp
00281 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends