Tpetra Matrix/Vector Services Version of the Day
Tpetra_DistObject.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 // 
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00038 // 
00039 // ************************************************************************
00040 // @HEADER
00041 
00042 #ifndef TPETRA_DISTOBJECT_HPP
00043 #define TPETRA_DISTOBJECT_HPP
00044 
00045 #include "Tpetra_ConfigDefs.hpp"
00046 #include "Tpetra_Map.hpp"
00047 #include "Tpetra_Import.hpp"
00048 #include "Tpetra_Export.hpp"
00049 #include "Tpetra_Distributor.hpp"
00050 
00051 #include <Kokkos_DefaultNode.hpp>
00052 #include <Teuchos_Describable.hpp>
00053 
00054 namespace Tpetra {
00055 
00080   template <class Packet, class LocalOrdinal = int, class GlobalOrdinal = LocalOrdinal, class Node = Kokkos::DefaultNode::DefaultNodeType>
00081   class DistObject : virtual public Teuchos::Describable {
00082 
00083   public:
00084 
00086 
00087 
00089     explicit DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > &map);
00090 
00092     DistObject(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source);
00093 
00095     virtual ~DistObject();
00096 
00098 
00100 
00101 
00103     void 
00104     doImport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source, 
00105         const Import<LocalOrdinal,GlobalOrdinal,Node> &importer, 
00106         CombineMode CM);
00107 
00109     void doExport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest, 
00110        const Export<LocalOrdinal,GlobalOrdinal,Node> &exporter, 
00111        CombineMode CM);
00112 
00114     void doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00115                   const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter, CombineMode CM);
00116 
00118     void doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest,
00119                   const Import<LocalOrdinal,GlobalOrdinal,Node>& importer, CombineMode CM);
00120 
00122 
00124 
00125 
00130     inline bool isDistributed() const;
00131 
00133     inline const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& 
00134     getMap() const { return map_; }
00135 
00137 
00139 
00140 
00142 
00143     void print(std::ostream &os) const;
00144 
00146 
00147   protected:
00148 
00151     enum ReverseOption {
00152       DoForward, //*!< Perform the transfer in forward mode.
00153       DoReverse  //*!< Perform the transfer in reverse mode.
00154     };
00155 
00157     virtual void doTransfer(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00158                             CombineMode CM,
00159                             size_t numSameIDs,
00160                             const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00161                             const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00162                             const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00163                             const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00164                             Distributor &distor,
00165                             ReverseOption revOp);
00166 
00167     // The following four methods must be implemented by the derived class
00168 
00172     virtual bool 
00173     checkSizes (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source) = 0;
00174 
00176 
00192     virtual void copyAndPermute(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00193                                 size_t numSameIDs,
00194                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00195                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs) = 0;
00196 
00198 
00216     virtual void packAndPrepare(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00217                                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00218                                 Teuchos::Array<Packet> &exports,
00219                                 const Teuchos::ArrayView<size_t> & numPacketsPerLID,
00220                                 size_t& constantNumPackets,
00221                                 Distributor &distor) = 0;
00222 
00224 
00241     virtual void 
00242     unpackAndCombine (const Teuchos::ArrayView<const LocalOrdinal> &importLIDs,
00243           const Teuchos::ArrayView<const Packet> &imports,
00244           const Teuchos::ArrayView<size_t> &numPacketsPerLID,
00245           size_t constantNumPackets,
00246           Distributor &distor,
00247           CombineMode CM) = 0;
00248 
00249     virtual void createViews() const {}
00250     virtual void createViewsNonConst(Kokkos::ReadWriteOption rwo) {}
00251     virtual void releaseViews() const {}
00252 
00253   private:
00255     Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > map_;
00256     // buffers into which packed data is imported
00257     Teuchos::Array<Packet> imports_;
00258     Teuchos::Array<size_t> numImportPacketsPerLID_;
00259     // buffers from which packed data is exported
00260     Teuchos::Array<Packet> exports_;
00261     Teuchos::Array<size_t> numExportPacketsPerLID_;
00262 
00263   }; // class DistObject
00264 
00265   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00266   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & map)
00267   : map_(map)
00268   {}
00269 
00270   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00271   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source)
00272   : map_(source.map_)
00273   {}
00274 
00275   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00276   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::~DistObject() 
00277   {}
00278 
00279   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00280   void 
00281   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00282   doImport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00283       const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, 
00284       CombineMode CM) 
00285   {
00286     TEUCHOS_TEST_FOR_EXCEPTION(   *getMap() != *importer.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00287     TEUCHOS_TEST_FOR_EXCEPTION( *A.getMap() != *importer.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00288     size_t numSameIDs = importer.getNumSameIDs();
00289     const Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getExportLIDs();
00290     const Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getRemoteLIDs();
00291     const Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteToLIDs();
00292     const Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteFromLIDs();
00293     this->doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00294                      importer.getDistributor(), DoForward);
00295   }
00296 
00297   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00298   void 
00299   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00300   doExport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00301       const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, 
00302       CombineMode CM) 
00303   {
00304     TEUCHOS_TEST_FOR_EXCEPTION(   *getMap() != *exporter.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00305     TEUCHOS_TEST_FOR_EXCEPTION( *A.getMap() != *exporter.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00306     size_t numSameIDs = exporter.getNumSameIDs();
00307     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getExportLIDs();
00308     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getRemoteLIDs();
00309     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteToLIDs();
00310     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteFromLIDs();
00311     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00312                exporter.getDistributor(), DoForward);
00313   }
00314 
00315   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00316   void 
00317   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00318   doImport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00319       const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, 
00320       CombineMode CM) 
00321   {
00322     TEUCHOS_TEST_FOR_EXCEPTION(  * getMap() != *exporter.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00323     TEUCHOS_TEST_FOR_EXCEPTION( *A.getMap() != *exporter.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00324     size_t numSameIDs = exporter.getNumSameIDs();
00325     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getRemoteLIDs();
00326     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getExportLIDs();
00327     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteFromLIDs();
00328     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteToLIDs();
00329     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00330                exporter.getDistributor(), DoReverse);
00331   }
00332 
00333   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00334   void 
00335   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00336   doExport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00337       const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, 
00338       CombineMode CM) 
00339   {
00340     TEUCHOS_TEST_FOR_EXCEPTION( *  getMap() != *importer.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00341     TEUCHOS_TEST_FOR_EXCEPTION( *A.getMap() != *importer.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00342     size_t numSameIDs = importer.getNumSameIDs();
00343     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getRemoteLIDs();
00344     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getExportLIDs();
00345     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteFromLIDs();
00346     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteToLIDs();
00347     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00348                importer.getDistributor(), DoReverse);
00349   }
00350 
00351   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00352   bool DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::isDistributed() const {
00353     return map_->isDistributed();
00354   }
00355 
00356   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00357   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doTransfer(
00358       const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00359       CombineMode CM,
00360       size_t numSameIDs, 
00361       const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs, 
00362       const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00363       const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,    
00364       const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00365       Distributor &distor, ReverseOption revOp) 
00366   {
00367     TEUCHOS_TEST_FOR_EXCEPTION( checkSizes(source) == false, std::runtime_error, 
00368         "Tpetra::DistObject::doTransfer(): checkSizes() indicates that DistOjbects are not size-compatible.");
00369     Kokkos::ReadWriteOption rwo = Kokkos::ReadWrite;
00370     if (CM == INSERT || CM == REPLACE) {
00371       if (numSameIDs + permuteToLIDs.size() + remoteLIDs.size() == this->getMap()->getNodeNumElements()) {
00372         // we are overwriting, and everything will be overwritten
00373         rwo = Kokkos::WriteOnly;
00374       }
00375     }
00376     source.createViews();
00377     this->createViewsNonConst(rwo); 
00378     if (numSameIDs + permuteToLIDs.size()) {
00379       copyAndPermute(source,numSameIDs,permuteToLIDs,permuteFromLIDs);
00380     }
00381     size_t constantNumPackets = 0;
00382     numExportPacketsPerLID_.resize(exportLIDs.size());
00383     numImportPacketsPerLID_.resize(remoteLIDs.size());
00384     packAndPrepare(source,exportLIDs,exports_,numExportPacketsPerLID_(),constantNumPackets,distor);
00385     source.releaseViews();
00386     if (constantNumPackets != 0) {
00387       size_t rbufLen = remoteLIDs.size()*constantNumPackets;
00388       imports_.resize(rbufLen);
00389     }
00390     if ((isDistributed() && revOp == DoReverse) || (source.isDistributed() && revOp == DoForward)) 
00391     {
00392       // call one of the doPostsAndWaits functions
00393       if (revOp == DoReverse) {
00394         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00395           distor.doReversePostsAndWaits(numExportPacketsPerLID_().getConst(), 1,
00396                                         numImportPacketsPerLID_());
00397           size_t totalImportPackets = 0;
00398           for(Array_size_type i=0; i<numImportPacketsPerLID_.size(); ++i) {
00399             totalImportPackets += numImportPacketsPerLID_[i];
00400           }
00401           imports_.resize(totalImportPackets);
00402           distor.doReversePostsAndWaits(exports_().getConst(),numExportPacketsPerLID_(),
00403                                         imports_(), numImportPacketsPerLID_());
00404         }
00405         else {
00406           distor.doReversePostsAndWaits(exports_().getConst(),constantNumPackets,imports_());
00407         }
00408       }
00409       else {
00410         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00411           distor.doPostsAndWaits(numExportPacketsPerLID_().getConst(), 1,
00412                                  numImportPacketsPerLID_());
00413           size_t totalImportPackets = 0;
00414           for(Array_size_type i=0; i<numImportPacketsPerLID_.size(); ++i) {
00415             totalImportPackets += numImportPacketsPerLID_[i];
00416           }
00417           imports_.resize(totalImportPackets);
00418           distor.doPostsAndWaits(exports_().getConst(),numExportPacketsPerLID_(),
00419                                  imports_(), numImportPacketsPerLID_());
00420         }
00421         else {
00422           distor.doPostsAndWaits(exports_().getConst(),constantNumPackets,imports_());
00423         }
00424       }
00425       unpackAndCombine(remoteLIDs,imports_(),numImportPacketsPerLID_(), constantNumPackets, distor,CM);
00426     }
00427     this->releaseViews();
00428   }
00429 
00430   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00431   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::print(std::ostream &os) const
00432   {
00433     using std::endl;
00434     os << "Tpetra::DistObject" << endl
00435        << " export buffer size: " << exports_.size() << endl
00436        << " import buffer size: " << imports_.size() << endl
00437        << "Map:" << endl
00438        << map_;
00439   }
00440 
00441 } // namespace Tpetra
00442 
00443 #endif /* TPETRA_DISTOBJECT_HPP */
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines