Tpetra_DistObject.hpp

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_DISTOBJECT_HPP
00030 #define TPETRA_DISTOBJECT_HPP
00031 
00032 #include "Tpetra_ConfigDefs.hpp"
00033 #include "Tpetra_Map.hpp"
00034 #include "Tpetra_Import.hpp"
00035 #include "Tpetra_Export.hpp"
00036 #include "Tpetra_Distributor.hpp"
00037 
00038 #include <Kokkos_DefaultNode.hpp>
00039 #include <Teuchos_Describable.hpp>
00040 #include <Teuchos_OrdinalTraits.hpp>
00041 #include <Teuchos_RCP.hpp>
00042 #include <Teuchos_Comm.hpp>
00043 
00044 namespace Tpetra {
00045 
00047 
00067   template <class Packet, class LocalOrdinal = int, class GlobalOrdinal = LocalOrdinal, class Node = Kokkos::DefaultNode::DefaultNodeType>
00068   class DistObject : virtual public Teuchos::Describable {
00069 
00070   public:
00071 
00073 
00074 
00076     explicit DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > &map);
00077 
00079     DistObject(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source);
00080 
00082     virtual ~DistObject();
00083 
00085 
00087 
00088 
00090     void doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source, 
00091                   const Import<LocalOrdinal,GlobalOrdinal,Node> &importer, CombineMode CM);
00092 
00094     void doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest, 
00095                   const Export<LocalOrdinal,GlobalOrdinal,Node> &exporter, CombineMode CM);
00096 
00098     void doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00099                   const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter, CombineMode CM);
00100 
00102     void doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest,
00103                   const Import<LocalOrdinal,GlobalOrdinal,Node>& importer, CombineMode CM);
00104 
00106 
00108 
00109 
00111     inline bool isDistributed() const;
00112 
00114     inline const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & getMap() const { return map_; }
00115 
00117 
00119 
00120 
00122 
00123     void print(std::ostream &os) const;
00124 
00126 
00127   protected:
00128 
00130     enum ReverseOption {
00131       DoForward, //*!< Indicates that the transfer should be performed in forward mode.
00132       DoReverse  //*!< Indicates that the transfer should be performed in reverse mode.
00133     };
00134 
00136     virtual void doTransfer(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00137                             CombineMode CM,
00138                             size_t numSameIDs,
00139                             const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00140                             const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00141                             const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00142                             const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00143                             Distributor &distor,
00144                             ReverseOption revOp);
00145 
00146     // The following four methods must be implemented by the derived class
00147 
00149 
00150     virtual bool checkSizes(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source) = 0;
00151 
00153 
00169     virtual void copyAndPermute(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00170                                 size_t numSameIDs,
00171                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00172                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs) = 0;
00173 
00175 
00193     virtual void packAndPrepare(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00194                                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00195                                 Teuchos::Array<Packet> &exports,
00196                                 const Teuchos::ArrayView<size_t> & numPacketsPerLID,
00197                                 size_t& constantNumPackets,
00198                                 Distributor &distor) = 0;
00199 
00201 
00218     virtual void unpackAndCombine(const Teuchos::ArrayView<const LocalOrdinal> &importLIDs,
00219                                   const Teuchos::ArrayView<const Packet> &imports,
00220                                   const Teuchos::ArrayView<size_t> &numPacketsPerLID,
00221                                   size_t constantNumPackets,
00222                                   Distributor &distor,
00223                                   CombineMode CM) = 0;
00224 
00225     virtual void createViews() const {}
00226     virtual void createViewsNonConst(Kokkos::ReadWriteOption rwo) {}
00227     virtual void releaseViews() const {}
00228 
00229   private:
00230 
00231     Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > map_;
00232     // buffers into which packed data is imported
00233     Teuchos::Array<Packet> imports_;
00234     Teuchos::Array<size_t> numImportPacketsPerLID_;
00235     // buffers from which packed data is exported
00236     Teuchos::Array<Packet> exports_;
00237     Teuchos::Array<size_t> numExportPacketsPerLID_;
00238 
00239   }; // class DistObject
00240 
00241   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00242   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & map)
00243   : map_(map)
00244   {}
00245 
00246   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00247   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source)
00248   : map_(source.map_)
00249   {}
00250 
00251   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00252   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::~DistObject() 
00253   {}
00254 
00255   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00256   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00257                                             const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, CombineMode CM) 
00258   {
00259     TEST_FOR_EXCEPTION( getMap() != importer.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00260     TEST_FOR_EXCEPTION( A.getMap() != importer.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00261     size_t numSameIDs = importer.getNumSameIDs();
00262     const Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getExportLIDs();
00263     const Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getRemoteLIDs();
00264     const Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteToLIDs();
00265     const Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteFromLIDs();
00266     this->doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00267                      importer.getDistributor(), DoForward);
00268   }
00269 
00270   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00271   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00272                                             const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, CombineMode CM) 
00273   {
00274     TEST_FOR_EXCEPTION( getMap() != exporter.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00275     TEST_FOR_EXCEPTION( A.getMap() != exporter.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00276     size_t numSameIDs = exporter.getNumSameIDs();
00277     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getExportLIDs();
00278     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getRemoteLIDs();
00279     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteToLIDs();
00280     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteFromLIDs();
00281     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00282                exporter.getDistributor(), DoForward);
00283   }
00284 
00285   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00286   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doImport(
00287                                                 const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00288                                                 const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, 
00289                                                 CombineMode CM) {
00290     TEST_FOR_EXCEPTION( getMap() != exporter.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00291     TEST_FOR_EXCEPTION( A.getMap() != exporter.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00292     size_t numSameIDs = exporter.getNumSameIDs();
00293     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getRemoteLIDs();
00294     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getExportLIDs();
00295     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteFromLIDs();
00296     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteToLIDs();
00297     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00298                exporter.getDistributor(), DoReverse);
00299   }
00300 
00301   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00302   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doExport(
00303                                                 const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00304                                                 const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, CombineMode CM) {
00305     TEST_FOR_EXCEPTION( getMap() != importer.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00306     TEST_FOR_EXCEPTION( A.getMap() != importer.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00307     size_t numSameIDs = importer.getNumSameIDs();
00308     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getRemoteLIDs();
00309     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getExportLIDs();
00310     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteFromLIDs();
00311     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteToLIDs();
00312     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00313                importer.getDistributor(), DoReverse);
00314   }
00315 
00316   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00317   bool DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::isDistributed() const {
00318     return map_->isDistributed();
00319   }
00320 
00321   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00322   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doTransfer(
00323       const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00324       CombineMode CM,
00325       size_t numSameIDs, 
00326       const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs, 
00327       const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00328       const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,    
00329       const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00330       Distributor &distor, ReverseOption revOp) 
00331   {
00332     TEST_FOR_EXCEPTION( checkSizes(source) == false, std::runtime_error, 
00333         "Tpetra::DistObject::doTransfer(): checkSizes() indicates that DistOjbects are not size-compatible.");
00334     Kokkos::ReadWriteOption rwo = Kokkos::ReadWrite;
00335     if (CM == INSERT || CM == REPLACE) {
00336       if (numSameIDs + permuteToLIDs.size() + remoteLIDs.size() == this->getMap()->getNodeNumElements()) {
00337         // we are overwriting, and everything will be overwritten
00338         rwo = Kokkos::WriteOnly;
00339       }
00340     }
00341     source.createViews();
00342     this->createViewsNonConst(rwo); 
00343     if (numSameIDs + permuteToLIDs.size()) {
00344       copyAndPermute(source,numSameIDs,permuteToLIDs,permuteFromLIDs);
00345     }
00346     size_t constantNumPackets = 0;
00347     numExportPacketsPerLID_.resize(exportLIDs.size());
00348     numImportPacketsPerLID_.resize(remoteLIDs.size());
00349     packAndPrepare(source,exportLIDs,exports_,numExportPacketsPerLID_(),constantNumPackets,distor);
00350     source.releaseViews();
00351     if (constantNumPackets != 0) {
00352       size_t rbufLen = remoteLIDs.size()*constantNumPackets;
00353       imports_.resize(rbufLen);
00354     }
00355     if ((isDistributed() && revOp == DoReverse) || (source.isDistributed() && revOp == DoForward)) 
00356     {
00357       // call one of the doPostsAndWaits functions
00358       if (revOp == DoReverse) {
00359         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00360           distor.doReversePostsAndWaits(numExportPacketsPerLID_().getConst(), 1,
00361                                         numImportPacketsPerLID_());
00362           size_t totalImportPackets = 0;
00363           for(Array_size_type i=0; i<numImportPacketsPerLID_.size(); ++i) {
00364             totalImportPackets += numImportPacketsPerLID_[i];
00365           }
00366           imports_.resize(totalImportPackets);
00367           distor.doReversePostsAndWaits(exports_().getConst(),numExportPacketsPerLID_(),
00368                                         imports_(), numImportPacketsPerLID_());
00369         }
00370         else {
00371           distor.doReversePostsAndWaits(exports_().getConst(),constantNumPackets,imports_());
00372         }
00373       }
00374       else {
00375         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00376           distor.doPostsAndWaits(numExportPacketsPerLID_().getConst(), 1,
00377                                  numImportPacketsPerLID_());
00378           size_t totalImportPackets = 0;
00379           for(Array_size_type i=0; i<numImportPacketsPerLID_.size(); ++i) {
00380             totalImportPackets += numImportPacketsPerLID_[i];
00381           }
00382           imports_.resize(totalImportPackets);
00383           distor.doPostsAndWaits(exports_().getConst(),numExportPacketsPerLID_(),
00384                                  imports_(), numImportPacketsPerLID_());
00385         }
00386         else {
00387           distor.doPostsAndWaits(exports_().getConst(),constantNumPackets,imports_());
00388         }
00389       }
00390       unpackAndCombine(remoteLIDs,imports_(),numImportPacketsPerLID_(), constantNumPackets, distor,CM);
00391     }
00392     this->releaseViews();
00393   }
00394 
00395   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00396   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::print(std::ostream &os) const
00397   {
00398     using std::endl;
00399     os << "Tpetra::DistObject" << endl
00400        << " export buffer size: " << exports_.size() << endl
00401        << " import buffer size: " << imports_.size() << endl
00402        << "Map:" << endl
00403        << map_;
00404   }
00405 
00406 } // namespace Tpetra
00407 
00408 #endif /* TPETRA_DISTOBJECT_HPP */
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines
Generated on Wed Apr 13 10:21:41 2011 for Tpetra Matrix/Vector Services by  doxygen 1.6.3