Tpetra_DistObject.hpp

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_DISTOBJECT_HPP
00030 #define TPETRA_DISTOBJECT_HPP
00031 
00032 #include "Tpetra_ConfigDefs.hpp"
00033 #include "Tpetra_Map.hpp"
00034 #include "Tpetra_Import.hpp"
00035 #include "Tpetra_Export.hpp"
00036 #include "Tpetra_Distributor.hpp"
00037 
00038 #include <Kokkos_DefaultNode.hpp>
00039 #include <Teuchos_Describable.hpp>
00040 #include <Teuchos_OrdinalTraits.hpp>
00041 #include <Teuchos_RCP.hpp>
00042 #include <Teuchos_Comm.hpp>
00043 
00044 namespace Tpetra {
00045 
00047 
00067   template <class Packet, class LocalOrdinal = int, class GlobalOrdinal = LocalOrdinal, class Node = Kokkos::DefaultNode::DefaultNodeType>
00068   class DistObject : public Teuchos::Describable {
00069 
00070   public:
00071 
00073 
00074 
00076     explicit DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > &map);
00077 
00079     DistObject(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source);
00080 
00082     virtual ~DistObject();
00083 
00085 
00087 
00088 
00090     void doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source, 
00091                   const Import<LocalOrdinal,GlobalOrdinal,Node> &importer, CombineMode CM);
00092 
00094     void doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest, 
00095                   const Export<LocalOrdinal,GlobalOrdinal,Node> &exporter, CombineMode CM);
00096 
00098     void doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00099                   const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter, CombineMode CM);
00100 
00102     void doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest,
00103                   const Import<LocalOrdinal,GlobalOrdinal,Node>& importer, CombineMode CM);
00104 
00106 
00108 
00109 
00111     inline bool isDistributed() const;
00112 
00114     inline const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & getMap() const { return map_; }
00115 
00117 
00119 
00120 
00122 
00123     void print(std::ostream &os) const;
00124 
00126 
00127   protected:
00128 
00129     enum ReverseOption {
00130       DoForward,
00131       DoReverse
00132     };
00133 
00135     virtual void doTransfer(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00136                             CombineMode CM,
00137                             size_t numSameIDs,
00138                             const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00139                             const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00140                             const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00141                             const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00142                             Distributor &distor,
00143                             ReverseOption revOp);
00144 
00145     // The following four methods must be implemented by the derived class
00146 
00148 
00149     virtual bool checkSizes(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source) = 0;
00150 
00152 
00168     virtual void copyAndPermute(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00169                                 size_t numSameIDs,
00170                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00171                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs) = 0;
00172 
00174 
00192     virtual void packAndPrepare(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00193                                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00194                                 Teuchos::Array<Packet> &exports,
00195                                 const Teuchos::ArrayView<size_t> & numPacketsPerLID,
00196                                 size_t& constantNumPackets,
00197                                 Distributor &distor) = 0;
00198 
00200 
00217     virtual void unpackAndCombine(const Teuchos::ArrayView<const LocalOrdinal> &importLIDs,
00218                                   const Teuchos::ArrayView<const Packet> &imports,
00219                                   const Teuchos::ArrayView<size_t> &numPacketsPerLID,
00220                                   size_t constantNumPackets,
00221                                   Distributor &distor,
00222                                   CombineMode CM) = 0;
00223 
00224   private:
00225 
00226     Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > map_;
00227     // buffers into which packed data is imported
00228     Teuchos::Array<Packet> imports_;
00229     Teuchos::Array<size_t> numImportPacketsPerLID_;
00230     // buffers from which packed data is exported
00231     Teuchos::Array<Packet> exports_;
00232     Teuchos::Array<size_t> numExportPacketsPerLID_;
00233 
00234   }; // class DistObject
00235 
00236   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00237   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & map)
00238   : map_(map)
00239   {}
00240 
00241   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00242   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source)
00243   : map_(source.map_)
00244   {}
00245 
00246   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00247   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::~DistObject() 
00248   {}
00249 
00250   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00251   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00252                                             const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, CombineMode CM) 
00253   {
00254     TEST_FOR_EXCEPTION( getMap() != importer.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00255     TEST_FOR_EXCEPTION( A.getMap() != importer.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00256     size_t numSameIDs = importer.getNumSameIDs();
00257     const Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getExportLIDs();
00258     const Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getRemoteLIDs();
00259     const Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteToLIDs();
00260     const Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteFromLIDs();
00261     this->doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00262                      importer.getDistributor(), DoForward);
00263   }
00264 
00265   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00266   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00267                                             const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, CombineMode CM) 
00268   {
00269     TEST_FOR_EXCEPTION( getMap() != exporter.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00270     TEST_FOR_EXCEPTION( A.getMap() != exporter.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00271     size_t numSameIDs = exporter.getNumSameIDs();
00272     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getExportLIDs();
00273     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getRemoteLIDs();
00274     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteToLIDs();
00275     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteFromLIDs();
00276     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00277                exporter.getDistributor(), DoForward);
00278   }
00279 
00280   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00281   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doImport(
00282                                                 const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00283                                                 const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, 
00284                                                 CombineMode CM) {
00285     TEST_FOR_EXCEPTION( getMap() != exporter.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00286     TEST_FOR_EXCEPTION( A.getMap() != exporter.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00287     size_t numSameIDs = exporter.getNumSameIDs();
00288     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getRemoteLIDs();
00289     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getExportLIDs();
00290     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteFromLIDs();
00291     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteToLIDs();
00292     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00293                exporter.getDistributor(), DoReverse);
00294   }
00295 
00296   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00297   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doExport(
00298                                                 const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00299                                                 const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, CombineMode CM) {
00300     TEST_FOR_EXCEPTION( getMap() != importer.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00301     TEST_FOR_EXCEPTION( A.getMap() != importer.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00302     size_t numSameIDs = importer.getNumSameIDs();
00303     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getRemoteLIDs();
00304     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getExportLIDs();
00305     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteFromLIDs();
00306     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteToLIDs();
00307     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00308                importer.getDistributor(), DoReverse);
00309   }
00310 
00311   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00312   bool DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::isDistributed() const {
00313     return map_->isDistributed();
00314   }
00315 
00316   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00317   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doTransfer(
00318       const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00319       CombineMode CM,
00320       size_t numSameIDs, 
00321       const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs, 
00322       const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00323       const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,    
00324       const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00325       Distributor &distor, ReverseOption revOp) 
00326   {
00327     TEST_FOR_EXCEPTION( checkSizes(source) == false, std::runtime_error, 
00328         "Tpetra::DistObject::doTransfer(): checkSizes() indicates that DistOjbects are not size-compatible.");
00329     if (numSameIDs + permuteToLIDs.size()) {
00330       copyAndPermute(source,numSameIDs,permuteToLIDs,permuteFromLIDs);
00331     }
00332     size_t constantNumPackets = 0;
00333     numExportPacketsPerLID_.resize(exportLIDs.size());
00334     numImportPacketsPerLID_.resize(remoteLIDs.size());
00335     packAndPrepare(source,exportLIDs,exports_,numExportPacketsPerLID_(),constantNumPackets,distor);
00336     if (constantNumPackets != 0) {
00337       size_t rbufLen = remoteLIDs.size()*constantNumPackets;
00338       imports_.resize(rbufLen);
00339     }
00340     if ((isDistributed() && revOp == DoReverse) || (source.isDistributed() && revOp == DoForward)) 
00341     {
00342       // call one of the doPostsAndWaits functions
00343       if (revOp == DoReverse) {
00344         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00345           distor.doReversePostsAndWaits(numExportPacketsPerLID_().getConst(), 1,
00346                                         numImportPacketsPerLID_());
00347           size_t totalImportPackets = 0;
00348           for(Array_size_type i=0; i<numImportPacketsPerLID_.size(); ++i) {
00349             totalImportPackets += numImportPacketsPerLID_[i];
00350           }
00351           imports_.resize(totalImportPackets);
00352           distor.doReversePostsAndWaits(exports_().getConst(),numExportPacketsPerLID_(),
00353                                         imports_(), numImportPacketsPerLID_());
00354         }
00355         else {
00356           distor.doReversePostsAndWaits(exports_().getConst(),constantNumPackets,imports_());
00357         }
00358       }
00359       else {
00360         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00361           distor.doPostsAndWaits(numExportPacketsPerLID_().getConst(), 1,
00362                                  numImportPacketsPerLID_());
00363           size_t totalImportPackets = 0;
00364           for(Array_size_type i=0; i<numImportPacketsPerLID_.size(); ++i) {
00365             totalImportPackets += numImportPacketsPerLID_[i];
00366           }
00367           imports_.resize(totalImportPackets);
00368           distor.doPostsAndWaits(exports_().getConst(),numExportPacketsPerLID_(),
00369                                  imports_(), numImportPacketsPerLID_());
00370         }
00371         else {
00372           distor.doPostsAndWaits(exports_().getConst(),constantNumPackets,imports_());
00373         }
00374       }
00375       unpackAndCombine(remoteLIDs,imports_(),numImportPacketsPerLID_(), constantNumPackets, distor,CM);
00376     }
00377   }
00378 
00379   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00380   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::print(std::ostream &os) const
00381   {
00382     using std::endl;
00383     os << "Tpetra::DistObject" << endl
00384        << " export buffer size: " << exports_.size() << endl
00385        << " import buffer size: " << imports_.size() << endl
00386        << "Map:" << endl
00387        << map_;
00388   }
00389 
00390 } // namespace Tpetra
00391 
00392 #endif /* TPETRA_DISTOBJECT_HPP */

Generated on Tue Jul 13 09:39:06 2010 for Tpetra Matrix/Vector Services by  doxygen 1.4.7