Tpetra_DistObject.hpp

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_DISTOBJECT_HPP
00030 #define TPETRA_DISTOBJECT_HPP
00031 
00032 #include "Tpetra_ConfigDefs.hpp"
00033 #include "Tpetra_Map.hpp"
00034 #include "Tpetra_Import.hpp"
00035 #include "Tpetra_Export.hpp"
00036 #include "Tpetra_Distributor.hpp"
00037 
00038 #include <Kokkos_DefaultNode.hpp>
00039 #include <Teuchos_Describable.hpp>
00040 #include <Teuchos_OrdinalTraits.hpp>
00041 #include <Teuchos_RCP.hpp>
00042 #include <Teuchos_Comm.hpp>
00043 
00044 namespace Tpetra {
00045 
00047 
00067   template <class Scalar, class LocalOrdinal = int, class GlobalOrdinal = LocalOrdinal, class Node = Kokkos::DefaultNode::DefaultNodeType>
00068   class DistObject : public Teuchos::Describable {
00069 
00070   public:
00071 
00073 
00074 
00076     explicit DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > &map);
00077 
00079     DistObject(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> &source);
00080 
00082     virtual ~DistObject();
00083 
00085 
00087 
00088 
00090     void doImport(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> &source, 
00091                   const Import<LocalOrdinal,GlobalOrdinal,Node> &importer, CombineMode CM);
00092 
00094     void doExport(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> &dest, 
00095                   const Export<LocalOrdinal,GlobalOrdinal,Node> &exporter, CombineMode CM);
00096 
00098     void doImport(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> &source,
00099                   const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter, CombineMode CM);
00100 
00102     void doExport(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> &dest,
00103                   const Import<LocalOrdinal,GlobalOrdinal,Node>& importer, CombineMode CM);
00104 
00106 
00108 
00109 
00111     inline bool isDistributed() const;
00112 
00114     inline const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & getMap() const;
00115 
00117 
00119 
00120 
00122 
00123     void print(std::ostream &os) const;
00124 
00126 
00127   protected:
00128 
00129     enum ReverseOption {
00130       DoForward,
00131       DoReverse
00132     };
00133 
00135     virtual void doTransfer(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> &source,
00136                             CombineMode CM,
00137                             size_t numSameIDs,
00138                             const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00139                             const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00140                             const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00141                             const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00142                             Distributor &distor,
00143                             ReverseOption revOp);
00144 
00145     // The following four methods must be implemented by the derived class
00146 
00148 
00149     virtual bool checkSizes(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & source, size_t &packetSize) = 0;
00150 
00152 
00168     virtual void copyAndPermute(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & source,
00169                                 size_t numSameIDs,
00170                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00171                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs) = 0;
00172 
00174 
00185     virtual void packAndPrepare(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & source,
00186                                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00187                                 const Teuchos::ArrayView<Scalar> &exports,
00188                                 Distributor &distor) = 0;
00189 
00191 
00202     virtual void unpackAndCombine(const Teuchos::ArrayView<const LocalOrdinal> &importLIDs,
00203                                   const Teuchos::ArrayView<const Scalar> &imports,
00204                                   Distributor &distor,
00205                                   CombineMode CM) = 0;
00206 
00207   private:
00208 
00209     Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > map_;
00210     // buffers into which packed data is imported
00211     Teuchos::Array<Scalar> imports_;
00212     // buffers from which packed data is exported
00213     Teuchos::Array<Scalar> exports_;
00214 
00215   }; // class DistObject
00216 
00217   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00218   DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & map)
00219   : map_(map)
00220   {}
00221 
00222   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00223   DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & source)
00224   : map_(source.map_)
00225   {}
00226 
00227   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00228   DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::~DistObject() 
00229   {}
00230 
00231   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00232   void DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::doImport(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & A, 
00233                                             const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, CombineMode CM) 
00234   {
00235     TEST_FOR_EXCEPTION( getMap() != importer.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00236     TEST_FOR_EXCEPTION( A.getMap() != importer.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00237     size_t numSameIDs = importer.getNumSameIDs();
00238     const Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getExportLIDs();
00239     const Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getRemoteLIDs();
00240     const Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteToLIDs();
00241     const Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteFromLIDs();
00242     this->doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00243                      importer.getDistributor(), DoForward);
00244   }
00245 
00246   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00247   void DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::doExport(const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & A, 
00248                                             const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, CombineMode CM) 
00249   {
00250     TEST_FOR_EXCEPTION( getMap() != exporter.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00251     TEST_FOR_EXCEPTION( A.getMap() != exporter.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00252     size_t numSameIDs = exporter.getNumSameIDs();
00253     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getExportLIDs();
00254     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getRemoteLIDs();
00255     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteToLIDs();
00256     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteFromLIDs();
00257     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00258                exporter.getDistributor(), DoForward);
00259   }
00260 
00261   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00262   void DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::doImport(
00263                                                 const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & A,
00264                                                 const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, 
00265                                                 CombineMode CM) {
00266     TEST_FOR_EXCEPTION( getMap() != exporter.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00267     TEST_FOR_EXCEPTION( A.getMap() != exporter.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00268     size_t numSameIDs = exporter.getNumSameIDs();
00269     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getRemoteLIDs();
00270     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getExportLIDs();
00271     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteFromLIDs();
00272     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteToLIDs();
00273     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00274                exporter.getDistributor(), DoReverse);
00275   }
00276 
00277   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00278   void DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::doExport(
00279                                                 const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & A,
00280                                                 const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, CombineMode CM) {
00281     TEST_FOR_EXCEPTION( getMap() != importer.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00282     TEST_FOR_EXCEPTION( A.getMap() != importer.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00283     size_t numSameIDs = importer.getNumSameIDs();
00284     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getRemoteLIDs();
00285     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getExportLIDs();
00286     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteFromLIDs();
00287     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteToLIDs();
00288     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00289                importer.getDistributor(), DoReverse);
00290   }
00291 
00292   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00293   bool DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::isDistributed() const {
00294     return map_->isDistributed();
00295   }
00296 
00297   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00298   const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & 
00299   DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::getMap() const {
00300     return map_;
00301   }
00302 
00303   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00304   void DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::doTransfer(
00305       const DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node> & source,
00306       CombineMode CM,
00307       size_t numSameIDs, 
00308       const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs, 
00309       const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00310       const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,    
00311       const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00312       Distributor &distor, ReverseOption revOp) 
00313   {
00314     size_t packetSize;
00315     TEST_FOR_EXCEPTION( checkSizes(source,packetSize) == false, std::runtime_error, 
00316         "Tpetra::DistObject::doTransfer(): checkSizes() indicates that DistOjbects are not size-compatible.");
00317     size_t sbufLen = exportLIDs.size()*packetSize;
00318     size_t rbufLen = remoteLIDs.size()*packetSize;
00319     exports_.resize(sbufLen);
00320     imports_.resize(rbufLen);
00321     if (numSameIDs + permuteToLIDs.size()) {
00322       copyAndPermute(source,numSameIDs,permuteToLIDs,permuteFromLIDs);
00323     }
00324     packAndPrepare(source,exportLIDs,exports_(),distor);
00325     if ((isDistributed() && revOp == DoReverse) || (source.isDistributed() && revOp == DoForward)) 
00326     {
00327       // call one of the doPostsAndWaits functions
00328       if (revOp == DoReverse) {
00329         distor.doReversePostsAndWaits(exports_().getConst(),packetSize,imports_());
00330       }
00331       else {
00332         distor.doPostsAndWaits(exports_().getConst(),packetSize,imports_());
00333       }
00334       unpackAndCombine(remoteLIDs,imports_(),distor,CM);
00335     }
00336   }
00337 
00338   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
00339   void DistObject<Scalar,LocalOrdinal,GlobalOrdinal,Node>::print(std::ostream &os) const
00340   {
00341     using std::endl;
00342     os << "Tpetra::DistObject" << endl
00343        << " export buffer size: " << exports_.size() << endl
00344        << " import buffer size: " << imports_.size() << endl
00345        << "Map:" << endl
00346        << map_;
00347   }
00348 
00349 } // namespace Tpetra
00350 
00351 #endif /* TPETRA_DISTOBJECT_HPP */

Generated on Wed May 12 21:40:14 2010 for Tpetra Matrix/Vector Services by  doxygen 1.4.7