Tpetra Matrix/Vector Services Version of the Day
Tpetra_Export.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_EXPORT_HPP
00030 #define TPETRA_EXPORT_HPP
00031 
00032 #include <Kokkos_DefaultNode.hpp>
00033 #include <Teuchos_Describable.hpp>
00034 #include <Teuchos_as.hpp>
00035 #include "Tpetra_Map.hpp"
00036 #include "Tpetra_Util.hpp"
00037 #include "Tpetra_ImportExportData.hpp"
00038 #include <iterator>
00039 
00040 namespace Tpetra {
00041 
00043 
00055   template <class LocalOrdinal, class GlobalOrdinal = LocalOrdinal, class Node = Kokkos::DefaultNode::DefaultNodeType>
00056   class Export: public Teuchos::Describable {
00057 
00058   public:
00059 
00061 
00062 
00064     Export(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & source,  
00065            const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & target);
00066 
00068     Export(const Export<LocalOrdinal,GlobalOrdinal,Node> & rhs);
00069 
00071     ~Export();
00072 
00074 
00076 
00077 
00079     size_t getNumSameIDs() const;
00080 
00082     size_t getNumPermuteIDs() const;
00083 
00085     Teuchos::ArrayView<const LocalOrdinal> getPermuteFromLIDs() const;
00086 
00088     Teuchos::ArrayView<const LocalOrdinal> getPermuteToLIDs() const;
00089 
00091     size_t getNumRemoteIDs() const;
00092 
00094     Teuchos::ArrayView<const LocalOrdinal> getRemoteLIDs() const;
00095 
00097     size_t getNumExportIDs() const;
00098 
00100     Teuchos::ArrayView<const LocalOrdinal> getExportLIDs() const;
00101 
00103     Teuchos::ArrayView<const int> getExportImageIDs() const;
00104 
00106     const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & getSourceMap() const;
00107 
00109     const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & getTargetMap() const;
00110 
00111     Distributor & getDistributor() const;
00112 
00114     Export<LocalOrdinal,GlobalOrdinal,Node>& operator = (const Export<LocalOrdinal,GlobalOrdinal,Node> & Source);
00115 
00117 
00119 
00120 
00122     virtual void print(std::ostream& os) const;
00123 
00125 
00126   private:
00127 
00128     Teuchos::RCP<ImportExportData<LocalOrdinal,GlobalOrdinal,Node> > ExportData_;
00129 
00130     // subfunctions used by constructor
00131     //==============================================================================
00132     // sets up numSameIDs_, numPermuteIDs_, and numExportIDs_
00133     // these variables are already initialized to 0 by the ImportExportData ctr.
00134     // also sets up permuteToLIDs_, permuteFromLIDs_, exportGIDs_, and exportLIDs_
00135     void setupSamePermuteExport();
00136     void setupRemote();
00137   };
00138 
00139   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00140   Export<LocalOrdinal,GlobalOrdinal,Node>::Export(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & source,   
00141                                                   const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & target) {
00142     ExportData_ = Teuchos::rcp(new ImportExportData<LocalOrdinal,GlobalOrdinal,Node>(source, target));
00143     // call subfunctions
00144     setupSamePermuteExport();
00145     if (source->isDistributed()) {
00146       setupRemote();
00147     }
00148   }
00149 
00150   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00151   Export<LocalOrdinal,GlobalOrdinal,Node>::Export(const Export<LocalOrdinal,GlobalOrdinal,Node> & rhs)
00152   : ExportData_(rhs.ExportData_)
00153   {}
00154 
00155   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00156   Export<LocalOrdinal,GlobalOrdinal,Node>::~Export() 
00157   {}
00158 
00159   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00160   size_t Export<LocalOrdinal,GlobalOrdinal,Node>::getNumSameIDs() const {
00161     return ExportData_->numSameIDs_;
00162   }
00163 
00164   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00165   size_t Export<LocalOrdinal,GlobalOrdinal,Node>::getNumPermuteIDs() const {
00166     return ExportData_->permuteFromLIDs_.size();
00167   }
00168 
00169   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00170   Teuchos::ArrayView<const LocalOrdinal> 
00171   Export<LocalOrdinal,GlobalOrdinal,Node>::getPermuteFromLIDs() const {
00172     return ExportData_->permuteFromLIDs_();
00173   }
00174 
00175   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00176   Teuchos::ArrayView<const LocalOrdinal>
00177   Export<LocalOrdinal,GlobalOrdinal,Node>::getPermuteToLIDs() const {
00178     return ExportData_->permuteToLIDs_();
00179   }
00180 
00181   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00182   size_t Export<LocalOrdinal,GlobalOrdinal,Node>::getNumRemoteIDs() const {
00183     return ExportData_->remoteLIDs_.size();
00184   }
00185 
00186   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00187   Teuchos::ArrayView<const LocalOrdinal> 
00188   Export<LocalOrdinal,GlobalOrdinal,Node>::getRemoteLIDs() const {
00189     return ExportData_->remoteLIDs_();
00190   }
00191 
00192   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00193   size_t Export<LocalOrdinal,GlobalOrdinal,Node>::getNumExportIDs() const {
00194     return ExportData_->exportLIDs_.size();
00195   }
00196 
00197   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00198   Teuchos::ArrayView<const LocalOrdinal> 
00199   Export<LocalOrdinal,GlobalOrdinal,Node>::getExportLIDs() const {
00200     return ExportData_->exportLIDs_();
00201   }
00202 
00203   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00204   Teuchos::ArrayView<const int> 
00205   Export<LocalOrdinal,GlobalOrdinal,Node>::getExportImageIDs() const {
00206     return ExportData_->exportImageIDs_();
00207   }
00208 
00209   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00210   const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & 
00211   Export<LocalOrdinal,GlobalOrdinal,Node>::getSourceMap() const {
00212     return ExportData_->source_;
00213   }
00214 
00215   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00216   const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & 
00217   Export<LocalOrdinal,GlobalOrdinal,Node>::getTargetMap() const {
00218     return ExportData_->target_;
00219   }
00220 
00221   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00222   Distributor & 
00223   Export<LocalOrdinal,GlobalOrdinal,Node>::getDistributor() const { return ExportData_->distributor_;
00224   }
00225 
00226   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00227   Export<LocalOrdinal,GlobalOrdinal,Node>& 
00228   Export<LocalOrdinal,GlobalOrdinal,Node>::operator=(const Export<LocalOrdinal,GlobalOrdinal,Node> & source) {
00229     ExportData_ = source.ExportData_;
00230     return *this;
00231   }
00232 
00233   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00234   void Export<LocalOrdinal,GlobalOrdinal,Node>::print(std::ostream& os) const {
00235     using std::endl;
00236     Teuchos::ArrayView<const LocalOrdinal> av;
00237     Teuchos::ArrayView<const int> avi;
00238     const Teuchos::RCP<const Teuchos::Comm<int> > & comm = getSourceMap()->getComm();
00239     const int myImageID = comm->getRank();
00240     const int numImages = comm->getSize();
00241     for (int imageCtr = 0; imageCtr < numImages; ++imageCtr) {
00242       if(myImageID == imageCtr) {
00243         os << endl;
00244         if(myImageID == 0) { // this is the root node (only output this info once)
00245           os << "Export Data Members:" << endl;
00246         }
00247         os << "Image ID       : " << myImageID << endl;
00248         os << "permuteFromLIDs: {"; av = getPermuteFromLIDs(); std::copy(av.begin(),av.end(),std::ostream_iterator<LocalOrdinal>(os," ")); os << " }" << endl;
00249         os << "permuteToLIDs  : {"; av = getPermuteToLIDs();   std::copy(av.begin(),av.end(),std::ostream_iterator<LocalOrdinal>(os," ")); os << " }" << endl;
00250         os << "remoteLIDs     : {"; av = getRemoteLIDs();      std::copy(av.begin(),av.end(),std::ostream_iterator<LocalOrdinal>(os," ")); os << " }" << endl;
00251         os << "exportLIDs     : {"; av = getExportLIDs();      std::copy(av.begin(),av.end(),std::ostream_iterator<LocalOrdinal>(os," ")); os << " }" << endl;
00252         os << "exportImageIDs : {"; avi = getExportImageIDs();  std::copy(avi.begin(),avi.end(),std::ostream_iterator<int>(os," ")); os << " }" << endl;
00253         os << "numSameIDs     : " << getNumSameIDs() << endl;
00254         os << "numPermuteIDs  : " << getNumPermuteIDs() << endl;
00255         os << "numRemoteIDs   : " << getNumRemoteIDs() << endl;
00256         os << "numExportIDs   : " << getNumExportIDs() << endl;
00257       }
00258       // Do a few global ops to give I/O a chance to complete
00259       comm->barrier();
00260       comm->barrier();
00261       comm->barrier();
00262     }
00263     if (myImageID == 0) {
00264       os << "\nSource Map: " << endl; 
00265     }
00266     os << *getSourceMap();
00267     if (myImageID == 0) {
00268       os << "\nTarget Map: " << endl; 
00269     }
00270     os << *getTargetMap();
00271   }
00272 
00273 
00274   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00275   void Export<LocalOrdinal,GlobalOrdinal,Node>::setupSamePermuteExport() {
00276     const Map<LocalOrdinal,GlobalOrdinal,Node> & source = *getSourceMap();
00277     const Map<LocalOrdinal,GlobalOrdinal,Node> & target = *getTargetMap();
00278     Teuchos::ArrayView<const GlobalOrdinal> sourceGIDs = source.getNodeElementList();
00279     Teuchos::ArrayView<const GlobalOrdinal> targetGIDs = target.getNodeElementList();
00280 
00281     // -- compute numSameIDs_ ---
00282     // go through GID lists of source and target. if the ith GID on both is the same, 
00283     // increment numSameIDs_ and try the next. as soon as you come to a pair that don't
00284     // match, give up.
00285     typename Teuchos::ArrayView<const GlobalOrdinal>::iterator sourceIter = sourceGIDs.begin(),
00286                                                                targetIter = targetGIDs.begin();
00287     while( sourceIter != sourceGIDs.end() && targetIter != targetGIDs.end() && *sourceIter == *targetIter ) {
00288       ++ExportData_->numSameIDs_;
00289       ++sourceIter;
00290       ++targetIter;
00291     }
00292     // sourceIter should now point to the GID of the first non-same entry or at the end of targetGIDs
00293 
00294     // -- compute numPermuteIDs --
00295     // -- fill permuteToLIDs_, permuteFromLIDs_ --
00296     // go through remaining entries in sourceGIDs. if target owns that GID, 
00297     // increment numPermuteIDs_, and add entries to permuteToLIDs_ and permuteFromLIDs_.
00298     // otherwise increment numExportIDs_ and add entries to exportLIDs_ and exportGIDs_.
00299     for(; sourceIter != sourceGIDs.end(); ++sourceIter) {
00300       if(target.isNodeGlobalElement(*sourceIter)) {
00301         // both source and target list this GID (*targetIter)
00302         // determine the LIDs for this GID on both Maps and add them to the permutation lists
00303         ExportData_->permuteToLIDs_.push_back(  target.getLocalElement(*sourceIter));
00304         ExportData_->permuteFromLIDs_.push_back(source.getLocalElement(*sourceIter));
00305       }
00306       else {
00307         ExportData_->exportGIDs_.push_back(*sourceIter);
00308       }
00309     }
00310 
00311     TEST_FOR_EXCEPTION( (getNumExportIDs() > 0) && (!source.isDistributed()), std::runtime_error, 
00312         Teuchos::typeName(*this) << "::setupSamePermuteExport(): Source has export LIDs but Source is not distributed globally.");
00313 
00314     // -- compute exportImageIDs_ --
00315     // get list of images that own the GIDs in exportGIDs_ (in the target Map)
00316     ExportData_->exportImageIDs_ = Teuchos::arcp<int>(ExportData_->exportGIDs_.size());
00317     ExportData_->exportLIDs_     = Teuchos::arcp<LocalOrdinal>(ExportData_->exportGIDs_.size());
00318     {
00319       typename Teuchos::ArrayRCP<LocalOrdinal>::iterator liditer = ExportData_->exportLIDs_.begin();
00320       typename Teuchos::Array<GlobalOrdinal>::iterator    giditer = ExportData_->exportGIDs_.begin();
00321       for (; giditer != ExportData_->exportGIDs_.end(); ++liditer, ++giditer) {
00322         *liditer = source.getLocalElement(*giditer);
00323       }
00324     }
00325     TEST_FOR_EXCEPTION( target.getRemoteIndexList(ExportData_->exportGIDs_(), ExportData_->exportImageIDs_()) == IDNotPresent,
00326         std::runtime_error, "Tpetra::Export::setupSamePermuteExport(): Source has GIDs not found in Target.");
00327   }
00328 
00329 
00330   template <class LocalOrdinal, class GlobalOrdinal, class Node>
00331   void Export<LocalOrdinal,GlobalOrdinal,Node>::setupRemote() {
00332     const Map<LocalOrdinal,GlobalOrdinal,Node> & target = *getTargetMap();
00333 
00334     // make sure export IDs are ordered by image
00335     // sort exportImageIDs_ in ascending order,
00336     // and apply the same permutation to exportGIDs_ and exportLIDs_.
00337     sort3(ExportData_->exportImageIDs_.begin(), ExportData_->exportImageIDs_.end(), ExportData_->exportGIDs_.begin(), ExportData_->exportLIDs_.begin());
00338 
00339     // Construct list of entries that calling image needs to send as a result
00340     // of everyone asking for what it needs to receive.
00341     size_t numRemoteIDs;
00342     numRemoteIDs = ExportData_->distributor_.createFromSends(ExportData_->exportImageIDs_());
00343 
00344     // Use comm plan with ExportGIDs to find out who is sending to us and
00345     // get proper ordering of GIDs for remote entries 
00346     // (these will be converted to LIDs when done).
00347     Teuchos::Array<GlobalOrdinal> remoteGIDs(numRemoteIDs);
00348     ExportData_->distributor_.doPostsAndWaits(ExportData_->exportGIDs_().getConst(),1,remoteGIDs());
00349 
00350     // Remote IDs come in as GIDs, convert to LIDs
00351     ExportData_->remoteLIDs_.resize(numRemoteIDs);
00352     {
00353       typename Teuchos::Array<GlobalOrdinal>::const_iterator i = remoteGIDs.begin();
00354       typename Teuchos::Array<LocalOrdinal>::iterator       j = ExportData_->remoteLIDs_.begin();
00355       while (i != remoteGIDs.end()) 
00356       {
00357         *j++ = target.getLocalElement(*i++);
00358       }
00359     }
00360   }
00361 
00362 } // namespace Tpetra
00363 
00364 #endif // TPETRA_EXPORT_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines