Tpetra Matrix/Vector Services Version of the Day
Tpetra_DistObject_def.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 //
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ************************************************************************
00040 // @HEADER
00041 
00042 #ifndef TPETRA_DISTOBJECT_DEF_HPP
00043 #define TPETRA_DISTOBJECT_DEF_HPP
00044 
00045 #include "Tpetra_ConfigDefs.hpp"
00046 #include "Tpetra_Map.hpp"
00047 #include "Tpetra_Import.hpp"
00048 #include "Tpetra_Export.hpp"
00049 #include "Tpetra_Distributor.hpp"
00050 
00051 #ifdef DOXYGEN_USE_ONLY
00052 #  include "Tpetra_DistObject_decl.hpp"
00053 #endif // DOXYGEN_USE_ONLY
00054 
00055 
00056 namespace Tpetra {
00057   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00058   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00059   DistObject (const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& map)
00060     : map_ (map)
00061   {
00062 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00063     using Teuchos::RCP;
00064     using Teuchos::Time;
00065     using Teuchos::TimeMonitor;
00066 
00067     RCP<Time> doXferTimer =
00068       TimeMonitor::lookupCounter ("Tpetra::DistObject::doTransfer");
00069     if (doXferTimer.is_null ()) {
00070       doXferTimer =
00071         TimeMonitor::getNewCounter ("Tpetra::DistObject::doTransfer");
00072     }
00073     doXferTimer_ = doXferTimer;
00074 
00075     RCP<Time> copyAndPermuteTimer =
00076       TimeMonitor::lookupCounter ("Tpetra::DistObject::copyAndPermute");
00077     if (copyAndPermuteTimer.is_null ()) {
00078       copyAndPermuteTimer =
00079         TimeMonitor::getNewCounter ("Tpetra::DistObject::copyAndPermute");
00080     }
00081     copyAndPermuteTimer_ = copyAndPermuteTimer;
00082 
00083     RCP<Time> packAndPrepareTimer =
00084       TimeMonitor::lookupCounter ("Tpetra::DistObject::packAndPrepare");
00085     if (packAndPrepareTimer.is_null ()) {
00086       packAndPrepareTimer =
00087         TimeMonitor::getNewCounter ("Tpetra::DistObject::packAndPrepare");
00088     }
00089     packAndPrepareTimer_ = packAndPrepareTimer;
00090 
00091     RCP<Time> doPostsAndWaitsTimer =
00092       TimeMonitor::lookupCounter ("Tpetra::DistObject::doPostsAndWaits");
00093     if (doPostsAndWaitsTimer.is_null ()) {
00094       doPostsAndWaitsTimer =
00095         TimeMonitor::getNewCounter ("Tpetra::DistObject::doPostsAndWaits");
00096     }
00097     doPostsAndWaitsTimer_ = doPostsAndWaitsTimer;
00098 
00099     RCP<Time> unpackAndCombineTimer =
00100       TimeMonitor::lookupCounter ("Tpetra::DistObject::unpackAndCombine");
00101     if (unpackAndCombineTimer.is_null ()) {
00102       unpackAndCombineTimer =
00103         TimeMonitor::getNewCounter ("Tpetra::DistObject::unpackAndCombine");
00104     }
00105     unpackAndCombineTimer_ = unpackAndCombineTimer;
00106 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00107   }
00108 
00109   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00110   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00111   DistObject (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& rhs)
00112     : map_ (rhs.map_)
00113   {}
00114 
00115   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00116   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::~DistObject()
00117   {}
00118 
00119   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00120   std::string
00121   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::description () const
00122   {
00123     using Teuchos::TypeNameTraits;
00124 
00125     std::ostringstream os;
00126     os << "Tpetra::DistObject<"
00127        << TypeNameTraits<Packet>::name ()
00128        << ", " << TypeNameTraits<LocalOrdinal>::name ()
00129        << ", " << TypeNameTraits<GlobalOrdinal>::name ()
00130        << ", " << TypeNameTraits<Node>::name ()
00131        << ">";
00132     return os.str ();
00133   }
00134 
00135   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00136   void
00137   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00138   describe (Teuchos::FancyOStream &out,
00139             const Teuchos::EVerbosityLevel verbLevel) const
00140   {
00141     using Teuchos::rcpFromRef;
00142     using std::endl;
00143 
00144     const Teuchos::EVerbosityLevel vl = (verbLevel == Teuchos::VERB_DEFAULT) ?
00145       Teuchos::VERB_LOW : verbLevel;
00146 
00147     if (vl != Teuchos::VERB_NONE) {
00148       out << this->description () << endl;
00149       Teuchos::OSTab tab (rcpFromRef (out));
00150       out << "Export buffer size (in packets): " << exports_.size() << endl
00151           << "Import buffer size (in packets): " << imports_.size() << endl
00152           << "Map over which this object is distributed:" << endl;
00153       map_->describe (out, vl);
00154     }
00155   }
00156 
00157   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00158   void
00159   DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>::
00160   removeEmptyProcessesInPlace (const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& newMap)
00161   {
00162     TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error,
00163       "Tpetra::DistObject::removeEmptyProcessesInPlace: Not implemented");
00164   }
00165 
00166   template<class DistObjectType>
00167   void
00168   removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
00169                                const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
00170                                                             typename DistObjectType::global_ordinal_type,
00171                                                             typename DistObjectType::node_type> >& newMap)
00172   {
00173     input->removeEmptyProcessesInPlace (newMap);
00174     if (newMap.is_null ()) { // my process is excluded
00175       input = Teuchos::null;
00176     }
00177   }
00178 
00179   template<class DistObjectType>
00180   void
00181   removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input)
00182   {
00183     using Teuchos::RCP;
00184     typedef typename DistObjectType::local_ordinal_type LO;
00185     typedef typename DistObjectType::global_ordinal_type GO;
00186     typedef typename DistObjectType::node_type NT;
00187     typedef Map<LO, GO, NT> map_type;
00188 
00189     RCP<const map_type> newMap = input->getMap ()->removeEmptyProcesses ();
00190     removeEmptyProcessesInPlace<DistObjectType> (input, newMap);
00191   }
00192 
00193   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00194   void
00195   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00196   doImport (const SrcDistObject& source,
00197             const Import<LocalOrdinal,GlobalOrdinal,Node>& importer,
00198             CombineMode CM)
00199   {
00200     TEUCHOS_TEST_FOR_EXCEPTION(*getMap() != *importer.getTargetMap(),
00201       std::invalid_argument, "doImport: The target DistObject's Map is not "
00202       "identical to the Import's target Map.");
00203 #ifdef HAVE_TPETRA_DEBUG
00204     {
00205       typedef DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> this_type;
00206       const this_type* srcDistObj = dynamic_cast<const this_type*> (&source);
00207       TEUCHOS_TEST_FOR_EXCEPTION(
00208         srcDistObj != NULL && * (srcDistObj->getMap ()) != *importer.getSourceMap(),
00209         std::invalid_argument, "doImport: The source is a DistObject, yet its "
00210         "Map is not identical to the Import's source Map.");
00211     }
00212 #endif // HAVE_TPETRA_DEBUG
00213     size_t numSameIDs = importer.getNumSameIDs ();
00214 
00215     typedef Teuchos::ArrayView<const LocalOrdinal> view_type;
00216     const view_type exportLIDs      = importer.getExportLIDs();
00217     const view_type remoteLIDs      = importer.getRemoteLIDs();
00218     const view_type permuteToLIDs   = importer.getPermuteToLIDs();
00219     const view_type permuteFromLIDs = importer.getPermuteFromLIDs();
00220     this->doTransfer (source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs,
00221                       remoteLIDs, exportLIDs, importer.getDistributor (),
00222                       DoForward);
00223   }
00224 
00225   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00226   void
00227   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00228   doExport (const SrcDistObject& source,
00229             const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter,
00230             CombineMode CM)
00231   {
00232     TEUCHOS_TEST_FOR_EXCEPTION(
00233       *getMap() != *exporter.getTargetMap(), std::invalid_argument,
00234       "doExport: The target DistObject's Map is not identical to the Export's "
00235       "target Map.");
00236 #ifdef HAVE_TPETRA_DEBUG
00237     {
00238       typedef DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> this_type;
00239       const this_type* srcDistObj = dynamic_cast<const this_type*> (&source);
00240       TEUCHOS_TEST_FOR_EXCEPTION(
00241         srcDistObj != NULL && * (srcDistObj->getMap ()) != *exporter.getSourceMap(),
00242         std::invalid_argument, "doExport: The source is a DistObject, yet its "
00243         "Map is not identical to the Export's source Map.");
00244     }
00245 #endif // HAVE_TPETRA_DEBUG
00246     size_t numSameIDs = exporter.getNumSameIDs();
00247 
00248     typedef ArrayView<const LocalOrdinal> view_type;
00249     view_type exportLIDs      = exporter.getExportLIDs();
00250     view_type remoteLIDs      = exporter.getRemoteLIDs();
00251     view_type permuteToLIDs   = exporter.getPermuteToLIDs();
00252     view_type permuteFromLIDs = exporter.getPermuteFromLIDs();
00253     doTransfer (source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs,
00254                 exportLIDs, exporter.getDistributor (), DoForward);
00255   }
00256 
00257   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00258   void
00259   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00260   doImport (const SrcDistObject& source,
00261             const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter,
00262             CombineMode CM)
00263   {
00264     TEUCHOS_TEST_FOR_EXCEPTION(
00265       *getMap() != *exporter.getSourceMap(), std::invalid_argument,
00266       "doImport (reverse mode): The target DistObject's Map is not identical "
00267       "to the Export's source Map.");
00268 #ifdef HAVE_TPETRA_DEBUG
00269     {
00270       typedef DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> this_type;
00271       const this_type* srcDistObj = dynamic_cast<const this_type*> (&source);
00272       TEUCHOS_TEST_FOR_EXCEPTION(
00273         srcDistObj != NULL && * (srcDistObj->getMap ()) != *exporter.getTargetMap(),
00274         std::invalid_argument,
00275         "doImport (reverse mode): The source is a DistObject, yet its "
00276         "Map is not identical to the Export's target Map.");
00277     }
00278 #endif // HAVE_TPETRA_DEBUG
00279     size_t numSameIDs = exporter.getNumSameIDs();
00280 
00281     typedef ArrayView<const LocalOrdinal> view_type;
00282     view_type exportLIDs      = exporter.getRemoteLIDs();
00283     view_type remoteLIDs      = exporter.getExportLIDs();
00284     view_type permuteToLIDs   = exporter.getPermuteFromLIDs();
00285     view_type permuteFromLIDs = exporter.getPermuteToLIDs();
00286     doTransfer (source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs,
00287                 exportLIDs, exporter.getDistributor (), DoReverse);
00288   }
00289 
00290   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00291   void
00292   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00293   doExport (const SrcDistObject& source,
00294             const Import<LocalOrdinal,GlobalOrdinal,Node> & importer,
00295             CombineMode CM)
00296   {
00297     TEUCHOS_TEST_FOR_EXCEPTION(
00298       *getMap() != *importer.getSourceMap(), std::invalid_argument,
00299       "doExport (reverse mode): The target object's Map "
00300       "is not identical to the Import's source Map.");
00301 #ifdef HAVE_TPETRA_DEBUG
00302     {
00303       typedef DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> this_type;
00304       const this_type* srcDistObj = dynamic_cast<const this_type*> (&source);
00305       TEUCHOS_TEST_FOR_EXCEPTION(
00306         srcDistObj != NULL && * (srcDistObj->getMap ()) != *importer.getTargetMap(),
00307         std::invalid_argument,
00308         "doExport (reverse mode): The source is a DistObject, yet its "
00309         "Map is not identical to the Import's target Map.");
00310     }
00311 #endif // HAVE_TPETRA_DEBUG
00312     size_t numSameIDs = importer.getNumSameIDs();
00313 
00314     typedef ArrayView<const LocalOrdinal> view_type;
00315     view_type exportLIDs      = importer.getRemoteLIDs();
00316     view_type remoteLIDs      = importer.getExportLIDs();
00317     view_type permuteToLIDs   = importer.getPermuteFromLIDs();
00318     view_type permuteFromLIDs = importer.getPermuteToLIDs();
00319     doTransfer (source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs,
00320                 exportLIDs, importer.getDistributor (), DoReverse);
00321   }
00322 
00323   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00324   bool
00325   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::isDistributed() const {
00326     return map_->isDistributed ();
00327   }
00328 
00329   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00330   size_t
00331   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00332   constantNumberOfPackets () const {
00333     return 0; // default implementation; subclasses may override
00334   }
00335 
00336   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00337   void
00338   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00339   doTransfer (const SrcDistObject& src,
00340               CombineMode CM,
00341               size_t numSameIDs,
00342               const Teuchos::ArrayView<const LocalOrdinal>& permuteToLIDs,
00343               const Teuchos::ArrayView<const LocalOrdinal>& permuteFromLIDs,
00344               const Teuchos::ArrayView<const LocalOrdinal>& remoteLIDs,
00345               const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
00346               Distributor &distor,
00347               ReverseOption revOp)
00348   {
00349     using Teuchos::as;
00350 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00351     Teuchos::TimeMonitor doXferMon (*doXferTimer_);
00352 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00353 
00354     TEUCHOS_TEST_FOR_EXCEPTION(
00355       ! checkSizes (src), std::invalid_argument,
00356       "Tpetra::DistObject::doTransfer(): checkSizes() indicates that the "
00357       "destination object is not a legal target for redistribution from the "
00358       "source object.  This probably means that they do not have the same "
00359       "dimensions.  For example, MultiVectors must have the same number of "
00360       "rows and columns.");
00361     KokkosClassic::ReadWriteOption rwo = KokkosClassic::ReadWrite;
00362     if (CM == INSERT || CM == REPLACE) {
00363       const size_t numIDsToWrite = numSameIDs +
00364         as<size_t> (permuteToLIDs.size ()) +
00365         as<size_t> (remoteLIDs.size ());
00366       if (numIDsToWrite == this->getMap ()->getNodeNumElements ()) {
00367         // We're overwriting all of our local data in the destination
00368         // object, so a write-only view suffices.
00369         //
00370         // FIXME (mfh 10 Apr 2012) This doesn't make sense for a
00371         // CrsMatrix with a dynamic graph.  INSERT mode could mean
00372         // that we're adding new entries to the object, but we don't
00373         // want to get rid of the old ones.
00374         rwo = KokkosClassic::WriteOnly;
00375       }
00376     }
00377     // Tell the source to create a read-only view of its data.  On a
00378     // discrete accelerator such as a GPU, this brings EVERYTHING from
00379     // device memory to host memory.
00380     //
00381     // FIXME (mfh 23 Mar 2012) By passing in the list of GIDs (or
00382     // rather, local LIDs to send) and packet counts, createViews()
00383     // could create a "sparse view" that only brings in the necessary
00384     // data from device to host memory.
00385     typedef DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> this_type;
00386     const this_type* srcDistObj = dynamic_cast<const this_type*> (&src);
00387     if (srcDistObj != NULL) {
00388       srcDistObj->createViews ();
00389     }
00390 
00391     // Tell the target to create a view of its data.  Depending on
00392     // rwo, this could be a write-only view or a read-and-write view.
00393     // On a discrete accelerator such as a GPU, a write-only view only
00394     // requires a transfer from host to device memory.  A
00395     // read-and-write view requires a two-way transfer.  This has the
00396     // same problem as createViews(): it transfers EVERYTHING, not
00397     // just the necessary data.
00398     //
00399     // FIXME (mfh 23 Mar 2012) By passing in the list of GIDs (or
00400     // rather, local LIDs into which to receive) and packet counts,
00401     // createViewsNonConst() could create a "sparse view" that only
00402     // transfers the necessary data.
00403     this->createViewsNonConst (rwo);
00404 
00405     if (numSameIDs + permuteToLIDs.size()) {
00406 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00407       Teuchos::TimeMonitor copyAndPermuteMon (*copyAndPermuteTimer_);
00408 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00409       // There is at least one GID to copy or permute.
00410       copyAndPermute (src, numSameIDs, permuteToLIDs, permuteFromLIDs);
00411     }
00412 
00413     // The method may return zero even if the implementation actually
00414     // does have a constant number of packets per LID.  However, if it
00415     // returns nonzero, we may use this information to avoid
00416     // (re)allocating num{Ex,Im}portPacketsPerLID_.  packAndPrepare()
00417     // will set this to its final value.
00418     //
00419     // We only need this if CM != ZERO, but it has to be lifted out of
00420     // that scope because there are multiple tests for CM != ZERO.
00421     size_t constantNumPackets = this->constantNumberOfPackets ();
00422 
00423     // We only need to pack communication buffers if the combine mode
00424     // is not ZERO. A "ZERO combine mode" means that the results are
00425     // the same as if we had received all zeros, and added them to the
00426     // existing values. That means we don't need to communicate.
00427     if (CM != ZERO) {
00428       if (constantNumPackets == 0) {
00429         numExportPacketsPerLID_.resize (exportLIDs.size ());
00430         numImportPacketsPerLID_.resize (remoteLIDs.size ());
00431       }
00432 
00433       {
00434 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00435         Teuchos::TimeMonitor packAndPrepareMon (*packAndPrepareTimer_);
00436 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00437         // Ask the source to pack data.  Also ask it whether there are a
00438         // constant number of packets per element (constantNumPackets is
00439         // an output argument).  If there are, constantNumPackets will
00440         // come back nonzero.  Otherwise, the source will fill the
00441         // numExportPacketsPerLID_ array.
00442         packAndPrepare (src, exportLIDs, exports_, numExportPacketsPerLID_ (),
00443                         constantNumPackets, distor);
00444       }
00445     }
00446 
00447     // We don't need the source's data anymore, so it can let go of
00448     // its views.  On an accelerator device with a separate memory
00449     // space (like a GPU), this frees host memory, since device memory
00450     // has the "master" version of the data.
00451     if (srcDistObj != NULL) {
00452       srcDistObj->releaseViews ();
00453     }
00454 
00455     // We only need to send data if the combine mode is not ZERO.
00456     if (CM != ZERO) {
00457       if (constantNumPackets != 0) {
00458         // There are a constant number of packets per element.  We
00459         // already know (from the number of "remote" (incoming)
00460         // elements) how many incoming elements we expect, so we can
00461         // resize the buffer accordingly.
00462         const size_t rbufLen = remoteLIDs.size() * constantNumPackets;
00463         if (as<size_t> (imports_.size()) != rbufLen) {
00464           imports_.resize (rbufLen);
00465         }
00466       }
00467 
00468       // Do we need to do communication (via doPostsAndWaits)?
00469       bool needCommunication = true;
00470       if (revOp == DoReverse && ! isDistributed ()) {
00471         needCommunication = false;
00472       }
00473       // FIXME (mfh 30 Jun 2013): Checking whether the source object
00474       // is distributed requires a cast to DistObject.  If it's not a
00475       // DistObject, then I'm not quite sure what to do.  Perhaps it
00476       // would be more appropriate for SrcDistObject to have an
00477       // isDistributed() method.  For now, I'll just assume that we
00478       // need to do communication unless the cast succeeds and the
00479       // source is not distributed.
00480       else if (revOp == DoForward && srcDistObj != NULL &&
00481                ! srcDistObj->isDistributed ()) {
00482         needCommunication = false;
00483       }
00484 
00485       if (needCommunication) {
00486         if (revOp == DoReverse) {
00487 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00488           Teuchos::TimeMonitor doPostsAndWaitsMon (*doPostsAndWaitsTimer_);
00489 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00490           if (constantNumPackets == 0) { //variable num-packets-per-LID:
00491             distor.doReversePostsAndWaits (numExportPacketsPerLID_().getConst(),
00492                                            1,
00493                                            numImportPacketsPerLID_());
00494             size_t totalImportPackets = 0;
00495             for (Array_size_type i = 0; i < numImportPacketsPerLID_.size(); ++i) {
00496               totalImportPackets += numImportPacketsPerLID_[i];
00497             }
00498             imports_.resize(totalImportPackets);
00499             distor.doReversePostsAndWaits (exports_().getConst(),
00500                                            numExportPacketsPerLID_(),
00501                                            imports_(),
00502                                            numImportPacketsPerLID_());
00503           }
00504           else {
00505             distor.doReversePostsAndWaits (exports_ ().getConst (),
00506                                            constantNumPackets,
00507                                            imports_ ());
00508           }
00509         }
00510         else { // revOp == DoForward
00511 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00512           Teuchos::TimeMonitor doPostsAndWaitsMon (*doPostsAndWaitsTimer_);
00513 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00514           if (constantNumPackets == 0) { //variable num-packets-per-LID:
00515             distor.doPostsAndWaits (numExportPacketsPerLID_().getConst(), 1,
00516                                     numImportPacketsPerLID_());
00517             size_t totalImportPackets = 0;
00518             for (Array_size_type i = 0; i < numImportPacketsPerLID_.size(); ++i) {
00519               totalImportPackets += numImportPacketsPerLID_[i];
00520             }
00521             imports_.resize(totalImportPackets);
00522             distor.doPostsAndWaits (exports_().getConst(),
00523                                     numExportPacketsPerLID_(),
00524                                     imports_(),
00525                                     numImportPacketsPerLID_());
00526           }
00527           else {
00528             distor.doPostsAndWaits (exports_ ().getConst (),
00529                                     constantNumPackets,
00530                                     imports_ ());
00531           }
00532         }
00533         {
00534 #ifdef HAVE_TPETRA_TRANSFER_TIMERS
00535           Teuchos::TimeMonitor unpackAndCombineMon (*unpackAndCombineTimer_);
00536 #endif // HAVE_TPETRA_TRANSFER_TIMERS
00537           unpackAndCombine (remoteLIDs, imports_(), numImportPacketsPerLID_(),
00538                             constantNumPackets, distor, CM);
00539         }
00540       }
00541     } // if (CM != ZERO)
00542 
00543     this->releaseViews ();
00544   }
00545 
00546   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00547   void
00548   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::print (std::ostream &os) const
00549   {
00550     using Teuchos::FancyOStream;
00551     using Teuchos::getFancyOStream;
00552     using Teuchos::RCP;
00553     using Teuchos::rcpFromRef;
00554     using std::endl;
00555 
00556     RCP<FancyOStream> out = getFancyOStream (rcpFromRef (os));
00557     this->describe (*out, Teuchos::VERB_DEFAULT);
00558   }
00559 
00560   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00561   void
00562   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::createViews () const
00563   {}
00564 
00565   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00566   void
00567   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00568   createViewsNonConst (KokkosClassic::ReadWriteOption /*rwo*/)
00569   {}
00570 
00571   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00572   void
00573   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00574   releaseViews () const
00575   {}
00576 
00577 #define TPETRA_DISTOBJECT_INSTANT(SCALAR, LO, GO, NODE) \
00578   \
00579   template class DistObject< SCALAR , LO , GO , NODE >;
00580 
00581  // The "SLGN" stuff above doesn't work for Packet=char.
00582 #define TPETRA_DISTOBJECT_INSTANT_CHAR(LO, GO, NODE) \
00583   \
00584   template class DistObject< char , LO , GO , NODE >;
00585 
00586 
00587 } // namespace Tpetra
00588 
00589 // Include KokkosRefactor partial specialisation if enabled
00590 #if defined(TPETRA_HAVE_KOKKOS_REFACTOR)
00591 #include "Tpetra_KokkosRefactor_DistObject_def.hpp"
00592 #endif
00593 
00594 #endif /* TPETRA_DISTOBJECT_DEF_HPP */
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines