Tpetra Matrix/Vector Services Version of the Day
Tpetra_Distributor.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_DISTRIBUTOR_HPP
00030 #define TPETRA_DISTRIBUTOR_HPP
00031 
00032 #include "Tpetra_Util.hpp"
00033 #include <Teuchos_as.hpp>
00034 #include <Teuchos_Describable.hpp>
00035 
00036 
00037 // FINISH: some of the get accessors may not be necessary anymore. clean up.
00038 // FINISH: This class may not be const correct. doPosts() et al. perhaps should be const, with affected members made mutable.
00039 
00040 namespace Tpetra {
00041 
00043 
00048   class Distributor : public Teuchos::Describable {
00049   public:
00050 
00052 
00053 
00055     explicit Distributor(const Teuchos::RCP<const Teuchos::Comm<int> > & comm);
00056 
00058     Distributor(const Distributor &distributor);
00059 
00061     ~Distributor();
00062 
00064 
00065 
00067 
00068 
00070 
00087     size_t createFromSends(const Teuchos::ArrayView<const int> &exportNodeIDs);
00088 
00090 
00110     template <class Ordinal>
00111     void createFromRecvs(const Teuchos::ArrayView<const Ordinal> &remoteIDs, 
00112                          const Teuchos::ArrayView<const int> &remoteNodeIDs, 
00113                                Teuchos::ArrayRCP<Ordinal> &exportIDs, 
00114                                Teuchos::ArrayRCP<int> &exportNodeIDs);
00115 
00117 
00119 
00120 
00122     size_t getNumReceives() const;
00123 
00125     size_t getNumSends() const;
00126 
00128 
00129     bool hasSelfMessage() const;
00130 
00132     size_t getMaxSendLength() const;
00133 
00135     size_t getTotalReceiveLength() const;
00136 
00138     Teuchos::ArrayView<const int> getImagesFrom() const;
00139 
00141     Teuchos::ArrayView<const int> getImagesTo() const;
00142 
00144 
00145     Teuchos::ArrayView<const size_t> getLengthsFrom() const;
00146 
00148 
00149     Teuchos::ArrayView<const size_t> getLengthsTo() const;
00150 
00152 
00154 
00155 
00157 
00160     const Teuchos::RCP<Distributor> & getReverse() const;
00161 
00163 
00165 
00166 
00168 
00179     template <class Packet>
00180     void doPostsAndWaits(const Teuchos::ArrayView<const Packet> &exports,
00181                          size_t numPackets,
00182                          const Teuchos::ArrayView<Packet> &imports);
00183 
00185 
00196     template <class Packet>
00197     void doPostsAndWaits(const Teuchos::ArrayView<const Packet> &exports,
00198                          const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00199                          const Teuchos::ArrayView<Packet> &imports,
00200                          const Teuchos::ArrayView<size_t> &numImportPacketsPerLID);
00201 
00203 
00214     template <class Packet>
00215     void doPosts(const Teuchos::ArrayView<const Packet> &exports,
00216                  size_t numPackets,
00217                  const Teuchos::ArrayRCP<Packet> &imports);
00218 
00220 
00231     template <class Packet>
00232     void doPosts(const Teuchos::ArrayView<const Packet> &exports,
00233                  const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00234                  const Teuchos::ArrayRCP<Packet> &imports,
00235                  const Teuchos::ArrayView<size_t> &numImportPacketsPerLID);
00236 
00238     void doWaits();
00239 
00241 
00252     template <class Packet>
00253     void doReversePostsAndWaits(const Teuchos::ArrayView<const Packet> &exports,
00254                                 size_t numPackets,
00255                                 const Teuchos::ArrayView<Packet> &imports);
00256 
00258 
00269     template <class Packet>
00270     void doReversePostsAndWaits(const Teuchos::ArrayView<const Packet> &exports,
00271                                 const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00272                                 const Teuchos::ArrayView<Packet> &imports,
00273                                 const Teuchos::ArrayView<size_t> &numImportPacketsPerLID);
00274 
00276 
00287     template <class Packet>
00288     void doReversePosts(const Teuchos::ArrayView<const Packet> &exports,
00289                         size_t numPackets,
00290                         const Teuchos::ArrayRCP<Packet> &imports);
00291 
00293 
00304     template <class Packet>
00305     void doReversePosts(const Teuchos::ArrayView<const Packet> &exports,
00306                         const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00307                         const Teuchos::ArrayRCP<Packet> &imports,
00308                         const Teuchos::ArrayView<size_t> &numImportPacketsPerLID);
00309 
00311     void doReverseWaits();
00312 
00314 
00316 
00317 
00319     std::string description() const;
00320 
00322     void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const;
00323 
00325 
00326   private:
00327 
00328     // private data members
00329     Teuchos::RCP<const Teuchos::Comm<int> > comm_;
00330 
00331     size_t numExports_;
00332     // selfMessage_ is whether I have a send for myself
00333     bool selfMessage_;
00334     // numSends_ is number of sends to other nodes; is less than or equal to the number of nodes
00335     size_t numSends_;
00336     // imagesTo_, startsTo_ and lengthsTo_ each have size 
00337     //   numSends_ + selfMessage_
00338     Teuchos::Array<int> imagesTo_;
00339     /* Given an export buffer that contains all of the item being sent by this node,
00340        the block of values for node i will start at position startsTo_[i]  */
00341     Teuchos::Array<size_t> startsTo_;
00342     Teuchos::Array<size_t> lengthsTo_;
00343     // maxSendLength_ is the maximum send to another node: 
00344     //   max(lengthsTo_[i]) for i != me
00345     size_t maxSendLength_;
00346     Teuchos::Array<size_t> indicesTo_;
00347     // numReceives_ is the number of receives by me from other procs, not
00348     // counting self receives
00349     size_t numReceives_;
00350     // totalReceiveLength_ is the total number of Packet received, used to 
00351     // allocate the receive buffer
00352     size_t totalReceiveLength_;
00353     // imagesFrom_, startsFrom_ and lengthsFrom_ each have size 
00354     //   numReceives_ + selfMessage_
00355     Teuchos::Array<size_t> lengthsFrom_;
00356     Teuchos::Array<int> imagesFrom_;
00357     Teuchos::Array<size_t> startsFrom_;
00358     Teuchos::Array<size_t> indicesFrom_;
00359 
00360     // requests associated with non-blocking receives
00361     Teuchos::Array<Teuchos::RCP<Teuchos::CommRequest> > requests_;
00362 
00363     mutable Teuchos::RCP<Distributor> reverseDistributor_;
00364 
00365     // compute receive info from sends
00366     void computeReceives();
00367 
00368     // compute send info from receives
00369     template <class Ordinal>
00370     void computeSends(const Teuchos::ArrayView<const Ordinal> &importIDs,
00371                       const Teuchos::ArrayView<const int> &importNodeIDs,
00372                             Teuchos::ArrayRCP<Ordinal> &exportIDs,
00373                             Teuchos::ArrayRCP<int> &exportNodeIDs);
00374 
00375     // create a distributor for the reverse communciation pattern (pretty much
00376     // swap all send and receive info)
00377     void createReverseDistributor() const;
00378 
00379   }; // class Distributor
00380 
00381 
00382   template <class Packet>
00383   void Distributor::doPostsAndWaits(
00384       const Teuchos::ArrayView<const Packet>& exports,
00385       size_t numPackets,
00386       const Teuchos::ArrayView<Packet>& imports) 
00387   {
00388     TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
00389         Teuchos::typeName(*this) << "::doPostsAndWaits(): Cannot call with outstanding posts.");
00390     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00391     // however, it need only persist until doWaits is called, so it is safe for us to 
00392     // use a non-persisting reference in this case
00393     doPosts(exports, numPackets, Teuchos::arcp<Packet>(imports.getRawPtr(),0,imports.size(),false));
00394     doWaits();
00395   }
00396 
00397   template <class Packet>
00398   void Distributor::doPostsAndWaits(
00399       const Teuchos::ArrayView<const Packet>& exports,
00400       const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00401       const Teuchos::ArrayView<Packet> &imports,
00402       const Teuchos::ArrayView<size_t> &numImportPacketsPerLID)
00403   {
00404     TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
00405         Teuchos::typeName(*this) << "::doPostsAndWaits(): Cannot call with outstanding posts.");
00406     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00407     // however, it need only persist until doWaits is called, so it is safe for us to 
00408     // use a non-persisting reference in this case
00409     doPosts(exports, numExportPacketsPerLID, Teuchos::arcp<Packet>(imports.getRawPtr(),0,imports.size(),false), numImportPacketsPerLID);
00410     doWaits();
00411   }
00412 
00413 
00414   template <class Packet>
00415   void Distributor::doPosts(const Teuchos::ArrayView<const Packet>& exports,
00416                             size_t numPackets,
00417                             const Teuchos::ArrayRCP<Packet>& imports) {
00418     using Teuchos::ArrayRCP;
00419     // start of actual doPosts function
00420     const int myImageID = comm_->getRank();
00421     size_t selfReceiveOffset = 0;
00422 
00423 #ifdef HAVE_TEUCHOS_DEBUG
00424     TEST_FOR_EXCEPTION(Teuchos::as<size_t>(imports.size()) != totalReceiveLength_ * numPackets, std::runtime_error,
00425         Teuchos::typeName(*this) << "::doPosts(): imports must be large enough to store the imported data.");
00426 #endif
00427 
00428     // allocate space in requests
00429     requests_.resize(0);
00430     requests_.reserve(numReceives_);
00431 
00432     // start up the Irecv's
00433     {
00434       size_t curBufferOffset = 0;
00435       for (size_t i = 0; i < numReceives_ + (selfMessage_ ? 1 : 0); ++i) {
00436         if (imagesFrom_[i] != myImageID) { 
00437           // receiving this one from another image
00438           // setup reference into imports of the appropriate size and at the appropriate place
00439           ArrayRCP<Packet> impptr = imports.persistingView(curBufferOffset,lengthsFrom_[i]*numPackets);
00440           requests_.push_back( Teuchos::ireceive<int,Packet>(*comm_,impptr,imagesFrom_[i]) );
00441         }
00442         else {
00443           // receiving this one from myself 
00444           // note that offset
00445           selfReceiveOffset = curBufferOffset;
00446         }
00447         curBufferOffset += lengthsFrom_[i]*numPackets;
00448       }
00449     }
00450 
00451     // wait for everyone else before posting ready-sends below to ensure that 
00452     // all non-blocking receives above have been posted
00453     Teuchos::barrier(*comm_);
00454 
00455     // setup scan through imagesTo_ list starting with higher numbered images
00456     // (should help balance message traffic)
00457     size_t numBlocks = numSends_+ selfMessage_;
00458     size_t imageIndex = 0;
00459     while ((imageIndex < numBlocks) && (imagesTo_[imageIndex] < myImageID)) {
00460       ++imageIndex;
00461     }
00462     if (imageIndex == numBlocks) {
00463       imageIndex = 0;
00464     }
00465 
00466     size_t selfNum = 0;
00467     size_t selfIndex = 0;
00468 
00469     if (indicesTo_.empty()) { // data is already blocked by processor
00470       for (size_t i = 0; i < numBlocks; ++i) {
00471         size_t p = i + imageIndex;
00472         if (p > (numBlocks - 1)) {
00473           p -= numBlocks;
00474         }
00475 
00476         if (imagesTo_[p] != myImageID) {
00477           // sending it to another image
00478           Teuchos::ArrayView<const Packet> tmpSend(&exports[startsTo_[p]*numPackets],lengthsTo_[p]*numPackets);
00479           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00480         }
00481         else {
00482           // sending it to ourself
00483           selfNum = p;
00484         }
00485       }
00486 
00487       if (selfMessage_) {
00488         std::copy(exports.begin()+startsTo_[selfNum]*numPackets, exports.begin()+startsTo_[selfNum]*numPackets+lengthsTo_[selfNum]*numPackets, 
00489                   imports.begin()+selfReceiveOffset);
00490       }
00491     }
00492     else { // data is not blocked by image, use send buffer
00493       // allocate sendArray buffer
00494       Teuchos::Array<Packet> sendArray(maxSendLength_*numPackets); 
00495 
00496       for (size_t i = 0; i < numBlocks; ++i) {
00497         size_t p = i + imageIndex;
00498         if (p > (numBlocks - 1)) {
00499           p -= numBlocks;
00500         }
00501 
00502         if (imagesTo_[p] != myImageID) { 
00503           // sending it to another image
00504           typename Teuchos::ArrayView<const Packet>::iterator srcBegin, srcEnd;
00505           size_t sendArrayOffset = 0;
00506           size_t j = startsTo_[p];
00507           for (size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
00508             srcBegin = exports.begin() + indicesTo_[j]*numPackets;
00509             srcEnd   = srcBegin + numPackets;
00510             std::copy( srcBegin, srcEnd, sendArray.begin()+sendArrayOffset );
00511             sendArrayOffset += numPackets;
00512           }
00513           Teuchos::ArrayView<const Packet> tmpSend = sendArray(0,lengthsTo_[p]*numPackets);
00514           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00515         }
00516         else { 
00517           // sending it to myself
00518           selfNum = p;
00519           selfIndex = startsTo_[p];
00520         }
00521       }
00522 
00523       if (selfMessage_) {
00524         for (size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
00525           std::copy( exports.begin()+indicesTo_[selfIndex]*numPackets,
00526                      exports.begin()+indicesTo_[selfIndex]*numPackets + numPackets,
00527                      imports.begin() + selfReceiveOffset );
00528           ++selfIndex;
00529           selfReceiveOffset += numPackets;
00530         }
00531       }
00532     }
00533   }
00534 
00535   template <class Packet>
00536   void Distributor::doPosts(const Teuchos::ArrayView<const Packet>& exports,
00537                             const Teuchos::ArrayView<size_t>& numExportPacketsPerLID,
00538                             const Teuchos::ArrayRCP<Packet>& imports,
00539                             const Teuchos::ArrayView<size_t>& numImportPacketsPerLID) {
00540     using Teuchos::ArrayRCP;
00541     // start of actual doPosts function
00542     const int myImageID = comm_->getRank();
00543     size_t selfReceiveOffset = 0;
00544 
00545 #ifdef HAVE_TEUCHOS_DEBUG
00546     size_t totalNumPackets = 0;
00547     for(int ii=0; ii<numImportPacketsPerLID.size(); ++ii) {
00548       totalNumPackets += numImportPacketsPerLID[ii];
00549     }
00550     TEST_FOR_EXCEPTION(Teuchos::as<size_t>(imports.size()) != totalNumPackets, std::runtime_error,
00551         Teuchos::typeName(*this) << "::doPosts(): imports must be large enough to store the imported data.");
00552 #endif
00553 
00554     // allocate space in requests
00555     requests_.resize(0);
00556     requests_.reserve(numReceives_);
00557 
00558     // start up the Irecv's
00559     {
00560       size_t curBufferOffset = 0;
00561       size_t curLIDoffset = 0;
00562       for (size_t i = 0; i < numReceives_ + (selfMessage_ ? 1 : 0); ++i) {
00563         size_t totalPacketsFrom_i = 0;
00564         for(size_t j=0; j<lengthsFrom_[i]; ++j) {
00565           totalPacketsFrom_i += numImportPacketsPerLID[curLIDoffset+j];
00566         }
00567         curLIDoffset += lengthsFrom_[i];
00568         if (imagesFrom_[i] != myImageID) { 
00569           // receiving this one from another image
00570           // setup reference into imports of the appropriate size and at the appropriate place
00571           ArrayRCP<Packet> impptr = imports.persistingView(curBufferOffset,totalPacketsFrom_i);
00572           requests_.push_back( Teuchos::ireceive<int,Packet>(*comm_,impptr,imagesFrom_[i]) );
00573         }
00574         else {
00575           // receiving this one from myself 
00576           // note that offset
00577           selfReceiveOffset = curBufferOffset;
00578         }
00579         curBufferOffset += totalPacketsFrom_i;
00580       }
00581     }
00582 
00583     // wait for everyone else before posting ready-sends below to ensure that 
00584     // all non-blocking receives above have been posted
00585     Teuchos::barrier(*comm_);
00586 
00587     //setup arrays containing starting-offsets into exports for each send,
00588     //and num-packets-to-send for each send.
00589     Teuchos::Array<size_t> sendPacketOffsets(numSends_,0), packetsPerSend(numSends_,0);
00590     size_t maxNumPackets = 0;
00591     size_t curPKToffset = 0;
00592     for(size_t pp=0; pp<numSends_; ++pp) {
00593       sendPacketOffsets[pp] = curPKToffset;
00594       size_t numPackets = 0;
00595       for(size_t j=startsTo_[pp]; j<startsTo_[pp]+lengthsTo_[pp]; ++j) {
00596         numPackets += numExportPacketsPerLID[j];
00597       }
00598       if (numPackets > maxNumPackets) maxNumPackets = numPackets;
00599       packetsPerSend[pp] = numPackets;
00600       curPKToffset += numPackets;
00601     }
00602 
00603     // setup scan through imagesTo_ list starting with higher numbered images
00604     // (should help balance message traffic)
00605     size_t numBlocks = numSends_+ selfMessage_;
00606     size_t imageIndex = 0;
00607     while ((imageIndex < numBlocks) && (imagesTo_[imageIndex] < myImageID)) {
00608       ++imageIndex;
00609     }
00610     if (imageIndex == numBlocks) {
00611       imageIndex = 0;
00612     }
00613 
00614     size_t selfNum = 0;
00615     size_t selfIndex = 0;
00616 
00617     if (indicesTo_.empty()) { // data is already blocked by processor
00618       for (size_t i = 0; i < numBlocks; ++i) {
00619         size_t p = i + imageIndex;
00620         if (p > (numBlocks - 1)) {
00621           p -= numBlocks;
00622         }
00623 
00624         if (imagesTo_[p] != myImageID) {
00625           // sending it to another image
00626           Teuchos::ArrayView<const Packet> tmpSend(&exports[sendPacketOffsets[p]],packetsPerSend[p]);
00627           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00628         }
00629         else {
00630           // sending it to ourself
00631           selfNum = p;
00632         }
00633       }
00634 
00635       if (selfMessage_) {
00636         std::copy(exports.begin()+sendPacketOffsets[selfNum], exports.begin()+sendPacketOffsets[selfNum]+packetsPerSend[selfNum], 
00637                   imports.begin()+selfReceiveOffset);
00638       }
00639     }
00640     else { // data is not blocked by image, use send buffer
00641       // allocate sendArray buffer
00642       Teuchos::Array<Packet> sendArray(maxNumPackets); 
00643       Teuchos::Array<size_t> indicesOffsets(numExportPacketsPerLID.size(),0);
00644       size_t ioffset = 0;
00645       for(int j=0; j<numExportPacketsPerLID.size(); ++j) {
00646         indicesOffsets[j] = ioffset;
00647         ioffset += numExportPacketsPerLID[j];
00648       }
00649 
00650       for (size_t i = 0; i < numBlocks; ++i) {
00651         size_t p = i + imageIndex;
00652         if (p > (numBlocks - 1)) {
00653           p -= numBlocks;
00654         }
00655 
00656         if (imagesTo_[p] != myImageID) { 
00657           // sending it to another image
00658           typename Teuchos::ArrayView<const Packet>::iterator srcBegin, srcEnd;
00659           size_t sendArrayOffset = 0;
00660           size_t j = startsTo_[p];
00661           size_t numPacketsTo_p = 0;
00662           for (size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
00663             srcBegin = exports.begin() + indicesOffsets[j];
00664             srcEnd   = srcBegin + numExportPacketsPerLID[j];
00665             numPacketsTo_p += numExportPacketsPerLID[j];
00666             std::copy( srcBegin, srcEnd, sendArray.begin()+sendArrayOffset );
00667             sendArrayOffset += numExportPacketsPerLID[j];
00668           }
00669           Teuchos::ArrayView<const Packet> tmpSend = sendArray(0,numPacketsTo_p);
00670           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00671         }
00672         else { 
00673           // sending it to myself
00674           selfNum = p;
00675           selfIndex = startsTo_[p];
00676         }
00677       }
00678 
00679       if (selfMessage_) {
00680         for (size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
00681           std::copy( exports.begin()+indicesOffsets[selfIndex],
00682                      exports.begin()+indicesOffsets[selfIndex]+numExportPacketsPerLID[selfIndex],
00683                      imports.begin() + selfReceiveOffset );
00684           selfReceiveOffset += numExportPacketsPerLID[selfIndex];
00685           ++selfIndex;
00686         }
00687       }
00688     }
00689   }
00690 
00691 
00692   template <class Packet>
00693   void Distributor::doReversePostsAndWaits(
00694       const Teuchos::ArrayView<const Packet>& exports,
00695       size_t numPackets,
00696       const Teuchos::ArrayView<Packet>& imports) 
00697   {
00698     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00699     // however, it need only persist until doWaits is called, so it is safe for us to 
00700     // use a non-persisting reference in this case
00701     doReversePosts(exports, numPackets, Teuchos::arcp<Packet>(imports.getRawPtr(),0,imports.size(),false));
00702     doReverseWaits();
00703   }
00704 
00705   template <class Packet>
00706   void Distributor::doReversePostsAndWaits(
00707       const Teuchos::ArrayView<const Packet>& exports,
00708        const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00709        const Teuchos::ArrayView<Packet> &imports,
00710        const Teuchos::ArrayView<size_t> &numImportPacketsPerLID)
00711   {
00712     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00713     // however, it need only persist until doWaits is called, so it is safe for us to 
00714     // use a non-persisting reference in this case
00715     doReversePosts(exports, numExportPacketsPerLID, Teuchos::arcp<Packet>(imports.getRawPtr(),0,imports.size(),false),numImportPacketsPerLID);
00716     doReverseWaits();
00717   }
00718 
00719 
00720   template <class Packet>
00721   void Distributor::doReversePosts(
00722       const Teuchos::ArrayView<const Packet>& exports,
00723       size_t numPackets,
00724       const Teuchos::ArrayRCP<Packet>& imports) 
00725   {
00726     TEST_FOR_EXCEPTION(!indicesTo_.empty(),std::runtime_error,
00727         Teuchos::typeName(*this) << "::doReversePosts(): Can only do reverse comm when original data is blocked by image.");
00728     if (reverseDistributor_ == Teuchos::null) {
00729       createReverseDistributor();
00730     }
00731     reverseDistributor_->doPosts(exports,numPackets,imports);
00732   }
00733 
00734   template <class Packet>
00735   void Distributor::doReversePosts(
00736       const Teuchos::ArrayView<const Packet>& exports,
00737       const Teuchos::ArrayView<size_t>& numExportPacketsPerLID,
00738       const Teuchos::ArrayRCP<Packet>& imports,
00739       const Teuchos::ArrayView<size_t>& numImportPacketsPerLID) 
00740   {
00741     TEST_FOR_EXCEPTION(!indicesTo_.empty(),std::runtime_error,
00742         Teuchos::typeName(*this) << "::doReversePosts(): Can only do reverse comm when original data is blocked by image.");
00743     if (reverseDistributor_ == Teuchos::null) {
00744       createReverseDistributor();
00745     }
00746     reverseDistributor_->doPosts(exports,numExportPacketsPerLID,imports,numImportPacketsPerLID);
00747   }
00748 
00749 
00750   // note: assumes that size_t >= Ordinal
00752   template <class Ordinal>
00753   void Distributor::computeSends(
00754       const Teuchos::ArrayView<const Ordinal> & importIDs,
00755       const Teuchos::ArrayView<const int> & importNodeIDs,
00756             Teuchos::ArrayRCP<Ordinal>& exportIDs,
00757             Teuchos::ArrayRCP<int>& exportNodeIDs)
00758   {
00759     int myImageID = comm_->getRank();
00760 
00761     size_t numImports = importNodeIDs.size();
00762     Teuchos::Array<size_t> importObjs(2*numImports);
00763     for (size_t i = 0; i < numImports; ++i ) {  
00764       importObjs[2*i]   = Teuchos::as<size_t>(importIDs[i]);
00765       importObjs[2*i+1] = Teuchos::as<size_t>(myImageID);
00766     }
00767 
00768     size_t numExports;
00769     Distributor tempPlan(comm_);
00770     numExports = tempPlan.createFromSends(importNodeIDs);
00771     if (numExports > 0) {
00772       exportIDs = Teuchos::arcp<Ordinal>(numExports);
00773       exportNodeIDs = Teuchos::arcp<int>(numExports);
00774     }
00775 
00776     Teuchos::Array<size_t> exportObjs(tempPlan.getTotalReceiveLength()*2);
00777     tempPlan.doPostsAndWaits<size_t>(importObjs(),2,exportObjs());
00778 
00779     for (size_t i = 0; i < numExports; ++i) {
00780       exportIDs[i]     = Teuchos::as<Ordinal>(exportObjs[2*i]);
00781       exportNodeIDs[i] = exportObjs[2*i+1];
00782     }
00783   }
00784 
00785 
00787   template <class Ordinal>
00788   void Distributor::createFromRecvs(
00789       const Teuchos::ArrayView<const Ordinal> &remoteIDs, 
00790       const Teuchos::ArrayView<const int> &remoteImageIDs, 
00791             Teuchos::ArrayRCP<Ordinal> &exportGIDs, 
00792             Teuchos::ArrayRCP<int> &exportNodeIDs)
00793   {
00794     using Teuchos::outArg;
00795     {
00796       const int myImageID = comm_->getRank();
00797       int err_node = (remoteIDs.size() != remoteImageIDs.size()) ? myImageID : -1;
00798       int gbl_err;
00799       Teuchos::reduceAll(*comm_,Teuchos::REDUCE_MAX,err_node,outArg(gbl_err));
00800       TEST_FOR_EXCEPTION(gbl_err != -1, std::runtime_error,
00801           Teuchos::typeName(*this) 
00802           << "::createFromRecvs(): lists of remote IDs and remote node IDs must have the same size (error on node " 
00803           << gbl_err << ").");
00804     }
00805     computeSends(remoteIDs, remoteImageIDs, exportGIDs, exportNodeIDs);
00806     (void)createFromSends(exportNodeIDs());
00807   }
00808 
00809 } // namespace Tpetra
00810 
00811 #endif // TPETRA_DISTRIBUTOR_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines