Tpetra Matrix/Vector Services Version of the Day
Tpetra_Distributor.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 // 
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00038 // 
00039 // ************************************************************************
00040 // @HEADER
00041 
00042 #ifndef TPETRA_DISTRIBUTOR_HPP
00043 #define TPETRA_DISTRIBUTOR_HPP
00044 
00045 #include "Tpetra_Util.hpp"
00046 #include <Teuchos_as.hpp>
00047 #include <Teuchos_Describable.hpp>
00048 
00049 
00050 // FINISH: some of the get accessors may not be necessary anymore. clean up.
00051 // FINISH: This class may not be const correct. doPosts() et al. perhaps should be const, with affected members made mutable.
00052 
00053 namespace Tpetra {
00054 
00061   class Distributor : public Teuchos::Describable {
00062   public:
00063 
00065 
00066 
00072     explicit Distributor(const RCP<const Comm<int> > & comm);
00073 
00075     Distributor(const Distributor &distributor);
00076 
00078     ~Distributor();
00079 
00081 
00082 
00084 
00085 
00103     size_t createFromSends (const ArrayView<const int>& exportNodeIDs);
00104 
00137     template <class Ordinal>
00138     void createFromRecvs(const ArrayView<const Ordinal> &remoteIDs, 
00139                          const ArrayView<const int> &remoteNodeIDs, 
00140                                ArrayRCP<Ordinal> &exportIDs, 
00141                                ArrayRCP<int> &exportNodeIDs);
00142 
00144 
00146 
00147 
00149     size_t getNumReceives() const;
00150 
00152     size_t getNumSends() const;
00153 
00155 
00156     bool hasSelfMessage() const;
00157 
00159     size_t getMaxSendLength() const;
00160 
00162     size_t getTotalReceiveLength() const;
00163 
00165     ArrayView<const int> getImagesFrom() const;
00166 
00168     ArrayView<const int> getImagesTo() const;
00169 
00171 
00172     ArrayView<const size_t> getLengthsFrom() const;
00173 
00175 
00176     ArrayView<const size_t> getLengthsTo() const;
00177 
00179 
00181 
00182 
00184 
00187     const RCP<Distributor>& getReverse() const;
00188 
00190 
00192 
00193 
00195 
00206     template <class Packet>
00207     void doPostsAndWaits(const ArrayView<const Packet> &exports,
00208                          size_t numPackets,
00209                          const ArrayView<Packet> &imports);
00210 
00212 
00223     template <class Packet>
00224     void doPostsAndWaits(const ArrayView<const Packet> &exports,
00225                          const ArrayView<size_t> &numExportPacketsPerLID,
00226                          const ArrayView<Packet> &imports,
00227                          const ArrayView<size_t> &numImportPacketsPerLID);
00228 
00230 
00241     template <class Packet>
00242     void doPosts(const ArrayView<const Packet> &exports,
00243                  size_t numPackets,
00244                  const ArrayRCP<Packet> &imports);
00245 
00247 
00258     template <class Packet>
00259     void doPosts(const ArrayView<const Packet> &exports,
00260                  const ArrayView<size_t> &numExportPacketsPerLID,
00261                  const ArrayRCP<Packet> &imports,
00262                  const ArrayView<size_t> &numImportPacketsPerLID);
00263 
00265     void doWaits();
00266 
00268 
00279     template <class Packet>
00280     void doReversePostsAndWaits(const ArrayView<const Packet> &exports,
00281                                 size_t numPackets,
00282                                 const ArrayView<Packet> &imports);
00283 
00285 
00296     template <class Packet>
00297     void doReversePostsAndWaits(const ArrayView<const Packet> &exports,
00298                                 const ArrayView<size_t> &numExportPacketsPerLID,
00299                                 const ArrayView<Packet> &imports,
00300                                 const ArrayView<size_t> &numImportPacketsPerLID);
00301 
00303 
00314     template <class Packet>
00315     void doReversePosts(const ArrayView<const Packet> &exports,
00316                         size_t numPackets,
00317                         const ArrayRCP<Packet> &imports);
00318 
00320 
00331     template <class Packet>
00332     void doReversePosts(const ArrayView<const Packet> &exports,
00333                         const ArrayView<size_t> &numExportPacketsPerLID,
00334                         const ArrayRCP<Packet> &imports,
00335                         const ArrayView<size_t> &numImportPacketsPerLID);
00336 
00338     void doReverseWaits();
00339 
00341 
00343 
00344 
00346     std::string description() const;
00347 
00349     void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const;
00350 
00352 
00353   private:
00355     RCP<const Comm<int> > comm_;
00356 
00365     size_t numExports_;
00366 
00368     bool selfMessage_;
00369 
00378     size_t numSends_;
00379 
00380     // imagesTo_, startsTo_ and lengthsTo_ each have size 
00381     //   numSends_ + selfMessage_
00382     Array<int> imagesTo_;
00383 
00392     Array<size_t> startsTo_;
00393 
00399     Array<size_t> lengthsTo_;
00400 
00404     size_t maxSendLength_;
00405     Array<size_t> indicesTo_;
00406     
00416     size_t numReceives_;
00417 
00418     // totalReceiveLength_ is the total number of Packet received, used to 
00419     // allocate the receive buffer
00420     size_t totalReceiveLength_;
00421     // imagesFrom_, startsFrom_ and lengthsFrom_ each have size 
00422     //   numReceives_ + selfMessage_
00423     Array<size_t> lengthsFrom_;
00424     Array<int> imagesFrom_;
00425     Array<size_t> startsFrom_;
00426     Array<size_t> indicesFrom_;
00427 
00429     Array<RCP<Teuchos::CommRequest> > requests_;
00430 
00435     mutable RCP<Distributor> reverseDistributor_;
00436 
00438     void computeReceives();
00439 
00441     template <class Ordinal>
00442     void computeSends (const ArrayView<const Ordinal> &importIDs,
00443            const ArrayView<const int> &importNodeIDs,
00444            ArrayRCP<Ordinal> &exportIDs,
00445            ArrayRCP<int> &exportNodeIDs);
00446 
00448     void createReverseDistributor() const;
00449 
00450   }; // class Distributor
00451 
00452 
00453   template <class Packet>
00454   void Distributor::doPostsAndWaits(
00455       const ArrayView<const Packet>& exports,
00456       size_t numPackets,
00457       const ArrayView<Packet>& imports) 
00458   {
00459     TEUCHOS_TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
00460         Teuchos::typeName(*this) << "::doPostsAndWaits(): Cannot call with outstanding posts.");
00461     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00462     // however, it need only persist until doWaits is called, so it is safe for us to 
00463     // use a non-persisting reference in this case
00464     doPosts(exports, numPackets, arcp<Packet>(imports.getRawPtr(),0,imports.size(),false));
00465     doWaits();
00466   }
00467 
00468   template <class Packet>
00469   void Distributor::doPostsAndWaits(
00470       const ArrayView<const Packet>& exports,
00471       const ArrayView<size_t> &numExportPacketsPerLID,
00472       const ArrayView<Packet> &imports,
00473       const ArrayView<size_t> &numImportPacketsPerLID)
00474   {
00475     TEUCHOS_TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
00476         Teuchos::typeName(*this) << "::doPostsAndWaits(): Cannot call with outstanding posts.");
00477     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00478     // however, it need only persist until doWaits is called, so it is safe for us to 
00479     // use a non-persisting reference in this case
00480     doPosts(exports, numExportPacketsPerLID, arcp<Packet>(imports.getRawPtr(),0,imports.size(),false), numImportPacketsPerLID);
00481     doWaits();
00482   }
00483 
00484 
00485   template <class Packet>
00486   void Distributor::doPosts(const ArrayView<const Packet>& exports,
00487                             size_t numPackets,
00488                             const ArrayRCP<Packet>& imports) {
00489     // start of actual doPosts function
00490     const int myImageID = comm_->getRank();
00491     size_t selfReceiveOffset = 0;
00492 
00493 #ifdef HAVE_TEUCHOS_DEBUG
00494     TEUCHOS_TEST_FOR_EXCEPTION(Teuchos::as<size_t>(imports.size()) != totalReceiveLength_ * numPackets, std::runtime_error,
00495         Teuchos::typeName(*this) << "::doPosts(): imports must be large enough to store the imported data.");
00496 #endif
00497 
00498     // allocate space in requests
00499     requests_.resize(0);
00500     requests_.reserve(numReceives_);
00501 
00502     // start up the Irecv's
00503     {
00504       size_t curBufferOffset = 0;
00505       for (size_t i = 0; i < numReceives_ + (selfMessage_ ? 1 : 0); ++i) {
00506         if (imagesFrom_[i] != myImageID) { 
00507           // receiving this one from another image
00508           // setup reference into imports of the appropriate size and at the appropriate place
00509           ArrayRCP<Packet> impptr = imports.persistingView(curBufferOffset,lengthsFrom_[i]*numPackets);
00510           requests_.push_back( Teuchos::ireceive<int,Packet>(*comm_,impptr,imagesFrom_[i]) );
00511         }
00512         else {
00513           // receiving this one from myself 
00514           // note that offset
00515           selfReceiveOffset = curBufferOffset;
00516         }
00517         curBufferOffset += lengthsFrom_[i]*numPackets;
00518       }
00519     }
00520 
00521     // wait for everyone else before posting ready-sends below to ensure that 
00522     // all non-blocking receives above have been posted
00523     Teuchos::barrier(*comm_);
00524 
00525     // setup scan through imagesTo_ list starting with higher numbered images
00526     // (should help balance message traffic)
00527     size_t numBlocks = numSends_+ selfMessage_;
00528     size_t imageIndex = 0;
00529     while ((imageIndex < numBlocks) && (imagesTo_[imageIndex] < myImageID)) {
00530       ++imageIndex;
00531     }
00532     if (imageIndex == numBlocks) {
00533       imageIndex = 0;
00534     }
00535 
00536     size_t selfNum = 0;
00537     size_t selfIndex = 0;
00538 
00539     if (indicesTo_.empty()) { // data is already blocked by processor
00540       for (size_t i = 0; i < numBlocks; ++i) {
00541         size_t p = i + imageIndex;
00542         if (p > (numBlocks - 1)) {
00543           p -= numBlocks;
00544         }
00545 
00546         if (imagesTo_[p] != myImageID) {
00547           // sending it to another image
00548           ArrayView<const Packet> tmpSend(&exports[startsTo_[p]*numPackets],lengthsTo_[p]*numPackets);
00549           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00550         }
00551         else {
00552           // sending it to ourself
00553           selfNum = p;
00554         }
00555       }
00556 
00557       if (selfMessage_) {
00558         std::copy(exports.begin()+startsTo_[selfNum]*numPackets, exports.begin()+startsTo_[selfNum]*numPackets+lengthsTo_[selfNum]*numPackets, 
00559                   imports.begin()+selfReceiveOffset);
00560       }
00561     }
00562     else { // data is not blocked by image, use send buffer
00563       // allocate sendArray buffer
00564       Array<Packet> sendArray(maxSendLength_*numPackets); 
00565 
00566       for (size_t i = 0; i < numBlocks; ++i) {
00567         size_t p = i + imageIndex;
00568         if (p > (numBlocks - 1)) {
00569           p -= numBlocks;
00570         }
00571 
00572         if (imagesTo_[p] != myImageID) { 
00573           // sending it to another image
00574           typename ArrayView<const Packet>::iterator srcBegin, srcEnd;
00575           size_t sendArrayOffset = 0;
00576           size_t j = startsTo_[p];
00577           for (size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
00578             srcBegin = exports.begin() + indicesTo_[j]*numPackets;
00579             srcEnd   = srcBegin + numPackets;
00580             std::copy( srcBegin, srcEnd, sendArray.begin()+sendArrayOffset );
00581             sendArrayOffset += numPackets;
00582           }
00583           ArrayView<const Packet> tmpSend = sendArray(0,lengthsTo_[p]*numPackets);
00584           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00585         }
00586         else { 
00587           // sending it to myself
00588           selfNum = p;
00589           selfIndex = startsTo_[p];
00590         }
00591       }
00592 
00593       if (selfMessage_) {
00594         for (size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
00595           std::copy( exports.begin()+indicesTo_[selfIndex]*numPackets,
00596                      exports.begin()+indicesTo_[selfIndex]*numPackets + numPackets,
00597                      imports.begin() + selfReceiveOffset );
00598           ++selfIndex;
00599           selfReceiveOffset += numPackets;
00600         }
00601       }
00602     }
00603   }
00604 
00605   template <class Packet>
00606   void Distributor::doPosts(const ArrayView<const Packet>& exports,
00607                             const ArrayView<size_t>& numExportPacketsPerLID,
00608                             const ArrayRCP<Packet>& imports,
00609                             const ArrayView<size_t>& numImportPacketsPerLID) {
00610     // start of actual doPosts function
00611     const int myImageID = comm_->getRank();
00612     size_t selfReceiveOffset = 0;
00613 
00614 #ifdef HAVE_TEUCHOS_DEBUG
00615     size_t totalNumPackets = 0;
00616     for(int ii=0; ii<numImportPacketsPerLID.size(); ++ii) {
00617       totalNumPackets += numImportPacketsPerLID[ii];
00618     }
00619     TEUCHOS_TEST_FOR_EXCEPTION(Teuchos::as<size_t>(imports.size()) != totalNumPackets, std::runtime_error,
00620         Teuchos::typeName(*this) << "::doPosts(): imports must be large enough to store the imported data.");
00621 #endif
00622 
00623     // allocate space in requests
00624     requests_.resize(0);
00625     requests_.reserve(numReceives_);
00626 
00627     // start up the Irecv's
00628     {
00629       size_t curBufferOffset = 0;
00630       size_t curLIDoffset = 0;
00631       for (size_t i = 0; i < numReceives_ + (selfMessage_ ? 1 : 0); ++i) {
00632         size_t totalPacketsFrom_i = 0;
00633         for (size_t j=0; j<lengthsFrom_[i]; ++j) {
00634           totalPacketsFrom_i += numImportPacketsPerLID[curLIDoffset+j];
00635         }
00636         curLIDoffset += lengthsFrom_[i];
00637         if (imagesFrom_[i] != myImageID && totalPacketsFrom_i) { 
00638           // receiving this one from another image
00639           // setup reference into imports of the appropriate size and at the appropriate place
00640           ArrayRCP<Packet> impptr = imports.persistingView(curBufferOffset,totalPacketsFrom_i);
00641           requests_.push_back( Teuchos::ireceive<int,Packet>(*comm_,impptr,imagesFrom_[i]) );
00642         }
00643         else {
00644           // receiving this one from myself 
00645           // note that offset
00646           selfReceiveOffset = curBufferOffset;
00647         }
00648         curBufferOffset += totalPacketsFrom_i;
00649       }
00650     }
00651 
00652     // wait for everyone else before posting ready-sends below to ensure that 
00653     // all non-blocking receives above have been posted
00654     Teuchos::barrier(*comm_);
00655 
00656     // setup arrays containing starting-offsets into exports for each send,
00657     // and num-packets-to-send for each send.
00658     Array<size_t> sendPacketOffsets(numSends_,0), packetsPerSend(numSends_,0);
00659     size_t maxNumPackets = 0;
00660     size_t curPKToffset = 0;
00661     for (size_t pp=0; pp<numSends_; ++pp) {
00662       sendPacketOffsets[pp] = curPKToffset;
00663       size_t numPackets = 0;
00664       for (size_t j=startsTo_[pp]; j<startsTo_[pp]+lengthsTo_[pp]; ++j) {
00665         numPackets += numExportPacketsPerLID[j];
00666       }
00667       if (numPackets > maxNumPackets) maxNumPackets = numPackets;
00668       packetsPerSend[pp] = numPackets;
00669       curPKToffset += numPackets;
00670     }
00671 
00672     // setup scan through imagesTo_ list starting with higher numbered images
00673     // (should help balance message traffic)
00674     size_t numBlocks = numSends_+ selfMessage_;
00675     size_t imageIndex = 0;
00676     while ((imageIndex < numBlocks) && (imagesTo_[imageIndex] < myImageID)) {
00677       ++imageIndex;
00678     }
00679     if (imageIndex == numBlocks) {
00680       imageIndex = 0;
00681     }
00682 
00683     size_t selfNum = 0;
00684     size_t selfIndex = 0;
00685 
00686     if (indicesTo_.empty()) { // data is already laid out according to processor
00687       for (size_t i = 0; i < numBlocks; ++i) {
00688         size_t p = i + imageIndex;
00689         if (p > (numBlocks - 1)) {
00690           p -= numBlocks;
00691         }
00692 
00693         if (imagesTo_[p] != myImageID && packetsPerSend[p] > 0) {
00694           // sending it to another image
00695           ArrayView<const Packet> tmpSend(&exports[sendPacketOffsets[p]],packetsPerSend[p]);
00696           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00697         }
00698         else {
00699           // sending it to ourself
00700           selfNum = p;
00701         }
00702       }
00703 
00704       if (selfMessage_) {
00705         std::copy(exports.begin()+sendPacketOffsets[selfNum], exports.begin()+sendPacketOffsets[selfNum]+packetsPerSend[selfNum], 
00706                   imports.begin()+selfReceiveOffset);
00707       }
00708     }
00709     else { // data is not blocked by image, use send buffer
00710       // allocate sendArray buffer
00711       Array<Packet> sendArray(maxNumPackets); 
00712       Array<size_t> indicesOffsets(numExportPacketsPerLID.size(),0);
00713       size_t ioffset = 0;
00714       for (int j=0; j<numExportPacketsPerLID.size(); ++j) {
00715         indicesOffsets[j] = ioffset;
00716         ioffset += numExportPacketsPerLID[j];
00717       }
00718 
00719       for (size_t i = 0; i < numBlocks; ++i) {
00720         size_t p = i + imageIndex;
00721         if (p > (numBlocks - 1)) {
00722           p -= numBlocks;
00723         }
00724 
00725         if (imagesTo_[p] != myImageID) { 
00726           // sending it to another image
00727           typename ArrayView<const Packet>::iterator srcBegin, srcEnd;
00728           size_t sendArrayOffset = 0;
00729           size_t j = startsTo_[p];
00730           size_t numPacketsTo_p = 0;
00731           for (size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
00732             srcBegin = exports.begin() + indicesOffsets[j];
00733             srcEnd   = srcBegin + numExportPacketsPerLID[j];
00734             numPacketsTo_p += numExportPacketsPerLID[j];
00735             std::copy( srcBegin, srcEnd, sendArray.begin()+sendArrayOffset );
00736             sendArrayOffset += numExportPacketsPerLID[j];
00737           }
00738           if (numPacketsTo_p > 0) {
00739             ArrayView<const Packet> tmpSend = sendArray(0,numPacketsTo_p);
00740             Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00741           }
00742         }
00743         else { 
00744           // sending it to myself
00745           selfNum = p;
00746           selfIndex = startsTo_[p];
00747         }
00748       }
00749 
00750       if (selfMessage_) {
00751         for (size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
00752           std::copy( exports.begin()+indicesOffsets[selfIndex],
00753                      exports.begin()+indicesOffsets[selfIndex]+numExportPacketsPerLID[selfIndex],
00754                      imports.begin() + selfReceiveOffset );
00755           selfReceiveOffset += numExportPacketsPerLID[selfIndex];
00756           ++selfIndex;
00757         }
00758       }
00759     }
00760   }
00761 
00762 
00763   template <class Packet>
00764   void Distributor::doReversePostsAndWaits(
00765       const ArrayView<const Packet>& exports,
00766       size_t numPackets,
00767       const ArrayView<Packet>& imports) 
00768   {
00769     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00770     // however, it need only persist until doWaits is called, so it is safe for us to 
00771     // use a non-persisting reference in this case
00772     doReversePosts(exports, numPackets, arcp<Packet>(imports.getRawPtr(),0,imports.size(),false));
00773     doReverseWaits();
00774   }
00775 
00776   template <class Packet>
00777   void Distributor::doReversePostsAndWaits(
00778        const ArrayView<const Packet>& exports,
00779        const ArrayView<size_t> &numExportPacketsPerLID,
00780        const ArrayView<Packet> &imports,
00781        const ArrayView<size_t> &numImportPacketsPerLID)
00782   {
00783     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00784     // however, it need only persist until doWaits is called, so it is safe for us to 
00785     // use a non-persisting reference in this case
00786     doReversePosts(exports, numExportPacketsPerLID, arcp<Packet>(imports.getRawPtr(),0,imports.size(),false),numImportPacketsPerLID);
00787     doReverseWaits();
00788   }
00789 
00790 
00791   template <class Packet>
00792   void Distributor::doReversePosts(
00793       const ArrayView<const Packet>& exports,
00794       size_t numPackets,
00795       const ArrayRCP<Packet>& imports) 
00796   {
00797     TEUCHOS_TEST_FOR_EXCEPTION(!indicesTo_.empty(),std::runtime_error,
00798         Teuchos::typeName(*this) << "::doReversePosts(): Can only do reverse comm when original data is blocked by image.");
00799     if (reverseDistributor_ == null) {
00800       createReverseDistributor();
00801     }
00802     reverseDistributor_->doPosts(exports,numPackets,imports);
00803   }
00804 
00805   template <class Packet>
00806   void Distributor::doReversePosts(
00807       const ArrayView<const Packet>& exports,
00808       const ArrayView<size_t>& numExportPacketsPerLID,
00809       const ArrayRCP<Packet>& imports,
00810       const ArrayView<size_t>& numImportPacketsPerLID) 
00811   {
00812     TEUCHOS_TEST_FOR_EXCEPTION(!indicesTo_.empty(),std::runtime_error,
00813         Teuchos::typeName(*this) << "::doReversePosts(): Can only do reverse comm when original data is blocked by image.");
00814     if (reverseDistributor_ == null) {
00815       createReverseDistributor();
00816     }
00817     reverseDistributor_->doPosts(exports,numExportPacketsPerLID,imports,numImportPacketsPerLID);
00818   }
00819 
00820 
00821   // note: assumes that size_t >= Ordinal
00823   template <class Ordinal>
00824   void Distributor::computeSends(
00825       const ArrayView<const Ordinal> & importIDs,
00826       const ArrayView<const int> & importNodeIDs,
00827             ArrayRCP<Ordinal> & exportIDs,
00828             ArrayRCP<int> & exportNodeIDs)
00829   {
00830     int myImageID = comm_->getRank();
00831 
00832     size_t numImports = importNodeIDs.size();
00833     Array<size_t> importObjs(2*numImports);
00834     for (size_t i = 0; i < numImports; ++i ) {  
00835       importObjs[2*i]   = Teuchos::as<size_t>(importIDs[i]);
00836       importObjs[2*i+1] = Teuchos::as<size_t>(myImageID);
00837     }
00838 
00839     size_t numExports;
00840     Distributor tempPlan(comm_);
00841     numExports = tempPlan.createFromSends(importNodeIDs);
00842     if (numExports > 0) {
00843       exportIDs = arcp<Ordinal>(numExports);
00844       exportNodeIDs = arcp<int>(numExports);
00845     }
00846 
00847     Array<size_t> exportObjs(tempPlan.getTotalReceiveLength()*2);
00848     tempPlan.doPostsAndWaits<size_t>(importObjs(),2,exportObjs());
00849 
00850     for (size_t i = 0; i < numExports; ++i) {
00851       exportIDs[i]     = Teuchos::as<Ordinal>(exportObjs[2*i]);
00852       exportNodeIDs[i] = exportObjs[2*i+1];
00853     }
00854   }
00855 
00856 
00858   template <class Ordinal>
00859   void Distributor::createFromRecvs(
00860       const ArrayView<const Ordinal> &remoteIDs, 
00861       const ArrayView<const int> &remoteImageIDs, 
00862             ArrayRCP<Ordinal> &exportGIDs, 
00863             ArrayRCP<int> &exportNodeIDs)
00864   {
00865     using Teuchos::outArg;
00866     {
00867       const int myImageID = comm_->getRank();
00868       int err_node = (remoteIDs.size() != remoteImageIDs.size()) ? myImageID : -1;
00869       int gbl_err;
00870       Teuchos::reduceAll(*comm_,Teuchos::REDUCE_MAX,err_node,outArg(gbl_err));
00871       TEUCHOS_TEST_FOR_EXCEPTION(gbl_err != -1, std::runtime_error,
00872           Teuchos::typeName(*this) 
00873           << "::createFromRecvs(): lists of remote IDs and remote node IDs must have the same size (error on node " 
00874           << gbl_err << ").");
00875     }
00876     computeSends(remoteIDs, remoteImageIDs, exportGIDs, exportNodeIDs);
00877     (void)createFromSends(exportNodeIDs());
00878   }
00879 
00880 } // namespace Tpetra
00881 
00882 #endif // TPETRA_DISTRIBUTOR_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines