Tpetra_Distributor.hpp

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_DISTRIBUTOR_HPP
00030 #define TPETRA_DISTRIBUTOR_HPP
00031 
00032 #include "Tpetra_Util.hpp"
00033 #include <Teuchos_RCP.hpp>
00034 #include <Teuchos_as.hpp>
00035 #include <Teuchos_TypeNameTraits.hpp>
00036 #include <Teuchos_Describable.hpp>
00037 #include <Teuchos_Comm.hpp>
00038 #include <Teuchos_CommHelpers.hpp>
00039 #include <Teuchos_Array.hpp>
00040 #include <Teuchos_ArrayView.hpp>
00041 #include <Teuchos_ArrayRCP.hpp>
00042 
00043 
00044 // FINISH: some of the get accessors may not be necessary anymore. clean up.
00045 // FINISH: This class may not be const correct. doPosts() et al. perhaps should be const, with affected members made mutable.
00046 
00047 namespace Tpetra {
00048 
00050 
00055   class Distributor : public Teuchos::Describable {
00056   public:
00057 
00059 
00060 
00062     explicit Distributor(const Teuchos::RCP<const Teuchos::Comm<int> > & comm);
00063 
00065     Distributor(const Distributor &distributor);
00066 
00068     ~Distributor();
00069 
00071 
00072 
00074 
00075 
00077 
00094     size_t createFromSends(const Teuchos::ArrayView<const int> &exportNodeIDs);
00095 
00097 
00117     template <class Ordinal>
00118     void createFromRecvs(const Teuchos::ArrayView<const Ordinal> &remoteIDs, 
00119                          const Teuchos::ArrayView<const int> &remoteNodeIDs, 
00120                                Teuchos::ArrayRCP<Ordinal> &exportIDs, 
00121                                Teuchos::ArrayRCP<int> &exportNodeIDs);
00122 
00124 
00126 
00127 
00129     size_t getNumReceives() const;
00130 
00132     size_t getNumSends() const;
00133 
00135 
00136     bool hasSelfMessage() const;
00137 
00139     size_t getMaxSendLength() const;
00140 
00142     size_t getTotalReceiveLength() const;
00143 
00145     Teuchos::ArrayView<const int> getImagesFrom() const;
00146 
00148     Teuchos::ArrayView<const int> getImagesTo() const;
00149 
00151 
00152     Teuchos::ArrayView<const size_t> getLengthsFrom() const;
00153 
00155 
00156     Teuchos::ArrayView<const size_t> getLengthsTo() const;
00157 
00159 
00161 
00162 
00164 
00167     const Teuchos::RCP<Distributor> & getReverse() const;
00168 
00170 
00172 
00173 
00175 
00186     template <class Packet>
00187     void doPostsAndWaits(const Teuchos::ArrayView<const Packet> &exports,
00188                          size_t numPackets,
00189                          const Teuchos::ArrayView<Packet> &imports);
00190 
00192 
00203     template <class Packet>
00204     void doPostsAndWaits(const Teuchos::ArrayView<const Packet> &exports,
00205                          const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00206                          const Teuchos::ArrayView<Packet> &imports,
00207                          const Teuchos::ArrayView<size_t> &numImportPacketsPerLID);
00208 
00210 
00221     template <class Packet>
00222     void doPosts(const Teuchos::ArrayView<const Packet> &exports,
00223                  size_t numPackets,
00224                  const Teuchos::ArrayRCP<Packet> &imports);
00225 
00227 
00238     template <class Packet>
00239     void doPosts(const Teuchos::ArrayView<const Packet> &exports,
00240                  const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00241                  const Teuchos::ArrayRCP<Packet> &imports,
00242                  const Teuchos::ArrayView<size_t> &numImportPacketsPerLID);
00243 
00245     void doWaits();
00246 
00248 
00259     template <class Packet>
00260     void doReversePostsAndWaits(const Teuchos::ArrayView<const Packet> &exports,
00261                                 size_t numPackets,
00262                                 const Teuchos::ArrayView<Packet> &imports);
00263 
00265 
00276     template <class Packet>
00277     void doReversePostsAndWaits(const Teuchos::ArrayView<const Packet> &exports,
00278                                 const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00279                                 const Teuchos::ArrayView<Packet> &imports,
00280                                 const Teuchos::ArrayView<size_t> &numImportPacketsPerLID);
00281 
00283 
00294     template <class Packet>
00295     void doReversePosts(const Teuchos::ArrayView<const Packet> &exports,
00296                         size_t numPackets,
00297                         const Teuchos::ArrayRCP<Packet> &imports);
00298 
00300 
00311     template <class Packet>
00312     void doReversePosts(const Teuchos::ArrayView<const Packet> &exports,
00313                         const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00314                         const Teuchos::ArrayRCP<Packet> &imports,
00315                         const Teuchos::ArrayView<size_t> &numImportPacketsPerLID);
00316 
00318     void doReverseWaits();
00319 
00321 
00323 
00324 
00326     std::string description() const;
00327 
00329     void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const;
00330 
00332 
00333   private:
00334 
00335     // private data members
00336     Teuchos::RCP<const Teuchos::Comm<int> > comm_;
00337 
00338     size_t numExports_;
00339     // selfMessage_ is whether I have a send for myself
00340     bool selfMessage_;
00341     // numSends_ is number of sends to other nodes; is less than or equal to the number of nodes
00342     size_t numSends_;
00343     // imagesTo_, startsTo_ and lengthsTo_ each have size 
00344     //   numSends_ + selfMessage_
00345     Teuchos::Array<int> imagesTo_;
00346     /* Given an export buffer that contains all of the item being sent by this node,
00347        the block of values for node i will start at position startsTo_[i]  */
00348     Teuchos::Array<size_t> startsTo_;
00349     Teuchos::Array<size_t> lengthsTo_;
00350     // maxSendLength_ is the maximum send to another node: 
00351     //   max(lengthsTo_[i]) for i != me
00352     size_t maxSendLength_;
00353     Teuchos::Array<size_t> indicesTo_;
00354     // numReceives_ is the number of receives by me from other procs, not
00355     // counting self receives
00356     size_t numReceives_;
00357     // totalReceiveLength_ is the total number of Packet received, used to 
00358     // allocate the receive buffer
00359     size_t totalReceiveLength_;
00360     // imagesFrom_, startsFrom_ and lengthsFrom_ each have size 
00361     //   numReceives_ + selfMessage_
00362     Teuchos::Array<size_t> lengthsFrom_;
00363     Teuchos::Array<int> imagesFrom_;
00364     Teuchos::Array<size_t> startsFrom_;
00365     Teuchos::Array<size_t> indicesFrom_;
00366 
00367     // requests associated with non-blocking receives
00368     Teuchos::Array<Teuchos::RCP<Teuchos::CommRequest> > requests_;
00369 
00370     mutable Teuchos::RCP<Distributor> reverseDistributor_;
00371 
00372     // compute receive info from sends
00373     void computeReceives();
00374 
00375     // compute send info from receives
00376     template <class Ordinal>
00377     void computeSends(const Teuchos::ArrayView<const Ordinal> &importIDs,
00378                       const Teuchos::ArrayView<const int> &importNodeIDs,
00379                             Teuchos::ArrayRCP<Ordinal> &exportIDs,
00380                             Teuchos::ArrayRCP<int> &exportNodeIDs);
00381 
00382     // create a distributor for the reverse communciation pattern (pretty much
00383     // swap all send and receive info)
00384     void createReverseDistributor() const;
00385 
00386   }; // class Distributor
00387 
00388 
00389   template <class Packet>
00390   void Distributor::doPostsAndWaits(
00391       const Teuchos::ArrayView<const Packet>& exports,
00392       size_t numPackets,
00393       const Teuchos::ArrayView<Packet>& imports) 
00394   {
00395     TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
00396         Teuchos::typeName(*this) << "::doPostsAndWaits(): Cannot call with outstanding posts.");
00397     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00398     // however, it need only persist until doWaits is called, so it is safe for us to 
00399     // use a non-persisting reference in this case
00400     doPosts(exports, numPackets, Teuchos::arcp<Packet>(imports.getRawPtr(),0,imports.size(),false));
00401     doWaits();
00402   }
00403 
00404   template <class Packet>
00405   void Distributor::doPostsAndWaits(
00406       const Teuchos::ArrayView<const Packet>& exports,
00407       const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00408       const Teuchos::ArrayView<Packet> &imports,
00409       const Teuchos::ArrayView<size_t> &numImportPacketsPerLID)
00410   {
00411     TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
00412         Teuchos::typeName(*this) << "::doPostsAndWaits(): Cannot call with outstanding posts.");
00413     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00414     // however, it need only persist until doWaits is called, so it is safe for us to 
00415     // use a non-persisting reference in this case
00416     doPosts(exports, numExportPacketsPerLID, Teuchos::arcp<Packet>(imports.getRawPtr(),0,imports.size(),false), numImportPacketsPerLID);
00417     doWaits();
00418   }
00419 
00420 
00421   template <class Packet>
00422   void Distributor::doPosts(const Teuchos::ArrayView<const Packet>& exports,
00423                             size_t numPackets,
00424                             const Teuchos::ArrayRCP<Packet>& imports) {
00425     using Teuchos::ArrayRCP;
00426     // start of actual doPosts function
00427     const int myImageID = comm_->getRank();
00428     size_t selfReceiveOffset = 0;
00429 
00430 #ifdef HAVE_TEUCHOS_DEBUG
00431     TEST_FOR_EXCEPTION(Teuchos::as<size_t>(imports.size()) != totalReceiveLength_ * numPackets, std::runtime_error,
00432         Teuchos::typeName(*this) << "::doPosts(): imports must be large enough to store the imported data.");
00433 #endif
00434 
00435     // allocate space in requests
00436     requests_.resize(0);
00437     requests_.reserve(numReceives_);
00438 
00439     // start up the Irecv's
00440     {
00441       size_t curBufferOffset = 0;
00442       for (size_t i = 0; i < numReceives_ + (selfMessage_ ? 1 : 0); ++i) {
00443         if (imagesFrom_[i] != myImageID) { 
00444           // receiving this one from another image
00445           // setup reference into imports of the appropriate size and at the appropriate place
00446           ArrayRCP<Packet> impptr = imports.persistingView(curBufferOffset,lengthsFrom_[i]*numPackets);
00447           requests_.push_back( Teuchos::ireceive<int,Packet>(*comm_,impptr,imagesFrom_[i]) );
00448         }
00449         else {
00450           // receiving this one from myself 
00451           // note that offset
00452           selfReceiveOffset = curBufferOffset;
00453         }
00454         curBufferOffset += lengthsFrom_[i]*numPackets;
00455       }
00456     }
00457 
00458     // wait for everyone else before posting ready-sends below to ensure that 
00459     // all non-blocking receives above have been posted
00460     Teuchos::barrier(*comm_);
00461 
00462     // setup scan through imagesTo_ list starting with higher numbered images
00463     // (should help balance message traffic)
00464     size_t numBlocks = numSends_+ selfMessage_;
00465     size_t imageIndex = 0;
00466     while ((imageIndex < numBlocks) && (imagesTo_[imageIndex] < myImageID)) {
00467       ++imageIndex;
00468     }
00469     if (imageIndex == numBlocks) {
00470       imageIndex = 0;
00471     }
00472 
00473     size_t selfNum = 0;
00474     size_t selfIndex = 0;
00475 
00476     if (indicesTo_.empty()) { // data is already blocked by processor
00477       for (size_t i = 0; i < numBlocks; ++i) {
00478         size_t p = i + imageIndex;
00479         if (p > (numBlocks - 1)) {
00480           p -= numBlocks;
00481         }
00482 
00483         if (imagesTo_[p] != myImageID) {
00484           // sending it to another image
00485           Teuchos::ArrayView<const Packet> tmpSend(&exports[startsTo_[p]*numPackets],lengthsTo_[p]*numPackets);
00486           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00487         }
00488         else {
00489           // sending it to ourself
00490           selfNum = p;
00491         }
00492       }
00493 
00494       if (selfMessage_) {
00495         std::copy(exports.begin()+startsTo_[selfNum]*numPackets, exports.begin()+startsTo_[selfNum]*numPackets+lengthsTo_[selfNum]*numPackets, 
00496                   imports.begin()+selfReceiveOffset);
00497       }
00498     }
00499     else { // data is not blocked by image, use send buffer
00500       // allocate sendArray buffer
00501       Teuchos::Array<Packet> sendArray(maxSendLength_*numPackets); 
00502 
00503       for (size_t i = 0; i < numBlocks; ++i) {
00504         size_t p = i + imageIndex;
00505         if (p > (numBlocks - 1)) {
00506           p -= numBlocks;
00507         }
00508 
00509         if (imagesTo_[p] != myImageID) { 
00510           // sending it to another image
00511           typename Teuchos::ArrayView<const Packet>::iterator srcBegin, srcEnd;
00512           size_t sendArrayOffset = 0;
00513           size_t j = startsTo_[p];
00514           for (size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
00515             srcBegin = exports.begin() + indicesTo_[j]*numPackets;
00516             srcEnd   = srcBegin + numPackets;
00517             std::copy( srcBegin, srcEnd, sendArray.begin()+sendArrayOffset );
00518             sendArrayOffset += numPackets;
00519           }
00520           Teuchos::ArrayView<const Packet> tmpSend = sendArray(0,lengthsTo_[p]*numPackets);
00521           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00522         }
00523         else { 
00524           // sending it to myself
00525           selfNum = p;
00526           selfIndex = startsTo_[p];
00527         }
00528       }
00529 
00530       if (selfMessage_) {
00531         for (size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
00532           std::copy( exports.begin()+indicesTo_[selfIndex]*numPackets,
00533                      exports.begin()+indicesTo_[selfIndex]*numPackets + numPackets,
00534                      imports.begin() + selfReceiveOffset );
00535           ++selfIndex;
00536           selfReceiveOffset += numPackets;
00537         }
00538       }
00539     }
00540   }
00541 
00542   template <class Packet>
00543   void Distributor::doPosts(const Teuchos::ArrayView<const Packet>& exports,
00544                             const Teuchos::ArrayView<size_t>& numExportPacketsPerLID,
00545                             const Teuchos::ArrayRCP<Packet>& imports,
00546                             const Teuchos::ArrayView<size_t>& numImportPacketsPerLID) {
00547     using Teuchos::ArrayRCP;
00548     // start of actual doPosts function
00549     const int myImageID = comm_->getRank();
00550     size_t selfReceiveOffset = 0;
00551 
00552 #ifdef HAVE_TEUCHOS_DEBUG
00553     size_t totalNumPackets = 0;
00554     for(int ii=0; ii<numImportPacketsPerLID.size(); ++ii) {
00555       totalNumPackets += numImportPacketsPerLID[ii];
00556     }
00557     TEST_FOR_EXCEPTION(Teuchos::as<size_t>(imports.size()) != totalNumPackets, std::runtime_error,
00558         Teuchos::typeName(*this) << "::doPosts(): imports must be large enough to store the imported data.");
00559 #endif
00560 
00561     // allocate space in requests
00562     requests_.resize(0);
00563     requests_.reserve(numReceives_);
00564 
00565     // start up the Irecv's
00566     {
00567       size_t curBufferOffset = 0;
00568       size_t curLIDoffset = 0;
00569       for (size_t i = 0; i < numReceives_ + (selfMessage_ ? 1 : 0); ++i) {
00570         size_t totalPacketsFrom_i = 0;
00571         for(size_t j=0; j<lengthsFrom_[i]; ++j) {
00572           totalPacketsFrom_i += numImportPacketsPerLID[curLIDoffset+j];
00573         }
00574         curLIDoffset += lengthsFrom_[i];
00575         if (imagesFrom_[i] != myImageID) { 
00576           // receiving this one from another image
00577           // setup reference into imports of the appropriate size and at the appropriate place
00578           ArrayRCP<Packet> impptr = imports.persistingView(curBufferOffset,totalPacketsFrom_i);
00579           requests_.push_back( Teuchos::ireceive<int,Packet>(*comm_,impptr,imagesFrom_[i]) );
00580         }
00581         else {
00582           // receiving this one from myself 
00583           // note that offset
00584           selfReceiveOffset = curBufferOffset;
00585         }
00586         curBufferOffset += totalPacketsFrom_i;
00587       }
00588     }
00589 
00590     // wait for everyone else before posting ready-sends below to ensure that 
00591     // all non-blocking receives above have been posted
00592     Teuchos::barrier(*comm_);
00593 
00594     //setup arrays containing starting-offsets into exports for each send,
00595     //and num-packets-to-send for each send.
00596     Teuchos::Array<size_t> sendPacketOffsets(numSends_,0), packetsPerSend(numSends_,0);
00597     size_t maxNumPackets = 0;
00598     size_t curPKToffset = 0;
00599     for(size_t pp=0; pp<numSends_; ++pp) {
00600       sendPacketOffsets[pp] = curPKToffset;
00601       size_t numPackets = 0;
00602       for(size_t j=startsTo_[pp]; j<startsTo_[pp]+lengthsTo_[pp]; ++j) {
00603         numPackets += numExportPacketsPerLID[j];
00604       }
00605       if (numPackets > maxNumPackets) maxNumPackets = numPackets;
00606       packetsPerSend[pp] = numPackets;
00607       curPKToffset += numPackets;
00608     }
00609 
00610     // setup scan through imagesTo_ list starting with higher numbered images
00611     // (should help balance message traffic)
00612     size_t numBlocks = numSends_+ selfMessage_;
00613     size_t imageIndex = 0;
00614     while ((imageIndex < numBlocks) && (imagesTo_[imageIndex] < myImageID)) {
00615       ++imageIndex;
00616     }
00617     if (imageIndex == numBlocks) {
00618       imageIndex = 0;
00619     }
00620 
00621     size_t selfNum = 0;
00622     size_t selfIndex = 0;
00623 
00624     if (indicesTo_.empty()) { // data is already blocked by processor
00625       for (size_t i = 0; i < numBlocks; ++i) {
00626         size_t p = i + imageIndex;
00627         if (p > (numBlocks - 1)) {
00628           p -= numBlocks;
00629         }
00630 
00631         if (imagesTo_[p] != myImageID) {
00632           // sending it to another image
00633           Teuchos::ArrayView<const Packet> tmpSend(&exports[sendPacketOffsets[p]],packetsPerSend[p]);
00634           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00635         }
00636         else {
00637           // sending it to ourself
00638           selfNum = p;
00639         }
00640       }
00641 
00642       if (selfMessage_) {
00643         std::copy(exports.begin()+sendPacketOffsets[selfNum], exports.begin()+sendPacketOffsets[selfNum]+packetsPerSend[selfNum], 
00644                   imports.begin()+selfReceiveOffset);
00645       }
00646     }
00647     else { // data is not blocked by image, use send buffer
00648       // allocate sendArray buffer
00649       Teuchos::Array<Packet> sendArray(maxNumPackets); 
00650       Teuchos::Array<size_t> indicesOffsets(numExportPacketsPerLID.size(),0);
00651       size_t ioffset = 0;
00652       for(int j=0; j<numExportPacketsPerLID.size(); ++j) {
00653         indicesOffsets[j] = ioffset;
00654         ioffset += numExportPacketsPerLID[j];
00655       }
00656 
00657       for (size_t i = 0; i < numBlocks; ++i) {
00658         size_t p = i + imageIndex;
00659         if (p > (numBlocks - 1)) {
00660           p -= numBlocks;
00661         }
00662 
00663         if (imagesTo_[p] != myImageID) { 
00664           // sending it to another image
00665           typename Teuchos::ArrayView<const Packet>::iterator srcBegin, srcEnd;
00666           size_t sendArrayOffset = 0;
00667           size_t j = startsTo_[p];
00668           size_t numPacketsTo_p = 0;
00669           for (size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
00670             srcBegin = exports.begin() + indicesOffsets[j];
00671             srcEnd   = srcBegin + numExportPacketsPerLID[j];
00672             numPacketsTo_p += numExportPacketsPerLID[j];
00673             std::copy( srcBegin, srcEnd, sendArray.begin()+sendArrayOffset );
00674             sendArrayOffset += numExportPacketsPerLID[j];
00675           }
00676           Teuchos::ArrayView<const Packet> tmpSend = sendArray(0,numPacketsTo_p);
00677           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00678         }
00679         else { 
00680           // sending it to myself
00681           selfNum = p;
00682           selfIndex = startsTo_[p];
00683         }
00684       }
00685 
00686       if (selfMessage_) {
00687         for (size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
00688           std::copy( exports.begin()+indicesOffsets[selfIndex],
00689                      exports.begin()+indicesOffsets[selfIndex]+numExportPacketsPerLID[selfIndex],
00690                      imports.begin() + selfReceiveOffset );
00691           selfReceiveOffset += numExportPacketsPerLID[selfIndex];
00692           ++selfIndex;
00693         }
00694       }
00695     }
00696   }
00697 
00698 
00699   template <class Packet>
00700   void Distributor::doReversePostsAndWaits(
00701       const Teuchos::ArrayView<const Packet>& exports,
00702       size_t numPackets,
00703       const Teuchos::ArrayView<Packet>& imports) 
00704   {
00705     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00706     // however, it need only persist until doWaits is called, so it is safe for us to 
00707     // use a non-persisting reference in this case
00708     doReversePosts(exports, numPackets, Teuchos::arcp<Packet>(imports.getRawPtr(),0,imports.size(),false));
00709     doReverseWaits();
00710   }
00711 
00712   template <class Packet>
00713   void Distributor::doReversePostsAndWaits(
00714       const Teuchos::ArrayView<const Packet>& exports,
00715        const Teuchos::ArrayView<size_t> &numExportPacketsPerLID,
00716        const Teuchos::ArrayView<Packet> &imports,
00717        const Teuchos::ArrayView<size_t> &numImportPacketsPerLID)
00718   {
00719     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00720     // however, it need only persist until doWaits is called, so it is safe for us to 
00721     // use a non-persisting reference in this case
00722     doReversePosts(exports, numExportPacketsPerLID, Teuchos::arcp<Packet>(imports.getRawPtr(),0,imports.size(),false),numImportPacketsPerLID);
00723     doReverseWaits();
00724   }
00725 
00726 
00727   template <class Packet>
00728   void Distributor::doReversePosts(
00729       const Teuchos::ArrayView<const Packet>& exports,
00730       size_t numPackets,
00731       const Teuchos::ArrayRCP<Packet>& imports) 
00732   {
00733     TEST_FOR_EXCEPTION(!indicesTo_.empty(),std::runtime_error,
00734         Teuchos::typeName(*this) << "::doReversePosts(): Can only do reverse comm when original data is blocked by image.");
00735     if (reverseDistributor_ == Teuchos::null) {
00736       createReverseDistributor();
00737     }
00738     reverseDistributor_->doPosts(exports,numPackets,imports);
00739   }
00740 
00741   template <class Packet>
00742   void Distributor::doReversePosts(
00743       const Teuchos::ArrayView<const Packet>& exports,
00744       const Teuchos::ArrayView<size_t>& numExportPacketsPerLID,
00745       const Teuchos::ArrayRCP<Packet>& imports,
00746       const Teuchos::ArrayView<size_t>& numImportPacketsPerLID) 
00747   {
00748     TEST_FOR_EXCEPTION(!indicesTo_.empty(),std::runtime_error,
00749         Teuchos::typeName(*this) << "::doReversePosts(): Can only do reverse comm when original data is blocked by image.");
00750     if (reverseDistributor_ == Teuchos::null) {
00751       createReverseDistributor();
00752     }
00753     reverseDistributor_->doPosts(exports,numExportPacketsPerLID,imports,numImportPacketsPerLID);
00754   }
00755 
00756 
00757   // note: assumes that size_t >= Ordinal
00759   template <class Ordinal>
00760   void Distributor::computeSends(
00761       const Teuchos::ArrayView<const Ordinal> & importIDs,
00762       const Teuchos::ArrayView<const int> & importNodeIDs,
00763             Teuchos::ArrayRCP<Ordinal>& exportIDs,
00764             Teuchos::ArrayRCP<int>& exportNodeIDs)
00765   {
00766     int myImageID = comm_->getRank();
00767 
00768     size_t numImports = importNodeIDs.size();
00769     Teuchos::Array<size_t> importObjs(2*numImports);
00770     for (size_t i = 0; i < numImports; ++i ) {  
00771       importObjs[2*i]   = Teuchos::as<size_t>(importIDs[i]);
00772       importObjs[2*i+1] = Teuchos::as<size_t>(myImageID);
00773     }
00774 
00775     size_t numExports;
00776     Distributor tempPlan(comm_);
00777     numExports = tempPlan.createFromSends(importNodeIDs);
00778     if (numExports > 0) {
00779       exportIDs = Teuchos::arcp<Ordinal>(numExports);
00780       exportNodeIDs = Teuchos::arcp<int>(numExports);
00781     }
00782 
00783     Teuchos::Array<size_t> exportObjs(tempPlan.getTotalReceiveLength()*2);
00784     tempPlan.doPostsAndWaits<size_t>(importObjs(),2,exportObjs());
00785 
00786     for (size_t i = 0; i < numExports; ++i) {
00787       exportIDs[i]     = Teuchos::as<Ordinal>(exportObjs[2*i]);
00788       exportNodeIDs[i] = exportObjs[2*i+1];
00789     }
00790   }
00791 
00792 
00794   template <class Ordinal>
00795   void Distributor::createFromRecvs(
00796       const Teuchos::ArrayView<const Ordinal> &remoteIDs, 
00797       const Teuchos::ArrayView<const int> &remoteImageIDs, 
00798             Teuchos::ArrayRCP<Ordinal> &exportGIDs, 
00799             Teuchos::ArrayRCP<int> &exportNodeIDs)
00800   {
00801     using Teuchos::outArg;
00802     {
00803       const int myImageID = comm_->getRank();
00804       int err_node = (remoteIDs.size() != remoteImageIDs.size()) ? myImageID : -1;
00805       int gbl_err;
00806       Teuchos::reduceAll(*comm_,Teuchos::REDUCE_MAX,err_node,outArg(gbl_err));
00807       TEST_FOR_EXCEPTION(gbl_err != -1, std::runtime_error,
00808           Teuchos::typeName(*this) 
00809           << "::createFromRecvs(): lists of remote IDs and remote node IDs must have the same size (error on node " 
00810           << gbl_err << ").");
00811     }
00812     computeSends(remoteIDs, remoteImageIDs, exportGIDs, exportNodeIDs);
00813     (void)createFromSends(exportNodeIDs());
00814   }
00815 
00816 } // namespace Tpetra
00817 
00818 #endif // TPETRA_DISTRIBUTOR_HPP

Generated on Tue Jul 13 09:39:06 2010 for Tpetra Matrix/Vector Services by  doxygen 1.4.7