Tpetra Matrix/Vector Services Version of the Day
Tpetra_Distributor.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_DISTRIBUTOR_HPP
00030 #define TPETRA_DISTRIBUTOR_HPP
00031 
00032 #include "Tpetra_Util.hpp"
00033 #include <Teuchos_as.hpp>
00034 #include <Teuchos_Describable.hpp>
00035 
00036 
00037 // FINISH: some of the get accessors may not be necessary anymore. clean up.
00038 // FINISH: This class may not be const correct. doPosts() et al. perhaps should be const, with affected members made mutable.
00039 
00040 namespace Tpetra {
00041 
00043 
00048   class Distributor : public Teuchos::Describable {
00049   public:
00050 
00052 
00053 
00055     explicit Distributor(const RCP<const Comm<int> > & comm);
00056 
00058     Distributor(const Distributor &distributor);
00059 
00061     ~Distributor();
00062 
00064 
00065 
00067 
00068 
00070 
00087     size_t createFromSends(const ArrayView<const int> &exportNodeIDs);
00088 
00090 
00110     template <class Ordinal>
00111     void createFromRecvs(const ArrayView<const Ordinal> &remoteIDs, 
00112                          const ArrayView<const int> &remoteNodeIDs, 
00113                                ArrayRCP<Ordinal> &exportIDs, 
00114                                ArrayRCP<int> &exportNodeIDs);
00115 
00117 
00119 
00120 
00122     size_t getNumReceives() const;
00123 
00125     size_t getNumSends() const;
00126 
00128 
00129     bool hasSelfMessage() const;
00130 
00132     size_t getMaxSendLength() const;
00133 
00135     size_t getTotalReceiveLength() const;
00136 
00138     ArrayView<const int> getImagesFrom() const;
00139 
00141     ArrayView<const int> getImagesTo() const;
00142 
00144 
00145     ArrayView<const size_t> getLengthsFrom() const;
00146 
00148 
00149     ArrayView<const size_t> getLengthsTo() const;
00150 
00152 
00154 
00155 
00157 
00160     const RCP<Distributor> & getReverse() const;
00161 
00163 
00165 
00166 
00168 
00179     template <class Packet>
00180     void doPostsAndWaits(const ArrayView<const Packet> &exports,
00181                          size_t numPackets,
00182                          const ArrayView<Packet> &imports);
00183 
00185 
00196     template <class Packet>
00197     void doPostsAndWaits(const ArrayView<const Packet> &exports,
00198                          const ArrayView<size_t> &numExportPacketsPerLID,
00199                          const ArrayView<Packet> &imports,
00200                          const ArrayView<size_t> &numImportPacketsPerLID);
00201 
00203 
00214     template <class Packet>
00215     void doPosts(const ArrayView<const Packet> &exports,
00216                  size_t numPackets,
00217                  const ArrayRCP<Packet> &imports);
00218 
00220 
00231     template <class Packet>
00232     void doPosts(const ArrayView<const Packet> &exports,
00233                  const ArrayView<size_t> &numExportPacketsPerLID,
00234                  const ArrayRCP<Packet> &imports,
00235                  const ArrayView<size_t> &numImportPacketsPerLID);
00236 
00238     void doWaits();
00239 
00241 
00252     template <class Packet>
00253     void doReversePostsAndWaits(const ArrayView<const Packet> &exports,
00254                                 size_t numPackets,
00255                                 const ArrayView<Packet> &imports);
00256 
00258 
00269     template <class Packet>
00270     void doReversePostsAndWaits(const ArrayView<const Packet> &exports,
00271                                 const ArrayView<size_t> &numExportPacketsPerLID,
00272                                 const ArrayView<Packet> &imports,
00273                                 const ArrayView<size_t> &numImportPacketsPerLID);
00274 
00276 
00287     template <class Packet>
00288     void doReversePosts(const ArrayView<const Packet> &exports,
00289                         size_t numPackets,
00290                         const ArrayRCP<Packet> &imports);
00291 
00293 
00304     template <class Packet>
00305     void doReversePosts(const ArrayView<const Packet> &exports,
00306                         const ArrayView<size_t> &numExportPacketsPerLID,
00307                         const ArrayRCP<Packet> &imports,
00308                         const ArrayView<size_t> &numImportPacketsPerLID);
00309 
00311     void doReverseWaits();
00312 
00314 
00316 
00317 
00319     std::string description() const;
00320 
00322     void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const;
00323 
00325 
00326   private:
00327 
00328     // private data members
00329     RCP<const Comm<int> > comm_;
00330 
00331     size_t numExports_;
00332     // selfMessage_ is whether I have a send for myself
00333     bool selfMessage_;
00334     // numSends_ is number of sends to other nodes; is less than or equal to the number of nodes
00335     size_t numSends_;
00336     // imagesTo_, startsTo_ and lengthsTo_ each have size 
00337     //   numSends_ + selfMessage_
00338     Array<int> imagesTo_;
00339     /* Given an export buffer that contains all of the item being sent by this node,
00340        the block of values for node i will start at position startsTo_[i]  */
00341     Array<size_t> startsTo_;
00342     Array<size_t> lengthsTo_;
00343     // maxSendLength_ is the maximum send to another node: 
00344     //   max(lengthsTo_[i]) for i != me
00345     size_t maxSendLength_;
00346     Array<size_t> indicesTo_;
00347     // numReceives_ is the number of receives by me from other procs, not
00348     // counting self receives
00349     size_t numReceives_;
00350     // totalReceiveLength_ is the total number of Packet received, used to 
00351     // allocate the receive buffer
00352     size_t totalReceiveLength_;
00353     // imagesFrom_, startsFrom_ and lengthsFrom_ each have size 
00354     //   numReceives_ + selfMessage_
00355     Array<size_t> lengthsFrom_;
00356     Array<int> imagesFrom_;
00357     Array<size_t> startsFrom_;
00358     Array<size_t> indicesFrom_;
00359 
00360     // requests associated with non-blocking receives
00361     Array<RCP<Teuchos::CommRequest> > requests_;
00362 
00363     mutable RCP<Distributor> reverseDistributor_;
00364 
00365     // compute receive info from sends
00366     void computeReceives();
00367 
00368     // compute send info from receives
00369     template <class Ordinal>
00370     void computeSends(const ArrayView<const Ordinal> &importIDs,
00371                       const ArrayView<const int> &importNodeIDs,
00372                             ArrayRCP<Ordinal> &exportIDs,
00373                             ArrayRCP<int> &exportNodeIDs);
00374 
00375     // create a distributor for the reverse communciation pattern (pretty much
00376     // swap all send and receive info)
00377     void createReverseDistributor() const;
00378 
00379   }; // class Distributor
00380 
00381 
00382   template <class Packet>
00383   void Distributor::doPostsAndWaits(
00384       const ArrayView<const Packet>& exports,
00385       size_t numPackets,
00386       const ArrayView<Packet>& imports) 
00387   {
00388     TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
00389         Teuchos::typeName(*this) << "::doPostsAndWaits(): Cannot call with outstanding posts.");
00390     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00391     // however, it need only persist until doWaits is called, so it is safe for us to 
00392     // use a non-persisting reference in this case
00393     doPosts(exports, numPackets, arcp<Packet>(imports.getRawPtr(),0,imports.size(),false));
00394     doWaits();
00395   }
00396 
00397   template <class Packet>
00398   void Distributor::doPostsAndWaits(
00399       const ArrayView<const Packet>& exports,
00400       const ArrayView<size_t> &numExportPacketsPerLID,
00401       const ArrayView<Packet> &imports,
00402       const ArrayView<size_t> &numImportPacketsPerLID)
00403   {
00404     TEST_FOR_EXCEPTION(requests_.size() != 0, std::runtime_error,
00405         Teuchos::typeName(*this) << "::doPostsAndWaits(): Cannot call with outstanding posts.");
00406     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00407     // however, it need only persist until doWaits is called, so it is safe for us to 
00408     // use a non-persisting reference in this case
00409     doPosts(exports, numExportPacketsPerLID, arcp<Packet>(imports.getRawPtr(),0,imports.size(),false), numImportPacketsPerLID);
00410     doWaits();
00411   }
00412 
00413 
00414   template <class Packet>
00415   void Distributor::doPosts(const ArrayView<const Packet>& exports,
00416                             size_t numPackets,
00417                             const ArrayRCP<Packet>& imports) {
00418     // start of actual doPosts function
00419     const int myImageID = comm_->getRank();
00420     size_t selfReceiveOffset = 0;
00421 
00422 #ifdef HAVE_TEUCHOS_DEBUG
00423     TEST_FOR_EXCEPTION(Teuchos::as<size_t>(imports.size()) != totalReceiveLength_ * numPackets, std::runtime_error,
00424         Teuchos::typeName(*this) << "::doPosts(): imports must be large enough to store the imported data.");
00425 #endif
00426 
00427     // allocate space in requests
00428     requests_.resize(0);
00429     requests_.reserve(numReceives_);
00430 
00431     // start up the Irecv's
00432     {
00433       size_t curBufferOffset = 0;
00434       for (size_t i = 0; i < numReceives_ + (selfMessage_ ? 1 : 0); ++i) {
00435         if (imagesFrom_[i] != myImageID) { 
00436           // receiving this one from another image
00437           // setup reference into imports of the appropriate size and at the appropriate place
00438           ArrayRCP<Packet> impptr = imports.persistingView(curBufferOffset,lengthsFrom_[i]*numPackets);
00439           requests_.push_back( Teuchos::ireceive<int,Packet>(*comm_,impptr,imagesFrom_[i]) );
00440         }
00441         else {
00442           // receiving this one from myself 
00443           // note that offset
00444           selfReceiveOffset = curBufferOffset;
00445         }
00446         curBufferOffset += lengthsFrom_[i]*numPackets;
00447       }
00448     }
00449 
00450     // wait for everyone else before posting ready-sends below to ensure that 
00451     // all non-blocking receives above have been posted
00452     Teuchos::barrier(*comm_);
00453 
00454     // setup scan through imagesTo_ list starting with higher numbered images
00455     // (should help balance message traffic)
00456     size_t numBlocks = numSends_+ selfMessage_;
00457     size_t imageIndex = 0;
00458     while ((imageIndex < numBlocks) && (imagesTo_[imageIndex] < myImageID)) {
00459       ++imageIndex;
00460     }
00461     if (imageIndex == numBlocks) {
00462       imageIndex = 0;
00463     }
00464 
00465     size_t selfNum = 0;
00466     size_t selfIndex = 0;
00467 
00468     if (indicesTo_.empty()) { // data is already blocked by processor
00469       for (size_t i = 0; i < numBlocks; ++i) {
00470         size_t p = i + imageIndex;
00471         if (p > (numBlocks - 1)) {
00472           p -= numBlocks;
00473         }
00474 
00475         if (imagesTo_[p] != myImageID) {
00476           // sending it to another image
00477           ArrayView<const Packet> tmpSend(&exports[startsTo_[p]*numPackets],lengthsTo_[p]*numPackets);
00478           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00479         }
00480         else {
00481           // sending it to ourself
00482           selfNum = p;
00483         }
00484       }
00485 
00486       if (selfMessage_) {
00487         std::copy(exports.begin()+startsTo_[selfNum]*numPackets, exports.begin()+startsTo_[selfNum]*numPackets+lengthsTo_[selfNum]*numPackets, 
00488                   imports.begin()+selfReceiveOffset);
00489       }
00490     }
00491     else { // data is not blocked by image, use send buffer
00492       // allocate sendArray buffer
00493       Array<Packet> sendArray(maxSendLength_*numPackets); 
00494 
00495       for (size_t i = 0; i < numBlocks; ++i) {
00496         size_t p = i + imageIndex;
00497         if (p > (numBlocks - 1)) {
00498           p -= numBlocks;
00499         }
00500 
00501         if (imagesTo_[p] != myImageID) { 
00502           // sending it to another image
00503           typename ArrayView<const Packet>::iterator srcBegin, srcEnd;
00504           size_t sendArrayOffset = 0;
00505           size_t j = startsTo_[p];
00506           for (size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
00507             srcBegin = exports.begin() + indicesTo_[j]*numPackets;
00508             srcEnd   = srcBegin + numPackets;
00509             std::copy( srcBegin, srcEnd, sendArray.begin()+sendArrayOffset );
00510             sendArrayOffset += numPackets;
00511           }
00512           ArrayView<const Packet> tmpSend = sendArray(0,lengthsTo_[p]*numPackets);
00513           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00514         }
00515         else { 
00516           // sending it to myself
00517           selfNum = p;
00518           selfIndex = startsTo_[p];
00519         }
00520       }
00521 
00522       if (selfMessage_) {
00523         for (size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
00524           std::copy( exports.begin()+indicesTo_[selfIndex]*numPackets,
00525                      exports.begin()+indicesTo_[selfIndex]*numPackets + numPackets,
00526                      imports.begin() + selfReceiveOffset );
00527           ++selfIndex;
00528           selfReceiveOffset += numPackets;
00529         }
00530       }
00531     }
00532   }
00533 
00534   template <class Packet>
00535   void Distributor::doPosts(const ArrayView<const Packet>& exports,
00536                             const ArrayView<size_t>& numExportPacketsPerLID,
00537                             const ArrayRCP<Packet>& imports,
00538                             const ArrayView<size_t>& numImportPacketsPerLID) {
00539     // start of actual doPosts function
00540     const int myImageID = comm_->getRank();
00541     size_t selfReceiveOffset = 0;
00542 
00543 #ifdef HAVE_TEUCHOS_DEBUG
00544     size_t totalNumPackets = 0;
00545     for(int ii=0; ii<numImportPacketsPerLID.size(); ++ii) {
00546       totalNumPackets += numImportPacketsPerLID[ii];
00547     }
00548     TEST_FOR_EXCEPTION(Teuchos::as<size_t>(imports.size()) != totalNumPackets, std::runtime_error,
00549         Teuchos::typeName(*this) << "::doPosts(): imports must be large enough to store the imported data.");
00550 #endif
00551 
00552     // allocate space in requests
00553     requests_.resize(0);
00554     requests_.reserve(numReceives_);
00555 
00556     // start up the Irecv's
00557     {
00558       size_t curBufferOffset = 0;
00559       size_t curLIDoffset = 0;
00560       for (size_t i = 0; i < numReceives_ + (selfMessage_ ? 1 : 0); ++i) {
00561         size_t totalPacketsFrom_i = 0;
00562         for (size_t j=0; j<lengthsFrom_[i]; ++j) {
00563           totalPacketsFrom_i += numImportPacketsPerLID[curLIDoffset+j];
00564         }
00565         curLIDoffset += lengthsFrom_[i];
00566         if (imagesFrom_[i] != myImageID && totalPacketsFrom_i) { 
00567           // receiving this one from another image
00568           // setup reference into imports of the appropriate size and at the appropriate place
00569           ArrayRCP<Packet> impptr = imports.persistingView(curBufferOffset,totalPacketsFrom_i);
00570           requests_.push_back( Teuchos::ireceive<int,Packet>(*comm_,impptr,imagesFrom_[i]) );
00571         }
00572         else {
00573           // receiving this one from myself 
00574           // note that offset
00575           selfReceiveOffset = curBufferOffset;
00576         }
00577         curBufferOffset += totalPacketsFrom_i;
00578       }
00579     }
00580 
00581     // wait for everyone else before posting ready-sends below to ensure that 
00582     // all non-blocking receives above have been posted
00583     Teuchos::barrier(*comm_);
00584 
00585     // setup arrays containing starting-offsets into exports for each send,
00586     // and num-packets-to-send for each send.
00587     Array<size_t> sendPacketOffsets(numSends_,0), packetsPerSend(numSends_,0);
00588     size_t maxNumPackets = 0;
00589     size_t curPKToffset = 0;
00590     for (size_t pp=0; pp<numSends_; ++pp) {
00591       sendPacketOffsets[pp] = curPKToffset;
00592       size_t numPackets = 0;
00593       for (size_t j=startsTo_[pp]; j<startsTo_[pp]+lengthsTo_[pp]; ++j) {
00594         numPackets += numExportPacketsPerLID[j];
00595       }
00596       if (numPackets > maxNumPackets) maxNumPackets = numPackets;
00597       packetsPerSend[pp] = numPackets;
00598       curPKToffset += numPackets;
00599     }
00600 
00601     // setup scan through imagesTo_ list starting with higher numbered images
00602     // (should help balance message traffic)
00603     size_t numBlocks = numSends_+ selfMessage_;
00604     size_t imageIndex = 0;
00605     while ((imageIndex < numBlocks) && (imagesTo_[imageIndex] < myImageID)) {
00606       ++imageIndex;
00607     }
00608     if (imageIndex == numBlocks) {
00609       imageIndex = 0;
00610     }
00611 
00612     size_t selfNum = 0;
00613     size_t selfIndex = 0;
00614 
00615     if (indicesTo_.empty()) { // data is already laid out according to processor
00616       for (size_t i = 0; i < numBlocks; ++i) {
00617         size_t p = i + imageIndex;
00618         if (p > (numBlocks - 1)) {
00619           p -= numBlocks;
00620         }
00621 
00622         if (imagesTo_[p] != myImageID && packetsPerSend[p] > 0) {
00623           // sending it to another image
00624           ArrayView<const Packet> tmpSend(&exports[sendPacketOffsets[p]],packetsPerSend[p]);
00625           Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00626         }
00627         else {
00628           // sending it to ourself
00629           selfNum = p;
00630         }
00631       }
00632 
00633       if (selfMessage_) {
00634         std::copy(exports.begin()+sendPacketOffsets[selfNum], exports.begin()+sendPacketOffsets[selfNum]+packetsPerSend[selfNum], 
00635                   imports.begin()+selfReceiveOffset);
00636       }
00637     }
00638     else { // data is not blocked by image, use send buffer
00639       // allocate sendArray buffer
00640       Array<Packet> sendArray(maxNumPackets); 
00641       Array<size_t> indicesOffsets(numExportPacketsPerLID.size(),0);
00642       size_t ioffset = 0;
00643       for (int j=0; j<numExportPacketsPerLID.size(); ++j) {
00644         indicesOffsets[j] = ioffset;
00645         ioffset += numExportPacketsPerLID[j];
00646       }
00647 
00648       for (size_t i = 0; i < numBlocks; ++i) {
00649         size_t p = i + imageIndex;
00650         if (p > (numBlocks - 1)) {
00651           p -= numBlocks;
00652         }
00653 
00654         if (imagesTo_[p] != myImageID) { 
00655           // sending it to another image
00656           typename ArrayView<const Packet>::iterator srcBegin, srcEnd;
00657           size_t sendArrayOffset = 0;
00658           size_t j = startsTo_[p];
00659           size_t numPacketsTo_p = 0;
00660           for (size_t k = 0; k < lengthsTo_[p]; ++k, ++j) {
00661             srcBegin = exports.begin() + indicesOffsets[j];
00662             srcEnd   = srcBegin + numExportPacketsPerLID[j];
00663             numPacketsTo_p += numExportPacketsPerLID[j];
00664             std::copy( srcBegin, srcEnd, sendArray.begin()+sendArrayOffset );
00665             sendArrayOffset += numExportPacketsPerLID[j];
00666           }
00667           if (numPacketsTo_p > 0) {
00668             ArrayView<const Packet> tmpSend = sendArray(0,numPacketsTo_p);
00669             Teuchos::readySend<int,Packet>(*comm_,tmpSend,imagesTo_[p]);
00670           }
00671         }
00672         else { 
00673           // sending it to myself
00674           selfNum = p;
00675           selfIndex = startsTo_[p];
00676         }
00677       }
00678 
00679       if (selfMessage_) {
00680         for (size_t k = 0; k < lengthsTo_[selfNum]; ++k) {
00681           std::copy( exports.begin()+indicesOffsets[selfIndex],
00682                      exports.begin()+indicesOffsets[selfIndex]+numExportPacketsPerLID[selfIndex],
00683                      imports.begin() + selfReceiveOffset );
00684           selfReceiveOffset += numExportPacketsPerLID[selfIndex];
00685           ++selfIndex;
00686         }
00687       }
00688     }
00689   }
00690 
00691 
00692   template <class Packet>
00693   void Distributor::doReversePostsAndWaits(
00694       const ArrayView<const Packet>& exports,
00695       size_t numPackets,
00696       const ArrayView<Packet>& imports) 
00697   {
00698     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00699     // however, it need only persist until doWaits is called, so it is safe for us to 
00700     // use a non-persisting reference in this case
00701     doReversePosts(exports, numPackets, arcp<Packet>(imports.getRawPtr(),0,imports.size(),false));
00702     doReverseWaits();
00703   }
00704 
00705   template <class Packet>
00706   void Distributor::doReversePostsAndWaits(
00707        const ArrayView<const Packet>& exports,
00708        const ArrayView<size_t> &numExportPacketsPerLID,
00709        const ArrayView<Packet> &imports,
00710        const ArrayView<size_t> &numImportPacketsPerLID)
00711   {
00712     // doPosts takes imports as an ArrayRCP, requiring that the memory location is persisting
00713     // however, it need only persist until doWaits is called, so it is safe for us to 
00714     // use a non-persisting reference in this case
00715     doReversePosts(exports, numExportPacketsPerLID, arcp<Packet>(imports.getRawPtr(),0,imports.size(),false),numImportPacketsPerLID);
00716     doReverseWaits();
00717   }
00718 
00719 
00720   template <class Packet>
00721   void Distributor::doReversePosts(
00722       const ArrayView<const Packet>& exports,
00723       size_t numPackets,
00724       const ArrayRCP<Packet>& imports) 
00725   {
00726     TEST_FOR_EXCEPTION(!indicesTo_.empty(),std::runtime_error,
00727         Teuchos::typeName(*this) << "::doReversePosts(): Can only do reverse comm when original data is blocked by image.");
00728     if (reverseDistributor_ == null) {
00729       createReverseDistributor();
00730     }
00731     reverseDistributor_->doPosts(exports,numPackets,imports);
00732   }
00733 
00734   template <class Packet>
00735   void Distributor::doReversePosts(
00736       const ArrayView<const Packet>& exports,
00737       const ArrayView<size_t>& numExportPacketsPerLID,
00738       const ArrayRCP<Packet>& imports,
00739       const ArrayView<size_t>& numImportPacketsPerLID) 
00740   {
00741     TEST_FOR_EXCEPTION(!indicesTo_.empty(),std::runtime_error,
00742         Teuchos::typeName(*this) << "::doReversePosts(): Can only do reverse comm when original data is blocked by image.");
00743     if (reverseDistributor_ == null) {
00744       createReverseDistributor();
00745     }
00746     reverseDistributor_->doPosts(exports,numExportPacketsPerLID,imports,numImportPacketsPerLID);
00747   }
00748 
00749 
00750   // note: assumes that size_t >= Ordinal
00752   template <class Ordinal>
00753   void Distributor::computeSends(
00754       const ArrayView<const Ordinal> & importIDs,
00755       const ArrayView<const int> & importNodeIDs,
00756             ArrayRCP<Ordinal> & exportIDs,
00757             ArrayRCP<int> & exportNodeIDs)
00758   {
00759     int myImageID = comm_->getRank();
00760 
00761     size_t numImports = importNodeIDs.size();
00762     Array<size_t> importObjs(2*numImports);
00763     for (size_t i = 0; i < numImports; ++i ) {  
00764       importObjs[2*i]   = Teuchos::as<size_t>(importIDs[i]);
00765       importObjs[2*i+1] = Teuchos::as<size_t>(myImageID);
00766     }
00767 
00768     size_t numExports;
00769     Distributor tempPlan(comm_);
00770     numExports = tempPlan.createFromSends(importNodeIDs);
00771     if (numExports > 0) {
00772       exportIDs = arcp<Ordinal>(numExports);
00773       exportNodeIDs = arcp<int>(numExports);
00774     }
00775 
00776     Array<size_t> exportObjs(tempPlan.getTotalReceiveLength()*2);
00777     tempPlan.doPostsAndWaits<size_t>(importObjs(),2,exportObjs());
00778 
00779     for (size_t i = 0; i < numExports; ++i) {
00780       exportIDs[i]     = Teuchos::as<Ordinal>(exportObjs[2*i]);
00781       exportNodeIDs[i] = exportObjs[2*i+1];
00782     }
00783   }
00784 
00785 
00787   template <class Ordinal>
00788   void Distributor::createFromRecvs(
00789       const ArrayView<const Ordinal> &remoteIDs, 
00790       const ArrayView<const int> &remoteImageIDs, 
00791             ArrayRCP<Ordinal> &exportGIDs, 
00792             ArrayRCP<int> &exportNodeIDs)
00793   {
00794     using Teuchos::outArg;
00795     {
00796       const int myImageID = comm_->getRank();
00797       int err_node = (remoteIDs.size() != remoteImageIDs.size()) ? myImageID : -1;
00798       int gbl_err;
00799       Teuchos::reduceAll(*comm_,Teuchos::REDUCE_MAX,err_node,outArg(gbl_err));
00800       TEST_FOR_EXCEPTION(gbl_err != -1, std::runtime_error,
00801           Teuchos::typeName(*this) 
00802           << "::createFromRecvs(): lists of remote IDs and remote node IDs must have the same size (error on node " 
00803           << gbl_err << ").");
00804     }
00805     computeSends(remoteIDs, remoteImageIDs, exportGIDs, exportNodeIDs);
00806     (void)createFromSends(exportNodeIDs());
00807   }
00808 
00809 } // namespace Tpetra
00810 
00811 #endif // TPETRA_DISTRIBUTOR_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines