FEI Version of the Day
fei_NodeCommMgr.cpp
00001 /*
00002 // @HEADER
00003 // ************************************************************************
00004 //             FEI: Finite Element Interface to Linear Solvers
00005 //                  Copyright (2005) Sandia Corporation.
00006 //
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, the
00008 // U.S. Government retains certain rights in this software.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Alan Williams (william@sandia.gov) 
00038 //
00039 // ************************************************************************
00040 // @HEADER
00041 */
00042 
00043 
00044 #include <fei_macros.hpp>
00045 
00046 #include <fei_mpi.h>
00047 
00048 #include <fei_defs.h>
00049 
00050 #include <fei_TemplateUtils.hpp>
00051 #include <fei_mpiTraits.hpp>
00052 #include <fei_CommUtils.hpp>
00053 #include <fei_NodeDescriptor.hpp>
00054 #include <fei_NodeCommMgr.hpp>
00055 #include <SNL_FEI_Structure.hpp>
00056 
00057 #include <fei_NodeDatabase.hpp>
00058 
00059 #undef fei_file
00060 #define fei_file "fei_NodeCommMgr.cpp"
00061 #include <fei_ErrMacros.hpp>
00062 
00063 //------Constructor-------------------------------------------------------------
00064 NodeCommMgr::NodeCommMgr(MPI_Comm comm, const SNL_FEI_Structure& problemStructure, int sharedNodeOwnership)
00065   : sharedNodes_(NULL),
00066     sharedNodesAllocated_(false),
00067     sharedNodeOwnership_(sharedNodeOwnership),
00068     localNodeIDs(),
00069     remoteNodeIDs(),
00070     sharedNodeIDs(),
00071     sharedNodeSubdomains(),
00072     trivialSubdomainList(1),
00073     sharingProcs_(),
00074     sharedNodeNumbers(),
00075     remoteOwnerProcs_(),
00076     remoteSharingProcs_(),
00077     nodesPerOwnerProc_(),
00078     nodesPerSharingProc_(),
00079     comm_(comm),
00080     numProcs_(1),
00081     localProc_(0),
00082     maxFields_(0),
00083     maxBlocks_(0),
00084     maxSubdomains_(0),
00085     initCompleteCalled_(false),
00086     probStruc(problemStructure)
00087 {
00088   numProcs_ = fei::numProcs(comm_);
00089   localProc_= fei::localProc(comm_);
00090   trivialSubdomainList[0] = localProc_;
00091 }
00092 
00093 //-----Destructor---------------------------------------------------------------
00094 NodeCommMgr::~NodeCommMgr() {
00095 
00096    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00097       delete sharingProcs_[i];
00098    }
00099 
00100    delete [] sharedNodes_;
00101    sharedNodesAllocated_ = false;
00102 }
00103 
00104 //------------------------------------------------------------------------------
00105 int NodeCommMgr::getSharedNodeIndex(GlobalID nodeID)
00106 {
00107   return( fei::binarySearch(nodeID, &sharedNodeIDs[0], sharedNodeIDs.size()) );
00108 }
00109 
00110 //------------------------------------------------------------------------------
00111 int NodeCommMgr::getSharedNodeNumSubdomains(GlobalID nodeID)
00112 {
00113   int index = getSharedNodeIndex(nodeID);
00114 
00115   //If the node isn't one of our shared nodes, then return 1, signifying that
00116   //the node is in 1 subdomain, that being the local subdomain.
00117   if (index < 0) return(1);
00118 
00119   //Since the node is one of our shared nodes, it should have an entry in our
00120   //sharedNodeNumSubdomains array indicating how many subdomains it
00121   //appears in. So return this number.
00122   return(sharedNodeSubdomains[index].size());
00123 }
00124 
00125 //------------------------------------------------------------------------------
00126 std::vector<int>* NodeCommMgr::getSharedNodeSubdomainList(GlobalID nodeID)
00127 {
00128   int index = getSharedNodeIndex(nodeID);
00129 
00130   //If the node isn't one of our shared nodes, then return 1, signifying that
00131   //the node is in 1 subdomain, that being the local subdomain.
00132   if (index < 0) return( &trivialSubdomainList );
00133 
00134   //Since the node is one of our shared nodes, it should have an entry in our
00135   //sharedNodeSubdomains array.
00136   return( &(sharedNodeSubdomains[index]) );
00137 }
00138 
00139 //------------------------------------------------------------------------------
00140 int NodeCommMgr::informLocal(const NodeDescriptor& node) {
00141 //
00142 //NodeCommMgr is being informed that 'node' is present in the local
00143 //active node list.
00144 //
00145 //This means that either:
00146 // 1. it is a locally-owned shared node, or
00147 // 3. it isn't even a shared node (it's a purely local node), in which
00148 //    case we'll do nothing.
00149 //
00150 
00151    if (numProcs_ == 1) return(0);
00152 
00153    GlobalID nodeID = node.getGlobalNodeID();
00154 
00155    int sharedIndex = getSharedNodeIndex(nodeID);
00156 
00157    //if this node isn't a shared node, then simply return.
00158    if (sharedIndex < 0) return(0);
00159 
00160    //Since this node is present as a shared node, let's put nodeID in the
00161    //localNodeIDs list if it isn't already there.
00162 
00163    int index = fei::sortedListInsert(nodeID, localNodeIDs);
00164 
00165    //if index is -2, it means the localNodeIDs array had an allocation failure.
00166    if (index == -2) return(-2);
00167 
00168    return(0);
00169 }
00170 
00171 //------------------------------------------------------------------------------
00172 int NodeCommMgr::getGlobalMaxFieldsBlocks(int& maxFields, int& maxBlocks)
00173 {
00174   std::vector<int> localMax(2, 0), globalMax(2, 0);
00175 
00176   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00177     int numFlds = sharedNodes_[i]->getNumFields();
00178     if (numFlds > localMax[0]) localMax[0] = numFlds;
00179 
00180     int numBlks = sharedNodes_[i]->getNumBlocks();
00181     if (numBlks > localMax[1]) localMax[1] = numBlks;
00182   }
00183 
00184   int err = fei::GlobalMax(comm_, localMax, globalMax);
00185   if (err != 0) return(err);
00186 
00187   maxFields = globalMax[0];
00188   maxBlocks = globalMax[1];
00189 
00190   return(0);
00191 }
00192 
00193 //------------------------------------------------------------------------------
00194 int NodeCommMgr::getGlobalMaxFieldsBlocksSubdomains()
00195 {
00196   std::vector<int> localMax(3, 0), globalMax(3, 0);
00197 
00198   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00199     int numFlds = sharedNodes_[i]->getNumFields();
00200     if (numFlds > localMax[0]) localMax[0] = numFlds;
00201 
00202     int numBlks = sharedNodes_[i]->getNumBlocks();
00203     if (numBlks > localMax[1]) localMax[1] = numBlks;
00204 
00205     int numShrd = sharingProcs_[i]->size();
00206     if (numShrd > localMax[2]) localMax[2] = numShrd;
00207   }
00208 
00209   int err = fei::GlobalMax(comm_, localMax, globalMax);
00210   if (err != 0) return(err);
00211 
00212   maxFields_     = globalMax[0];
00213   maxBlocks_     = globalMax[1];
00214   maxSubdomains_ = globalMax[2];
00215 
00216   return(0);
00217 }
00218 
00219 //------------------------------------------------------------------------------
00220 std::vector<int>& NodeCommMgr::getSendProcs()
00221 {
00222   return( remoteSharingProcs_ );
00223 }
00224 
00225 //------------------------------------------------------------------------------
00226 std::vector<int>& NodeCommMgr::getRecvProcs()
00227 {
00228   return( remoteOwnerProcs_ );
00229 }
00230 
00231 //------------------------------------------------------------------------------
00232 int NodeCommMgr::getSendMessageLength(int destProc, int& messageLength)
00233 {
00234   std::vector<int>::iterator
00235    rs_iter = std::lower_bound(remoteSharingProcs_.begin(),
00236                               remoteSharingProcs_.end(), destProc);
00237   if (rs_iter == remoteSharingProcs_.end() || destProc != *rs_iter) {
00238     ERReturn(-1);
00239   }
00240 
00241   int idx = rs_iter - remoteSharingProcs_.begin();
00242 
00243   int len = 7+maxFields_*2 + maxBlocks_ + maxSubdomains_;
00244   messageLength = nodesPerSharingProc_[idx] * (len+1);
00245   return(0);
00246 }
00247 
00248 //------------------------------------------------------------------------------
00249 int NodeCommMgr::getSendMessage(int destProc, std::vector<int>& message)
00250 {
00251   std::vector<int>::iterator
00252    rs_iter = std::lower_bound(remoteSharingProcs_.begin(),
00253                               remoteSharingProcs_.end(), destProc);
00254   if (rs_iter == remoteSharingProcs_.end() || destProc != *rs_iter) {
00255     ERReturn(-1);
00256   }
00257 
00258   int idx = rs_iter - remoteSharingProcs_.begin();
00259   int len = 0;
00260   CHK_ERR( getSendMessageLength(destProc, len) );
00261   message.resize(len);
00262 
00263   packLocalNodesAndData(&message[0], destProc,
00264       nodesPerSharingProc_[idx], len);
00265   return(0);
00266 }
00267 
00268 //------------------------------------------------------------------------------
00269 int NodeCommMgr::processRecvMessage(int srcProc, std::vector<int>& message)
00270 {
00271   int idx = fei::binarySearch(srcProc, &remoteOwnerProcs_[0],
00272                                   remoteOwnerProcs_.size());
00273   int numNodes = nodesPerOwnerProc_[idx];
00274   int* msgPtr = &message[0];
00275   int offset = 0;
00276 
00277   for(int j=0; j<numNodes; j++) {
00278     int nIndex = fei::binarySearch(msgPtr[j], &sharedNodeIDs[0], sharedNodeIDs.size());
00279     if (nIndex < 0) return(-1);
00280     NodeDescriptor* node = sharedNodes_[nIndex];
00281 
00282     int nodeNum         = msgPtr[numNodes+offset++];
00283     int numFields       = msgPtr[numNodes+offset++];
00284     int numBlocks       = msgPtr[numNodes+offset++];
00285     int numSubdomains   = msgPtr[numNodes+offset++];
00286 
00287     node->setNodeNumber(nodeNum);
00288     node->setNumNodalDOF( msgPtr[numNodes+offset++]);
00289     node->setBlkEqnNumber(msgPtr[numNodes+offset++]);
00290 
00291     for(int fld=0; fld<numFields; fld++) {
00292       int fieldID = msgPtr[numNodes+offset++];
00293       int eqnNum  = msgPtr[numNodes+offset++];
00294       node->addField(fieldID);
00295       node->setFieldEqnNumber(fieldID, eqnNum);
00296     }
00297 
00298     for(int blk=0; blk<numBlocks; blk++) {
00299       int blk_idx = probStruc.getIndexOfBlock(msgPtr[numNodes+offset++]);
00300       //if blk_idx < 0 it means the incoming blockID doesn't exist on this proc
00301       if (blk_idx >= 0) {
00302         node->addBlockIndex(blk_idx);
00303       }
00304     }
00305 
00306     sharedNodeSubdomains[nIndex].resize(numSubdomains);
00307     for(int sd=0; sd<numSubdomains; sd++) {
00308       (sharedNodeSubdomains[nIndex])[sd] =
00309   msgPtr[numNodes+offset++];
00310     }
00311   }
00312 
00313   return(0);
00314 }
00315 
00316 //------------------------------------------------------------------------------
00317 int NodeCommMgr::exchangeEqnInfo()
00318 {
00319   //
00320   //This function will perform the communication necessary to:
00321   //
00322   //   1. For each locally owned node, send to each remote sharing proc:
00323   //      the node's nodeNumber, whether or not it appears in local elements,
00324   //      fieldID(s), the field-size(s), the first global equation numbers
00325   //      for those fields, and the processor-subdomains that contain the node.
00326   //   2. For each remotely owned node, receive the above information from
00327   //      the owners of these nodes.
00328   //
00329   //This is a collective function. All procs must enter it before any can
00330   //exit it.
00331   //
00332   //Most of this function is #ifdef'd according to whether FEI_SER is
00333   //defined.
00334 
00335 #ifndef FEI_SER
00336    if (numProcs_ == 1) return(0);
00337 
00338    //each proc will find out a max. number of fields, blocks
00339    //and subdomains to expect per node.
00340 
00341    CHK_ERR( getGlobalMaxFieldsBlocksSubdomains() );
00342 
00343    CHK_ERR( fei::exchange(comm_, this) );
00344 
00345    setNodeNumbersArray();
00346 
00347 #endif //#ifndef FEI_SER
00348 
00349    return(0);
00350 }
00351 
00352 //------------------------------------------------------------------------------
00353 void NodeCommMgr::packLocalNodesAndData(int* data, 
00354                                        int proc, int numNodes, int len)
00355 {
00356 //This function packs up nodeIDs, as well as the list containing, for
00357 //each node, the following:
00358 //   node-number
00359 //   numFields
00360 //   numBlocks
00361 //   numSubdomains
00362 //   num-nodal-dof
00363 //   blk-eqn-number
00364 //     'numFields' pairs of (fieldID,eqnNumber)
00365 //     subdomain list, length 'numSubdomains'
00366 //
00367 //Incoming parameter len is:
00368 //  numNodes * (7 + maxFields*2 + maxBlocks + maxSubdomains),
00369 //where maxFields is the maximum number of fields associated with any node,
00370 //maxBlocks is the maximum number of blocks associated with any node, and
00371 //maxSubdomains is the maximum number of subdomains containing any node.
00372 //data is of length numNodes*(len+1).
00373 //
00374 //The above data will all be packed into the 'data' list, with nodeIDs 
00375 //occupying the first numNodes positions, followed by the rest of the data.
00376 //
00377    int nodeCounter = 0;
00378    int offset = 0;
00379 
00380    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00381      if (sharedNodes_[i]->getOwnerProc() != localProc_) continue;
00382 
00383       NodeDescriptor* node = sharedNodes_[i];
00384 
00385       //is this local node associated with processor 'proc'?
00386 
00387       std::vector<int>& sProcs = *(sharingProcs_[i]);
00388       int index = fei::binarySearch(proc, &sProcs[0], sProcs.size());
00389 
00390       //if not, skip to the next iteration...
00391       if (index < 0) continue;
00392 
00393       if (nodeCounter >= numNodes) {
00394   fei::console_out() << "NodeCommMgr::packLocalNodesAndData: ERROR,"
00395        << " nodeCounter >= numNodes." << FEI_ENDL;
00396       }
00397 
00398       data[nodeCounter++] = (int)(node->getGlobalNodeID());
00399 
00400       int nodeNum = node->getNodeNumber();
00401       int numFields = node->getNumFields();
00402       int numBlocks = node->getNumBlocks();
00403       const int* fieldIDsPtr = node->getFieldIDList();
00404       const int* fieldEqnNums = node->getFieldEqnNumbers();
00405       int blkEqnNumber = node->getBlkEqnNumber();
00406 
00407       const std::vector<unsigned>& nodeBlocks = node->getBlockIndexList();
00408       std::vector<int>& subdomains = sharedNodeSubdomains[i];
00409 
00410       data[numNodes+offset++] = nodeNum;
00411       data[numNodes+offset++] = numFields;
00412       data[numNodes+offset++] = numBlocks;
00413       data[numNodes+offset++] = subdomains.size();
00414       data[numNodes+offset++] = node->getNumNodalDOF();
00415       data[numNodes+offset++] = blkEqnNumber;
00416 
00417       for(int j=0; j<numFields; j++) {
00418   data[numNodes+offset++] = fieldIDsPtr[j];
00419 
00420   if (offset >= len) {
00421     fei::console_out() << "NodeCommMgr::packLocalNodesAndData: ERROR,"
00422          << " offset >= len." << FEI_ENDL;
00423   }
00424 
00425   data[numNodes+offset++] = fieldEqnNums[j];
00426       }
00427 
00428       for(int kk=0; kk<numBlocks; kk++) {
00429         GlobalID blkID = probStruc.getBlockID(nodeBlocks[kk]);
00430         data[numNodes+offset++] = blkID;
00431       }
00432 
00433       for(unsigned k=0; k<subdomains.size(); k++) {
00434   data[numNodes+offset++] = subdomains[k];
00435       }
00436    }
00437 }
00438 
00439 //------------------------------------------------------------------------------
00440 void NodeCommMgr::packRemoteNodesAndData(GlobalID* data,
00441            int proc, int numNodes, int len)
00442 {
00443 //
00444 //This function packs up the nodeIDs owned by proc, as well as the list
00445 //containing, for each node, the following:
00446 //   residesLocally (0 or 1) indicating whether it appears in the local
00447 //       processor's element domain.
00448 //   numFields
00449 //   numBlocks
00450 //   numNodalDOF
00451 //     'numFields' entries of (fieldID)
00452 //     'numBlocks' entries of (block)
00453 //
00454 //Incoming parameter len is numNodes * (3 + maxFields + maxBlocks),
00455 //where maxFields is the maximum number of fields associated with any node,
00456 //and maxBlocks is the maximum number of blocks associated with any node.
00457 //nodeIDs is of length numNodes, and
00458 //data is of length numNodes*len.
00459 //
00460 //The above data will all be put in the 'data' list, with the nodeIDs
00461 //occupying the first numNodes positions, followed by the rest of the data.
00462 //
00463    int nodeCounter = 0;
00464    int offset = 0;
00465 
00466    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00467       NodeDescriptor* node = sharedNodes_[i];
00468 
00469       int thisProc = node->getOwnerProc();
00470       if (thisProc != proc) continue;
00471 
00472       if (nodeCounter >= numNodes) {
00473          fei::console_out() << localProc_ << ": NodeCommMgr::packRemoteNodesAndData: ERROR,"
00474               << " nodeCounter >= numNodes: " << numNodes << FEI_ENDL;
00475       }
00476 
00477       data[nodeCounter++] = node->getGlobalNodeID();
00478 
00479       int numFields = node->getNumFields();
00480       int numBlocks = node->getNumBlocks();
00481       const int* fieldIDsPtr = node->getFieldIDList();
00482 
00483       const std::vector<unsigned>& nodeBlocks = node->getBlockIndexList();
00484       int lindex = fei::binarySearch(sharedNodeIDs[i], &localNodeIDs[0], localNodeIDs.size());
00485 
00486       data[numNodes+offset++] = (lindex >= 0) ? 1 : 0;
00487       data[numNodes+offset++] = (GlobalID)numFields;
00488       data[numNodes+offset++] = (GlobalID)numBlocks;
00489       data[numNodes+offset++] = (GlobalID)node->getNumNodalDOF();
00490 
00491       for(int j=0; j<numFields; j++) {
00492          if (offset >= len) {
00493             fei::console_out() << "NodeCommMgr::packRemoteNodesAndData: ERROR,"
00494                  << " offset >= len." << FEI_ENDL;
00495          }
00496 
00497          data[numNodes+offset++] = (GlobalID)fieldIDsPtr[j];
00498       }
00499 
00500       for(int k=0; k<numBlocks; k++) {
00501          if (offset >= len) {
00502             fei::console_out() << "NodeCommMgr::packRemoteNodesAndData: ERROR,"
00503                  << " offset >= len." << FEI_ENDL;
00504          }
00505 
00506          data[numNodes+offset++] = probStruc.getBlockID(nodeBlocks[k]);
00507       }
00508    }
00509 }
00510 
00511 //------------------------------------------------------------------------------
00512 int NodeCommMgr::createProcList(std::vector<int>& itemsPerProc,
00513                                 std::vector<int>& procs)
00514 {
00515 //
00516 //This function looks through the itemsPerProc list and counts how many
00517 //positions in this list are greater than 0. Then it creates a list of
00518 //the indices of those positions. i.e., itemsPerProc is a list of how many
00519 //items are to be sent to or recvd from each proc. When itemsPerProc is
00520 //greater than 0, that proc is put in the sharingProcs list.
00521 //
00522    int numProcs = 0;
00523    int len = itemsPerProc.size();
00524 
00525    for(int i=0; i<len; i++) {
00526       if (itemsPerProc[i] > 0) numProcs++;
00527    }
00528 
00529    procs.resize(numProcs);
00530 
00531    int offset = 0;
00532 
00533    for(int i=0; i<len; i++) {
00534       if (itemsPerProc[i] > 0) procs[offset++] = i;
00535    }
00536    return(0);
00537 }
00538 
00539 //------------------------------------------------------------------------------
00540 int NodeCommMgr::getSharedNodeIndex_num(int nodeNumber)
00541 {
00542   for(unsigned i=0; i<sharedNodeNumbers.size(); ++i) {
00543     if (sharedNodeNumbers[i] == nodeNumber) return(i);
00544   }
00545 
00546   return(-1);
00547 }
00548 
00549 //------------------------------------------------------------------------------
00550 int NodeCommMgr::addSharedNodes( const GlobalID* nodeIDs,
00551          int numNodes, 
00552          const int* const* procs,
00553          const int* numProcs )
00554 {
00555   //
00556   //Store the incoming nodeIDs and proc-numbers in the sharedNodeIDs array and
00557   //sharingProcs_ table.
00558   //
00559 
00560   try {
00561 
00562   for(int i=0; i<numNodes; i++) {
00563     int insertPoint = -1;
00564     int index = fei::binarySearch(nodeIDs[i], sharedNodeIDs, insertPoint);
00565     if (index < 0) {
00566       sharingProcs_.insert(sharingProcs_.begin()+insertPoint, new std::vector<int>);
00567 
00568       sharedNodeIDs.insert(sharedNodeIDs.begin()+insertPoint, nodeIDs[i]);
00569 
00570       index = insertPoint;
00571     }
00572     
00573     int err = storeNodeProcs(index, sharingProcs_, procs[i], numProcs[i]);
00574     if (err != 0) return(err);
00575   }
00576 
00577   }
00578   catch(std::runtime_error& exc) {
00579     fei::console_out() << exc.what() << FEI_ENDL;
00580     ERReturn(-1);
00581   }
00582 
00583   return(0);
00584 }
00585 
00586 //------------------------------------------------------------------------------
00587 int NodeCommMgr::allocateNodeDescriptorPtrs(NodeDatabase& nodeDB)
00588 {
00589   //This function is called when all shared nodes have been added. We now
00590   //allocate a list of pointer-to-NodeDescriptor, of length 
00591   //sharedNodeIDs.length(), and fill that list with NodeDescriptor-pointers
00592   //from the node-database.
00593 
00594   if (sharedNodeIDs.size() == 0) return(0);
00595 
00596   if (sharedNodes_ != NULL) delete [] sharedNodes_;
00597   sharedNodes_ = new NodeDescriptor*[sharedNodeIDs.size()];
00598   if (sharedNodes_ == NULL) return(-1);
00599 
00600   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00601     NodeDescriptor* node = NULL;
00602     int err = nodeDB.getNodeWithID(sharedNodeIDs[i], node);
00603     if (err != 0) return(-1);
00604 
00605     sharedNodes_[i] = node;
00606   }
00607 
00608   sharedNodesAllocated_ = true;
00609   return(0);
00610 }
00611 
00612 //------------------------------------------------------------------------------
00613 int NodeCommMgr::initComplete(NodeDatabase& nodeDB, bool safetyCheck)
00614 {
00615 //
00616 //This function is called when initialization is complete (i.e., when
00617 //all sharedNodes have been added, allocatedNodeDescriptorPtrs() has been
00618 //called, and informLocal() has been called for all nodes that appear in
00619 //the local finite-element structure.
00620 //
00621 //The task of this function is to assign owner-procs to nodes.
00622 //
00623 //if 'safetyCheck' is true, a global consistency check of the shared node info
00624 //will be performed before the communication is attempted.
00625 //
00626 //return value is 0 if successful, non-zero if an error was encountered
00627 //
00628   int err = allocateNodeDescriptorPtrs(nodeDB);
00629   if (err != 0) return(err);
00630 
00631   //Run through the shared nodes, and for each one that has been
00632   //identified as local, assign its owner to be the lowest-numbered sharing
00633   //processor, which may or may not be localProc_.
00634 
00635   for(unsigned ii=0; ii<sharedNodeIDs.size(); ii++) {
00636     std::vector<int>& shProcs = *(sharingProcs_[ii]);
00637 
00638     //first, insert localProc_ in this node's list of sharing proc, since the
00639     //FEI's initSharedNodes function doesn't mandate that the local processor be
00640     //included in the list of sharing processors. (i.e., localProc_ may not be
00641     //in this list yet...)
00642     std::vector<int>::iterator sh_iter =
00643       std::lower_bound(shProcs.begin(), shProcs.end(), localProc_);
00644     if (sh_iter == shProcs.end() || localProc_ != *sh_iter) {
00645       shProcs.insert(sh_iter, localProc_);
00646     }
00647 
00648     int proc = shProcs[0];
00649 
00650     sharedNodes_[ii]->setOwnerProc(proc);
00651   }
00652 
00653   //One of the tasks of this object is to gather information on the number
00654   //of subdomains each shared node appears in. So one thing we'll do here is
00655   //size and zero the array that will hold that information.
00656   sharedNodeSubdomains.resize(sharedNodeIDs.size());
00657 
00658   for(unsigned i=0; i<sharedNodeSubdomains.size(); ++i) {
00659     sharedNodeSubdomains[i].resize(0);
00660   }
00661 
00662   //now add the local processor to the sharedNodeSubdomains for each node that
00663   //appears in our localNodeIDs list.
00664   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00665     int index = fei::binarySearch(sharedNodeIDs[i], &localNodeIDs[0], localNodeIDs.size());
00666     if (index >= 0) {
00667       sharedNodeSubdomains[i].push_back(localProc_);
00668     }
00669   }
00670 
00671   if (sharedNodeOwnership_ != STRICTLY_LOW_PROC) {
00672     err = adjustSharedOwnership();
00673     if (err != 0) return(err);
00674   }
00675 
00676   err = createProcLists();
00677 
00678   if (safetyCheck) {
00679     err = checkSharedNodeInfo();
00680     if (err != 0) return(-1);
00681   }
00682 
00683   exchangeSharedRemoteFieldsBlks();
00684 
00685   initCompleteCalled_ = true;
00686 
00687   return(0);
00688 }
00689 
00690 //------------------------------------------------------------------------------
00691 #undef _feiFunc_
00692 #define _feiFunc_ "NodeCommMgr::checkSharedNodeInfo"
00693 int NodeCommMgr::checkSharedNodeInfo()
00694 {
00695   //This function's task is to "audit" the shared-node info. I.e., to make sure
00696   //that the info is globally symmetric (e.g., if the local processor thinks it
00697   //shares a node with another processor, does that other processor also think
00698   //it shares a node with the local proc?).
00699   //If this function finds that the shared-node info is consistent/correct, then
00700   //the return-value is 0. If the shared-node info is found to be wrong, then
00701   //one or more messages will be written to stderr, and the return-value is -1.
00702   //
00703   //This is a collective function, which is relatively expensive. It does a
00704   //few global reductions...
00705   //
00706 
00707   if (numProcs_==1) return(0);
00708 
00709   //Make sure that the processors we think are "remote owner"
00710   //procs, think we are a "remote sharing" proc, and vice-versa.
00711 
00712   std::vector<int> globalOwnerProcs, globalSharingProcs;
00713   std::vector<int> recvOwnerLengths, recvSharingLengths;
00714 
00715   std::vector<int> globalNodesPerOwnerProcs, globalNodesPerSharingProcs;
00716   std::vector<int> recvNodesPerOwnerLengths, recvNodesPerSharingLengths;
00717 
00718   //First, gather up each processor's list of remote procs and nodes-per-proc
00719   //onto all other processors...
00720 
00721   CHK_ERR( fei::Allgatherv(comm_, remoteOwnerProcs_,
00722          recvOwnerLengths, globalOwnerProcs) );
00723 
00724   CHK_ERR( fei::Allgatherv(comm_, nodesPerOwnerProc_,
00725          recvNodesPerOwnerLengths,
00726          globalNodesPerOwnerProcs) );
00727 
00728   CHK_ERR( fei::Allgatherv(comm_, remoteSharingProcs_,
00729          recvSharingLengths, globalSharingProcs) );
00730 
00731   CHK_ERR( fei::Allgatherv(comm_, nodesPerSharingProc_,
00732          recvNodesPerSharingLengths,
00733          globalNodesPerSharingProcs) );
00734 
00735   //Now check the consistency of the global "owners" data against local "sharing"
00736   //data.
00737   int err =  checkCommArrays( "owners",
00738             globalOwnerProcs, globalNodesPerOwnerProcs,
00739             recvOwnerLengths,
00740             nodesPerSharingProc_, remoteSharingProcs_ );
00741 
00742   //Now check the consistency of the global "sharing" data against local "owners"
00743   //data.
00744   err +=  checkCommArrays( "sharing",
00745          globalSharingProcs, globalNodesPerSharingProcs,
00746          recvSharingLengths,
00747          nodesPerOwnerProc_, remoteOwnerProcs_ );
00748 
00749   int globalErr = 0;
00750 
00751   CHK_ERR( fei::GlobalSum(comm_, err, globalErr) );
00752 
00753   return(globalErr);
00754 }
00755 
00756 //------------------------------------------------------------------------------
00757 int NodeCommMgr::checkCommArrays(const char* whichCheck,
00758          std::vector<int>& globalRemoteProcs,
00759          std::vector<int>& globalNodesPerRemoteProc,
00760          std::vector<int>& globalRemoteProcLengths,
00761          std::vector<int>& nodesPerRemoteProc,
00762          std::vector<int>& remoteProcs)
00763 {
00764   int offset = 0;
00765 
00766   for(int i=0; i<numProcs_; i++) {
00767     int length = globalRemoteProcLengths[i];
00768 
00769     if (i==localProc_) { offset += length; continue; }
00770 
00771     for(int j=0; j<length; j++) {
00772       if (globalRemoteProcs[offset+j] == localProc_) {
00773   //proc i says that we (localProc_) own nodes that it shares.
00774   int numShared = globalNodesPerRemoteProc[offset+j];
00775 
00776   int index = fei::binarySearch(i, &remoteProcs[0], remoteProcs.size());
00777   if (index < 0) {
00778     //we don't think proc i shares any nodes that we own.
00779     fei::console_out() << "FEI NodeCommMgr::checkSharedNodeInfo "<<whichCheck
00780          << " ERROR. Local proc (" << localProc_ 
00781          << ") doesn't share nodes with proc " << i << " but proc " << i
00782          << " thinks it shares nodes with proc " << localProc_ << FEI_ENDL;
00783     return(-1);
00784   }
00785 
00786   //We think that we own nodesPerRemoteProc[index] nodes that proc i
00787   //shares.
00788   int numWeThinkWeShare = nodesPerRemoteProc[index];
00789   if (numWeThinkWeShare != numShared) {
00790     fei::console_out() << "FEI NodeCommMgr::checkSharedNodeInfo "<<whichCheck
00791          << " ERROR. Local proc (" << localProc_ << ") thinks it shares "
00792          << numWeThinkWeShare << " nodes with proc " << i << ", but proc " 
00793          << i << " thinks it shares " << numShared << " nodes with proc "
00794          << localProc_ << "." << FEI_ENDL;
00795     return(-1);
00796   }
00797       }
00798     }
00799 
00800     offset += length;
00801   }
00802 
00803   return(0);
00804 }
00805 
00806 //------------------------------------------------------------------------------
00807 int NodeCommMgr::adjustSharedOwnership()
00808 {
00809   //For each shared node that has not been identified as local, assign its
00810   //owner to be the next lowest-numbered sharing proc. (Each list of sharing
00811   //procs is sorted by processor-number, so we just assign the owner to be the
00812   //next one in the list.)
00813   //
00814   //If a node is not local, and localProc_ is the lowest sharing proc, then we
00815   //also need to flag that node as remote and tell other processors that we
00816   //don't own it.
00817   //
00818   remoteNodeIDs.resize(0);
00819   int err;
00820   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00821     GlobalID nodeID = sharedNodeIDs[i];
00822 
00823     std::vector<int>& shProcs = *(sharingProcs_[i]);
00824 
00825     if (fei::binarySearch(nodeID, &localNodeIDs[0], localNodeIDs.size()) >= 0) continue;
00826 
00827     int proc = shProcs[0];
00828 
00829     if (proc == localProc_) {
00830       sharedNodes_[i]->setOwnerProc(shProcs[1]);
00831       err = fei::sortedListInsert(nodeID, remoteNodeIDs);
00832       if (err == -2) return(err);
00833     }
00834   }
00835 
00836   //Now we need to let the other processors know that the remote nodes 
00837   //aren't owned by us. This is going to require some communication. We'll
00838   //gather the nodeIDs onto all processors, after which each processor
00839   //will reset the owner proc for that node. (Later, as an optimization, I'll
00840   //do this without all-to-all communication.)
00841 
00842   std::vector<GlobalID> allRemoteNodeIDs;
00843   std::vector<int> numPerProc;
00844 
00845   err = fei::Allgatherv(comm_, remoteNodeIDs, numPerProc, allRemoteNodeIDs);
00846   if (err != 0) return(-1);
00847 
00848   //Now we need to run through the global list of 'special' nodes, and for the ones
00849   //that we do own (appear locally), add them to a new list that will be once again
00850   //globally gathered. That new list will then be used by each processor in setting
00851   //the nodes' real owners.
00852 
00853   //we'll keep the 'new' list in remoteNodeIDs.
00854   remoteNodeIDs.resize(0);
00855 
00856   int offset = 0;
00857   for(unsigned i=0; i<numPerProc.size(); i++) {
00858     for(int j=0; j<numPerProc[i]; j++) {
00859 
00860       //skip the nodes that we sent, we already know we don't own those.
00861       if ((int)i==localProc_) {offset++; continue;}
00862 
00863       GlobalID nodeID = allRemoteNodeIDs[offset++];
00864       int index = getSharedNodeIndex(nodeID);
00865 
00866       //if it's not even one of our shared nodes, then continue.
00867       if (index < 0) continue;
00868 
00869       if (fei::binarySearch(nodeID, &localNodeIDs[0], localNodeIDs.size()) >= 0) {
00870         fei::sortedListInsert(nodeID, remoteNodeIDs);
00871       }
00872     }
00873   }
00874 
00875   //now re-gather the remoteNodeIDs list to all processors. This time, we should only
00876   //receive nodeIDs from processors that can be valid owners. i.e., processors that
00877   //have those nodes in at least one local element.
00878   err = fei::Allgatherv(comm_, remoteNodeIDs, numPerProc, allRemoteNodeIDs);
00879   if (err != 0) return(-1);
00880 
00881   //Now we run the 'allRemoteNodeIDs' list for the last time, setting the owner-proc
00882   //for each node. We'll run the list from the back to the front so that if multiple
00883   //processors are possible owners, the lowest-numbered one will be the last one that
00884   //get's set.
00885   offset = allRemoteNodeIDs.size()-1;
00886   for(int i=(int)numPerProc.size()-1; i>=0; i--) {
00887     for(int j=0; j<numPerProc[i]; j++) {
00888       GlobalID nodeID = allRemoteNodeIDs[offset--];
00889       int index = getSharedNodeIndex(nodeID);
00890 
00891       if (index < 0) continue;
00892 
00893       sharedNodes_[index]->setOwnerProc(i);
00894     }
00895   }
00896 
00897   return(0);
00898 }
00899 
00900 //------------------------------------------------------------------------------
00901 void NodeCommMgr::setNodeNumbersArray()
00902 {
00903   sharedNodeNumbers.resize(sharedNodeIDs.size());
00904 
00905   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00906     sharedNodeNumbers[i] = sharedNodes_[i]->getNodeNumber();
00907   }
00908 }
00909 
00910 //------------------------------------------------------------------------------
00911 int NodeCommMgr::createProcLists()
00912 {
00913   std::vector<int> localNodesPerProc(numProcs_, 0);
00914   std::vector<int> remoteNodesPerProc(numProcs_, 0);
00915 
00916   //first, figure out how many locally-owned nodes each remote processor is
00917   //associated with, and how many remotely-owned nodes we'll be recv'ing info
00918   //about from each remote processor.
00919 
00920   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00921     int proc = sharedNodes_[i]->getOwnerProc();
00922 
00923     if (proc != localProc_) {
00924       remoteNodesPerProc[proc]++;
00925     }
00926     else {
00927       std::vector<int>& shProcs = *(sharingProcs_[i]);
00928       for(unsigned j=0; j<shProcs.size(); j++) {
00929   int sproc = shProcs[j];
00930 
00931   if (sproc != localProc_) {
00932     localNodesPerProc[sproc]++;
00933   }
00934       }
00935     }
00936   }
00937 
00938   //now create condensed lists of remote owner procs, and
00939   //remote sharing procs.
00940   int err = createProcList(remoteNodesPerProc, remoteOwnerProcs_);
00941   if (err != 0) return(err);
00942 
00943   err = createProcList(localNodesPerProc, remoteSharingProcs_);
00944   if (err != 0) return(err);
00945 
00946 
00947   nodesPerOwnerProc_.resize(remoteOwnerProcs_.size());
00948 
00949   nodesPerSharingProc_.resize(remoteSharingProcs_.size());
00950 
00951   int offset = 0;
00952   for(int i=0; i<numProcs_; i++) {
00953     if (remoteNodesPerProc[i] > 0) 
00954       nodesPerOwnerProc_[offset++] = remoteNodesPerProc[i];
00955   }
00956 
00957   offset = 0;
00958   for(int i=0; i<numProcs_; i++) {
00959     if (localNodesPerProc[i] > 0) 
00960       nodesPerSharingProc_[offset++] = localNodesPerProc[i];
00961   }
00962 
00963   return(0);
00964 }
00965 
00966 //------------------------------------------------------------------------------
00967 int NodeCommMgr::exchangeSharedRemoteFieldsBlks()
00968 {
00969   //first each proc will find out a max. number of fields and blocks to expect
00970   //per node.
00971 
00972   //most of this function is #ifdef'd according to whether FEI_SER is
00973   //defined.
00974 #ifndef FEI_SER
00975   int maxFields, maxBlocks;
00976   int err = getGlobalMaxFieldsBlocks(maxFields, maxBlocks);
00977   if (err) return(-1);
00978 
00979   //now we can allocate lists to recv into and launch the irecv's.
00980   //from each processor, we'll recv a list of length:
00981   //            num-nodes*(4+ maxFields + maxBlocks)
00982 
00983   int len = 4 + maxFields + maxBlocks;
00984 
00985   GlobalID** recvData = NULL;
00986   MPI_Request* recvDataReqs = NULL;
00987 
00988   unsigned i, numRecvProcs = remoteSharingProcs_.size();
00989 
00990   if (numRecvProcs > 0) {
00991     recvData = new GlobalID*[numRecvProcs];
00992     recvDataReqs = new MPI_Request[numRecvProcs];
00993   }
00994 
00995   int dataTag = 19904;
00996 
00997   int numRcvStarted = 0;
00998   for(i=0; i<remoteSharingProcs_.size(); i++) {
00999     int numRecvNodes = nodesPerSharingProc_[i];
01000     recvData[i] = new GlobalID[numRecvNodes*(len+1)];
01001     MPI_Irecv(recvData[i], numRecvNodes*(len+1),
01002         fei::mpiTraits<GlobalID>::mpi_type(),
01003         remoteSharingProcs_[i], dataTag, comm_, &recvDataReqs[i]);
01004     numRcvStarted++;
01005   }
01006 
01007   //next, send all outgoing messages.
01008 
01009   fei::Barrier(comm_);
01010 
01011   for(i=0; i<remoteOwnerProcs_.size(); i++) {
01012     int numSendNodes = nodesPerOwnerProc_[i];
01013 
01014     std::vector<GlobalID> sendData(numSendNodes*(len+1), 0);
01015 
01016     packRemoteNodesAndData(&sendData[0], remoteOwnerProcs_[i],
01017                            numSendNodes, numSendNodes*len);
01018 
01019     MPI_Send(&sendData[0], sendData.size(),
01020         fei::mpiTraits<GlobalID>::mpi_type(),
01021        remoteOwnerProcs_[i], dataTag, comm_);
01022   }
01023 
01024   //finally, complete the irecvs and put away the node field info.
01025   int numCompleted = 0;
01026   for(i=0; i<remoteSharingProcs_.size(); i++) {
01027     MPI_Status status;
01028     int index = i;
01029     MPI_Wait(&recvDataReqs[index], &status);
01030     numCompleted++;
01031     int remoteProc = status.MPI_SOURCE;
01032 
01033     int offset = 0;
01034     int numNodes = nodesPerSharingProc_[index];
01035 
01036     for(int j=0; j<numNodes; j++) {
01037       int nIndex = fei::binarySearch(recvData[index][j], &sharedNodeIDs[0], sharedNodeIDs.size());
01038       if (nIndex < 0) {
01039   fei::console_out() << "NodeCommMgr::exchangeSharedRemote...: error, unknown nodeID "
01040        << (int)recvData[index][j] << ", " << j
01041        << "th node recvd from proc "
01042        <<remoteSharingProcs_[index]
01043        << ". Probably a communication mis-match, we expected " 
01044        << numNodes
01045        << " nodes from that proc, but recvd less than that." << FEI_ENDL;
01046   std::abort();
01047       }
01048 
01049       int residesRemotely = (int)recvData[index][numNodes+offset++];
01050 
01051       if (residesRemotely) {
01052         std::vector<int>& snSubd = sharedNodeSubdomains[nIndex];
01053         std::vector<int>::iterator sn_iter =
01054           std::lower_bound(snSubd.begin(), snSubd.end(), remoteProc);
01055         if (sn_iter == snSubd.end() || remoteProc != *sn_iter) {
01056           snSubd.insert(sn_iter, remoteProc);
01057         }
01058       }
01059       int numFields       = (int)recvData[index][numNodes+offset++];
01060       int numBlocks       = (int)recvData[index][numNodes+offset++];
01061       sharedNodes_[nIndex]->
01062        setNumNodalDOF((int)recvData[index][numNodes+offset++]);
01063 
01064       for(int fld=0; fld<numFields; fld++) {
01065         int fieldID = (int)recvData[index][numNodes+offset++];
01066 
01067         sharedNodes_[nIndex]->addField(fieldID);
01068       }
01069 
01070       for(int blk=0; blk<numBlocks; blk++) {
01071         int blk_idx = probStruc.getIndexOfBlock(recvData[index][numNodes+offset++]);
01072         //if blk_idx < 0 it means the incoming blockID doesn't exist on this proc
01073         if (blk_idx >= 0) {
01074           sharedNodes_[nIndex]->addBlockIndex(blk_idx);
01075         }
01076       }
01077     }
01078   }
01079 
01080   if (numRcvStarted != numCompleted) {
01081     fei::console_out() << "NodeCommMgr::exchangeSharedRemote...: recv-send mismatch;"
01082          << " numRcvStarted: " << numRcvStarted << ", numCompleted: "
01083          << numCompleted << FEI_ENDL;
01084     std::abort();
01085   }
01086 
01087   for(i=0; i<numRecvProcs; i++) {
01088     delete [] recvData[i];
01089   }
01090 
01091   delete [] recvData;
01092   delete [] recvDataReqs;
01093 
01094 #endif //#ifndef FEI_SER
01095 
01096   return(0);
01097 }
01098 
01099 //------------------------------------------------------------------------------
01100 int NodeCommMgr::storeNodeProcs(int index,
01101         std::vector<std::vector<int>*>& procTable,
01102         const int* procs, int numProcs)
01103 {
01104 //Private NodeCommMgr function.
01105 //
01106 //This function stores 'procs' in row 'index' of procTable, maintaining order
01107 //in that row.
01108 //
01109   std::vector<int>& row_index = *(procTable[index]);
01110   for(int i=0; i<numProcs; i++) {
01111     std::vector<int>::iterator r_iter =
01112       std::lower_bound(row_index.begin(), row_index.end(), procs[i]);
01113     if (r_iter == row_index.end() || procs[i] != *r_iter) {
01114       row_index.insert(r_iter, procs[i]);
01115     }
01116   }
01117 
01118   return(0);
01119 }
01120 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Friends