FEI Version of the Day
fei_NodeCommMgr.cpp
00001 /*--------------------------------------------------------------------*/
00002 /*    Copyright 2005 Sandia Corporation.                              */
00003 /*    Under the terms of Contract DE-AC04-94AL85000, there is a       */
00004 /*    non-exclusive license for use of this work by or on behalf      */
00005 /*    of the U.S. Government.  Export of this program may require     */
00006 /*    a license from the United States Government.                    */
00007 /*--------------------------------------------------------------------*/
00008 
00009 #include <fei_macros.hpp>
00010 
00011 #include <fei_mpi.h>
00012 
00013 #include <fei_defs.h>
00014 
00015 #include <fei_TemplateUtils.hpp>
00016 #include <fei_mpiTraits.hpp>
00017 #include <fei_CommUtils.hpp>
00018 #include <fei_NodeDescriptor.hpp>
00019 #include <fei_NodeCommMgr.hpp>
00020 #include <SNL_FEI_Structure.hpp>
00021 
00022 #include <fei_NodeDatabase.hpp>
00023 
00024 #undef fei_file
00025 #define fei_file "fei_NodeCommMgr.cpp"
00026 #include <fei_ErrMacros.hpp>
00027 
00028 //------Constructor-------------------------------------------------------------
00029 NodeCommMgr::NodeCommMgr(MPI_Comm comm, const SNL_FEI_Structure& problemStructure, int sharedNodeOwnership)
00030   : sharedNodes_(NULL),
00031     sharedNodesAllocated_(false),
00032     sharedNodeOwnership_(sharedNodeOwnership),
00033     localNodeIDs(),
00034     remoteNodeIDs(),
00035     sharedNodeIDs(),
00036     sharedNodeSubdomains(),
00037     trivialSubdomainList(1),
00038     sharingProcs_(),
00039     sharedNodeNumbers(),
00040     remoteOwnerProcs_(),
00041     remoteSharingProcs_(),
00042     nodesPerOwnerProc_(),
00043     nodesPerSharingProc_(),
00044     comm_(comm),
00045     numProcs_(1),
00046     localProc_(0),
00047     maxFields_(0),
00048     maxBlocks_(0),
00049     maxSubdomains_(0),
00050     initCompleteCalled_(false),
00051     probStruc(problemStructure)
00052 {
00053   numProcs_ = fei::numProcs(comm_);
00054   localProc_= fei::localProc(comm_);
00055   trivialSubdomainList[0] = localProc_;
00056 }
00057 
00058 //-----Destructor---------------------------------------------------------------
00059 NodeCommMgr::~NodeCommMgr() {
00060 
00061    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00062       delete sharingProcs_[i];
00063    }
00064 
00065    delete [] sharedNodes_;
00066    sharedNodesAllocated_ = false;
00067 }
00068 
00069 //------------------------------------------------------------------------------
00070 int NodeCommMgr::getSharedNodeIndex(GlobalID nodeID)
00071 {
00072   return( fei::binarySearch(nodeID, &sharedNodeIDs[0], sharedNodeIDs.size()) );
00073 }
00074 
00075 //------------------------------------------------------------------------------
00076 int NodeCommMgr::getSharedNodeNumSubdomains(GlobalID nodeID)
00077 {
00078   int index = getSharedNodeIndex(nodeID);
00079 
00080   //If the node isn't one of our shared nodes, then return 1, signifying that
00081   //the node is in 1 subdomain, that being the local subdomain.
00082   if (index < 0) return(1);
00083 
00084   //Since the node is one of our shared nodes, it should have an entry in our
00085   //sharedNodeNumSubdomains array indicating how many subdomains it
00086   //appears in. So return this number.
00087   return(sharedNodeSubdomains[index].size());
00088 }
00089 
00090 //------------------------------------------------------------------------------
00091 std::vector<int>* NodeCommMgr::getSharedNodeSubdomainList(GlobalID nodeID)
00092 {
00093   int index = getSharedNodeIndex(nodeID);
00094 
00095   //If the node isn't one of our shared nodes, then return 1, signifying that
00096   //the node is in 1 subdomain, that being the local subdomain.
00097   if (index < 0) return( &trivialSubdomainList );
00098 
00099   //Since the node is one of our shared nodes, it should have an entry in our
00100   //sharedNodeSubdomains array.
00101   return( &(sharedNodeSubdomains[index]) );
00102 }
00103 
00104 //------------------------------------------------------------------------------
00105 int NodeCommMgr::informLocal(const NodeDescriptor& node) {
00106 //
00107 //NodeCommMgr is being informed that 'node' is present in the local
00108 //active node list.
00109 //
00110 //This means that either:
00111 // 1. it is a locally-owned shared node, or
00112 // 3. it isn't even a shared node (it's a purely local node), in which
00113 //    case we'll do nothing.
00114 //
00115 
00116    if (numProcs_ == 1) return(0);
00117 
00118    GlobalID nodeID = node.getGlobalNodeID();
00119 
00120    int sharedIndex = getSharedNodeIndex(nodeID);
00121 
00122    //if this node isn't a shared node, then simply return.
00123    if (sharedIndex < 0) return(0);
00124 
00125    //Since this node is present as a shared node, let's put nodeID in the
00126    //localNodeIDs list if it isn't already there.
00127 
00128    int index = fei::sortedListInsert(nodeID, localNodeIDs);
00129 
00130    //if index is -2, it means the localNodeIDs array had an allocation failure.
00131    if (index == -2) return(-2);
00132 
00133    return(0);
00134 }
00135 
00136 //------------------------------------------------------------------------------
00137 int NodeCommMgr::getGlobalMaxFieldsBlocks(int& maxFields, int& maxBlocks)
00138 {
00139   std::vector<int> localMax(2, 0), globalMax(2, 0);
00140 
00141   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00142     int numFlds = sharedNodes_[i]->getNumFields();
00143     if (numFlds > localMax[0]) localMax[0] = numFlds;
00144 
00145     int numBlks = sharedNodes_[i]->getNumBlocks();
00146     if (numBlks > localMax[1]) localMax[1] = numBlks;
00147   }
00148 
00149   int err = fei::GlobalMax(comm_, localMax, globalMax);
00150   if (err != 0) return(err);
00151 
00152   maxFields = globalMax[0];
00153   maxBlocks = globalMax[1];
00154 
00155   return(0);
00156 }
00157 
00158 //------------------------------------------------------------------------------
00159 int NodeCommMgr::getGlobalMaxFieldsBlocksSubdomains()
00160 {
00161   std::vector<int> localMax(3, 0), globalMax(3, 0);
00162 
00163   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00164     int numFlds = sharedNodes_[i]->getNumFields();
00165     if (numFlds > localMax[0]) localMax[0] = numFlds;
00166 
00167     int numBlks = sharedNodes_[i]->getNumBlocks();
00168     if (numBlks > localMax[1]) localMax[1] = numBlks;
00169 
00170     int numShrd = sharingProcs_[i]->size();
00171     if (numShrd > localMax[2]) localMax[2] = numShrd;
00172   }
00173 
00174   int err = fei::GlobalMax(comm_, localMax, globalMax);
00175   if (err != 0) return(err);
00176 
00177   maxFields_     = globalMax[0];
00178   maxBlocks_     = globalMax[1];
00179   maxSubdomains_ = globalMax[2];
00180 
00181   return(0);
00182 }
00183 
00184 //------------------------------------------------------------------------------
00185 std::vector<int>& NodeCommMgr::getSendProcs()
00186 {
00187   return( remoteSharingProcs_ );
00188 }
00189 
00190 //------------------------------------------------------------------------------
00191 std::vector<int>& NodeCommMgr::getRecvProcs()
00192 {
00193   return( remoteOwnerProcs_ );
00194 }
00195 
00196 //------------------------------------------------------------------------------
00197 int NodeCommMgr::getSendMessageLength(int destProc, int& messageLength)
00198 {
00199   std::vector<int>::iterator
00200    rs_iter = std::lower_bound(remoteSharingProcs_.begin(),
00201                               remoteSharingProcs_.end(), destProc);
00202   if (rs_iter == remoteSharingProcs_.end() || destProc != *rs_iter) {
00203     ERReturn(-1);
00204   }
00205 
00206   int idx = rs_iter - remoteSharingProcs_.begin();
00207 
00208   int len = 7+maxFields_*2 + maxBlocks_ + maxSubdomains_;
00209   messageLength = nodesPerSharingProc_[idx] * (len+1);
00210   return(0);
00211 }
00212 
00213 //------------------------------------------------------------------------------
00214 int NodeCommMgr::getSendMessage(int destProc, std::vector<int>& message)
00215 {
00216   std::vector<int>::iterator
00217    rs_iter = std::lower_bound(remoteSharingProcs_.begin(),
00218                               remoteSharingProcs_.end(), destProc);
00219   if (rs_iter == remoteSharingProcs_.end() || destProc != *rs_iter) {
00220     ERReturn(-1);
00221   }
00222 
00223   int idx = rs_iter - remoteSharingProcs_.begin();
00224   int len = 0;
00225   CHK_ERR( getSendMessageLength(destProc, len) );
00226   message.resize(len);
00227 
00228   packLocalNodesAndData(&message[0], destProc,
00229       nodesPerSharingProc_[idx], len);
00230   return(0);
00231 }
00232 
00233 //------------------------------------------------------------------------------
00234 int NodeCommMgr::processRecvMessage(int srcProc, std::vector<int>& message)
00235 {
00236   int idx = fei::binarySearch(srcProc, &remoteOwnerProcs_[0],
00237                                   remoteOwnerProcs_.size());
00238   int numNodes = nodesPerOwnerProc_[idx];
00239   int* msgPtr = &message[0];
00240   int offset = 0;
00241 
00242   for(int j=0; j<numNodes; j++) {
00243     int nIndex = fei::binarySearch(msgPtr[j], &sharedNodeIDs[0], sharedNodeIDs.size());
00244     if (nIndex < 0) return(-1);
00245     NodeDescriptor* node = sharedNodes_[nIndex];
00246 
00247     int nodeNum         = msgPtr[numNodes+offset++];
00248     int numFields       = msgPtr[numNodes+offset++];
00249     int numBlocks       = msgPtr[numNodes+offset++];
00250     int numSubdomains   = msgPtr[numNodes+offset++];
00251 
00252     node->setNodeNumber(nodeNum);
00253     node->setNumNodalDOF( msgPtr[numNodes+offset++]);
00254     node->setBlkEqnNumber(msgPtr[numNodes+offset++]);
00255 
00256     for(int fld=0; fld<numFields; fld++) {
00257       int fieldID = msgPtr[numNodes+offset++];
00258       int eqnNum  = msgPtr[numNodes+offset++];
00259       node->addField(fieldID);
00260       node->setFieldEqnNumber(fieldID, eqnNum);
00261     }
00262 
00263     for(int blk=0; blk<numBlocks; blk++) {
00264       int blk_idx = probStruc.getIndexOfBlock(msgPtr[numNodes+offset++]);
00265       //if blk_idx < 0 it means the incoming blockID doesn't exist on this proc
00266       if (blk_idx >= 0) {
00267         node->addBlockIndex(blk_idx);
00268       }
00269     }
00270 
00271     sharedNodeSubdomains[nIndex].resize(numSubdomains);
00272     for(int sd=0; sd<numSubdomains; sd++) {
00273       (sharedNodeSubdomains[nIndex])[sd] =
00274   msgPtr[numNodes+offset++];
00275     }
00276   }
00277 
00278   return(0);
00279 }
00280 
00281 //------------------------------------------------------------------------------
00282 int NodeCommMgr::exchangeEqnInfo()
00283 {
00284   //
00285   //This function will perform the communication necessary to:
00286   //
00287   //   1. For each locally owned node, send to each remote sharing proc:
00288   //      the node's nodeNumber, whether or not it appears in local elements,
00289   //      fieldID(s), the field-size(s), the first global equation numbers
00290   //      for those fields, and the processor-subdomains that contain the node.
00291   //   2. For each remotely owned node, receive the above information from
00292   //      the owners of these nodes.
00293   //
00294   //This is a collective function. All procs must enter it before any can
00295   //exit it.
00296   //
00297   //Most of this function is #ifdef'd according to whether FEI_SER is
00298   //defined.
00299 
00300 #ifndef FEI_SER
00301    if (numProcs_ == 1) return(0);
00302 
00303    //each proc will find out a max. number of fields, blocks
00304    //and subdomains to expect per node.
00305 
00306    CHK_ERR( getGlobalMaxFieldsBlocksSubdomains() );
00307 
00308    CHK_ERR( fei::exchange(comm_, this) );
00309 
00310    setNodeNumbersArray();
00311 
00312 #endif //#ifndef FEI_SER
00313 
00314    return(0);
00315 }
00316 
00317 //------------------------------------------------------------------------------
00318 void NodeCommMgr::packLocalNodesAndData(int* data, 
00319                                        int proc, int numNodes, int len)
00320 {
00321 //This function packs up nodeIDs, as well as the list containing, for
00322 //each node, the following:
00323 //   node-number
00324 //   numFields
00325 //   numBlocks
00326 //   numSubdomains
00327 //   num-nodal-dof
00328 //   blk-eqn-number
00329 //     'numFields' pairs of (fieldID,eqnNumber)
00330 //     subdomain list, length 'numSubdomains'
00331 //
00332 //Incoming parameter len is:
00333 //  numNodes * (7 + maxFields*2 + maxBlocks + maxSubdomains),
00334 //where maxFields is the maximum number of fields associated with any node,
00335 //maxBlocks is the maximum number of blocks associated with any node, and
00336 //maxSubdomains is the maximum number of subdomains containing any node.
00337 //data is of length numNodes*(len+1).
00338 //
00339 //The above data will all be packed into the 'data' list, with nodeIDs 
00340 //occupying the first numNodes positions, followed by the rest of the data.
00341 //
00342    int nodeCounter = 0;
00343    int offset = 0;
00344 
00345    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00346      if (sharedNodes_[i]->getOwnerProc() != localProc_) continue;
00347 
00348       NodeDescriptor* node = sharedNodes_[i];
00349 
00350       //is this local node associated with processor 'proc'?
00351 
00352       std::vector<int>& sProcs = *(sharingProcs_[i]);
00353       int index = fei::binarySearch(proc, &sProcs[0], sProcs.size());
00354 
00355       //if not, skip to the next iteration...
00356       if (index < 0) continue;
00357 
00358       if (nodeCounter >= numNodes) {
00359   fei::console_out() << "NodeCommMgr::packLocalNodesAndData: ERROR,"
00360        << " nodeCounter >= numNodes." << FEI_ENDL;
00361       }
00362 
00363       data[nodeCounter++] = (int)(node->getGlobalNodeID());
00364 
00365       int nodeNum = node->getNodeNumber();
00366       int numFields = node->getNumFields();
00367       int numBlocks = node->getNumBlocks();
00368       const int* fieldIDsPtr = node->getFieldIDList();
00369       const int* fieldEqnNums = node->getFieldEqnNumbers();
00370       int blkEqnNumber = node->getBlkEqnNumber();
00371 
00372       const std::vector<unsigned>& nodeBlocks = node->getBlockIndexList();
00373       std::vector<int>& subdomains = sharedNodeSubdomains[i];
00374 
00375       data[numNodes+offset++] = nodeNum;
00376       data[numNodes+offset++] = numFields;
00377       data[numNodes+offset++] = numBlocks;
00378       data[numNodes+offset++] = subdomains.size();
00379       data[numNodes+offset++] = node->getNumNodalDOF();
00380       data[numNodes+offset++] = blkEqnNumber;
00381 
00382       for(int j=0; j<numFields; j++) {
00383   data[numNodes+offset++] = fieldIDsPtr[j];
00384 
00385   if (offset >= len) {
00386     fei::console_out() << "NodeCommMgr::packLocalNodesAndData: ERROR,"
00387          << " offset >= len." << FEI_ENDL;
00388   }
00389 
00390   data[numNodes+offset++] = fieldEqnNums[j];
00391       }
00392 
00393       for(int kk=0; kk<numBlocks; kk++) {
00394         GlobalID blkID = probStruc.getBlockID(nodeBlocks[kk]);
00395         data[numNodes+offset++] = blkID;
00396       }
00397 
00398       for(unsigned k=0; k<subdomains.size(); k++) {
00399   data[numNodes+offset++] = subdomains[k];
00400       }
00401    }
00402 }
00403 
00404 //------------------------------------------------------------------------------
00405 void NodeCommMgr::packRemoteNodesAndData(GlobalID* data,
00406            int proc, int numNodes, int len)
00407 {
00408 //
00409 //This function packs up the nodeIDs owned by proc, as well as the list
00410 //containing, for each node, the following:
00411 //   residesLocally (0 or 1) indicating whether it appears in the local
00412 //       processor's element domain.
00413 //   numFields
00414 //   numBlocks
00415 //   numNodalDOF
00416 //     'numFields' entries of (fieldID)
00417 //     'numBlocks' entries of (block)
00418 //
00419 //Incoming parameter len is numNodes * (3 + maxFields + maxBlocks),
00420 //where maxFields is the maximum number of fields associated with any node,
00421 //and maxBlocks is the maximum number of blocks associated with any node.
00422 //nodeIDs is of length numNodes, and
00423 //data is of length numNodes*len.
00424 //
00425 //The above data will all be put in the 'data' list, with the nodeIDs
00426 //occupying the first numNodes positions, followed by the rest of the data.
00427 //
00428    int nodeCounter = 0;
00429    int offset = 0;
00430 
00431    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00432       NodeDescriptor* node = sharedNodes_[i];
00433 
00434       int thisProc = node->getOwnerProc();
00435       if (thisProc != proc) continue;
00436 
00437       if (nodeCounter >= numNodes) {
00438          fei::console_out() << localProc_ << ": NodeCommMgr::packRemoteNodesAndData: ERROR,"
00439               << " nodeCounter >= numNodes: " << numNodes << FEI_ENDL;
00440       }
00441 
00442       data[nodeCounter++] = node->getGlobalNodeID();
00443 
00444       int numFields = node->getNumFields();
00445       int numBlocks = node->getNumBlocks();
00446       const int* fieldIDsPtr = node->getFieldIDList();
00447 
00448       const std::vector<unsigned>& nodeBlocks = node->getBlockIndexList();
00449       int lindex = fei::binarySearch(sharedNodeIDs[i], &localNodeIDs[0], localNodeIDs.size());
00450 
00451       data[numNodes+offset++] = (lindex >= 0) ? 1 : 0;
00452       data[numNodes+offset++] = (GlobalID)numFields;
00453       data[numNodes+offset++] = (GlobalID)numBlocks;
00454       data[numNodes+offset++] = (GlobalID)node->getNumNodalDOF();
00455 
00456       for(int j=0; j<numFields; j++) {
00457          if (offset >= len) {
00458             fei::console_out() << "NodeCommMgr::packRemoteNodesAndData: ERROR,"
00459                  << " offset >= len." << FEI_ENDL;
00460          }
00461 
00462          data[numNodes+offset++] = (GlobalID)fieldIDsPtr[j];
00463       }
00464 
00465       for(int k=0; k<numBlocks; k++) {
00466          if (offset >= len) {
00467             fei::console_out() << "NodeCommMgr::packRemoteNodesAndData: ERROR,"
00468                  << " offset >= len." << FEI_ENDL;
00469          }
00470 
00471          data[numNodes+offset++] = probStruc.getBlockID(nodeBlocks[k]);
00472       }
00473    }
00474 }
00475 
00476 //------------------------------------------------------------------------------
00477 int NodeCommMgr::createProcList(std::vector<int>& itemsPerProc,
00478                                 std::vector<int>& procs)
00479 {
00480 //
00481 //This function looks through the itemsPerProc list and counts how many
00482 //positions in this list are greater than 0. Then it creates a list of
00483 //the indices of those positions. i.e., itemsPerProc is a list of how many
00484 //items are to be sent to or recvd from each proc. When itemsPerProc is
00485 //greater than 0, that proc is put in the sharingProcs list.
00486 //
00487    int numProcs = 0;
00488    int len = itemsPerProc.size();
00489 
00490    for(int i=0; i<len; i++) {
00491       if (itemsPerProc[i] > 0) numProcs++;
00492    }
00493 
00494    procs.resize(numProcs);
00495 
00496    int offset = 0;
00497 
00498    for(int i=0; i<len; i++) {
00499       if (itemsPerProc[i] > 0) procs[offset++] = i;
00500    }
00501    return(0);
00502 }
00503 
00504 //------------------------------------------------------------------------------
00505 int NodeCommMgr::getSharedNodeIndex_num(int nodeNumber)
00506 {
00507   for(unsigned i=0; i<sharedNodeNumbers.size(); ++i) {
00508     if (sharedNodeNumbers[i] == nodeNumber) return(i);
00509   }
00510 
00511   return(-1);
00512 }
00513 
00514 //------------------------------------------------------------------------------
00515 int NodeCommMgr::addSharedNodes( const GlobalID* nodeIDs,
00516          int numNodes, 
00517          const int* const* procs,
00518          const int* numProcs )
00519 {
00520   //
00521   //Store the incoming nodeIDs and proc-numbers in the sharedNodeIDs array and
00522   //sharingProcs_ table.
00523   //
00524 
00525   try {
00526 
00527   for(int i=0; i<numNodes; i++) {
00528     int insertPoint = -1;
00529     int index = fei::binarySearch(nodeIDs[i], sharedNodeIDs, insertPoint);
00530     if (index < 0) {
00531       sharingProcs_.insert(sharingProcs_.begin()+insertPoint, new std::vector<int>);
00532 
00533       sharedNodeIDs.insert(sharedNodeIDs.begin()+insertPoint, nodeIDs[i]);
00534 
00535       index = insertPoint;
00536     }
00537     
00538     int err = storeNodeProcs(index, sharingProcs_, procs[i], numProcs[i]);
00539     if (err != 0) return(err);
00540   }
00541 
00542   }
00543   catch(std::runtime_error& exc) {
00544     fei::console_out() << exc.what() << FEI_ENDL;
00545     ERReturn(-1);
00546   }
00547 
00548   return(0);
00549 }
00550 
00551 //------------------------------------------------------------------------------
00552 int NodeCommMgr::allocateNodeDescriptorPtrs(NodeDatabase& nodeDB)
00553 {
00554   //This function is called when all shared nodes have been added. We now
00555   //allocate a list of pointer-to-NodeDescriptor, of length 
00556   //sharedNodeIDs.length(), and fill that list with NodeDescriptor-pointers
00557   //from the node-database.
00558 
00559   if (sharedNodeIDs.size() == 0) return(0);
00560 
00561   if (sharedNodes_ != NULL) delete [] sharedNodes_;
00562   sharedNodes_ = new NodeDescriptor*[sharedNodeIDs.size()];
00563   if (sharedNodes_ == NULL) return(-1);
00564 
00565   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00566     NodeDescriptor* node = NULL;
00567     int err = nodeDB.getNodeWithID(sharedNodeIDs[i], node);
00568     if (err != 0) return(-1);
00569 
00570     sharedNodes_[i] = node;
00571   }
00572 
00573   sharedNodesAllocated_ = true;
00574   return(0);
00575 }
00576 
00577 //------------------------------------------------------------------------------
00578 int NodeCommMgr::initComplete(NodeDatabase& nodeDB, bool safetyCheck)
00579 {
00580 //
00581 //This function is called when initialization is complete (i.e., when
00582 //all sharedNodes have been added, allocatedNodeDescriptorPtrs() has been
00583 //called, and informLocal() has been called for all nodes that appear in
00584 //the local finite-element structure.
00585 //
00586 //The task of this function is to assign owner-procs to nodes.
00587 //
00588 //if 'safetyCheck' is true, a global consistency check of the shared node info
00589 //will be performed before the communication is attempted.
00590 //
00591 //return value is 0 if successful, non-zero if an error was encountered
00592 //
00593   int err = allocateNodeDescriptorPtrs(nodeDB);
00594   if (err != 0) return(err);
00595 
00596   //Run through the shared nodes, and for each one that has been
00597   //identified as local, assign its owner to be the lowest-numbered sharing
00598   //processor, which may or may not be localProc_.
00599 
00600   for(unsigned ii=0; ii<sharedNodeIDs.size(); ii++) {
00601     std::vector<int>& shProcs = *(sharingProcs_[ii]);
00602 
00603     //first, insert localProc_ in this node's list of sharing proc, since the
00604     //FEI's initSharedNodes function doesn't mandate that the local processor be
00605     //included in the list of sharing processors. (i.e., localProc_ may not be
00606     //in this list yet...)
00607     std::vector<int>::iterator sh_iter =
00608       std::lower_bound(shProcs.begin(), shProcs.end(), localProc_);
00609     if (sh_iter == shProcs.end() || localProc_ != *sh_iter) {
00610       shProcs.insert(sh_iter, localProc_);
00611     }
00612 
00613     int proc = shProcs[0];
00614 
00615     sharedNodes_[ii]->setOwnerProc(proc);
00616   }
00617 
00618   //One of the tasks of this object is to gather information on the number
00619   //of subdomains each shared node appears in. So one thing we'll do here is
00620   //size and zero the array that will hold that information.
00621   sharedNodeSubdomains.resize(sharedNodeIDs.size());
00622 
00623   for(unsigned i=0; i<sharedNodeSubdomains.size(); ++i) {
00624     sharedNodeSubdomains[i].resize(0);
00625   }
00626 
00627   //now add the local processor to the sharedNodeSubdomains for each node that
00628   //appears in our localNodeIDs list.
00629   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00630     int index = fei::binarySearch(sharedNodeIDs[i], &localNodeIDs[0], localNodeIDs.size());
00631     if (index >= 0) {
00632       sharedNodeSubdomains[i].push_back(localProc_);
00633     }
00634   }
00635 
00636   if (sharedNodeOwnership_ == PROC_WITH_LOCAL_ELEM) {
00637     err = adjustSharedOwnership();
00638     if (err != 0) return(err);
00639   }
00640 
00641   err = createProcLists();
00642 
00643   if (safetyCheck) {
00644     err = checkSharedNodeInfo();
00645     if (err != 0) return(-1);
00646   }
00647 
00648   exchangeSharedRemoteFieldsBlks();
00649 
00650   initCompleteCalled_ = true;
00651 
00652   return(0);
00653 }
00654 
00655 //------------------------------------------------------------------------------
00656 #undef _feiFunc_
00657 #define _feiFunc_ "NodeCommMgr::checkSharedNodeInfo"
00658 int NodeCommMgr::checkSharedNodeInfo()
00659 {
00660   //This function's task is to "audit" the shared-node info. I.e., to make sure
00661   //that the info is globally symmetric (e.g., if the local processor thinks it
00662   //shares a node with another processor, does that other processor also think
00663   //it shares a node with the local proc?).
00664   //If this function finds that the shared-node info is consistent/correct, then
00665   //the return-value is 0. If the shared-node info is found to be wrong, then
00666   //one or more messages will be written to stderr, and the return-value is -1.
00667   //
00668   //This is a collective function, which is relatively expensive. It does a
00669   //few global reductions...
00670   //
00671 
00672   if (numProcs_==1) return(0);
00673 
00674   //Make sure that the processors we think are "remote owner"
00675   //procs, think we are a "remote sharing" proc, and vice-versa.
00676 
00677   std::vector<int> globalOwnerProcs, globalSharingProcs;
00678   std::vector<int> recvOwnerLengths, recvSharingLengths;
00679 
00680   std::vector<int> globalNodesPerOwnerProcs, globalNodesPerSharingProcs;
00681   std::vector<int> recvNodesPerOwnerLengths, recvNodesPerSharingLengths;
00682 
00683   //First, gather up each processor's list of remote procs and nodes-per-proc
00684   //onto all other processors...
00685 
00686   CHK_ERR( fei::Allgatherv(comm_, remoteOwnerProcs_,
00687          recvOwnerLengths, globalOwnerProcs) );
00688 
00689   CHK_ERR( fei::Allgatherv(comm_, nodesPerOwnerProc_,
00690          recvNodesPerOwnerLengths,
00691          globalNodesPerOwnerProcs) );
00692 
00693   CHK_ERR( fei::Allgatherv(comm_, remoteSharingProcs_,
00694          recvSharingLengths, globalSharingProcs) );
00695 
00696   CHK_ERR( fei::Allgatherv(comm_, nodesPerSharingProc_,
00697          recvNodesPerSharingLengths,
00698          globalNodesPerSharingProcs) );
00699 
00700   //Now check the consistency of the global "owners" data against local "sharing"
00701   //data.
00702   int err =  checkCommArrays( "owners",
00703             globalOwnerProcs, globalNodesPerOwnerProcs,
00704             recvOwnerLengths,
00705             nodesPerSharingProc_, remoteSharingProcs_ );
00706 
00707   //Now check the consistency of the global "sharing" data against local "owners"
00708   //data.
00709   err +=  checkCommArrays( "sharing",
00710          globalSharingProcs, globalNodesPerSharingProcs,
00711          recvSharingLengths,
00712          nodesPerOwnerProc_, remoteOwnerProcs_ );
00713 
00714   int globalErr = 0;
00715 
00716   CHK_ERR( fei::GlobalSum(comm_, err, globalErr) );
00717 
00718   return(globalErr);
00719 }
00720 
00721 //------------------------------------------------------------------------------
00722 int NodeCommMgr::checkCommArrays(const char* whichCheck,
00723          std::vector<int>& globalRemoteProcs,
00724          std::vector<int>& globalNodesPerRemoteProc,
00725          std::vector<int>& globalRemoteProcLengths,
00726          std::vector<int>& nodesPerRemoteProc,
00727          std::vector<int>& remoteProcs)
00728 {
00729   int offset = 0;
00730 
00731   for(int i=0; i<numProcs_; i++) {
00732     int length = globalRemoteProcLengths[i];
00733 
00734     if (i==localProc_) { offset += length; continue; }
00735 
00736     for(int j=0; j<length; j++) {
00737       if (globalRemoteProcs[offset+j] == localProc_) {
00738   //proc i says that we (localProc_) own nodes that it shares.
00739   int numShared = globalNodesPerRemoteProc[offset+j];
00740 
00741   int index = fei::binarySearch(i, &remoteProcs[0], remoteProcs.size());
00742   if (index < 0) {
00743     //we don't think proc i shares any nodes that we own.
00744     fei::console_out() << "FEI NodeCommMgr::checkSharedNodeInfo "<<whichCheck
00745          << " ERROR. Local proc (" << localProc_ 
00746          << ") doesn't share nodes with proc " << i << " but proc " << i
00747          << " thinks it shares nodes with proc " << localProc_ << FEI_ENDL;
00748     return(-1);
00749   }
00750 
00751   //We think that we own nodesPerRemoteProc[index] nodes that proc i
00752   //shares.
00753   int numWeThinkWeShare = nodesPerRemoteProc[index];
00754   if (numWeThinkWeShare != numShared) {
00755     fei::console_out() << "FEI NodeCommMgr::checkSharedNodeInfo "<<whichCheck
00756          << " ERROR. Local proc (" << localProc_ << ") thinks it shares "
00757          << numWeThinkWeShare << " nodes with proc " << i << ", but proc " 
00758          << i << " thinks it shares " << numShared << " nodes with proc "
00759          << localProc_ << "." << FEI_ENDL;
00760     return(-1);
00761   }
00762       }
00763     }
00764 
00765     offset += length;
00766   }
00767 
00768   return(0);
00769 }
00770 
00771 //------------------------------------------------------------------------------
00772 int NodeCommMgr::adjustSharedOwnership()
00773 {
00774   //For each shared node that has not been identified as local, assign its
00775   //owner to be the next lowest-numbered sharing proc. (Each list of sharing
00776   //procs is sorted by processor-number, so we just assign the owner to be the
00777   //next one in the list.)
00778   //
00779   //If a node is not local, and localProc_ is the lowest sharing proc, then we
00780   //also need to flag that node as remote and tell other processors that we
00781   //don't own it.
00782   //
00783   remoteNodeIDs.resize(0);
00784   int err;
00785   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00786     GlobalID nodeID = sharedNodeIDs[i];
00787 
00788     std::vector<int>& shProcs = *(sharingProcs_[i]);
00789 
00790     if (fei::binarySearch(nodeID, &localNodeIDs[0], localNodeIDs.size()) >= 0) continue;
00791 
00792     int proc = shProcs[0];
00793 
00794     if (proc == localProc_) {
00795       sharedNodes_[i]->setOwnerProc(shProcs[1]);
00796       err = fei::sortedListInsert(nodeID, remoteNodeIDs);
00797       if (err == -2) return(err);
00798     }
00799   }
00800 
00801   //Now we need to let the other processors know that the remote nodes 
00802   //aren't owned by us. This is going to require some communication. We'll
00803   //gather the nodeIDs onto all processors, after which each processor
00804   //will reset the owner proc for that node. (Later, as an optimization, I'll
00805   //do this without all-to-all communication.)
00806 
00807   std::vector<GlobalID> allRemoteNodeIDs;
00808   std::vector<int> numPerProc;
00809 
00810   err = fei::Allgatherv(comm_, remoteNodeIDs, numPerProc, allRemoteNodeIDs);
00811   if (err != 0) return(-1);
00812 
00813   //Now we need to run through the global list of 'special' nodes, and for the ones
00814   //that we do own (appear locally), add them to a new list that will be once again
00815   //globally gathered. That new list will then be used by each processor in setting
00816   //the nodes' real owners.
00817 
00818   //we'll keep the 'new' list in remoteNodeIDs.
00819   remoteNodeIDs.resize(0);
00820 
00821   int offset = 0;
00822   for(unsigned i=0; i<numPerProc.size(); i++) {
00823     for(int j=0; j<numPerProc[i]; j++) {
00824 
00825       //skip the nodes that we sent, we already know we don't own those.
00826       if ((int)i==localProc_) {offset++; continue;}
00827 
00828       GlobalID nodeID = allRemoteNodeIDs[offset++];
00829       int index = getSharedNodeIndex(nodeID);
00830 
00831       //if it's not even one of our shared nodes, then continue.
00832       if (index < 0) continue;
00833 
00834       if (fei::binarySearch(nodeID, &localNodeIDs[0], localNodeIDs.size()) >= 0) {
00835         fei::sortedListInsert(nodeID, remoteNodeIDs);
00836       }
00837     }
00838   }
00839 
00840   //now re-gather the remoteNodeIDs list to all processors. This time, we should only
00841   //receive nodeIDs from processors that can be valid owners. i.e., processors that
00842   //have those nodes in at least one local element.
00843   err = fei::Allgatherv(comm_, remoteNodeIDs, numPerProc, allRemoteNodeIDs);
00844   if (err != 0) return(-1);
00845 
00846   //Now we run the 'allRemoteNodeIDs' list for the last time, setting the owner-proc
00847   //for each node. We'll run the list from the back to the front so that if multiple
00848   //processors are possible owners, the lowest-numbered one will be the last one that
00849   //get's set.
00850   offset = allRemoteNodeIDs.size()-1;
00851   for(int i=(int)numPerProc.size()-1; i>=0; i--) {
00852     for(int j=0; j<numPerProc[i]; j++) {
00853       GlobalID nodeID = allRemoteNodeIDs[offset--];
00854       int index = getSharedNodeIndex(nodeID);
00855 
00856       if (index < 0) continue;
00857 
00858       sharedNodes_[index]->setOwnerProc(i);
00859     }
00860   }
00861 
00862   return(0);
00863 }
00864 
00865 //------------------------------------------------------------------------------
00866 void NodeCommMgr::setNodeNumbersArray()
00867 {
00868   sharedNodeNumbers.resize(sharedNodeIDs.size());
00869 
00870   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00871     sharedNodeNumbers[i] = sharedNodes_[i]->getNodeNumber();
00872   }
00873 }
00874 
00875 //------------------------------------------------------------------------------
00876 int NodeCommMgr::createProcLists()
00877 {
00878   std::vector<int> localNodesPerProc(numProcs_, 0);
00879   std::vector<int> remoteNodesPerProc(numProcs_, 0);
00880 
00881   //first, figure out how many locally-owned nodes each remote processor is
00882   //associated with, and how many remotely-owned nodes we'll be recv'ing info
00883   //about from each remote processor.
00884 
00885   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00886     int proc = sharedNodes_[i]->getOwnerProc();
00887 
00888     if (proc != localProc_) {
00889       remoteNodesPerProc[proc]++;
00890     }
00891     else {
00892       std::vector<int>& shProcs = *(sharingProcs_[i]);
00893       for(unsigned j=0; j<shProcs.size(); j++) {
00894   int sproc = shProcs[j];
00895 
00896   if (sproc != localProc_) {
00897     localNodesPerProc[sproc]++;
00898   }
00899       }
00900     }
00901   }
00902 
00903   //now create condensed lists of remote owner procs, and
00904   //remote sharing procs.
00905   int err = createProcList(remoteNodesPerProc, remoteOwnerProcs_);
00906   if (err != 0) return(err);
00907 
00908   err = createProcList(localNodesPerProc, remoteSharingProcs_);
00909   if (err != 0) return(err);
00910 
00911 
00912   nodesPerOwnerProc_.resize(remoteOwnerProcs_.size());
00913 
00914   nodesPerSharingProc_.resize(remoteSharingProcs_.size());
00915 
00916   int offset = 0;
00917   for(int i=0; i<numProcs_; i++) {
00918     if (remoteNodesPerProc[i] > 0) 
00919       nodesPerOwnerProc_[offset++] = remoteNodesPerProc[i];
00920   }
00921 
00922   offset = 0;
00923   for(int i=0; i<numProcs_; i++) {
00924     if (localNodesPerProc[i] > 0) 
00925       nodesPerSharingProc_[offset++] = localNodesPerProc[i];
00926   }
00927 
00928   return(0);
00929 }
00930 
00931 //------------------------------------------------------------------------------
00932 int NodeCommMgr::exchangeSharedRemoteFieldsBlks()
00933 {
00934   //first each proc will find out a max. number of fields and blocks to expect
00935   //per node.
00936 
00937   //most of this function is #ifdef'd according to whether FEI_SER is
00938   //defined.
00939 #ifndef FEI_SER
00940   int maxFields, maxBlocks;
00941   int err = getGlobalMaxFieldsBlocks(maxFields, maxBlocks);
00942   if (err) return(-1);
00943 
00944   //now we can allocate lists to recv into and launch the irecv's.
00945   //from each processor, we'll recv a list of length:
00946   //            num-nodes*(4+ maxFields + maxBlocks)
00947 
00948   int len = 4 + maxFields + maxBlocks;
00949 
00950   GlobalID** recvData = NULL;
00951   MPI_Request* recvDataReqs = NULL;
00952 
00953   unsigned i, numRecvProcs = remoteSharingProcs_.size();
00954 
00955   if (numRecvProcs > 0) {
00956     recvData = new GlobalID*[numRecvProcs];
00957     recvDataReqs = new MPI_Request[numRecvProcs];
00958   }
00959 
00960   int dataTag = 19904;
00961 
00962   int numRcvStarted = 0;
00963   for(i=0; i<remoteSharingProcs_.size(); i++) {
00964     int numRecvNodes = nodesPerSharingProc_[i];
00965     recvData[i] = new GlobalID[numRecvNodes*(len+1)];
00966     MPI_Irecv(recvData[i], numRecvNodes*(len+1),
00967         fei::mpiTraits<GlobalID>::mpi_type(),
00968         remoteSharingProcs_[i], dataTag, comm_, &recvDataReqs[i]);
00969     numRcvStarted++;
00970   }
00971 
00972   //next, send all outgoing messages.
00973 
00974   fei::Barrier(comm_);
00975 
00976   for(i=0; i<remoteOwnerProcs_.size(); i++) {
00977     int numSendNodes = nodesPerOwnerProc_[i];
00978 
00979     std::vector<GlobalID> sendData(numSendNodes*(len+1), 0);
00980 
00981     packRemoteNodesAndData(&sendData[0], remoteOwnerProcs_[i],
00982                            numSendNodes, numSendNodes*len);
00983 
00984     MPI_Send(&sendData[0], sendData.size(),
00985         fei::mpiTraits<GlobalID>::mpi_type(),
00986        remoteOwnerProcs_[i], dataTag, comm_);
00987   }
00988 
00989   //finally, complete the irecvs and put away the node field info.
00990   int numCompleted = 0;
00991   for(i=0; i<remoteSharingProcs_.size(); i++) {
00992     MPI_Status status;
00993     int index = i;
00994     MPI_Wait(&recvDataReqs[index], &status);
00995     numCompleted++;
00996     int remoteProc = status.MPI_SOURCE;
00997 
00998     int offset = 0;
00999     int numNodes = nodesPerSharingProc_[index];
01000 
01001     for(int j=0; j<numNodes; j++) {
01002       int nIndex = fei::binarySearch(recvData[index][j], &sharedNodeIDs[0], sharedNodeIDs.size());
01003       if (nIndex < 0) {
01004   fei::console_out() << "NodeCommMgr::exchangeSharedRemote...: error, unknown nodeID "
01005        << (int)recvData[index][j] << ", " << j
01006        << "th node recvd from proc "
01007        <<remoteSharingProcs_[index]
01008        << ". Probably a communication mis-match, we expected " 
01009        << numNodes
01010        << " nodes from that proc, but recvd less than that." << FEI_ENDL;
01011   std::abort();
01012       }
01013 
01014       int residesRemotely = (int)recvData[index][numNodes+offset++];
01015 
01016       if (residesRemotely) {
01017         std::vector<int>& snSubd = sharedNodeSubdomains[nIndex];
01018         std::vector<int>::iterator sn_iter =
01019           std::lower_bound(snSubd.begin(), snSubd.end(), remoteProc);
01020         if (sn_iter == snSubd.end() || remoteProc != *sn_iter) {
01021           snSubd.insert(sn_iter, remoteProc);
01022         }
01023       }
01024       int numFields       = (int)recvData[index][numNodes+offset++];
01025       int numBlocks       = (int)recvData[index][numNodes+offset++];
01026       sharedNodes_[nIndex]->
01027        setNumNodalDOF((int)recvData[index][numNodes+offset++]);
01028 
01029       for(int fld=0; fld<numFields; fld++) {
01030         int fieldID = (int)recvData[index][numNodes+offset++];
01031 
01032         sharedNodes_[nIndex]->addField(fieldID);
01033       }
01034 
01035       for(int blk=0; blk<numBlocks; blk++) {
01036         int blk_idx = probStruc.getIndexOfBlock(recvData[index][numNodes+offset++]);
01037         //if blk_idx < 0 it means the incoming blockID doesn't exist on this proc
01038         if (blk_idx >= 0) {
01039           sharedNodes_[nIndex]->addBlockIndex(blk_idx);
01040         }
01041       }
01042     }
01043   }
01044 
01045   if (numRcvStarted != numCompleted) {
01046     fei::console_out() << "NodeCommMgr::exchangeSharedRemote...: recv-send mismatch;"
01047          << " numRcvStarted: " << numRcvStarted << ", numCompleted: "
01048          << numCompleted << FEI_ENDL;
01049     std::abort();
01050   }
01051 
01052   for(i=0; i<numRecvProcs; i++) {
01053     delete [] recvData[i];
01054   }
01055 
01056   delete [] recvData;
01057   delete [] recvDataReqs;
01058 
01059 #endif //#ifndef FEI_SER
01060 
01061   return(0);
01062 }
01063 
01064 //------------------------------------------------------------------------------
01065 int NodeCommMgr::storeNodeProcs(int index,
01066         std::vector<std::vector<int>*>& procTable,
01067         const int* procs, int numProcs)
01068 {
01069 //Private NodeCommMgr function.
01070 //
01071 //This function stores 'procs' in row 'index' of procTable, maintaining order
01072 //in that row.
01073 //
01074   std::vector<int>& row_index = *(procTable[index]);
01075   for(int i=0; i<numProcs; i++) {
01076     std::vector<int>::iterator r_iter =
01077       std::lower_bound(row_index.begin(), row_index.end(), procs[i]);
01078     if (r_iter == row_index.end() || procs[i] != *r_iter) {
01079       row_index.insert(r_iter, procs[i]);
01080     }
01081   }
01082 
01083   return(0);
01084 }
01085 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Friends