FEI Version of the Day
fei_NodeCommMgr.cpp
00001 /*--------------------------------------------------------------------*/
00002 /*    Copyright 2005 Sandia Corporation.                              */
00003 /*    Under the terms of Contract DE-AC04-94AL85000, there is a       */
00004 /*    non-exclusive license for use of this work by or on behalf      */
00005 /*    of the U.S. Government.  Export of this program may require     */
00006 /*    a license from the United States Government.                    */
00007 /*--------------------------------------------------------------------*/
00008 
00009 #include <fei_macros.hpp>
00010 
00011 #include <fei_mpi.h>
00012 
00013 #include <fei_defs.h>
00014 
00015 #include <fei_TemplateUtils.hpp>
00016 #include <fei_mpiTraits.hpp>
00017 #include <fei_CommUtils.hpp>
00018 #include <fei_NodeDescriptor.hpp>
00019 #include <fei_NodeCommMgr.hpp>
00020 
00021 #include <fei_NodeDatabase.hpp>
00022 
00023 #undef fei_file
00024 #define fei_file "fei_NodeCommMgr.cpp"
00025 #include <fei_ErrMacros.hpp>
00026 
00027 //------Constructor-------------------------------------------------------------
00028 NodeCommMgr::NodeCommMgr(MPI_Comm comm, int sharedNodeOwnership)
00029   : sharedNodes_(NULL),
00030     sharedNodesAllocated_(false),
00031     sharedNodeOwnership_(sharedNodeOwnership),
00032     localNodeIDs(),
00033     remoteNodeIDs(),
00034     sharedNodeIDs(),
00035     sharedNodeSubdomains(),
00036     trivialSubdomainList(1),
00037     sharingProcs_(),
00038     sharedNodeNumbers(),
00039     remoteOwnerProcs_(),
00040     remoteSharingProcs_(),
00041     nodesPerOwnerProc_(),
00042     nodesPerSharingProc_(),
00043     comm_(comm),
00044     numProcs_(1),
00045     localProc_(0),
00046     maxFields_(0),
00047     maxBlocks_(0),
00048     maxSubdomains_(0),
00049     initCompleteCalled_(false)
00050 {
00051   numProcs_ = fei::numProcs(comm_);
00052   localProc_= fei::localProc(comm_);
00053   trivialSubdomainList[0] = localProc_;
00054 }
00055 
00056 //-----Destructor---------------------------------------------------------------
00057 NodeCommMgr::~NodeCommMgr() {
00058 
00059    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00060       delete sharingProcs_[i];
00061    }
00062 
00063    delete [] sharedNodes_;
00064    sharedNodesAllocated_ = false;
00065 }
00066 
00067 //------------------------------------------------------------------------------
00068 int NodeCommMgr::getSharedNodeIndex(GlobalID nodeID)
00069 {
00070   return( fei::binarySearch(nodeID, &sharedNodeIDs[0], sharedNodeIDs.size()) );
00071 }
00072 
00073 //------------------------------------------------------------------------------
00074 int NodeCommMgr::getSharedNodeNumSubdomains(GlobalID nodeID)
00075 {
00076   int index = getSharedNodeIndex(nodeID);
00077 
00078   //If the node isn't one of our shared nodes, then return 1, signifying that
00079   //the node is in 1 subdomain, that being the local subdomain.
00080   if (index < 0) return(1);
00081 
00082   //Since the node is one of our shared nodes, it should have an entry in our
00083   //sharedNodeNumSubdomains array indicating how many subdomains it
00084   //appears in. So return this number.
00085   return(sharedNodeSubdomains[index].size());
00086 }
00087 
00088 //------------------------------------------------------------------------------
00089 std::vector<int>* NodeCommMgr::getSharedNodeSubdomainList(GlobalID nodeID)
00090 {
00091   int index = getSharedNodeIndex(nodeID);
00092 
00093   //If the node isn't one of our shared nodes, then return 1, signifying that
00094   //the node is in 1 subdomain, that being the local subdomain.
00095   if (index < 0) return( &trivialSubdomainList );
00096 
00097   //Since the node is one of our shared nodes, it should have an entry in our
00098   //sharedNodeSubdomains array.
00099   return( &(sharedNodeSubdomains[index]) );
00100 }
00101 
00102 //------------------------------------------------------------------------------
00103 int NodeCommMgr::informLocal(const NodeDescriptor& node) {
00104 //
00105 //NodeCommMgr is being informed that 'node' is present in the local
00106 //active node list.
00107 //
00108 //This means that either:
00109 // 1. it is a locally-owned shared node, or
00110 // 3. it isn't even a shared node (it's a purely local node), in which
00111 //    case we'll do nothing.
00112 //
00113 
00114    if (numProcs_ == 1) return(0);
00115 
00116    GlobalID nodeID = node.getGlobalNodeID();
00117 
00118    int sharedIndex = getSharedNodeIndex(nodeID);
00119 
00120    //if this node isn't a shared node, then simply return.
00121    if (sharedIndex < 0) return(0);
00122 
00123    //Since this node is present as a shared node, let's put nodeID in the
00124    //localNodeIDs list if it isn't already there.
00125 
00126    int index = fei::sortedListInsert(nodeID, localNodeIDs);
00127 
00128    //if index is -2, it means the localNodeIDs array had an allocation failure.
00129    if (index == -2) return(-2);
00130 
00131    return(0);
00132 }
00133 
00134 //------------------------------------------------------------------------------
00135 int NodeCommMgr::getGlobalMaxFieldsBlocks(int& maxFields, int& maxBlocks)
00136 {
00137   std::vector<int> localMax(2, 0), globalMax(2, 0);
00138 
00139   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00140     int numFlds = sharedNodes_[i]->getNumFields();
00141     if (numFlds > localMax[0]) localMax[0] = numFlds;
00142 
00143     int numBlks = sharedNodes_[i]->getNumBlocks();
00144     if (numBlks > localMax[1]) localMax[1] = numBlks;
00145   }
00146 
00147   int err = fei::GlobalMax(comm_, localMax, globalMax);
00148   if (err != 0) return(err);
00149 
00150   maxFields = globalMax[0];
00151   maxBlocks = globalMax[1];
00152 
00153   return(0);
00154 }
00155 
00156 //------------------------------------------------------------------------------
00157 int NodeCommMgr::getGlobalMaxFieldsBlocksSubdomains()
00158 {
00159   std::vector<int> localMax(3, 0), globalMax(3, 0);
00160 
00161   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00162     int numFlds = sharedNodes_[i]->getNumFields();
00163     if (numFlds > localMax[0]) localMax[0] = numFlds;
00164 
00165     int numBlks = sharedNodes_[i]->getNumBlocks();
00166     if (numBlks > localMax[1]) localMax[1] = numBlks;
00167 
00168     int numShrd = sharingProcs_[i]->size();
00169     if (numShrd > localMax[2]) localMax[2] = numShrd;
00170   }
00171 
00172   int err = fei::GlobalMax(comm_, localMax, globalMax);
00173   if (err != 0) return(err);
00174 
00175   maxFields_     = globalMax[0];
00176   maxBlocks_     = globalMax[1];
00177   maxSubdomains_ = globalMax[2];
00178 
00179   return(0);
00180 }
00181 
00182 //------------------------------------------------------------------------------
00183 std::vector<int>& NodeCommMgr::getSendProcs()
00184 {
00185   return( remoteSharingProcs_ );
00186 }
00187 
00188 //------------------------------------------------------------------------------
00189 std::vector<int>& NodeCommMgr::getRecvProcs()
00190 {
00191   return( remoteOwnerProcs_ );
00192 }
00193 
00194 //------------------------------------------------------------------------------
00195 int NodeCommMgr::getSendMessageLength(int destProc, int& messageLength)
00196 {
00197   std::vector<int>::iterator
00198    rs_iter = std::lower_bound(remoteSharingProcs_.begin(),
00199                               remoteSharingProcs_.end(), destProc);
00200   if (rs_iter == remoteSharingProcs_.end() || destProc != *rs_iter) {
00201     ERReturn(-1);
00202   }
00203 
00204   int idx = rs_iter - remoteSharingProcs_.begin();
00205 
00206   int len = 7+maxFields_*2 + maxBlocks_ + maxSubdomains_;
00207   messageLength = nodesPerSharingProc_[idx] * (len+1);
00208   return(0);
00209 }
00210 
00211 //------------------------------------------------------------------------------
00212 int NodeCommMgr::getSendMessage(int destProc, std::vector<int>& message)
00213 {
00214   std::vector<int>::iterator
00215    rs_iter = std::lower_bound(remoteSharingProcs_.begin(),
00216                               remoteSharingProcs_.end(), destProc);
00217   if (rs_iter == remoteSharingProcs_.end() || destProc != *rs_iter) {
00218     ERReturn(-1);
00219   }
00220 
00221   int idx = rs_iter - remoteSharingProcs_.begin();
00222   int len = 0;
00223   CHK_ERR( getSendMessageLength(destProc, len) );
00224   message.resize(len);
00225 
00226   packLocalNodesAndData(&message[0], destProc,
00227       nodesPerSharingProc_[idx], len);
00228   return(0);
00229 }
00230 
00231 //------------------------------------------------------------------------------
00232 int NodeCommMgr::processRecvMessage(int srcProc, std::vector<int>& message)
00233 {
00234   int idx = fei::binarySearch(srcProc, &remoteOwnerProcs_[0],
00235                                   remoteOwnerProcs_.size());
00236   int numNodes = nodesPerOwnerProc_[idx];
00237   int* msgPtr = &message[0];
00238   int offset = 0;
00239 
00240   for(int j=0; j<numNodes; j++) {
00241     int nIndex = fei::binarySearch(msgPtr[j], &sharedNodeIDs[0], sharedNodeIDs.size());
00242     if (nIndex < 0) return(-1);
00243     NodeDescriptor* node = sharedNodes_[nIndex];
00244 
00245     int nodeNum         = msgPtr[numNodes+offset++];
00246     int numFields       = msgPtr[numNodes+offset++];
00247     int numBlocks       = msgPtr[numNodes+offset++];
00248     int numSubdomains   = msgPtr[numNodes+offset++];
00249 
00250     node->setNodeNumber(nodeNum);
00251     node->setNumNodalDOF( msgPtr[numNodes+offset++]);
00252     node->setBlkEqnNumber(msgPtr[numNodes+offset++]);
00253 
00254     for(int fld=0; fld<numFields; fld++) {
00255       int fieldID = msgPtr[numNodes+offset++];
00256       int eqnNum  = msgPtr[numNodes+offset++];
00257       node->addField(fieldID);
00258       node->setFieldEqnNumber(fieldID, eqnNum);
00259     }
00260 
00261     for(int blk=0; blk<numBlocks; blk++) {
00262       node->addBlock(msgPtr[numNodes+offset++]);
00263     }
00264 
00265     sharedNodeSubdomains[nIndex].resize(numSubdomains);
00266     for(int sd=0; sd<numSubdomains; sd++) {
00267       (sharedNodeSubdomains[nIndex])[sd] =
00268   msgPtr[numNodes+offset++];
00269     }
00270   }
00271 
00272   return(0);
00273 }
00274 
00275 //------------------------------------------------------------------------------
00276 int NodeCommMgr::exchangeEqnInfo()
00277 {
00278   //
00279   //This function will perform the communication necessary to:
00280   //
00281   //   1. For each locally owned node, send to each remote sharing proc:
00282   //      the node's nodeNumber, whether or not it appears in local elements,
00283   //      fieldID(s), the field-size(s), the first global equation numbers
00284   //      for those fields, and the processor-subdomains that contain the node.
00285   //   2. For each remotely owned node, receive the above information from
00286   //      the owners of these nodes.
00287   //
00288   //This is a collective function. All procs must enter it before any can
00289   //exit it.
00290   //
00291   //Most of this function is #ifdef'd according to whether FEI_SER is
00292   //defined.
00293 
00294 #ifndef FEI_SER
00295    if (numProcs_ == 1) return(0);
00296 
00297    //each proc will find out a max. number of fields, blocks
00298    //and subdomains to expect per node.
00299 
00300    CHK_ERR( getGlobalMaxFieldsBlocksSubdomains() );
00301 
00302    CHK_ERR( fei::exchange(comm_, this) );
00303 
00304    setNodeNumbersArray();
00305 
00306 #endif //#ifndef FEI_SER
00307 
00308    return(0);
00309 }
00310 
00311 //------------------------------------------------------------------------------
00312 void NodeCommMgr::packLocalNodesAndData(int* data, 
00313                                        int proc, int numNodes, int len)
00314 {
00315 //This function packs up nodeIDs, as well as the list containing, for
00316 //each node, the following:
00317 //   node-number
00318 //   numFields
00319 //   numBlocks
00320 //   numSubdomains
00321 //   num-nodal-dof
00322 //   blk-eqn-number
00323 //     'numFields' pairs of (fieldID,eqnNumber)
00324 //     subdomain list, length 'numSubdomains'
00325 //
00326 //Incoming parameter len is:
00327 //  numNodes * (7 + maxFields*2 + maxBlocks + maxSubdomains),
00328 //where maxFields is the maximum number of fields associated with any node,
00329 //maxBlocks is the maximum number of blocks associated with any node, and
00330 //maxSubdomains is the maximum number of subdomains containing any node.
00331 //data is of length numNodes*(len+1).
00332 //
00333 //The above data will all be packed into the 'data' list, with nodeIDs 
00334 //occupying the first numNodes positions, followed by the rest of the data.
00335 //
00336    int nodeCounter = 0;
00337    int offset = 0;
00338 
00339    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00340      if (sharedNodes_[i]->getOwnerProc() != localProc_) continue;
00341 
00342       NodeDescriptor* node = sharedNodes_[i];
00343 
00344       //is this local node associated with processor 'proc'?
00345 
00346       std::vector<int>& sProcs = *(sharingProcs_[i]);
00347       int index = fei::binarySearch(proc, &sProcs[0], sProcs.size());
00348 
00349       //if not, skip to the next iteration...
00350       if (index < 0) continue;
00351 
00352       if (nodeCounter >= numNodes) {
00353   FEI_CERR << "NodeCommMgr::packLocalNodesAndData: ERROR,"
00354        << " nodeCounter >= numNodes." << FEI_ENDL;
00355       }
00356 
00357       data[nodeCounter++] = (int)(node->getGlobalNodeID());
00358 
00359       int nodeNum = node->getNodeNumber();
00360       int numFields = node->getNumFields();
00361       int numBlocks = node->getNumBlocks();
00362       const int* fieldIDsPtr = node->getFieldIDList();
00363       const int* fieldEqnNums = node->getFieldEqnNumbers();
00364       int blkEqnNumber = node->getBlkEqnNumber();
00365 
00366       const GlobalID* nodeBlocks = node->getBlockList();
00367       std::vector<int>& subdomains = sharedNodeSubdomains[i];
00368 
00369       data[numNodes+offset++] = nodeNum;
00370       data[numNodes+offset++] = numFields;
00371       data[numNodes+offset++] = numBlocks;
00372       data[numNodes+offset++] = subdomains.size();
00373       data[numNodes+offset++] = node->getNumNodalDOF();
00374       data[numNodes+offset++] = blkEqnNumber;
00375 
00376       for(int j=0; j<numFields; j++) {
00377   data[numNodes+offset++] = fieldIDsPtr[j];
00378 
00379   if (offset >= len) {
00380     FEI_CERR << "NodeCommMgr::packLocalNodesAndData: ERROR,"
00381          << " offset >= len." << FEI_ENDL;
00382   }
00383 
00384   data[numNodes+offset++] = fieldEqnNums[j];
00385       }
00386 
00387       for(int kk=0; kk<numBlocks; kk++) {
00388   data[numNodes+offset++] = nodeBlocks[kk];
00389       }
00390 
00391       for(unsigned k=0; k<subdomains.size(); k++) {
00392   data[numNodes+offset++] = subdomains[k];
00393       }
00394    }
00395 }
00396 
00397 //------------------------------------------------------------------------------
00398 void NodeCommMgr::packRemoteNodesAndData(GlobalID* data,
00399            int proc, int numNodes, int len)
00400 {
00401 //
00402 //This function packs up the nodeIDs owned by proc, as well as the list
00403 //containing, for each node, the following:
00404 //   residesLocally (0 or 1) indicating whether it appears in the local
00405 //       processor's element domain.
00406 //   numFields
00407 //   numBlocks
00408 //   numNodalDOF
00409 //     'numFields' entries of (fieldID)
00410 //     'numBlocks' entries of (block)
00411 //
00412 //Incoming parameter len is numNodes * (3 + maxFields + maxBlocks),
00413 //where maxFields is the maximum number of fields associated with any node,
00414 //and maxBlocks is the maximum number of blocks associated with any node.
00415 //nodeIDs is of length numNodes, and
00416 //data is of length numNodes*len.
00417 //
00418 //The above data will all be put in the 'data' list, with the nodeIDs
00419 //occupying the first numNodes positions, followed by the rest of the data.
00420 //
00421    int nodeCounter = 0;
00422    int offset = 0;
00423 
00424    for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00425       NodeDescriptor* node = sharedNodes_[i];
00426 
00427       int thisProc = node->getOwnerProc();
00428       if (thisProc != proc) continue;
00429 
00430       if (nodeCounter >= numNodes) {
00431          FEI_CERR << localProc_ << ": NodeCommMgr::packRemoteNodesAndData: ERROR,"
00432               << " nodeCounter >= numNodes: " << numNodes << FEI_ENDL;
00433       }
00434 
00435       data[nodeCounter++] = node->getGlobalNodeID();
00436 
00437       int numFields = node->getNumFields();
00438       int numBlocks = node->getNumBlocks();
00439       const int* fieldIDsPtr = node->getFieldIDList();
00440 
00441       const GlobalID* nodeBlocks = node->getBlockList();
00442       int lindex = fei::binarySearch(sharedNodeIDs[i], &localNodeIDs[0], localNodeIDs.size());
00443 
00444       data[numNodes+offset++] = (lindex >= 0) ? 1 : 0;
00445       data[numNodes+offset++] = (GlobalID)numFields;
00446       data[numNodes+offset++] = (GlobalID)numBlocks;
00447       data[numNodes+offset++] = (GlobalID)node->getNumNodalDOF();
00448 
00449       for(int j=0; j<numFields; j++) {
00450          if (offset >= len) {
00451             FEI_CERR << "NodeCommMgr::packRemoteNodesAndData: ERROR,"
00452                  << " offset >= len." << FEI_ENDL;
00453          }
00454 
00455          data[numNodes+offset++] = (GlobalID)fieldIDsPtr[j];
00456       }
00457 
00458       for(int k=0; k<numBlocks; k++) {
00459          if (offset >= len) {
00460             FEI_CERR << "NodeCommMgr::packRemoteNodesAndData: ERROR,"
00461                  << " offset >= len." << FEI_ENDL;
00462          }
00463 
00464          data[numNodes+offset++] = nodeBlocks[k];
00465       }
00466    }
00467 }
00468 
00469 //------------------------------------------------------------------------------
00470 int NodeCommMgr::createProcList(std::vector<int>& itemsPerProc,
00471                                 std::vector<int>& procs)
00472 {
00473 //
00474 //This function looks through the itemsPerProc list and counts how many
00475 //positions in this list are greater than 0. Then it creates a list of
00476 //the indices of those positions. i.e., itemsPerProc is a list of how many
00477 //items are to be sent to or recvd from each proc. When itemsPerProc is
00478 //greater than 0, that proc is put in the sharingProcs list.
00479 //
00480    int numProcs = 0;
00481    int len = itemsPerProc.size();
00482 
00483    for(int i=0; i<len; i++) {
00484       if (itemsPerProc[i] > 0) numProcs++;
00485    }
00486 
00487    procs.resize(numProcs);
00488 
00489    int offset = 0;
00490 
00491    for(int i=0; i<len; i++) {
00492       if (itemsPerProc[i] > 0) procs[offset++] = i;
00493    }
00494    return(0);
00495 }
00496 
00497 //------------------------------------------------------------------------------
00498 int NodeCommMgr::getSharedNodeIndex_num(int nodeNumber)
00499 {
00500   for(unsigned i=0; i<sharedNodeNumbers.size(); ++i) {
00501     if (sharedNodeNumbers[i] == nodeNumber) return(i);
00502   }
00503 
00504   return(-1);
00505 }
00506 
00507 //------------------------------------------------------------------------------
00508 int NodeCommMgr::addSharedNodes( const GlobalID* nodeIDs,
00509          int numNodes, 
00510          const int* const* procs,
00511          const int* numProcs )
00512 {
00513   //
00514   //Store the incoming nodeIDs and proc-numbers in the sharedNodeIDs array and
00515   //sharingProcs_ table.
00516   //
00517 
00518   try {
00519 
00520   for(int i=0; i<numNodes; i++) {
00521     int insertPoint = -1;
00522     int index = fei::binarySearch(nodeIDs[i], sharedNodeIDs, insertPoint);
00523     if (index < 0) {
00524       sharingProcs_.insert(sharingProcs_.begin()+insertPoint, new std::vector<int>);
00525 
00526       sharedNodeIDs.insert(sharedNodeIDs.begin()+insertPoint, nodeIDs[i]);
00527 
00528       index = insertPoint;
00529     }
00530     
00531     int err = storeNodeProcs(index, sharingProcs_, procs[i], numProcs[i]);
00532     if (err != 0) return(err);
00533   }
00534 
00535   }
00536   catch(std::runtime_error& exc) {
00537     FEI_CERR << exc.what() << FEI_ENDL;
00538     ERReturn(-1);
00539   }
00540 
00541   return(0);
00542 }
00543 
00544 //------------------------------------------------------------------------------
00545 int NodeCommMgr::allocateNodeDescriptorPtrs(NodeDatabase& nodeDB)
00546 {
00547   //This function is called when all shared nodes have been added. We now
00548   //allocate a list of pointer-to-NodeDescriptor, of length 
00549   //sharedNodeIDs.length(), and fill that list with NodeDescriptor-pointers
00550   //from the node-database.
00551 
00552   if (sharedNodeIDs.size() == 0) return(0);
00553 
00554   if (sharedNodes_ != NULL) delete [] sharedNodes_;
00555   sharedNodes_ = new NodeDescriptor*[sharedNodeIDs.size()];
00556   if (sharedNodes_ == NULL) return(-1);
00557 
00558   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00559     NodeDescriptor* node = NULL;
00560     int err = nodeDB.getNodeWithID(sharedNodeIDs[i], node);
00561     if (err != 0) return(-1);
00562 
00563     sharedNodes_[i] = node;
00564   }
00565 
00566   sharedNodesAllocated_ = true;
00567   return(0);
00568 }
00569 
00570 //------------------------------------------------------------------------------
00571 int NodeCommMgr::initComplete(NodeDatabase& nodeDB, bool safetyCheck)
00572 {
00573 //
00574 //This function is called when initialization is complete (i.e., when
00575 //all sharedNodes have been added, allocatedNodeDescriptorPtrs() has been
00576 //called, and informLocal() has been called for all nodes that appear in
00577 //the local finite-element structure.
00578 //
00579 //The task of this function is to assign owner-procs to nodes.
00580 //
00581 //if 'safetyCheck' is true, a global consistency check of the shared node info
00582 //will be performed before the communication is attempted.
00583 //
00584 //return value is 0 if successful, non-zero if an error was encountered
00585 //
00586   int err = allocateNodeDescriptorPtrs(nodeDB);
00587   if (err != 0) return(err);
00588 
00589   //Run through the shared nodes, and for each one that has been
00590   //identified as local, assign its owner to be the lowest-numbered sharing
00591   //processor, which may or may not be localProc_.
00592 
00593   for(unsigned ii=0; ii<sharedNodeIDs.size(); ii++) {
00594     std::vector<int>& shProcs = *(sharingProcs_[ii]);
00595 
00596     //first, insert localProc_ in this node's list of sharing proc, since the
00597     //FEI's initSharedNodes function doesn't mandate that the local processor be
00598     //included in the list of sharing processors. (i.e., localProc_ may not be
00599     //in this list yet...)
00600     std::vector<int>::iterator sh_iter =
00601       std::lower_bound(shProcs.begin(), shProcs.end(), localProc_);
00602     if (sh_iter == shProcs.end() || localProc_ != *sh_iter) {
00603       shProcs.insert(sh_iter, localProc_);
00604     }
00605 
00606     int proc = shProcs[0];
00607 
00608     sharedNodes_[ii]->setOwnerProc(proc);
00609   }
00610 
00611   //One of the tasks of this object is to gather information on the number
00612   //of subdomains each shared node appears in. So one thing we'll do here is
00613   //size and zero the array that will hold that information.
00614   sharedNodeSubdomains.resize(sharedNodeIDs.size());
00615 
00616   for(unsigned i=0; i<sharedNodeSubdomains.size(); ++i) {
00617     sharedNodeSubdomains[i].resize(0);
00618   }
00619 
00620   //now add the local processor to the sharedNodeSubdomains for each node that
00621   //appears in our localNodeIDs list.
00622   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00623     int index = fei::binarySearch(sharedNodeIDs[i], &localNodeIDs[0], localNodeIDs.size());
00624     if (index >= 0) {
00625       sharedNodeSubdomains[i].push_back(localProc_);
00626     }
00627   }
00628 
00629   if (sharedNodeOwnership_ != STRICTLY_LOW_PROC) {
00630     err = adjustSharedOwnership();
00631     if (err != 0) return(err);
00632   }
00633 
00634   err = createProcLists();
00635 
00636   if (safetyCheck) {
00637     err = checkSharedNodeInfo();
00638     if (err != 0) return(-1);
00639   }
00640 
00641   exchangeSharedRemoteFieldsBlks();
00642 
00643   initCompleteCalled_ = true;
00644 
00645   return(0);
00646 }
00647 
00648 //------------------------------------------------------------------------------
00649 #undef _feiFunc_
00650 #define _feiFunc_ "NodeCommMgr::checkSharedNodeInfo"
00651 int NodeCommMgr::checkSharedNodeInfo()
00652 {
00653   //This function's task is to "audit" the shared-node info. I.e., to make sure
00654   //that the info is globally symmetric (e.g., if the local processor thinks it
00655   //shares a node with another processor, does that other processor also think
00656   //it shares a node with the local proc?).
00657   //If this function finds that the shared-node info is consistent/correct, then
00658   //the return-value is 0. If the shared-node info is found to be wrong, then
00659   //one or more messages will be written to stderr, and the return-value is -1.
00660   //
00661   //This is a collective function, which is relatively expensive. It does a
00662   //few global reductions...
00663   //
00664 
00665   if (numProcs_==1) return(0);
00666 
00667   //Make sure that the processors we think are "remote owner"
00668   //procs, think we are a "remote sharing" proc, and vice-versa.
00669 
00670   std::vector<int> globalOwnerProcs, globalSharingProcs;
00671   std::vector<int> recvOwnerLengths, recvSharingLengths;
00672 
00673   std::vector<int> globalNodesPerOwnerProcs, globalNodesPerSharingProcs;
00674   std::vector<int> recvNodesPerOwnerLengths, recvNodesPerSharingLengths;
00675 
00676   //First, gather up each processor's list of remote procs and nodes-per-proc
00677   //onto all other processors...
00678 
00679   CHK_ERR( fei::Allgatherv(comm_, remoteOwnerProcs_,
00680          recvOwnerLengths, globalOwnerProcs) );
00681 
00682   CHK_ERR( fei::Allgatherv(comm_, nodesPerOwnerProc_,
00683          recvNodesPerOwnerLengths,
00684          globalNodesPerOwnerProcs) );
00685 
00686   CHK_ERR( fei::Allgatherv(comm_, remoteSharingProcs_,
00687          recvSharingLengths, globalSharingProcs) );
00688 
00689   CHK_ERR( fei::Allgatherv(comm_, nodesPerSharingProc_,
00690          recvNodesPerSharingLengths,
00691          globalNodesPerSharingProcs) );
00692 
00693   //Now check the consistency of the global "owners" data against local "sharing"
00694   //data.
00695   int err =  checkCommArrays( "owners",
00696             globalOwnerProcs, globalNodesPerOwnerProcs,
00697             recvOwnerLengths,
00698             nodesPerSharingProc_, remoteSharingProcs_ );
00699 
00700   //Now check the consistency of the global "sharing" data against local "owners"
00701   //data.
00702   err +=  checkCommArrays( "sharing",
00703          globalSharingProcs, globalNodesPerSharingProcs,
00704          recvSharingLengths,
00705          nodesPerOwnerProc_, remoteOwnerProcs_ );
00706 
00707   int globalErr = 0;
00708 
00709   CHK_ERR( fei::GlobalSum(comm_, err, globalErr) );
00710 
00711   return(globalErr);
00712 }
00713 
00714 //------------------------------------------------------------------------------
00715 int NodeCommMgr::checkCommArrays(const char* whichCheck,
00716          std::vector<int>& globalRemoteProcs,
00717          std::vector<int>& globalNodesPerRemoteProc,
00718          std::vector<int>& globalRemoteProcLengths,
00719          std::vector<int>& nodesPerRemoteProc,
00720          std::vector<int>& remoteProcs)
00721 {
00722   int offset = 0;
00723 
00724   for(int i=0; i<numProcs_; i++) {
00725     int length = globalRemoteProcLengths[i];
00726 
00727     if (i==localProc_) { offset += length; continue; }
00728 
00729     for(int j=0; j<length; j++) {
00730       if (globalRemoteProcs[offset+j] == localProc_) {
00731   //proc i says that we (localProc_) own nodes that it shares.
00732   int numShared = globalNodesPerRemoteProc[offset+j];
00733 
00734   int index = fei::binarySearch(i, &remoteProcs[0], remoteProcs.size());
00735   if (index < 0) {
00736     //we don't think proc i shares any nodes that we own.
00737     FEI_CERR << "FEI NodeCommMgr::checkSharedNodeInfo "<<whichCheck
00738          << " ERROR. Local proc (" << localProc_ 
00739          << ") doesn't share nodes with proc " << i << " but proc " << i
00740          << " thinks it shares nodes with proc " << localProc_ << FEI_ENDL;
00741     return(-1);
00742   }
00743 
00744   //We think that we own nodesPerRemoteProc[index] nodes that proc i
00745   //shares.
00746   int numWeThinkWeShare = nodesPerRemoteProc[index];
00747   if (numWeThinkWeShare != numShared) {
00748     FEI_CERR << "FEI NodeCommMgr::checkSharedNodeInfo "<<whichCheck
00749          << " ERROR. Local proc (" << localProc_ << ") thinks it shares "
00750          << numWeThinkWeShare << " nodes with proc " << i << ", but proc " 
00751          << i << " thinks it shares " << numShared << " nodes with proc "
00752          << localProc_ << "." << FEI_ENDL;
00753     return(-1);
00754   }
00755       }
00756     }
00757 
00758     offset += length;
00759   }
00760 
00761   return(0);
00762 }
00763 
00764 //------------------------------------------------------------------------------
00765 int NodeCommMgr::adjustSharedOwnership()
00766 {
00767   //For each shared node that has not been identified as local, assign its
00768   //owner to be the next lowest-numbered sharing proc. (Each list of sharing
00769   //procs is sorted by processor-number, so we just assign the owner to be the
00770   //next one in the list.)
00771   //
00772   //If a node is not local, and localProc_ is the lowest sharing proc, then we
00773   //also need to flag that node as remote and tell other processors that we
00774   //don't own it.
00775   //
00776   remoteNodeIDs.resize(0);
00777   int err;
00778   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00779     GlobalID nodeID = sharedNodeIDs[i];
00780 
00781     std::vector<int>& shProcs = *(sharingProcs_[i]);
00782 
00783     if (fei::binarySearch(nodeID, &localNodeIDs[0], localNodeIDs.size()) >= 0) continue;
00784 
00785     int proc = shProcs[0];
00786 
00787     if (proc == localProc_) {
00788       sharedNodes_[i]->setOwnerProc(shProcs[1]);
00789       err = fei::sortedListInsert(nodeID, remoteNodeIDs);
00790       if (err == -2) return(err);
00791     }
00792   }
00793 
00794   //Now we need to let the other processors know that the remote nodes 
00795   //aren't owned by us. This is going to require some communication. We'll
00796   //gather the nodeIDs onto all processors, after which each processor
00797   //will reset the owner proc for that node. (Later, as an optimization, I'll
00798   //do this without all-to-all communication.)
00799 
00800   std::vector<GlobalID> allRemoteNodeIDs;
00801   std::vector<int> numPerProc;
00802 
00803   err = fei::Allgatherv(comm_, remoteNodeIDs, numPerProc, allRemoteNodeIDs);
00804   if (err != 0) return(-1);
00805 
00806   //Now we need to run through the global list of 'special' nodes, and for the ones
00807   //that we do own (appear locally), add them to a new list that will be once again
00808   //globally gathered. That new list will then be used by each processor in setting
00809   //the nodes' real owners.
00810 
00811   //we'll keep the 'new' list in remoteNodeIDs.
00812   remoteNodeIDs.resize(0);
00813 
00814   int offset = 0;
00815   for(unsigned i=0; i<numPerProc.size(); i++) {
00816     for(int j=0; j<numPerProc[i]; j++) {
00817 
00818       //skip the nodes that we sent, we already know we don't own those.
00819       if ((int)i==localProc_) {offset++; continue;}
00820 
00821       GlobalID nodeID = allRemoteNodeIDs[offset++];
00822       int index = getSharedNodeIndex(nodeID);
00823 
00824       //if it's not even one of our shared nodes, then continue.
00825       if (index < 0) continue;
00826 
00827       if (fei::binarySearch(nodeID, &localNodeIDs[0], localNodeIDs.size()) >= 0) {
00828         fei::sortedListInsert(nodeID, remoteNodeIDs);
00829       }
00830     }
00831   }
00832 
00833   //now re-gather the remoteNodeIDs list to all processors. This time, we should only
00834   //receive nodeIDs from processors that can be valid owners. i.e., processors that
00835   //have those nodes in at least one local element.
00836   err = fei::Allgatherv(comm_, remoteNodeIDs, numPerProc, allRemoteNodeIDs);
00837   if (err != 0) return(-1);
00838 
00839   //Now we run the 'allRemoteNodeIDs' list for the last time, setting the owner-proc
00840   //for each node. We'll run the list from the back to the front so that if multiple
00841   //processors are possible owners, the lowest-numbered one will be the last one that
00842   //get's set.
00843   offset = allRemoteNodeIDs.size()-1;
00844   for(int i=(int)numPerProc.size()-1; i>=0; i--) {
00845     for(int j=0; j<numPerProc[i]; j++) {
00846       GlobalID nodeID = allRemoteNodeIDs[offset--];
00847       int index = getSharedNodeIndex(nodeID);
00848 
00849       if (index < 0) continue;
00850 
00851       sharedNodes_[index]->setOwnerProc(i);
00852     }
00853   }
00854 
00855   return(0);
00856 }
00857 
00858 //------------------------------------------------------------------------------
00859 void NodeCommMgr::setNodeNumbersArray()
00860 {
00861   sharedNodeNumbers.resize(sharedNodeIDs.size());
00862 
00863   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00864     sharedNodeNumbers[i] = sharedNodes_[i]->getNodeNumber();
00865   }
00866 }
00867 
00868 //------------------------------------------------------------------------------
00869 int NodeCommMgr::createProcLists()
00870 {
00871   std::vector<int> localNodesPerProc(numProcs_, 0);
00872   std::vector<int> remoteNodesPerProc(numProcs_, 0);
00873 
00874   //first, figure out how many locally-owned nodes each remote processor is
00875   //associated with, and how many remotely-owned nodes we'll be recv'ing info
00876   //about from each remote processor.
00877 
00878   for(unsigned i=0; i<sharedNodeIDs.size(); i++) {
00879     int proc = sharedNodes_[i]->getOwnerProc();
00880 
00881     if (proc != localProc_) {
00882       remoteNodesPerProc[proc]++;
00883     }
00884     else {
00885       std::vector<int>& shProcs = *(sharingProcs_[i]);
00886       for(unsigned j=0; j<shProcs.size(); j++) {
00887   int sproc = shProcs[j];
00888 
00889   if (sproc != localProc_) {
00890     localNodesPerProc[sproc]++;
00891   }
00892       }
00893     }
00894   }
00895 
00896   //now create condensed lists of remote owner procs, and
00897   //remote sharing procs.
00898   int err = createProcList(remoteNodesPerProc, remoteOwnerProcs_);
00899   if (err != 0) return(err);
00900 
00901   err = createProcList(localNodesPerProc, remoteSharingProcs_);
00902   if (err != 0) return(err);
00903 
00904 
00905   nodesPerOwnerProc_.resize(remoteOwnerProcs_.size());
00906 
00907   nodesPerSharingProc_.resize(remoteSharingProcs_.size());
00908 
00909   int offset = 0;
00910   for(int i=0; i<numProcs_; i++) {
00911     if (remoteNodesPerProc[i] > 0) 
00912       nodesPerOwnerProc_[offset++] = remoteNodesPerProc[i];
00913   }
00914 
00915   offset = 0;
00916   for(int i=0; i<numProcs_; i++) {
00917     if (localNodesPerProc[i] > 0) 
00918       nodesPerSharingProc_[offset++] = localNodesPerProc[i];
00919   }
00920 
00921   return(0);
00922 }
00923 
00924 //------------------------------------------------------------------------------
00925 int NodeCommMgr::exchangeSharedRemoteFieldsBlks()
00926 {
00927   //first each proc will find out a max. number of fields and blocks to expect
00928   //per node.
00929 
00930   //most of this function is #ifdef'd according to whether FEI_SER is
00931   //defined.
00932 #ifndef FEI_SER
00933   int maxFields, maxBlocks;
00934   int err = getGlobalMaxFieldsBlocks(maxFields, maxBlocks);
00935   if (err) return(-1);
00936 
00937   //now we can allocate lists to recv into and launch the irecv's.
00938   //from each processor, we'll recv a list of length:
00939   //            num-nodes*(4+ maxFields + maxBlocks)
00940 
00941   int len = 4 + maxFields + maxBlocks;
00942 
00943   GlobalID** recvData = NULL;
00944   MPI_Request* recvDataReqs = NULL;
00945 
00946   unsigned i, numRecvProcs = remoteSharingProcs_.size();
00947 
00948   if (numRecvProcs > 0) {
00949     recvData = new GlobalID*[numRecvProcs];
00950     recvDataReqs = new MPI_Request[numRecvProcs];
00951   }
00952 
00953   int dataTag = 199904;
00954 
00955   int numRcvStarted = 0;
00956   for(i=0; i<remoteSharingProcs_.size(); i++) {
00957     int numRecvNodes = nodesPerSharingProc_[i];
00958     recvData[i] = new GlobalID[numRecvNodes*(len+1)];
00959     MPI_Irecv(recvData[i], numRecvNodes*(len+1),
00960         fei::mpiTraits<GlobalID>::mpi_type(),
00961         remoteSharingProcs_[i], dataTag, comm_, &recvDataReqs[i]);
00962     numRcvStarted++;
00963   }
00964 
00965   //next, send all outgoing messages.
00966 
00967   fei::Barrier(comm_);
00968 
00969   for(i=0; i<remoteOwnerProcs_.size(); i++) {
00970     int numSendNodes = nodesPerOwnerProc_[i];
00971 
00972     std::vector<GlobalID> sendData(numSendNodes*(len+1), 0);
00973 
00974     packRemoteNodesAndData(&sendData[0], remoteOwnerProcs_[i],
00975                            numSendNodes, numSendNodes*len);
00976 
00977     MPI_Send(&sendData[0], sendData.size(),
00978         fei::mpiTraits<GlobalID>::mpi_type(),
00979        remoteOwnerProcs_[i], dataTag, comm_);
00980   }
00981 
00982   //finally, complete the irecvs and put away the node field info.
00983   int numCompleted = 0;
00984   for(i=0; i<remoteSharingProcs_.size(); i++) {
00985     MPI_Status status;
00986     int index = i;
00987     MPI_Wait(&recvDataReqs[index], &status);
00988     numCompleted++;
00989     int remoteProc = status.MPI_SOURCE;
00990 
00991     int offset = 0;
00992     int numNodes = nodesPerSharingProc_[index];
00993 
00994     for(int j=0; j<numNodes; j++) {
00995       int nIndex = fei::binarySearch(recvData[index][j], &sharedNodeIDs[0], sharedNodeIDs.size());
00996       if (nIndex < 0) {
00997   FEI_CERR << "NodeCommMgr::exchangeSharedRemote...: error, unknown nodeID "
00998        << (int)recvData[index][j] << ", " << j
00999        << "th node recvd from proc "
01000        <<remoteSharingProcs_[index]
01001        << ". Probably a communication mis-match, we expected " 
01002        << numNodes
01003        << " nodes from that proc, but recvd less than that." << FEI_ENDL;
01004   std::abort();
01005       }
01006 
01007       int residesRemotely = (int)recvData[index][numNodes+offset++];
01008 
01009       if (residesRemotely) {
01010         std::vector<int>& snSubd = sharedNodeSubdomains[nIndex];
01011         std::vector<int>::iterator sn_iter =
01012           std::lower_bound(snSubd.begin(), snSubd.end(), remoteProc);
01013         if (sn_iter == snSubd.end() || remoteProc != *sn_iter) {
01014           snSubd.insert(sn_iter, remoteProc);
01015         }
01016       }
01017       int numFields       = (int)recvData[index][numNodes+offset++];
01018       int numBlocks       = (int)recvData[index][numNodes+offset++];
01019       sharedNodes_[nIndex]->
01020        setNumNodalDOF((int)recvData[index][numNodes+offset++]);
01021 
01022       for(int fld=0; fld<numFields; fld++) {
01023   int fieldID = (int)recvData[index][numNodes+offset++];
01024 
01025   sharedNodes_[nIndex]->addField(fieldID);
01026       }
01027 
01028       for(int blk=0; blk<numBlocks; blk++) {
01029   sharedNodes_[nIndex]->addBlock(recvData[index][numNodes+offset++]);
01030       }
01031     }
01032   }
01033 
01034   if (numRcvStarted != numCompleted) {
01035     FEI_CERR << "NodeCommMgr::exchangeSharedRemote...: recv-send mismatch;"
01036          << " numRcvStarted: " << numRcvStarted << ", numCompleted: "
01037          << numCompleted << FEI_ENDL;
01038     std::abort();
01039   }
01040 
01041   for(i=0; i<numRecvProcs; i++) {
01042     delete [] recvData[i];
01043   }
01044 
01045   delete [] recvData;
01046   delete [] recvDataReqs;
01047 
01048 #endif //#ifndef FEI_SER
01049 
01050   return(0);
01051 }
01052 
01053 //------------------------------------------------------------------------------
01054 int NodeCommMgr::storeNodeProcs(int index,
01055         std::vector<std::vector<int>*>& procTable,
01056         const int* procs, int numProcs)
01057 {
01058 //Private NodeCommMgr function.
01059 //
01060 //This function stores 'procs' in row 'index' of procTable, maintaining order
01061 //in that row.
01062 //
01063   std::vector<int>& row_index = *(procTable[index]);
01064   for(int i=0; i<numProcs; i++) {
01065     std::vector<int>::iterator r_iter =
01066       std::lower_bound(row_index.begin(), row_index.end(), procs[i]);
01067     if (r_iter == row_index.end() || procs[i] != *r_iter) {
01068       row_index.insert(r_iter, procs[i]);
01069     }
01070   }
01071 
01072   return(0);
01073 }
01074 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Friends