FEI Version of the Day
fei_NodeCommMgr.hpp
00001 /*
00002 // @HEADER
00003 // ************************************************************************
00004 //             FEI: Finite Element Interface to Linear Solvers
00005 //                  Copyright (2005) Sandia Corporation.
00006 //
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, the
00008 // U.S. Government retains certain rights in this software.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Alan Williams (william@sandia.gov) 
00038 //
00039 // ************************************************************************
00040 // @HEADER
00041 */
00042 
00043 #ifndef _NodeCommMgr_hpp_
00044 #define _NodeCommMgr_hpp_
00045 
00046 
00047 #include <fei_fwd.hpp>
00048 
00049 #include <fei_CommUtils.hpp>
00050 
00072 class NodeCommMgr : public fei::MessageHandler<int> {
00073  public:
00074    enum { STRICTLY_LOW_PROC, PROC_WITH_LOCAL_ELEM };
00075 
00076    NodeCommMgr(MPI_Comm comm, const SNL_FEI_Structure& problemStructure, int sharedNodeOwnership=STRICTLY_LOW_PROC);
00077    virtual ~NodeCommMgr();
00078 
00079    size_t getNumSharedNodes() {return(sharedNodeIDs.size());}
00080    std::vector<GlobalID>& getLocalNodeIDs() {return(localNodeIDs);}
00081    std::vector<GlobalID>& getSharedNodeIDs() {return(sharedNodeIDs);}
00082    std::vector<int>& getSharedNodeNumbers() {return(sharedNodeNumbers);}
00083 
00084    int getSharedNodeIndex_num(int nodeNumber);
00085 
00086    int addSharedNodes(const GlobalID* nodeIDs, int numNodes,
00087           const int* const* procs, const int* numProcs);
00088 
00089    int initComplete(NodeDatabase& nodeDB, bool safetyCheck);
00090 
00091    int informLocal(const NodeDescriptor& node);
00092 
00093    int exchangeEqnInfo();
00094 
00095    int getSharedNodeIndex(GlobalID nodeID);
00096 
00097    int getSharedNodeNumSubdomains(GlobalID nodeID);
00098    std::vector<int>* getSharedNodeSubdomainList(GlobalID nodeID);
00099 
00100    NodeDescriptor& getSharedNodeAtIndex(int index)
00101      {return(*(sharedNodes_[index]));}
00102 
00103    std::vector<int>& getSharedNodeProcs(int index) 
00104      {return(*(sharingProcs_[index]));};
00105 
00106    void setSharedOwnershipRule(int ownershipRule)
00107      { sharedNodeOwnership_ = ownershipRule; }
00108 
00109    std::vector<int>& getSendProcs();
00110    std::vector<int>& getRecvProcs();
00111 
00112    int getSendMessageLength(int destProc, int& messageLength);
00113    int getSendMessage(int destProc, std::vector<int>& message);
00114    int processRecvMessage(int srcProc, std::vector<int>& message);
00115 
00116  private:
00117    NodeCommMgr(const NodeCommMgr& src);
00118    NodeCommMgr& operator=(const NodeCommMgr& src);
00119 
00120    int allocateNodeDescriptorPtrs(NodeDatabase& nodeDB);
00121 
00122    int storeNodeProcs(int index, std::vector<std::vector<int>*>& procTable,
00123           const int* procs, int numProcs);
00124 
00125    int checkSharedNodeInfo();
00126 
00127    int checkCommArrays(const char* whichCheck,
00128            std::vector<int>& globalRemoteProcs,
00129            std::vector<int>& globalNodesPerRemoteProc,
00130            std::vector<int>& globalRemoteProcLengths,
00131            std::vector<int>& nodesPerRemoteProc,
00132            std::vector<int>& remoteProcs);
00133 
00134    void setNodeNumbersArray();
00135 
00136    void packLocalNodesAndData(int* data, int proc,
00137                              int numNodes, int len);
00138    void packRemoteNodesAndData(GlobalID* data, int proc,
00139                              int numNodes, int len);
00140 
00141    int adjustSharedOwnership();
00142 
00143    int createProcLists();
00144 
00145    int createProcList(std::vector<int>& itemsPerProc,
00146            std::vector<int>& procs);
00147 
00148    int exchangeSharedRemoteFieldsBlks();
00149 
00150    int getGlobalMaxFieldsBlocks(int& maxFields, int& maxBlocks);
00151 
00152    int getGlobalMaxFieldsBlocksSubdomains();
00153 
00154    NodeDescriptor** sharedNodes_;
00155    bool sharedNodesAllocated_;
00156 
00157    int sharedNodeOwnership_;
00158 
00159    std::vector<GlobalID> localNodeIDs;
00160    std::vector<GlobalID> remoteNodeIDs;
00161 
00162    std::vector<GlobalID> sharedNodeIDs; //global node identifiers
00163 
00164    std::vector<std::vector<int> > sharedNodeSubdomains;
00165                                           //subdomains each shared node
00166                                           //appears in.
00167    std::vector<int> trivialSubdomainList;
00168 
00169    std::vector<std::vector<int>*> sharingProcs_;//table, i-th row is a list of procs
00170                                           //associated with i-th shared node
00171 
00172    std::vector<int> sharedNodeNumbers;
00173 
00174    std::vector<int> remoteOwnerProcs_, remoteSharingProcs_;
00175    std::vector<int> nodesPerOwnerProc_, nodesPerSharingProc_;
00176 
00177    MPI_Comm comm_;
00178    int numProcs_, localProc_;
00179 
00180    int maxFields_;
00181    int maxBlocks_;
00182    int maxSubdomains_;
00183 
00184    bool initCompleteCalled_;
00185    const SNL_FEI_Structure& probStruc;
00186 };
00187 
00188 #endif
00189 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Friends