FEI Version of the Day
fei_set_shared_ids.cpp
00001 /*
00002 // @HEADER
00003 // ************************************************************************
00004 //             FEI: Finite Element Interface to Linear Solvers
00005 //                  Copyright (2005) Sandia Corporation.
00006 //
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, the
00008 // U.S. Government retains certain rights in this software.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Alan Williams (william@sandia.gov) 
00038 //
00039 // ************************************************************************
00040 // @HEADER
00041 */
00042 
00043 
00044 #include <fei_set_shared_ids.hpp>
00045 #include <fei_CommMap.hpp>
00046 #include <fei_CommUtils.hpp>
00047 #include <fei_TemplateUtils.hpp>
00048 #include <snl_fei_RecordCollection.hpp>
00049 #include <fei_SharedIDs.hpp>
00050 #include <fei_LinearDecomposition.hpp>
00051 
00052 namespace fei {
00053 
00054 void copy_into_shared_ids(const fei::CommMap<int>::Type& procs_to_ids_and_sharing_procs,
00055                           const snl_fei::RecordCollection& records,
00056                           fei::SharedIDs<int>& sharedIDs)
00057 {
00058   //Given a CommMap object which maps procs to packed vectors of ids and sharing
00059   //procs, and a RecordCollection of records that appear in the local subdomain:
00060   //Loop through the CommMap and for each id that exists in 'records', copy that
00061   //id along with the associated sharing procs into the sharedIDs object.
00062 
00063   fei::CommMap<int>::Type::const_iterator
00064     ip_iter = procs_to_ids_and_sharing_procs.begin(),
00065     ip_end = procs_to_ids_and_sharing_procs.end();
00066 
00067   for(; ip_iter != ip_end; ++ip_iter) {
00068     const std::vector<int>& ids_and_procs = ip_iter->second;
00069     size_t vsize = ids_and_procs.size();
00070     size_t offset = 0;
00071     while(offset < vsize) {
00072       int id = ids_and_procs[offset++];
00073       int num_procs = ids_and_procs[offset++];
00074       if (records.getRecordWithID(id) != NULL) {
00075         sharedIDs.addSharedID(id, num_procs, &ids_and_procs[offset]);
00076       }
00077       offset += num_procs;
00078     }
00079   }
00080 }
00081 
00082 void copy_remotelyowned_ids_into_CommMap(int myProc,
00083                                          const fei::LinearDecomposition<int>& lindecomp,
00084                                          const snl_fei::RecordCollection& records,
00085                                          fei::CommMap<int>::Type& procs_to_shared_ids)
00086 {
00087   for(size_t i=0; i<records.getNumRecords(); ++i) {
00088     int ID = records.getRecordWithLocalID(i)->getID();
00089     int proc = lindecomp.which_proc(ID);
00090     if (proc != myProc) {
00091       addItemsToCommMap(proc, 1, &ID, procs_to_shared_ids);
00092     }
00093   }
00094 }
00095 
00096 void set_shared_ids(MPI_Comm comm,
00097                     const snl_fei::RecordCollection& records,
00098                     fei::SharedIDs<int>& sharedIDs)
00099 {
00100   sharedIDs.getSharedIDs().clear();
00101   sharedIDs.getOwningProcs().clear();
00102 
00103   int numProcs = fei::numProcs(comm);
00104   if (numProcs < 2) return;
00105   int myProc = fei::localProc(comm);
00106 
00107   const fei::IndexType<int,int>& rmap = records.getNativeGlobalToLocalMap();
00108   int local_rmap_size = rmap.size();
00109   int global_rmap_size = 0;
00110   fei::GlobalMax(comm, local_rmap_size, global_rmap_size);
00111   if (global_rmap_size == 0) return;
00112 
00113   int lowest_local_id  = local_rmap_size>0 ? rmap.getMinKey() : 0;
00114   int highest_local_id = local_rmap_size>0 ? rmap.getMaxKey() : 0;
00115 
00116   int lowest_global_id = 0;
00117   int highest_global_id = 0;
00118 
00119   fei::GlobalMax(comm, highest_local_id, highest_global_id);
00120   fei::GlobalMin(comm, lowest_local_id, lowest_global_id);
00121 
00122   fei::LinearDecomposition<int> lindecomp(myProc,numProcs,
00123                                      lowest_global_id, highest_global_id);
00124 
00125   //Fill a CommMap (procs_to_shared_ids) that maps other procs to ids which we hold.
00126   //These are ids that appear locally, but which are *not* in our portion
00127   //of the linear-decomposition.
00128   fei::CommMap<int>::Type procs_to_shared_ids;
00129   copy_remotelyowned_ids_into_CommMap(myProc, lindecomp, records, procs_to_shared_ids);
00130 
00131   //Do a global-exchange where we send ids we share to procs that own them,
00132   //and receive IDs that we own from other procs that share them.
00133   fei::CommMap<int>::Type procs_to_owned_ids;
00134   fei::exchangeCommMapData<int>(comm, procs_to_shared_ids, procs_to_owned_ids);
00135 
00136   //transpose procs_to_owned_ids:
00137   fei::CommMap<int>::Type owned_ids_to_procs;
00138 
00139   fei::CommMap<int>::Type::iterator
00140     o_iter = procs_to_owned_ids.begin(), o_end = procs_to_owned_ids.end();
00141 
00142   for(; o_iter != o_end; ++o_iter) {
00143     int proc = o_iter->first;
00144     std::vector<int>& ids = o_iter->second;
00145     for(size_t i=0; i<ids.size(); ++i) {
00146       addItemsToCommMap(ids[i], 1, &proc, owned_ids_to_procs);
00147       if (records.getRecordWithID(ids[i]) != NULL) {
00148         addItemsToCommMap(ids[i], 1, &myProc, owned_ids_to_procs);
00149       }
00150     }
00151   }
00152 
00153   fei::CommMap<int>::Type procs_to_owned_ids_and_sharing_procs;
00154 
00155   for(o_iter=procs_to_owned_ids.begin(); o_iter!=procs_to_owned_ids.end(); ++o_iter) {
00156     int proc = o_iter->first;
00157     std::vector<int>& ids = o_iter->second;
00158     for(size_t i=0; i<ids.size(); ++i) {
00159       std::vector<int>& sharing_procs = owned_ids_to_procs[ids[i]];
00160       addItemsToCommMap(proc, 1, &ids[i], procs_to_owned_ids_and_sharing_procs, false);
00161       int num_sharing_procs = sharing_procs.size();
00162       addItemsToCommMap(proc, 1, &num_sharing_procs, procs_to_owned_ids_and_sharing_procs, false);
00163       addItemsToCommMap(proc, num_sharing_procs, &sharing_procs[0], procs_to_owned_ids_and_sharing_procs, false);
00164     }
00165   }
00166 
00167   fei::CommMap<int>::Type procs_to_shared_ids_and_sharing_procs;
00168   fei::exchangeCommMapData<int>(comm, procs_to_owned_ids_and_sharing_procs,
00169                                  procs_to_shared_ids_and_sharing_procs);
00170 
00171   copy_into_shared_ids(procs_to_owned_ids_and_sharing_procs, records, sharedIDs);
00172   copy_into_shared_ids(procs_to_shared_ids_and_sharing_procs, records, sharedIDs);
00173 }
00174 
00175 }//namespace fei
00176 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Friends