FEI Version of the Day
fei_set_shared_ids.cpp
00001 /*
00002 // @HEADER
00003 // ************************************************************************
00004 //             FEI: Finite Element Interface to Linear Solvers
00005 //                  Copyright (2005) Sandia Corporation.
00006 //
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, the
00008 // U.S. Government retains certain rights in this software.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Alan Williams (william@sandia.gov) 
00038 //
00039 // ************************************************************************
00040 // @HEADER
00041 */
00042 
00043 
00044 #include <fei_set_shared_ids.hpp>
00045 #include <fei_CommMap.hpp>
00046 #include <fei_CommUtils.hpp>
00047 #include <fei_TemplateUtils.hpp>
00048 #include <snl_fei_RecordCollection.hpp>
00049 #include <fei_SharedIDs.hpp>
00050 #include <fei_LinearDecomposition.hpp>
00051 
00052 namespace fei {
00053 
00054 void copy_into_shared_ids(const fei::CommMap<int>::Type& procs_to_ids_and_sharing_procs,
00055                           const snl_fei::RecordCollection& records,
00056                           fei::SharedIDs<int>& sharedIDs)
00057 {
00058   //Given a CommMap object which maps procs to packed vectors of ids and sharing
00059   //procs, and a RecordCollection of records that appear in the local subdomain:
00060   //Loop through the CommMap and for each id that exists in 'records', copy that
00061   //id along with the associated sharing procs into the sharedIDs object.
00062 
00063   fei::CommMap<int>::Type::const_iterator
00064     ip_iter = procs_to_ids_and_sharing_procs.begin(),
00065     ip_end = procs_to_ids_and_sharing_procs.end();
00066 
00067   for(; ip_iter != ip_end; ++ip_iter) {
00068     const std::vector<int>& ids_and_procs = ip_iter->second;
00069     size_t vsize = ids_and_procs.size();
00070     size_t offset = 0;
00071     while(offset < vsize) {
00072       int id = ids_and_procs[offset++];
00073       int num_procs = ids_and_procs[offset++];
00074       if (records.getRecordWithID(id) != NULL) {
00075         sharedIDs.addSharedID(id, num_procs, &ids_and_procs[offset]);
00076       }
00077       offset += num_procs;
00078     }
00079   }
00080 }
00081 
00082 void copy_remotelyowned_ids_into_CommMap(int myProc,
00083                                          const fei::LinearDecomposition<int>& lindecomp,
00084                                          const snl_fei::RecordCollection& records,
00085                                          fei::CommMap<int>::Type& procs_to_shared_ids)
00086 {
00087   for(size_t i=0; i<records.getNumRecords(); ++i) {
00088     int ID = records.getRecordWithLocalID(i)->getID();
00089     int proc = lindecomp.which_proc(ID);
00090     if (proc != myProc) {
00091       addItemsToCommMap(proc, 1, &ID, procs_to_shared_ids);
00092     }
00093   }
00094 }
00095 
00096 void set_shared_ids(MPI_Comm comm,
00097                     const snl_fei::RecordCollection& records,
00098                     fei::SharedIDs<int>& sharedIDs)
00099 {
00100   sharedIDs.getSharedIDs().clear();
00101   sharedIDs.getOwningProcs().clear();
00102 
00103   int numProcs = fei::numProcs(comm);
00104   if (numProcs < 2) return;
00105   int myProc = fei::localProc(comm);
00106 
00107   int lowest_local_id  = records.getMinID();
00108   int highest_local_id = records.getMaxID();
00109 
00110   int lowest_global_id = 0;
00111   int highest_global_id = 0;
00112 
00113   fei::GlobalMax(comm, highest_local_id, highest_global_id);
00114   fei::GlobalMin(comm, lowest_local_id, lowest_global_id);
00115 
00116   fei::LinearDecomposition<int> lindecomp(myProc,numProcs,
00117                                      lowest_global_id, highest_global_id);
00118 
00119   //Fill a CommMap (procs_to_shared_ids) that maps other procs to ids which we hold.
00120   //These are ids that appear locally, but which are *not* in our portion
00121   //of the linear-decomposition.
00122   fei::CommMap<int>::Type procs_to_shared_ids;
00123   copy_remotelyowned_ids_into_CommMap(myProc, lindecomp, records, procs_to_shared_ids);
00124 
00125   //Do a global-exchange where we send ids we share to procs that own them,
00126   //and receive IDs that we own from other procs that share them.
00127   fei::CommMap<int>::Type procs_to_owned_ids;
00128   fei::exchangeCommMapData<int>(comm, procs_to_shared_ids, procs_to_owned_ids);
00129 
00130   //transpose procs_to_owned_ids:
00131   fei::CommMap<int>::Type owned_ids_to_procs;
00132 
00133   fei::CommMap<int>::Type::iterator
00134     o_iter = procs_to_owned_ids.begin(), o_end = procs_to_owned_ids.end();
00135 
00136   for(; o_iter != o_end; ++o_iter) {
00137     int proc = o_iter->first;
00138     std::vector<int>& ids = o_iter->second;
00139     for(size_t i=0; i<ids.size(); ++i) {
00140       addItemsToCommMap(ids[i], 1, &proc, owned_ids_to_procs);
00141       if (records.getRecordWithID(ids[i]) != NULL) {
00142         addItemsToCommMap(ids[i], 1, &myProc, owned_ids_to_procs);
00143       }
00144     }
00145   }
00146 
00147   fei::CommMap<int>::Type procs_to_owned_ids_and_sharing_procs;
00148 
00149   for(o_iter=procs_to_owned_ids.begin(); o_iter!=procs_to_owned_ids.end(); ++o_iter) {
00150     int proc = o_iter->first;
00151     std::vector<int>& ids = o_iter->second;
00152     for(size_t i=0; i<ids.size(); ++i) {
00153       std::vector<int>& sharing_procs = owned_ids_to_procs[ids[i]];
00154       addItemsToCommMap(proc, 1, &ids[i], procs_to_owned_ids_and_sharing_procs, false);
00155       int num_sharing_procs = sharing_procs.size();
00156       addItemsToCommMap(proc, 1, &num_sharing_procs, procs_to_owned_ids_and_sharing_procs, false);
00157       addItemsToCommMap(proc, num_sharing_procs, &sharing_procs[0], procs_to_owned_ids_and_sharing_procs, false);
00158     }
00159   }
00160 
00161   fei::CommMap<int>::Type procs_to_shared_ids_and_sharing_procs;
00162   fei::exchangeCommMapData<int>(comm, procs_to_owned_ids_and_sharing_procs,
00163                                  procs_to_shared_ids_and_sharing_procs);
00164 
00165   copy_into_shared_ids(procs_to_owned_ids_and_sharing_procs, records, sharedIDs);
00166   copy_into_shared_ids(procs_to_shared_ids_and_sharing_procs, records, sharedIDs);
00167 }
00168 
00169 }//namespace fei
00170 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Friends