Sierra Toolkit Version of the Day
Rebalance.cpp
00001 /*--------------------------------------------------------------------*/
00002 /*    Copyright 2001, 2008, 2009, 2010 Sandia Corporation.                              */
00003 /*    Under the terms of Contract DE-AC04-94AL85000, there is a       */
00004 /*    non-exclusive license for use of this work by or on behalf      */
00005 /*    of the U.S. Government.  Export of this program may require     */
00006 /*    a license from the United States Government.                    */
00007 /*--------------------------------------------------------------------*/
00008 
00009 // Copyright 2001,2002 Sandia Corporation, Albuquerque, NM.
00010 
00011 #include <memory>
00012 #include <stdexcept>
00013 #include <vector>
00014 #include <string>
00015 
00016 #include <stk_util/environment/ReportHandler.hpp>
00017 #include <stk_util/parallel/ParallelReduce.hpp>
00018 
00019 #include <stk_mesh/base/BulkData.hpp>
00020 #include <stk_mesh/base/MetaData.hpp>
00021 #include <stk_mesh/base/GetEntities.hpp>
00022 #include <stk_mesh/base/FieldData.hpp>
00023 #include <stk_mesh/fem/FEMMetaData.hpp>
00024 
00025 #include <stk_rebalance/Rebalance.hpp>
00026 #include <stk_rebalance/Partition.hpp>
00027 
00028 using namespace stk;
00029 using namespace stk::rebalance;
00030 
00031 namespace {
00032 
00033 bool balance_comm_spec_domain( Partition * partition,
00034                                mesh::EntityProcVec & rebal_spec )
00035 {
00036   bool rebalancingHasOccurred = false;
00037   {
00038     int num_elems = partition->num_elems();
00039     int tot_elems;
00040     all_reduce_sum(partition->parallel(), &num_elems, &tot_elems, 1);
00041 
00042     if (tot_elems) {
00043       partition->determine_new_partition(rebalancingHasOccurred);
00044     }
00045   }
00046   if (rebalancingHasOccurred) partition->get_new_partition(rebal_spec);
00047 
00048   return rebalancingHasOccurred;
00049 }
00050 
00051 
00052 /*
00053  * Traversing the migrating elements in reverse order produces a simplistic
00054  * attempt at lowest-rank element proc greedy partitioning of dependents
00055  * which seems to often work in practice.  Some logic could be added here
00056  * as needed to enforce more deterministic dependent partitions.
00057  */
00058 
00059 void rebalance_dependent_entities( const mesh::BulkData    & bulk_data ,
00060                                    const Partition         * partition,
00061                                    const mesh::EntityRank  & dep_rank,
00062                                    mesh::EntityProcVec     & entity_procs,
00063                                    const stk::mesh::EntityRank rank)
00064 {
00065 
00066   stk::mesh::fem::FEMMetaData & fem_meta = stk::mesh::fem::FEMMetaData::get(bulk_data);
00067   const stk::mesh::EntityRank element_rank = (rank != stk::mesh::InvalidEntityRank) ? rank :
00068                                              fem_meta.element_rank();
00069 
00070   if (dep_rank == element_rank) return;
00071   // Create a map of ids of migrating elements to their owner proc and a vector of the migrating elements.
00072   std::map<mesh::EntityId, unsigned> elem_procs;
00073   mesh::EntityVector owned_moving_elems;
00074   mesh::EntityProcVec::iterator ep_iter = entity_procs.begin(),
00075                                  ep_end = entity_procs.end();
00076   for( ; ep_end != ep_iter; ++ep_iter ) {
00077     if( element_rank == ep_iter->first->entity_rank() )
00078     {
00079       const mesh::EntityId elem_id = ep_iter->first->identifier();
00080       elem_procs[elem_id] = ep_iter->second;
00081       owned_moving_elems.push_back(ep_iter->first);
00082     }
00083   }
00084   // TODO: Determine if this "dumb" greedy approach is adequate and the cost/benefit
00085   //       of doing something more sophisticated
00086 
00087   // This reverse traversal of elements overwrites assignment of procs for
00088   // dependents resulting in the last assignment winning.
00089 
00090   // For all dep-rank entities related to migrating elements, pack their info in to
00091   // dep_entity_procs.
00092   std::map<mesh::EntityId, unsigned> dep_entity_procs;
00093   mesh::EntityVector::reverse_iterator r_iter = owned_moving_elems.rbegin(),
00094                                         r_end = owned_moving_elems.rend();
00095   for( ; r_end != r_iter; ++r_iter )
00096   {
00097     const mesh::EntityId elem_id = (*r_iter)->identifier();
00098     mesh::EntityVector related_entities;
00099     mesh::EntityVector elems(1);
00100     elems[0] = *r_iter;
00101     stk::mesh::get_entities_through_relations(elems, dep_rank, related_entities);
00102     for( size_t j = 0; j < related_entities.size(); ++j ) {
00103       dep_entity_procs[related_entities[j]->identifier()] = elem_procs[elem_id];
00104     }
00105   }
00106 
00107 
00108   std::map<mesh::EntityId, unsigned>::const_iterator c_iter = dep_entity_procs.begin(),
00109                                                       c_end = dep_entity_procs.end();
00110   for( ; c_end != c_iter; ++c_iter )
00111   {
00112     mesh::Entity * de = bulk_data.get_entity( dep_rank, c_iter->first );
00113     if( parallel_machine_rank(partition->parallel()) == de->owner_rank() )
00114     {
00115       stk::mesh::EntityProc dep_proc(de, c_iter->second);
00116       entity_procs.push_back(dep_proc);
00117     }
00118   }
00119 }
00120 
00121 
00122 bool full_rebalance(mesh::BulkData  & bulk_data ,
00123                     Partition       * partition,
00124                     const stk::mesh::EntityRank rank)
00125 {
00126   mesh::EntityProcVec cs_elem;
00127   bool rebalancingHasOccurred =  balance_comm_spec_domain( partition, cs_elem );
00128 
00129   if(rebalancingHasOccurred && partition->partition_dependents_needed() )
00130   {
00131     stk::mesh::fem::FEMMetaData & fem_meta = stk::mesh::fem::FEMMetaData::get(bulk_data);
00132 
00133     const stk::mesh::EntityRank node_rank = fem_meta.node_rank();
00134     const stk::mesh::EntityRank edge_rank = fem_meta.edge_rank();
00135     const stk::mesh::EntityRank face_rank = fem_meta.face_rank();
00136     const stk::mesh::EntityRank elem_rank = fem_meta.element_rank();
00137     const stk::mesh::EntityRank cons_rank = elem_rank+1;
00138 
00139     // Don't know the rank of the elements rebalanced, assume all are dependent.
00140     rebalance_dependent_entities( bulk_data, partition, node_rank, cs_elem, rank );
00141     if (stk::mesh::InvalidEntityRank != edge_rank && rank != edge_rank)
00142       rebalance_dependent_entities( bulk_data, partition, edge_rank, cs_elem, rank );
00143     if (stk::mesh::InvalidEntityRank != face_rank && rank != face_rank)
00144       rebalance_dependent_entities( bulk_data, partition, face_rank, cs_elem, rank );
00145     if (stk::mesh::InvalidEntityRank != elem_rank && rank != elem_rank)
00146       rebalance_dependent_entities( bulk_data, partition, elem_rank, cs_elem, rank );
00147     if (stk::mesh::InvalidEntityRank != cons_rank && rank != cons_rank)
00148       rebalance_dependent_entities( bulk_data, partition, cons_rank, cs_elem, rank );
00149   }
00150 
00151   if ( rebalancingHasOccurred )
00152   {
00153     bulk_data.modification_begin();
00154     bulk_data.change_entity_owner( cs_elem );
00155     bulk_data.modification_end();
00156   }
00157 
00158   //: Finished
00159   return rebalancingHasOccurred;
00160 }
00161 } // namespace
00162 
00163 
00164 bool stk::rebalance::rebalance(mesh::BulkData   & bulk_data  ,
00165                                const mesh::Selector  & selector ,
00166                                const VectorField     * rebal_coord_ref ,
00167                                const ScalarField     * rebal_elem_weight_ref ,
00168                                Partition & partition,
00169                                const stk::mesh::EntityRank rank)
00170 {
00171   stk::mesh::fem::FEMMetaData &fem_meta = stk::mesh::fem::FEMMetaData::get(bulk_data);
00172   const stk::mesh::EntityRank element_rank = (rank != stk::mesh::InvalidEntityRank) ? rank :
00173                                              fem_meta.element_rank();
00174 
00175   mesh::EntityVector rebal_elem_ptrs;
00176   mesh::EntityVector entities;
00177 
00178   mesh::get_selected_entities(selector,
00179                               bulk_data.buckets(element_rank),
00180                               entities);
00181 
00182   for (mesh::EntityVector::iterator iA = entities.begin() ; iA != entities.end() ; ++iA ) {
00183     if(rebal_elem_weight_ref)
00184     {
00185       double * const w = mesh::field_data( *rebal_elem_weight_ref, **iA );
00186       ThrowRequireMsg( NULL != w,
00187         "Rebalance weight field is not defined on entities but should be defined on all entities.");
00188       // Should this be a throw instead???
00189       if ( *w <= 0.0 ) {
00190         *w = 1.0 ;
00191       }
00192     }
00193     rebal_elem_ptrs.push_back( *iA );
00194   }
00195 
00196   (&partition)->set_mesh_info(
00197       rebal_elem_ptrs,
00198       rebal_coord_ref,
00199       rebal_elem_weight_ref);
00200 
00201   bool rebalancingHasOccurred = full_rebalance(bulk_data, &partition, rank);
00202 
00203   return rebalancingHasOccurred;
00204 }
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends