Sierra Toolkit Version of the Day
UseCase_Rebal_1.cpp
00001 /*------------------------------------------------------------------------*/
00002 /*                 Copyright 2010 Sandia Corporation.                     */
00003 /*  Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive   */
00004 /*  license for use of this work by or on behalf of the U.S. Government.  */
00005 /*  Export of this program may require a license from the                 */
00006 /*  United States Government.                                             */
00007 /*------------------------------------------------------------------------*/
00008 
00009 #include <use_cases/UseCase_Rebal_1.hpp>
00010 
00011 #include <stk_util/parallel/Parallel.hpp>
00012 #include <stk_util/parallel/ParallelReduce.hpp>
00013 
00014 #include <stk_mesh/base/FieldData.hpp>
00015 #include <stk_mesh/base/GetEntities.hpp>
00016 
00017 #include <stk_rebalance/Rebalance.hpp>
00018 #include <stk_rebalance/Partition.hpp>
00019 #include <stk_rebalance/ZoltanPartition.hpp>
00020 
00021 #include <stk_rebalance_utils/RebalanceUtils.hpp>
00022 
00023 //----------------------------------------------------------------------
00024 
00025 using namespace stk::mesh::fixtures;
00026 
00027 typedef stk::mesh::Field<double> ScalarField ;
00028 
00029 namespace stk {
00030 namespace rebalance {
00031 namespace use_cases {
00032 
00033 bool test_unequal_weights( stk::ParallelMachine pm )
00034 {
00035   const unsigned p_size = stk::parallel_machine_size(pm);
00036   const unsigned p_rank = stk::parallel_machine_rank(pm);
00037 
00038   const unsigned ngx = p_size*(p_size+1)/2;
00039 
00040   unsigned nx = 0;
00041   if( 0 == p_rank )
00042     nx = ngx;
00043   unsigned ny = 1;
00044   unsigned nz = 1;
00045 
00046   stk::mesh::fixtures::HexFixture fixture(pm, nx, ny, nz);
00047 
00048   stk::mesh::fem::FEMMetaData & fem_meta  = fixture.m_fem_meta;
00049   stk::mesh::BulkData & bulk  = fixture.m_bulk_data;
00050 
00051   // Put weights field on all elements
00052   const stk::mesh::EntityRank element_rank = fem_meta.element_rank();
00053   ScalarField & weight_field( fem_meta.declare_field< ScalarField >( "element_weights" ) );
00054   stk::mesh::put_field(weight_field , element_rank , fem_meta.universal_part() );
00055 
00056   fem_meta.commit();
00057 
00058   bulk.modification_begin();
00059 
00060   // Initially put all elements on proc 0
00061   std::vector<stk::mesh::EntityId> my_element_ids;
00062   for ( unsigned i = 0 ; i < nx*ny*nz; ++i )
00063     my_element_ids.push_back(i+1);
00064 
00065   fixture.generate_mesh(my_element_ids);
00066 
00067   // Assign weights so that a perfect rebalance is possible so long as the rebalancer can figure out
00068   // to put p_rank+1 elements on proc = p_rank based on these weights.
00069   unsigned nslabs = 0;
00070   if( 0 == p_rank ) {
00071     for ( unsigned l = 1 ; l <= p_size ; ++l ) {
00072       for ( unsigned k = 0 ; k < nz ; ++k ) {
00073         for ( unsigned j = 0 ; j < ny ; ++j ) {
00074           for ( unsigned i = 0 ; i < l ; ++i ) {
00075             const stk::mesh::EntityId elem_id = 1 + nslabs + i + j*ngx + k*ngx*ny; 
00076             stk::mesh::Entity * elem = bulk.get_entity(element_rank, elem_id);
00077             double * const e_weight = stk::mesh::field_data( weight_field , *elem );
00078             *e_weight = double(ngx) / double(l);
00079           }
00080         }
00081       }
00082       nslabs += l;
00083     }
00084   }
00085   // end assign weights
00086 
00087   bulk.modification_end();
00088 
00089   // Use Zoltan to determine new partition
00090   Teuchos::ParameterList emptyList;
00091   stk::rebalance::Zoltan zoltan_partition(pm, fixture.m_spatial_dimension, emptyList);
00092 
00093   stk::mesh::Selector selector(fem_meta.universal_part());
00094 
00095   stk::rebalance::rebalance(bulk, selector, &fixture.m_coord_field, &weight_field, zoltan_partition);
00096 
00097   const double imbalance_threshold = stk::rebalance::check_balance(bulk, &weight_field, element_rank);
00098   const bool do_rebal = 1.5 < imbalance_threshold;
00099 
00100   if( 0 == p_rank )
00101     std::cerr << std::endl 
00102      << "imbalance_threshold after rebalance = " << imbalance_threshold << ", " << do_rebal << std::endl;
00103 
00104   stk::mesh::Selector owned_selector = fem_meta.locally_owned_part();
00105   size_t num_local_elems = stk::mesh::count_selected_entities(owned_selector, bulk.buckets(element_rank));
00106 
00107   // Check that we satisfy our threshhold
00108   bool result = true;
00109   if( 4 > p_size )
00110   {
00111     result = (fabs(imbalance_threshold - 1.0) < 1.e-8);
00112     result = result & (num_local_elems == p_rank+1);
00113   }
00114   else
00115   {
00116     // Would like to put something here, but Zoltan using its default algorithm (RCB)
00117     // isn't able to do an adequate job rebalancing
00118     result = !do_rebal;
00119   }
00120 
00121   return result;
00122 }
00123 
00124 } //namespace use_cases
00125 } //namespace rebalance
00126 } //namespace stk
00127 
00128 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends