Sierra Toolkit Version of the Day
BulkDataOwner.cpp
00001 /*------------------------------------------------------------------------*/
00002 /*                 Copyright 2010 Sandia Corporation.                     */
00003 /*  Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive   */
00004 /*  license for use of this work by or on behalf of the U.S. Government.  */
00005 /*  Export of this program may require a license from the                 */
00006 /*  United States Government.                                             */
00007 /*------------------------------------------------------------------------*/
00008 
00013 #include <stdexcept>
00014 #include <iostream>
00015 #include <sstream>
00016 #include <set>
00017 #include <vector>
00018 #include <algorithm>
00019 
00020 #include <stk_util/parallel/ParallelComm.hpp>
00021 #include <stk_util/parallel/ParallelReduce.hpp>
00022 
00023 #include <stk_mesh/base/BulkData.hpp>
00024 #include <stk_mesh/base/MetaData.hpp>
00025 #include <stk_mesh/base/Entity.hpp>
00026 #include <stk_mesh/base/EntityComm.hpp>
00027 #include <stk_mesh/base/Trace.hpp>
00028 
00029 namespace stk {
00030 namespace mesh {
00031 
00032 //----------------------------------------------------------------------
00033 
00034 namespace {
00035 
00036 // Given an entity, if it's a ghost, insert the closure of the entity
00037 // into work_list.
00038 void insert_closure_ghost( Entity * const entity ,
00039                            const unsigned proc_local ,
00040                            std::set<Entity*,EntityLess> & work_list )
00041 {
00042   if ( ! in_owned_closure( *entity , proc_local ) ) {
00043     // This entity is a ghost, put it on the work_list
00044     // along with all ghosts in its closure
00045 
00046     std::pair< std::set<Entity*,EntityLess>::iterator , bool >
00047       result = work_list.insert( entity );
00048 
00049     if ( result.second ) {
00050       // This ghost entity is new to the list, traverse its closure.
00051 
00052       const unsigned entity_rank = entity->entity_rank();
00053 
00054       // Recurse over downward relations
00055       for ( PairIterRelation
00056             irel = entity->relations() ; ! irel.empty() ; ++irel ) {
00057         if ( irel->entity_rank() < entity_rank ) {
00058           insert_closure_ghost( irel->entity() , proc_local ,work_list );
00059         }
00060       }
00061     }
00062   }
00063 }
00064 
00065 // Given an entity, insert the closures of every entity that has this entity
00066 // in its closure. Only ghosts will be inserted.
00067 void insert_transitive_ghost( Entity * const entity ,
00068                               const unsigned proc_local ,
00069                               std::set<Entity*,EntityLess> & work_list )
00070 {
00071   insert_closure_ghost( entity , proc_local , work_list );
00072 
00073   // Transitive:
00074   // If this entity is a member of another entity's closure
00075   // then that other entity is part of the traversal.
00076 
00077   const unsigned entity_rank = entity->entity_rank();
00078 
00079   // Recurse over upward relations
00080   for ( PairIterRelation rel = entity->relations(); ! rel.empty() ; ++rel ) {
00081     if ( entity_rank < rel->entity_rank() ) {
00082       insert_transitive_ghost( rel->entity() , proc_local , work_list );
00083     }
00084   }
00085 }
00086 
00087 //----------------------------------------------------------------------
00088 
00089 // Add EntityProc pairs to send_list for every entity in the closure of the
00090 // entity in send_entry. All these entities will be sent to the same proc as
00091 // the original send_entry.
00092 void insert_closure_send(
00093   const EntityProc                  send_entry ,
00094   std::set<EntityProc,EntityLess> & send_list )
00095 {
00096   ThrowRequireMsg( send_entry.first->log_query() != EntityLogDeleted,
00097       "Cannot send destroyed entity " << print_entity_key(send_entry.first));
00098 
00099   std::pair< std::set<EntityProc,EntityLess>::iterator , bool >
00100     result = send_list.insert( send_entry );
00101 
00102   if ( result.second ) {
00103     // First time this entity was inserted into the send_list.
00104 
00105     const unsigned erank  = send_entry.first->entity_rank();
00106     PairIterRelation irel = send_entry.first->relations();
00107 
00108     // Recurse over downward relations
00109     for ( ; ! irel.empty() ; ++irel ) {
00110       if ( irel->entity_rank() < erank ) {
00111         const EntityProc rel_send_entry( irel->entity(), send_entry.second );
00112 
00113         insert_closure_send( rel_send_entry , send_list );
00114       }
00115     }
00116   }
00117 }
00118 
00119 //----------------------------------------------------------------------
00120 
00121 bool member_of_owned_closure( const Entity & e , const unsigned p_rank )
00122 {
00123   bool result = p_rank == e.owner_rank();
00124 
00125   const unsigned entity_rank = e.entity_rank();
00126 
00127   // Any higher ranking entities locally owned?
00128   for ( PairIterRelation
00129         irel = e.relations(); ! result && ! irel.empty() ; ++irel ) {
00130     result = entity_rank  <  irel->entity_rank() &&
00131              p_rank == irel->entity()->owner_rank();
00132   }
00133 
00134   // Any higher ranking entity member of an owned closure?
00135   for ( PairIterRelation
00136         irel = e.relations(); ! result && ! irel.empty() ; ++irel ) {
00137     result = entity_rank < irel->entity_rank() &&
00138              member_of_owned_closure( * irel->entity() , p_rank );
00139   }
00140 
00141   return result ;
00142 }
00143 
00144 //----------------------------------------------------------------------
00145 
00146 // Given a vector of local ownership changes, remove duplicates and
00147 // sanity check.
00148 void clean_and_verify_parallel_change(
00149   const BulkData & mesh ,
00150   std::vector<EntityProc> & local_change )
00151 {
00152   const MetaData      & meta   = MetaData::get(mesh);
00153   const unsigned        p_rank = mesh.parallel_rank();
00154   const unsigned        p_size = mesh.parallel_size();
00155   const ParallelMachine p_comm = mesh.parallel();
00156 
00157   size_t error_count = 0 ;
00158 
00159   std::ostringstream error_msg ;
00160 
00161   // Order and eliminate redundancies:
00162   {
00163     std::vector<EntityProc>::iterator i = local_change.begin() ,
00164                                       j = local_change.end() ;
00165     std::sort( i , j , EntityLess() );
00166     i = std::unique( i , j );
00167     local_change.erase( i , j );
00168   }
00169 
00170   for ( std::vector<EntityProc>::iterator
00171         i = local_change.begin() ; i != local_change.end() ; ++i ) {
00172     std::vector<EntityProc>::iterator next = i+1 ;
00173     Entity * const entity    = i->first ;
00174     const unsigned new_owner = i->second ;
00175 
00176     // Verification:
00177     // 1) Cannot change the ownership of an entity that you've already marked as deleted
00178     // 2) Cannot change the ownership of an entity you do not own
00179     // 3) New owner must exist
00180     // 4) Cannot grant ownership to two different owners
00181 
00182     const bool bad_null = NULL == entity ;
00183 
00184     // Cannot change the ownership of an entity that you've already marked as deleted
00185     const bool bad_marked_deleted = ! bad_null && EntityLogDeleted == entity->log_query();
00186 
00187     // Cannot change the ownership of an entity you do not own
00188     const bool bad_process_not_entity_owner = ! bad_null && entity->owner_rank() != p_rank ;
00189 
00190     // New owner must exist
00191     const bool bad_new_owner_does_not_exist = p_size <= new_owner ;
00192 
00193     // Cannot grant ownership to two different owners
00194     const bool bad_inconsistent_change = ! bad_null && next != local_change.end() && entity == next->first ;
00195 
00196     if ( bad_null ||
00197          bad_process_not_entity_owner ||
00198          bad_new_owner_does_not_exist ||
00199          bad_inconsistent_change ||
00200          bad_marked_deleted)
00201     {
00202       ++error_count ;
00203 
00204       error_msg << "  P" << p_rank << ": " ;
00205       if ( bad_null ) { error_msg << " NULL ENTITY" ; }
00206       else { print_entity_key( error_msg , meta , entity->key() ); }
00207       if ( bad_marked_deleted ) { error_msg << " HAS_BEEN_DELETED" ; }
00208       if ( bad_process_not_entity_owner ) { error_msg << " NOT_CURRENT_OWNER" ; }
00209       if ( bad_new_owner_does_not_exist ) {
00210         error_msg << " BAD_NEW_OWNER( " << new_owner << " )" ;
00211       }
00212       if ( bad_inconsistent_change ) {
00213         error_msg << " CONFLICTING_NEW_OWNER( " << new_owner ;
00214         error_msg << " != " << next->second << " )" ;
00215       }
00216       error_msg << std::endl ;
00217     }
00218     else if ( new_owner == p_rank ) {
00219       // Eliminate non-changes
00220       i->first = NULL ;
00221       i->second = 0 ;
00222     }
00223   }
00224 
00225   all_reduce( p_comm , ReduceSum<1>( & error_count ) );
00226 
00227   if ( error_count ) {
00228     all_write_string( p_comm , std::cerr , error_msg.str() );
00229 
00230     ThrowErrorMsg("Bad change ownership directives\n");
00231   }
00232 
00233   // Filter out non-changes (entity will be NULL
00234   {
00235     std::vector<EntityProc>::iterator i = local_change.begin(),
00236                                       j = local_change.end();
00237     i = std::remove( i , j , EntityProc(NULL,0) );
00238     local_change.erase( i , j );
00239   }
00240 }
00241 
00242 //----------------------------------------------------------------------
00243 // Generate a parallel consistent list of ownership changes:
00244 // 1) Shared entities (not owned but in closure of an owned entity),
00245 // 2) Ghosted entities (not owned and not in closure of an owned entity), and
00246 // 3) Parallel index.
00247 
00248 void generate_parallel_change( const BulkData & mesh ,
00249                                const std::vector<EntityProc> & local_change ,
00250                                      std::vector<EntityProc> & shared_change ,
00251                                      std::vector<EntityProc> & ghosted_change )
00252 {
00253   const unsigned p_size = mesh.parallel_size();
00254 
00255   CommAll comm( mesh.parallel() );
00256 
00257   std::vector<unsigned> procs ;
00258 
00259   // pack and communicate change owner information to all
00260   // processes that know about the entity
00261   for ( int phase = 0; phase < 2; ++phase) {
00262     for ( std::vector<EntityProc>::const_iterator
00263           ip = local_change.begin() ; ip != local_change.end() ; ++ip ) {
00264       Entity & entity    = * ip->first ;
00265       unsigned new_owner = ip->second;
00266       comm_procs( entity , procs );
00267       for ( std::vector<unsigned>::iterator
00268             j = procs.begin() ; j != procs.end() ; ++j )
00269       {
00270         comm.send_buffer( *j )
00271           .pack<EntityKey>( entity.key() )
00272           .pack<unsigned>(  new_owner );
00273       }
00274     }
00275     if (phase == 0) { // allocation phase
00276       comm.allocate_buffers( p_size / 4 , 0 );
00277     }
00278     else { // communication phase
00279       comm.communicate();
00280     }
00281   }
00282 
00283   // unpack communicated owner information into the
00284   // ghosted and shared change vectors.
00285   for ( unsigned ip = 0 ; ip < p_size ; ++ip ) {
00286     CommBuffer & buf = comm.recv_buffer( ip );
00287     while ( buf.remaining() ) {
00288       EntityProc entry ;
00289       EntityKey key ;
00290       buf.unpack<EntityKey>( key )
00291          .unpack<unsigned>( entry.second );
00292 
00293       entry.first = mesh.get_entity( key );
00294 
00295       if ( in_receive_ghost( * entry.first ) ) {
00296         ghosted_change.push_back( entry );
00297       }
00298       else {
00299         shared_change.push_back( entry );
00300       }
00301     }
00302   }
00303 
00304   std::sort( shared_change.begin() , shared_change.end() , EntityLess() );
00305   std::sort( ghosted_change.begin() , ghosted_change.end() , EntityLess() );
00306 }
00307 
00308 }
00309 
00310 //----------------------------------------------------------------------
00311 //----------------------------------------------------------------------
00312 
00313 void BulkData::change_entity_owner( const std::vector<EntityProc> & arg_change )
00314 {
00315   Trace_("stk::mesh::BulkData::change_entity_owner");
00316   DiagIf(LOG_ENTITY, "arg_change: " << arg_change);
00317 
00318   const MetaData  & meta = m_mesh_meta_data ;
00319   const unsigned  p_rank = m_parallel_rank ;
00320   const unsigned  p_size = m_parallel_size ;
00321   ParallelMachine p_comm = m_parallel_machine ;
00322 
00323   //------------------------------
00324   // Verify the input changes, generate a clean local change list, and
00325   // generate the remote change list so that all processes know about
00326   // pending changes.
00327 
00328   std::vector<EntityProc> local_change( arg_change );
00329 
00330   // Parallel synchronous clean up and verify the requested changes:
00331   clean_and_verify_parallel_change( *this , local_change );
00332 
00333   //----------------------------------------
00334   // Parallel synchronous determination of changing shared and ghosted.
00335 
00336   // The two vectors below will contain changes to ghosted and shared
00337   // entities on this process coming from change-entity-owner requests
00338   // on other processes.
00339   std::vector<EntityProc> ghosted_change ;
00340   std::vector<EntityProc> shared_change ;
00341 
00342   generate_parallel_change( *this , local_change ,
00343                             shared_change , ghosted_change );
00344 
00345   //------------------------------
00346   // Have enough information to delete all effected ghosts.
00347   // If the closure of a ghost contains a changing entity
00348   // then that ghost must be deleted.
00349   // Request that all ghost entities in the closure of the ghost be deleted.
00350 
00351   typedef std::set<EntityProc,EntityLess> EntityProcSet;
00352   typedef std::set<Entity*,EntityLess> EntitySet;
00353 
00354   // Compute the closure of all the locally changing entities
00355   EntityProcSet send_closure ;
00356   for ( std::vector<EntityProc>::iterator
00357         i = local_change.begin() ; i != local_change.end() ; ++i ) {
00358     insert_closure_send( *i , send_closure );
00359   }
00360 
00361   // Calculate all the ghosts that are impacted by the set of ownership
00362   // changes. We look at ghosted, shared, and local changes looking for ghosts
00363   // that are either in the closure of the changing entity, or have the
00364   // changing entity in their closure. All modified ghosts will be removed.
00365   {
00366     EntitySet modified_ghosts ;
00367 
00368     for ( std::vector<EntityProc>::const_iterator
00369           i = ghosted_change.begin() ; i != ghosted_change.end() ; ++i ) {
00370       insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
00371     }
00372 
00373     for ( std::vector<EntityProc>::const_iterator
00374           i = shared_change.begin() ; i != shared_change.end() ; ++i ) {
00375       insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
00376     }
00377 
00378     for ( EntityProcSet::iterator
00379           i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
00380       insert_transitive_ghost( i->first , m_parallel_rank , modified_ghosts );
00381     }
00382 
00383     // The ghosted change list will become invalid
00384     ghosted_change.clear();
00385 
00386     std::vector<EntityProc> empty_add ;
00387     std::vector<Entity*> remove_modified_ghosts( modified_ghosts.begin() ,
00388                                                  modified_ghosts.end() );
00389 
00390     // Skip 'm_ghosting[0]' which is the shared subset.
00391     for ( std::vector<Ghosting*>::iterator
00392           ig = m_ghosting.begin() + 1; ig != m_ghosting.end() ; ++ig ) {
00393       // parallel synchronous:
00394       internal_change_ghosting( **ig , empty_add , remove_modified_ghosts );
00395     }
00396   }
00397 
00398   //------------------------------
00399   // Consistently change the owner on all processes.
00400   // 1) The local_change list is giving away ownership.
00401   // 2) The shared_change may or may not be receiving ownership
00402 
00403   {
00404     PartVector owned;
00405     owned.push_back(& meta.locally_owned_part());
00406 
00407     for ( std::vector<EntityProc>::iterator
00408           i = local_change.begin() ; i != local_change.end() ; ++i ) {
00409       // Giving ownership, change the parts first and then
00410       // the owner rank to pass the ownership test.
00411       Entity* entity = i->first;
00412 
00413       change_entity_parts( *entity , PartVector() , owned );
00414 
00415       m_entity_repo.set_entity_owner_rank( *entity, i->second );
00416     }
00417 
00418     for ( std::vector<EntityProc>::iterator
00419           i = shared_change.begin() ; i != shared_change.end() ; ++i ) {
00420       Entity* entity = i->first;
00421       m_entity_repo.set_entity_owner_rank( *entity, i->second);
00422       if ( p_rank == i->second ) { // I receive ownership
00423         change_entity_parts( *entity , owned , PartVector() );
00424       }
00425     }
00426   }
00427 
00428   //------------------------------
00429   // Send entities, along with their closure, to the new owner processes
00430   {
00431     std::ostringstream error_msg ;
00432     int error_count = 0 ;
00433 
00434     CommAll comm( p_comm );
00435 
00436     for ( std::set<EntityProc,EntityLess>::iterator
00437           i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
00438       CommBuffer & buffer = comm.send_buffer( i->second );
00439       Entity     & entity = * i->first ;
00440       pack_entity_info( buffer , entity );
00441       pack_field_values( buffer , entity );
00442     }
00443 
00444     comm.allocate_buffers( p_size / 4 );
00445 
00446     for ( std::set<EntityProc,EntityLess>::iterator
00447           i = send_closure.begin() ; i != send_closure.end() ; ++i ) {
00448       CommBuffer & buffer = comm.send_buffer( i->second );
00449       Entity     & entity = * i->first ;
00450       pack_entity_info( buffer , entity );
00451       pack_field_values( buffer , entity );
00452     }
00453 
00454     comm.communicate();
00455 
00456     for ( unsigned p = 0 ; p < p_size ; ++p ) {
00457       CommBuffer & buf = comm.recv_buffer(p);
00458       while ( buf.remaining() ) {
00459         PartVector parts ;
00460         std::vector<Relation> relations ;
00461         EntityKey key ;
00462         unsigned  owner = ~0u ;
00463 
00464         unpack_entity_info( buf, *this, key, owner, parts, relations );
00465 
00466         // Received entity information will be correct,
00467         // modulo the owned and shared parts
00468 
00469         remove( parts , meta.globally_shared_part() );
00470 
00471         if ( owner == p_rank ) {
00472           // Must have the locally_owned_part
00473           insert( parts , meta.locally_owned_part() );
00474         }
00475         else {
00476           // Must not have the locally_owned_part
00477           remove( parts , meta.locally_owned_part() );
00478         }
00479 
00480         std::pair<Entity*,bool> result =
00481           m_entity_repo.internal_create_entity( key );
00482 
00483         Entity* entity = result.first;
00484 
00485         m_entity_repo.log_created_parallel_copy( *entity );
00486 
00487         // The entity was copied and not created.
00488 
00489         m_entity_repo.set_entity_owner_rank( *entity, owner);
00490 
00491         internal_change_entity_parts( *entity , parts , PartVector() );
00492 
00493         declare_relation( *entity , relations );
00494 
00495         if ( ! unpack_field_values( buf , *entity , error_msg ) ) {
00496           ++error_count ;
00497         }
00498       }
00499     }
00500 
00501     all_reduce( p_comm , ReduceSum<1>( & error_count ) );
00502     ThrowErrorMsgIf( error_count, error_msg.str() );
00503 
00504     // Any entity that I sent and is not in an owned closure is deleted.
00505     // The owned closure will be effected by received entities, so can
00506     // only clean up after the newly owned entities have been received.
00507     // Destroy backwards so as not to invalidate closures in the process.
00508 
00509     {
00510       Entity * entity = NULL ;
00511 
00512       for ( std::set<EntityProc,EntityLess>::iterator
00513             i = send_closure.end() ; i != send_closure.begin() ; ) {
00514 
00515         Entity * e = (--i)->first ;
00516 
00517         // The same entity may be sent to more than one process.
00518         // Only evaluate it once.
00519 
00520         if ( entity != e ) {
00521           entity = e ;
00522           if ( ! member_of_owned_closure( *e , p_rank ) ) {
00523             ThrowRequireMsg( destroy_entity( e ),
00524                 "Failed to destroy entity " << print_entity_key(e) );
00525           }
00526         }
00527       }
00528     }
00529 
00530     send_closure.clear(); // Has been invalidated
00531   }
00532 }
00533 
00534 //----------------------------------------------------------------------
00535 
00536 } // namespace mesh
00537 } // namespace stk
00538 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines