Comm.cpp

00001 /*------------------------------------------------------------------------*/
00002 /*                 Copyright 2010 Sandia Corporation.                     */
00003 /*  Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive   */
00004 /*  license for use of this work by or on behalf of the U.S. Government.  */
00005 /*  Export of this program may require a license from the                 */
00006 /*  United States Government.                                             */
00007 /*------------------------------------------------------------------------*/
00008 
00009 #include <stdexcept>
00010 #include <sstream>
00011 #include <algorithm>
00012 
00013 #include <stk_util/parallel/ParallelComm.hpp>
00014 #include <stk_util/parallel/ParallelReduce.hpp>
00015 
00016 #include <stk_mesh/base/MetaData.hpp>
00017 #include <stk_mesh/base/BulkData.hpp>
00018 #include <stk_mesh/base/FieldData.hpp>
00019 #include <stk_mesh/base/Comm.hpp>
00020 #include <stk_mesh/base/EntityComm.hpp>
00021 
00022 
00023 namespace stk {
00024 namespace mesh {
00025 
00026 //----------------------------------------------------------------------
00027 
00028 namespace {
00029 
00030 void print_entry( std::ostream & msg , const EntityProc & e )
00031 {
00032   msg << "( " ;
00033   if ( e.first == NULL ) {
00034     msg << "NULL" ;
00035   }
00036   else {
00037     const MetaData & meta_data = e.first->bucket().mesh().mesh_meta_data();
00038     print_entity_key( msg , meta_data , e.first->key() );
00039   }
00040   msg << " , " << e.second << " )" ;
00041 }
00042 
00043 }
00044 
00045 //----------------------------------------------------------------------
00046 
00047 struct LessEntityProc {
00048   EntityLess compare ;
00049   LessEntityProc() {}
00050 
00051   bool operator()( const EntityProc & lhs , const EntityProc & rhs ) const
00052   { return compare( lhs , rhs ); }
00053 
00054   bool operator()( const EntityProc & lhs , const Entity & rhs ) const
00055   { return compare( lhs , rhs ); }
00056 
00057   bool operator()( const EntityProc & lhs , const unsigned rhs ) const
00058   { return lhs.first->entity_rank() < rhs ; }
00059 };
00060 
00061 struct EqualEntityProc {
00062   bool operator()( const EntityProc & lhs , const EntityProc & rhs ) const
00063   { return lhs.first == rhs.first && lhs.second == rhs.second ; }
00064 };
00065 
00066 //----------------------------------------------------------------------
00067 
00068 void sort_unique( std::vector<EntityProc> & v )
00069 {
00070   std::vector<EntityProc>::iterator i = v.begin();
00071   std::vector<EntityProc>::iterator e = v.end();
00072 
00073   std::sort( i , e , LessEntityProc() );
00074   i = std::unique( i , e , EqualEntityProc() );
00075   v.erase( i , e );
00076 }
00077 
00078 std::vector<EntityProc>::const_iterator
00079 lower_bound( const std::vector<EntityProc> & v , unsigned t )
00080 {
00081   const std::vector<EntityProc>::const_iterator i = v.begin();
00082   const std::vector<EntityProc>::const_iterator e = v.end();
00083   return lower_bound( i , e , t , LessEntityProc() );
00084 }
00085 
00086 std::vector<EntityProc>::const_iterator
00087 lower_bound( const std::vector<EntityProc> & v , Entity & m )
00088 {
00089   const std::vector<EntityProc>::const_iterator i = v.begin();
00090   const std::vector<EntityProc>::const_iterator e = v.end();
00091   return lower_bound( i , e , m , LessEntityProc() );
00092 }
00093 
00094 std::vector<EntityProc>::const_iterator
00095 lower_bound( const std::vector<EntityProc> & v , const EntityProc & m )
00096 {
00097   const std::vector<EntityProc>::const_iterator i = v.begin();
00098   const std::vector<EntityProc>::const_iterator e = v.end();
00099   return lower_bound( i , e , m , LessEntityProc() );
00100 }
00101 
00102 std::vector<EntityProc>::iterator
00103 lower_bound( std::vector<EntityProc> & v , Entity & m )
00104 {
00105   const std::vector<EntityProc>::iterator i = v.begin();
00106   const std::vector<EntityProc>::iterator e = v.end();
00107   return lower_bound( i , e , m , LessEntityProc() );
00108 }
00109 
00110 std::vector<EntityProc>::iterator
00111 lower_bound( std::vector<EntityProc> & v , const EntityProc & m )
00112 {
00113   const std::vector<EntityProc>::iterator i = v.begin();
00114   const std::vector<EntityProc>::iterator e = v.end();
00115   return lower_bound( i , e , m , LessEntityProc() );
00116 }
00117 
00118 //----------------------------------------------------------------------
00119 
00120 #if 0
00121 
00122 void procs( const std::vector<EntityProc> & v ,
00123             const std::vector< Entity * > & m ,
00124             std::vector<unsigned> & p )
00125 {
00126   typedef std::vector<EntityProc>::const_iterator Iter ;
00127 
00128   p.clear();
00129 
00130   std::vector<Iter> tmp ; tmp.reserve( m.size() );
00131 
00132   for ( unsigned j = 0 ; j < tmp.size() ; ++j ) {
00133     tmp.push_back( lower_bound( v , *m[j] ) );
00134   }
00135 
00136   if ( ! tmp.empty() ) {
00137     const Iter ve = v.end();
00138     Entity * const e0 = m[0] ;
00139 
00140     for ( ; tmp[0] != ve && tmp[0]->first == e0 ; ++tmp[0] ) {
00141       const unsigned proc = tmp[0]->second ;
00142       bool flag = true ;
00143 
00144       // Iterate remaining up to proc
00145 
00146       for ( unsigned j = 1 ; j < tmp.size() ; ++j ) {
00147         Entity * const ej = m[j] ;
00148         for ( ; tmp[j] != ve &&
00149                 tmp[j]->first == ej &&
00150                 tmp[j]->second < proc ; ++tmp[j] );
00151         if ( tmp[j] == ve ||
00152              tmp[j]->first != ej ||
00153              tmp[j]->second != proc ) {
00154           flag = false ;
00155         }
00156       }
00157 
00158       if ( flag ) { p.push_back( proc ); }
00159     }
00160   }
00161 }
00162 
00163 #endif
00164 
00165 //----------------------------------------------------------------------
00166 //----------------------------------------------------------------------
00167 
00168 bool verify( const std::vector<EntityProc> & v , std::string & msg )
00169 {
00170   const LessEntityProc less_op ;
00171 
00172   bool result = true ;
00173 
00174   if ( ! v.empty() ) {
00175 
00176     const std::vector<EntityProc>::const_iterator e = v.end();
00177     std::vector<EntityProc>::const_iterator i ;
00178 
00179     std::ostringstream os ;
00180 
00181     for ( i = v.begin() ; result && i != e ; ++i ) {
00182       if ( ! ( result = i->first != NULL ) ) {
00183         msg.append( "Contains NULL entries" );
00184       }
00185     }
00186 
00187     if ( result ) {
00188       BulkData   & M = v[0].first->bucket().mesh();
00189       const unsigned p_size = M.parallel_size();
00190 
00191       for ( i = v.begin() ; result && i != e ; ++i ) {
00192         if ( ! ( result = & M == & i->first->bucket().mesh() ) ) {
00193           msg.append( "Contains entries from different meshes" );
00194         }
00195       }
00196 
00197       for ( i = v.begin() ; result && i != e ; ++i ) {
00198         const unsigned p = i->second ;
00199         if ( ! ( result = p < p_size ) ) {
00200           os << "Contains entry with bad processor " ;
00201           print_entry( os , *i );
00202           msg.append( os.str() );
00203         }
00204       }
00205 
00206       EntityProc old( v[0] ) ;
00207 
00208       for ( i = v.begin() ; result && ++i != e ; ) {
00209         if ( ! ( result = less_op( old , *i ) ) ) {
00210           os << "Contains out-of-order entries " ;
00211           print_entry( os , old );
00212           print_entry( os , *i );
00213           msg.append( os.str() );
00214         }
00215         old = *i ;
00216       }
00217     }
00218   }
00219   return result ;
00220 }
00221 
00222 
00223 // REFACTOR: The alignment and preallocation of buffers doesn't mix.
00224 //           The ParallelComm alignment and packing are the problem here.
00225 //           See "x" variable hacks below
00226 //           The Marshall code simply packs tightly.
00227 //
00228 //           The buffer sizing and packing originally matched properly.
00229 //           A recently introduced hacking of this code to make the
00230 //           EntityKey refactor work broke this correctness.
00231 //           This function should have a proper refactoring
00232 //           instead of the hacking that it was given.
00233 //
00234 bool comm_verify( ParallelMachine comm ,
00235                   const std::vector<EntityProc> & v ,
00236                   std::string & msg )
00237 {
00238   static const char method[] = "stk::mesh::comm_verify[symmetric]" ;
00239 
00240   // Verify local ordering, the result flag is parallel inconsistent
00241 
00242   bool result = verify( v , msg );
00243 
00244   // Verify parallel consistency
00245   // Communicate asymmetric and compare keys and owners
00246 
00247   const int p_rank = parallel_machine_rank( comm );
00248   const int p_size = parallel_machine_size( comm );
00249 
00250   std::ostringstream os ;
00251 
00252   const unsigned zero = 0 ;
00253   std::vector<unsigned> send_size( p_size , zero );
00254 
00255   CommAll comm_sparse(comm) ;
00256 
00257   const std::vector<EntityProc>::const_iterator i_end = v.end();
00258   std::vector<EntityProc>::const_iterator i ;
00259 
00260   if ( result ) {
00261     for ( i = v.begin() ; i != i_end ; ++i ) {
00262       Entity & e = * i->first ;
00263       const unsigned  proc  = i->second ;
00264 
00265       CommBuffer & buf = comm_sparse.send_buffer( proc );
00266       buf.pack(e.key());
00267       buf.pack(e.owner_rank());
00268     }
00269   }
00270 
00271   {
00272     result = ! comm_sparse.allocate_buffers( p_size / 4 );
00273   }
00274 
00275   // The result flag is now parallel consistent
00276 
00277   if ( result ) {
00278     // Fill buffer
00279     for ( i = v.begin() ; i != i_end ; ++i ) {
00280       Entity & e = * i->first ;
00281       const unsigned  proc  = i->second ;
00282 
00283       CommBuffer & buf = comm_sparse.send_buffer( proc );
00284       buf.pack(e.key());
00285       buf.pack(e.owner_rank());
00286     }
00287 
00288     comm_sparse.communicate();
00289 
00290     // Verify symmetry of sizes
00291 //     for ( int j = 0 ; result && j < p_size ; ++j ) {
00292 //       const size_t nrecv = comm_sparse.recv_buffer( j ).remaining();
00293 //       if ( nrecv != send_size[j] ) {
00294 //         os << method ;
00295 //         os << " parallel inconsistency, P" << p_rank ;
00296 //         os << " expected from P" << j ;
00297 //         os << " " << send_size[j] ;
00298 //         os << " but received " ;
00299 //         os << nrecv ;
00300 //         os << " instead" ;
00301 //         result = false ;
00302 //         msg.append( os.str() );
00303 //       }
00304 //     }
00305 
00306     // Verify symmetry of content
00307     for ( i = v.begin() ; result && i != i_end ; ++i ) {
00308       Entity & e = * i->first ;
00309 
00310       const EntityKey this_key   = e.key();
00311       const unsigned        this_owner = e.owner_rank();
00312       const unsigned        proc  = i->second ;
00313 
00314       CommBuffer & buf = comm_sparse.recv_buffer( proc );
00315       EntityKey entity_key;
00316       unsigned owner = ~0u ;
00317       
00318       buf.unpack(entity_key);
00319       buf.unpack( owner );
00320       
00321       if ( this_key   != entity_key ||
00322            this_owner != owner ) {
00323         const MetaData & meta_data = e.bucket().mesh().mesh_meta_data();
00324         os << method ;
00325         os << " parallel inconsistency, P" << p_rank << " has " ;
00326         print_entity_key( os , meta_data , this_key );
00327         os << "].owner(P" << this_owner ;
00328         os << ") versus " ;
00329         print_entity_key( os , meta_data , entity_key );
00330         os << "].owner(P" << owner ;
00331         os << ")" ;
00332         result = false ;
00333         msg.append( os.str() );
00334       }
00335     }
00336 
00337     // The result flag is now parallel inconsistent
00338 
00339     {
00340       unsigned flag = result ;
00341       all_reduce( comm , ReduceMin<1>( & flag ) );
00342       result = flag ;
00343     }
00344   }
00345 
00346   return result ;
00347 }
00348 
00349 //----------------------------------------------------------------------
00350 
00351 bool comm_verify( ParallelMachine comm ,
00352                   const std::vector<EntityProc> & send ,
00353                   const std::vector<EntityProc> & recv ,
00354                   std::string & msg )
00355 {
00356   static const char method[] = "stk::mesh::comm_verify[asymmetric]" ;
00357 
00358   // Verify local ordering:
00359 
00360   bool result = verify( send , msg );
00361 
00362   const int p_rank = parallel_machine_rank( comm );
00363   const int p_size = parallel_machine_size( comm );
00364 
00365   std::ostringstream os ;
00366 
00367   const unsigned zero = 0 ;
00368   std::vector<unsigned> send_size( p_size , zero );
00369   std::vector<unsigned> recv_size( p_size , zero );
00370 
00371   std::vector<EntityProc>::const_iterator i ;
00372 
00373   if ( result ) {
00374     for ( i = send.begin() ; i != send.end() ; ++i ) {
00375       ++( send_size[ i->second ] );
00376     }
00377   }
00378 
00379   {
00380     unsigned msg_max = 0 ;
00381     unsigned * const p_send = & send_size[0] ;
00382     unsigned * const p_recv = & recv_size[0] ;
00383     result = ! comm_sizes( comm , p_size / 4 , msg_max ,
00384                            p_send , p_recv , ! result );
00385   }
00386 
00387   // Result flag is now parallel consistent
00388 
00389   if ( result ) {
00390     send_size.assign( p_size , zero );
00391 
00392     for ( i = recv.begin() ; i != recv.end() ; ++i ) {
00393       ++( send_size[ i->second ] );
00394     }
00395 
00396     for ( int j = 0 ; result && j < p_size ; ++j ) {
00397       const unsigned nrecv = recv_size[j] ;
00398       if ( nrecv != send_size[j] ) {
00399         os << method ;
00400         os << " parallel inconsistency, P" << p_rank ;
00401         os << " expected from P" << j ;
00402         os << " " << send_size[j] ;
00403         os << " but received " ;
00404         os << nrecv ;
00405         os << " instead" ;
00406         result = false ;
00407         msg.append( os.str() );
00408       }
00409     }
00410 
00411     // The result flag is now parallel inconsitent
00412 
00413     {
00414       unsigned flag = result ;
00415       all_reduce( comm , ReduceBitAnd<1>( & flag ) );
00416       result = flag ;
00417     }
00418   }
00419   return result ;
00420 }
00421 
00422 //----------------------------------------------------------------------
00423 //----------------------------------------------------------------------
00424 
00425 bool comm_mesh_counts( BulkData & M ,
00426                        std::vector<size_t> & counts ,
00427                        bool local_flag )
00428 {
00429   const size_t zero = 0 ;
00430 
00431   // Count locally owned entities
00432 
00433   const MetaData & S = M.mesh_meta_data();
00434   const unsigned entity_type_count = S.entity_type_count();
00435   const size_t   comm_count        = entity_type_count + 1 ;
00436 
00437   std::vector<size_t> local(  comm_count , zero );
00438   std::vector<size_t> global( comm_count , zero );
00439 
00440   ParallelMachine comm = M.parallel();
00441   Part & owns = S.locally_owned_part();
00442 
00443   for ( unsigned i = 0 ; i < entity_type_count ; ++i ) {
00444     const std::vector<Bucket*> & ks = M.buckets( i );
00445 
00446     std::vector<Bucket*>::const_iterator ik ;
00447 
00448     for ( ik = ks.begin() ; ik != ks.end() ; ++ik ) {
00449       if ( has_superset( **ik , owns ) ) {
00450         local[i] += (*ik)->size();
00451       }
00452     }
00453   }
00454 
00455   local[ entity_type_count ] = local_flag ;
00456 
00457   all_reduce_sum( comm , & local[0] , & global[0] , comm_count );
00458 
00459   counts.assign( global.begin() , global.begin() + entity_type_count );
00460 
00461   return 0 < global[ entity_type_count ] ;
00462 }
00463 
00464 //----------------------------------------------------------------------
00465 
00466 namespace {
00467 
00468 struct EntityRankLess {
00469   bool operator()( const Entity * const lhs , const unsigned rhs ) const
00470     { return lhs->entity_rank() < rhs ; }
00471 };
00472 
00473 std::pair< std::vector<Entity*>::const_iterator ,
00474            std::vector<Entity*>::const_iterator >
00475 span( const std::vector<Entity*> & v , const unsigned entity_rank )
00476 {
00477   const unsigned entity_rank_end = entity_rank + 1 ;
00478 
00479   std::pair< std::vector<Entity*>::const_iterator ,
00480              std::vector<Entity*>::const_iterator > result ;
00481 
00482   result.first  = v.begin();
00483   result.second = v.end();
00484 
00485   result.first  = std::lower_bound( result.first, result.second,
00486                                     entity_rank , EntityRankLess() );
00487 
00488   result.second = std::lower_bound( result.first, result.second,
00489                                     entity_rank_end , EntityRankLess() );
00490 
00491   return result ;
00492 }
00493 
00494 }
00495 
00496 //----------------------------------------------------------------------
00497 //----------------------------------------------------------------------
00498 
00499 bool comm_verify_shared_entity_values(
00500   const BulkData & M , unsigned t , const FieldBase & f )
00501 {
00502   const unsigned parallel_size = M.parallel_size();
00503   const unsigned parallel_rank = M.parallel_rank();
00504 
00505   const std::pair< std::vector<Entity*>::const_iterator ,
00506                    std::vector<Entity*>::const_iterator >
00507     entity_comm = span( M.entity_comm() , t );
00508 
00509   std::vector<Entity*>::const_iterator ic ;
00510 
00511   ParallelMachine comm = M.parallel();
00512 
00513   CommAll sparse( M.parallel() );
00514 
00515   for ( ic = entity_comm.first ; ic != entity_comm.second ; ++ic ) {
00516     Entity & entity = **ic ;
00517     if ( entity.owner_rank() == parallel_rank ) {
00518 
00519       const unsigned size = field_data_size( f , entity );
00520 
00521       for ( PairIterEntityComm ec = entity.comm();
00522             ! ec.empty() && ec->ghost_id == 0 ; ++ec ) {
00523         sparse.send_buffer( ec->proc )
00524               .skip<unsigned>(1)
00525               .skip<unsigned char>( size );
00526       }
00527     }
00528   }
00529 
00530   sparse.allocate_buffers( parallel_size / 4 );
00531 
00532   for ( ic = entity_comm.first ; ic != entity_comm.second ; ++ic ) {
00533     Entity & entity = **ic ;
00534     if ( entity.owner_rank() == parallel_rank ) {
00535 
00536       const unsigned size = field_data_size( f , entity );
00537       unsigned char * ptr =
00538         reinterpret_cast<unsigned char *>( field_data( f , entity ) );
00539 
00540       for ( PairIterEntityComm ec = entity.comm();
00541             ! ec.empty() && ec->ghost_id == 0 ; ++ec ) {
00542         sparse.send_buffer( ec->proc )
00543               .pack<unsigned>( size )
00544               .pack<unsigned char>( ptr , size );
00545       }
00546     }
00547   }
00548 
00549   sparse.communicate();
00550 
00551   const unsigned max_size = f.max_size(t) * f.data_traits().size_of ;
00552 
00553   std::vector<unsigned char> scratch( max_size );
00554 
00555   unsigned char * const scr = & scratch[0] ;
00556 
00557   unsigned ok = 1 ;
00558 
00559   for ( ic = entity_comm.first ; ok && ic != entity_comm.second ; ++ic ) {
00560 
00561     Entity & entity = **ic ;
00562 
00563     if ( in_shared( entity , entity.owner_rank() ) ) {
00564 
00565       const unsigned size = field_data_size( f , entity );
00566       unsigned char * ptr =
00567         reinterpret_cast<unsigned char *>( field_data( f , entity ) );
00568 
00569       unsigned recv_size = 0 ;
00570       sparse.recv_buffer( entity.owner_rank() )
00571             .unpack<unsigned>( recv_size )
00572             .unpack<unsigned char>( scr , recv_size );
00573 
00574       ok = recv_size == size ;
00575 
00576       // Compare data and scratch
00577       for ( unsigned j = 0 ; ok && j < size ; ++j ) {
00578         ok = ptr[j] == scr[j] ;
00579       }
00580     }
00581   }
00582 
00583   all_reduce( comm , ReduceMin<1>( & ok ) );
00584 
00585   return ok ;
00586 }
00587 
00588 //----------------------------------------------------------------------
00589 //----------------------------------------------------------------------
00590 
00591 } // namespace mesh
00592 } // namespace stk
00593 

Generated on Tue Jul 13 09:27:31 2010 for Sierra Toolkit by  doxygen 1.4.7