Sierra Toolkit Version of the Day
ParallelComm.hpp
00001 /*------------------------------------------------------------------------*/
00002 /*                 Copyright 2010 Sandia Corporation.                     */
00003 /*  Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive   */
00004 /*  license for use of this work by or on behalf of the U.S. Government.  */
00005 /*  Export of this program may require a license from the                 */
00006 /*  United States Government.                                             */
00007 /*------------------------------------------------------------------------*/
00008 
00009 #ifndef stk_util_parallel_ParallelComm_hpp
00010 #define stk_util_parallel_ParallelComm_hpp
00011 
00012 #include <cstddef>
00013 #include <iosfwd>
00014 #include <stk_util/parallel/Parallel.hpp>
00015 
00016 //------------------------------------------------------------------------
00017 
00018 namespace stk {
00019 
00028 class CommAll ;
00029 
00032 class CommBuffer ;
00033 
00046 bool comm_sizes( ParallelMachine ,
00047                  const unsigned   num_msg_bound ,
00048                        unsigned & num_msg_maximum ,
00049                  const unsigned * const send_size ,
00050                        unsigned * const recv_size ,
00051                  bool local_flag = false );
00052 
00055 bool comm_dense_sizes( ParallelMachine ,
00056                        const unsigned * const send_size ,
00057                              unsigned * const recv_size ,
00058                        bool local_flag = false );
00059 
00060 //------------------------------------------------------------------------
00061 
00062 class CommBuffer {
00063 public:
00064 
00066   template<typename T> CommBuffer &pack( const T & value );
00067 
00069   template<typename T> CommBuffer &pack( const T * value , size_t number );
00070 
00072   template<typename T> CommBuffer &unpack( T & value );
00073 
00075   template<typename T> CommBuffer &unpack( T * value , size_t number );
00076 
00078   template<typename T> CommBuffer &peek( T & value );
00079 
00081   template<typename T> CommBuffer &peek( T * value , size_t number );
00082 
00084   template<typename T> CommBuffer &skip( size_t number );
00085 
00087   void reset();
00088 
00092   size_t capacity() const ;
00093 
00098   size_t size() const ;
00099 
00106   ptrdiff_t remaining() const ;
00107 
00109   void * buffer() const ;
00110 
00111   ~CommBuffer();
00112   CommBuffer();
00113 
00114 private:
00115   friend class CommAll ;
00116   friend class CommGather ;
00117   friend class CommBroadcast ;
00118 
00119   static CommBuffer * allocate( const unsigned, const unsigned * const );
00120   static void deallocate( const unsigned , CommBuffer * );
00121 
00122   void pack_overflow() const ;
00123   void unpack_overflow() const ;
00124 
00125   CommBuffer( const CommBuffer & );
00126   CommBuffer & operator = ( const CommBuffer & );
00127 
00128   typedef unsigned char * ucharp ;
00129 
00130   ucharp m_beg ;
00131   ucharp m_ptr ;
00132   ucharp m_end ;
00133 };
00134 
00135 //------------------------------------------------------------------------
00136 
00137 class CommAll {
00138 public:
00139 
00140   ParallelMachine parallel()      const { return m_comm ; }
00141   unsigned        parallel_size() const { return m_size ; }
00142   unsigned        parallel_rank() const { return m_rank ; }
00143 
00145   CommBuffer & send_buffer( unsigned ) const ;
00146 
00148   CommBuffer & recv_buffer( unsigned ) const ;
00149 
00150   //----------------------------------------
00154   CommAll();
00155 
00163   bool allocate_buffers( ParallelMachine ,
00164                          const unsigned num_msg_bounds ,
00165                          const unsigned * const send_size ,
00166                          const unsigned * const recv_size ,
00167                          const bool local_flag = false );
00168 
00169   //----------------------------------------
00182   explicit CommAll( ParallelMachine );
00183 
00190   bool allocate_buffers( const unsigned num_msg_bounds ,
00191                          const bool symmetric = false ,
00192                          const bool local_flag = false );
00193 
00194   //----------------------------------------
00196   void communicate();
00197 
00198   //----------------------------------------
00200   void swap_send_recv();
00201 
00205   void reset_buffers();
00206 
00207   ~CommAll();
00208 
00209 private:
00210 
00211   CommAll( const CommAll & );
00212   CommAll & operator = ( const CommAll & );
00213 
00214   void rank_error( const char * , unsigned ) const ;
00215 
00216   bool allocate_buffers( const unsigned * const send_size ,
00217                          const unsigned * const recv_size ,
00218                          bool local_flag );
00219 
00220   ParallelMachine m_comm ;
00221   unsigned        m_size ;
00222   unsigned        m_rank ;
00223   unsigned        m_bound ;
00224   unsigned        m_max ;
00225   CommBuffer    * m_send ;
00226   CommBuffer    * m_recv ;
00227 };
00228 
00229 //------------------------------------------------------------------------
00230 
00231 class CommBroadcast {
00232 public:
00233 
00234   ParallelMachine parallel()      const { return m_comm ; }
00235   unsigned        parallel_size() const { return m_size ; }
00236   unsigned        parallel_rank() const { return m_rank ; }
00237 
00239   CommBuffer & send_buffer();
00240 
00242   CommBuffer & recv_buffer();
00243 
00244   //----------------------------------------
00245 
00246   CommBroadcast( ParallelMachine , unsigned root_rank );
00247 
00248   void communicate();
00249 
00250   bool allocate_buffer( const bool local_flag = false );
00251 
00252   ~CommBroadcast();
00253 
00254 private:
00255 
00256   CommBroadcast();
00257   CommBroadcast( const CommBroadcast & );
00258   CommBroadcast & operator = ( const CommBroadcast & );
00259 
00260   ParallelMachine m_comm ;
00261   unsigned        m_size ;
00262   unsigned        m_rank ;
00263   unsigned        m_root_rank ;
00264   CommBuffer      m_buffer ;
00265 };
00266 
00267 //----------------------------------------------------------------------
00268 
00269 class CommGather {
00270 public:
00271 
00272   ParallelMachine parallel()      const { return m_comm ; }
00273   unsigned        parallel_size() const { return m_size ; }
00274   unsigned        parallel_rank() const { return m_rank ; }
00275 
00276   ~CommGather();
00277 
00278   CommGather( ParallelMachine , unsigned root_rank , unsigned send_size );
00279 
00280   CommBuffer & send_buffer() { return m_send ; }
00281 
00282   void communicate();
00283 
00284   CommBuffer & recv_buffer( unsigned );
00285 
00286   void reset(); 
00287 
00288 private:
00289 
00290   CommGather();
00291   CommGather( const CommBroadcast & );
00292   CommGather & operator = ( const CommBroadcast & );
00293 
00294   ParallelMachine m_comm ;
00295   unsigned        m_size ;
00296   unsigned        m_rank ;
00297   unsigned        m_root_rank ;
00298   CommBuffer      m_send ;
00299   CommBuffer    * m_recv ;
00300   int           * m_recv_count ;
00301   int           * m_recv_displ ;
00302 };
00303 
00304 }
00305 
00306 //----------------------------------------------------------------------
00307 //----------------------------------------------------------------------
00308 // Inlined template implementations for the CommBuffer
00309 
00310 namespace stk {
00311 
00312 template<unsigned N> struct CommBufferAlign ;
00313 
00314 template<>
00315 struct CommBufferAlign<1> {
00316   static size_t align( size_t ) { return 0 ; }
00317 };
00318 
00319 template<unsigned N>
00320 struct CommBufferAlign {
00321   static size_t align( size_t i ) { i %= N ; return i ? ( N - i ) : 0 ; }
00322 };
00323 
00324 template<typename T>
00325 inline
00326 CommBuffer &CommBuffer::pack( const T & value )
00327 {
00328   enum { Size = sizeof(T) };
00329   size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
00330   if ( m_beg ) {
00331     if ( m_end < m_ptr + nalign + Size ) { pack_overflow(); }
00332     while ( nalign ) { --nalign ; *m_ptr = 0 ; ++m_ptr ; }
00333     T * tmp = reinterpret_cast<T*>(m_ptr);
00334     *tmp = value ;
00335     m_ptr = reinterpret_cast<ucharp>( ++tmp );
00336   }
00337   else {
00338     m_ptr += nalign + Size ;
00339   }
00340   return *this;
00341 }
00342 
00343 template<typename T>
00344 inline
00345 CommBuffer &CommBuffer::pack( const T * value , size_t number )
00346 {
00347   enum { Size = sizeof(T) };
00348   size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
00349   if ( m_beg ) {
00350     if ( m_end < m_ptr + nalign + number * Size ) { pack_overflow(); }
00351     while ( nalign ) { --nalign ; *m_ptr = 0 ; ++m_ptr ; }
00352     T * tmp = reinterpret_cast<T*>(m_ptr);
00353     while ( number ) { --number ; *tmp = *value ; ++tmp ; ++value ; }
00354     m_ptr = reinterpret_cast<ucharp>( tmp );
00355   }
00356   else {
00357     m_ptr += nalign + number * Size ;
00358   }
00359   return *this;
00360 }
00361 
00362 template<typename T>
00363 inline
00364 CommBuffer &CommBuffer::skip( size_t number )
00365 {
00366   enum { Size = sizeof(T) };
00367   m_ptr += CommBufferAlign<Size>::align( m_ptr - m_beg ) + Size * number ;
00368   if ( m_beg && m_end < m_ptr ) { unpack_overflow(); }
00369   return *this;
00370 }
00371 
00372 template<typename T>
00373 inline
00374 CommBuffer &CommBuffer::unpack( T & value )
00375 {
00376   enum { Size = sizeof(T) };
00377   const size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
00378   T * tmp = reinterpret_cast<T*>( m_ptr + nalign );
00379   value = *tmp ;
00380   m_ptr = reinterpret_cast<ucharp>( ++tmp );
00381   if ( m_end < m_ptr ) { unpack_overflow(); }
00382   return *this;
00383 }
00384 
00385 template<typename T>
00386 inline
00387 CommBuffer &CommBuffer::unpack( T * value , size_t number )
00388 {
00389   enum { Size = sizeof(T) };
00390   const size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
00391   T * tmp = reinterpret_cast<T*>( m_ptr + nalign );
00392   while ( number ) { --number ; *value = *tmp ; ++tmp ; ++value ; }
00393   m_ptr = reinterpret_cast<ucharp>( tmp );
00394   if ( m_end < m_ptr ) { unpack_overflow(); }
00395   return *this;
00396 }
00397 
00398 template<typename T>
00399 inline
00400 CommBuffer &CommBuffer::peek( T & value )
00401 {
00402   enum { Size = sizeof(T) };
00403   const size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
00404   T * tmp = reinterpret_cast<T*>( m_ptr + nalign );
00405   value = *tmp ;
00406   if ( m_end < reinterpret_cast<ucharp>(++tmp) ) { unpack_overflow(); }
00407   return *this;
00408 }
00409 
00410 template<typename T>
00411 inline
00412 CommBuffer &CommBuffer::peek( T * value , size_t number )
00413 {
00414   enum { Size = sizeof(T) };
00415   const size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
00416   T * tmp = reinterpret_cast<T*>( m_ptr + nalign );
00417   while ( number ) { --number ; *value = *tmp ; ++tmp ; ++value ; }
00418   if ( m_end < reinterpret_cast<ucharp>(tmp) ) { unpack_overflow(); }
00419   return *this;
00420 }
00421 
00422 inline
00423 void CommBuffer::reset()
00424 { m_ptr = m_beg ; }
00425 
00426 inline
00427 size_t CommBuffer::capacity() const
00428 { return m_end - m_beg ; }
00429 
00430 inline
00431 size_t CommBuffer::size() const
00432 { return m_ptr - m_beg ; }
00433 
00434 inline
00435 ptrdiff_t CommBuffer::remaining() const
00436 { return m_end - m_ptr ; }
00437 
00438 inline
00439 void * CommBuffer::buffer() const
00440 { return static_cast<void*>( m_beg ); }
00441 
00442 //----------------------------------------------------------------------
00443 // Inline implementations for the CommAll
00444 
00445 inline
00446 CommBuffer & CommAll::send_buffer( unsigned p ) const
00447 {
00448   if ( m_size <= p ) { rank_error("send_buffer",p); }
00449   return m_send[p] ;
00450 }
00451 
00452 inline
00453 CommBuffer & CommAll::recv_buffer( unsigned p ) const
00454 {
00455   if ( m_size <= p ) { rank_error("recv_buffer",p); }
00456   return m_recv[p] ;
00457 }
00458 
00459 }
00460 
00461 //----------------------------------------------------------------------
00462 //----------------------------------------------------------------------
00463 
00464 #endif
00465 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends