RTOpPack_SPMD_apply_op.hpp

Go to the documentation of this file.
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 // RTOp: Interfaces and Support Software for Vector Reduction Transformation
00005 //       Operations
00006 //                Copyright (2006) Sandia Corporation
00007 // 
00008 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00009 // license for use of this work by or on behalf of the U.S. Government.
00010 // 
00011 // This library is free software; you can redistribute it and/or modify
00012 // it under the terms of the GNU Lesser General Public License as
00013 // published by the Free Software Foundation; either version 2.1 of the
00014 // License, or (at your option) any later version.
00015 //  
00016 // This library is distributed in the hope that it will be useful, but
00017 // WITHOUT ANY WARRANTY; without even the implied warranty of
00018 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00019 // Lesser General Public License for more details.
00020 //  
00021 // You should have received a copy of the GNU Lesser General Public
00022 // License along with this library; if not, write to the Free Software
00023 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00024 // USA
00025 // Questions? Contact Roscoe A. Bartlett (rabartl@sandia.gov) 
00026 // 
00027 // ***********************************************************************
00028 // @HEADER
00029 
00030 #ifndef RTOPPACK_SPMD_APPLY_OP_HPP
00031 #define RTOPPACK_SPMD_APPLY_OP_HPP
00032 
00033 #include "RTOpPack_SPMD_apply_op_decl.hpp"
00034 #include "Teuchos_Workspace.hpp"
00035 #include "Teuchos_CommHelpers.hpp"
00036 
00037 #ifdef RTOPPACK_SPMD_APPLY_OP_DUMP
00038 #  include "Teuchos_VerboseObject.hpp"
00039 #endif // RTOPPACK_SPMD_APPLY_OP_DUMP
00040 
00041 namespace RTOpPack {
00042 
00043 #ifdef RTOPPACK_SPMD_APPLY_OP_DUMP
00044 
00045 template<class Scalar>
00046 void print( const ConstSubVectorView<Scalar> &v, Teuchos::FancyOStream &out_arg )
00047 {
00048   Teuchos::RefCountPtr<Teuchos::FancyOStream> out = Teuchos::rcp(&out_arg,false);
00049   Teuchos::OSTab tab(out);
00050   *out << "globalOffset="<<v.globalOffset()<<"\n";
00051   *out << "subDim="<<v.subDim()<<"\n";
00052   *out << "values:\n";
00053   tab.incrTab();
00054   for( int i = 0; i < v.subDim(); ++i )
00055     *out << " " << v(i) << ":" << (v.globalOffset()+i);
00056   *out << "\n";
00057 }
00058 
00059 #  include "Teuchos_VerboseObject.hpp"
00060 
00061 #endif // RTOPPACK_SPMD_APPLY_OP_DUMP
00062 
00063 } // namespace RTOpPack
00064 
00065 // ///////////////////////////
00066 // Template implementations
00067 
00068 //
00069 // Misc Helper functions
00070 //
00071 
00072 template<class PrimitiveScalar>
00073 int RTOpPack::serializedSize(
00074   int   num_values
00075   ,int  num_indexes
00076   ,int  num_chars
00077   )
00078 {
00079   return 3 * sizeof(index_type)
00080     + num_values   * sizeof(PrimitiveScalar)
00081     + num_indexes  * sizeof(index_type)
00082     + num_chars    * sizeof(char_type);
00083 }
00084 
00085 template<class Scalar>
00086 void RTOpPack::serialize(
00087   const RTOpT<Scalar>    &op
00088   ,int                   num_values
00089   ,int                   num_indexes
00090   ,int                   num_chars
00091   ,const ReductTarget    &reduct_obj
00092   ,char                  reduct_obj_ext[]
00093   )
00094 {
00095   typedef typename RTOpT<Scalar>::primitive_value_type primitive_value_type;
00096   typedef Teuchos::SerializationTraits<int,primitive_value_type> PVTST;
00097   typedef Teuchos::SerializationTraits<int,index_type>           ITST;
00098   typedef Teuchos::SerializationTraits<int,char_type>            CTST;
00099   const int
00100     prim_value_type_size = PVTST::fromCountToIndirectBytes(1),
00101     index_type_size      = ITST::fromCountToIndirectBytes(1);
00102   //char_type_size       = CTST::fromCountToIndirectBytes(1);
00103   const int
00104     num_values_off  = 0,
00105     num_indexes_off = num_values_off  + index_type_size,
00106     num_chars_off   = num_indexes_off + index_type_size,
00107     values_off      = num_chars_off   + index_type_size,
00108     indexes_off     = values_off      + num_values  * prim_value_type_size,
00109     chars_off       = indexes_off     + num_indexes * index_type_size;
00110   ITST::serialize(1,&num_values,index_type_size,&reduct_obj_ext[num_values_off]);
00111   ITST::serialize(1,&num_indexes,index_type_size,&reduct_obj_ext[num_indexes_off]);
00112   ITST::serialize(1,&num_chars,index_type_size,&reduct_obj_ext[num_chars_off]);
00113   op.extract_reduct_obj_state(
00114     reduct_obj
00115     ,num_values,  num_values  ? PVTST::convertFromCharPtr(&reduct_obj_ext[values_off]) : 0
00116     ,num_indexes, num_indexes ? ITST::convertFromCharPtr(&reduct_obj_ext[indexes_off]) : 0
00117     ,num_chars,   num_chars   ? CTST::convertFromCharPtr(&reduct_obj_ext[chars_off])   : 0
00118     );
00119   // ToDo: Change above implementation to only require indirect serialization!
00120 }
00121 
00122 template<class Scalar>
00123 void RTOpPack::deserialize(
00124   const RTOpT<Scalar>    &op
00125   ,int                   num_values_in
00126   ,int                   num_indexes_in
00127   ,int                   num_chars_in
00128   ,const char            reduct_obj_ext[]
00129   ,ReductTarget          *reduct_obj
00130   )
00131 {
00132   typedef typename RTOpT<Scalar>::primitive_value_type primitive_value_type;
00133   typedef Teuchos::SerializationTraits<int,primitive_value_type> PVTST;
00134   typedef Teuchos::SerializationTraits<int,index_type>           ITST;
00135   typedef Teuchos::SerializationTraits<int,char_type>            CTST;
00136   const int
00137     prim_value_type_size = PVTST::fromCountToIndirectBytes(1),
00138     index_type_size      = ITST::fromCountToIndirectBytes(1);
00139   //char_type_size       = CTST::fromCountToIndirectBytes(1);
00140   const int
00141     num_values_off  = 0,
00142     num_indexes_off = num_values_off  + index_type_size,
00143     num_chars_off   = num_indexes_off + index_type_size,
00144     values_off      = num_chars_off   + index_type_size,
00145     indexes_off     = values_off      + num_values_in  * prim_value_type_size,
00146     chars_off       = indexes_off     + num_indexes_in * index_type_size;
00147 #ifdef TEUCHOS_DEBUG
00148   int num_values = -1, num_indexes = -1, num_chars = -1;
00149   ITST::deserialize(index_type_size,&reduct_obj_ext[num_values_off],1,&num_values);
00150   ITST::deserialize(index_type_size,&reduct_obj_ext[num_indexes_off],1,&num_indexes);
00151   ITST::deserialize(index_type_size,&reduct_obj_ext[num_chars_off],1,&num_chars);
00152   TEST_FOR_EXCEPT(
00153     !(
00154       num_values==num_values_in && num_indexes==num_indexes_in
00155       && num_chars==num_chars_in )
00156     );
00157 #endif
00158   op.load_reduct_obj_state(
00159     num_values_in,   
00160     num_values_in  ? PVTST::convertFromCharPtr(&reduct_obj_ext[values_off]) : 0
00161     ,num_indexes_in, num_indexes_in ? ITST::convertFromCharPtr(&reduct_obj_ext[indexes_off]) : 0
00162     ,num_chars_in,   num_chars_in   ? CTST::convertFromCharPtr(&reduct_obj_ext[chars_off])   : 0
00163     ,reduct_obj
00164     );
00165   // ToDo: Change above implementation to only require indirect serialization!
00166 }
00167 
00168 namespace RTOpPack {
00169 
00170 //
00171 // ReductTargetSerializer
00172 //
00173 
00174 template<class Scalar>
00175 ReductTargetSerializer<Scalar>::ReductTargetSerializer(
00176   const Teuchos::RefCountPtr<const RTOpT<Scalar> > &op
00177   )
00178   :op_(op.assert_not_null())
00179 {
00180   typedef typename RTOpT<Scalar>::primitive_value_type PrimitiveScalar;
00181   op_->get_reduct_type_num_entries(
00182     &num_values_,&num_indexes_,&num_chars_
00183     );
00184   reduct_obj_ext_size_
00185     = serializedSize<PrimitiveScalar>(num_values_,num_indexes_,num_chars_);
00186 }
00187 
00188 template<class Scalar>
00189 index_type
00190 ReductTargetSerializer<Scalar>::getBufferSize(const index_type count) const
00191 {
00192   return reduct_obj_ext_size_ * count;
00193 }
00194 
00195 template<class Scalar>
00196 void ReductTargetSerializer<Scalar>::serialize(
00197   const index_type              count
00198   ,const ReductTarget * const   reduct_objs[]
00199   ,const index_type             bytes
00200   ,char                         charBuffer[]
00201   ) const
00202 {
00203 #ifdef TEUCHOS_DEBUG
00204   TEST_FOR_EXCEPT( !(count > 0) );
00205   TEST_FOR_EXCEPT( !reduct_objs );
00206   TEST_FOR_EXCEPT( !(bytes==this->getBufferSize(count)) );
00207   TEST_FOR_EXCEPT( !charBuffer );
00208 #endif
00209   int offset = 0;
00210   for( int i = 0; i < count; ++i, offset += reduct_obj_ext_size_ ) {
00211     RTOpPack::serialize(
00212       *op_,num_values_,num_indexes_,num_chars_
00213       ,*reduct_objs[i],&charBuffer[offset]
00214       );
00215   }
00216 }
00217 
00218 template<class Scalar>
00219 Teuchos::RefCountPtr<ReductTarget>
00220 ReductTargetSerializer<Scalar>::createObj() const
00221 {
00222   return op_->reduct_obj_create();
00223 }
00224 
00225 template<class Scalar>
00226 void ReductTargetSerializer<Scalar>::deserialize(
00227   const index_type       bytes
00228   ,const char            charBuffer[]
00229   ,const index_type      count
00230   ,ReductTarget * const  reduct_objs[]
00231   ) const
00232 {
00233 #ifdef TEUCHOS_DEBUG
00234   TEST_FOR_EXCEPT( !(bytes > 0) );
00235   TEST_FOR_EXCEPT( !charBuffer );
00236   TEST_FOR_EXCEPT( !(bytes==getBufferSize(count)) );
00237   TEST_FOR_EXCEPT( !reduct_objs );
00238 #endif
00239   int offset = 0;
00240   for( int i = 0; i < count; ++i, offset += reduct_obj_ext_size_ ) {
00241     RTOpPack::deserialize(
00242       *op_,num_values_,num_indexes_,num_chars_
00243       ,&charBuffer[offset],reduct_objs[i]
00244       );
00245   }
00246 }
00247 
00248 //
00249 // ReductTargetReductionOp
00250 //
00251 
00252 template<class Scalar>
00253 ReductTargetReductionOp<Scalar>::ReductTargetReductionOp(
00254   const Teuchos::RefCountPtr<const RTOpT<Scalar> >  &op
00255   )
00256   :op_(op)
00257 {}
00258   
00259 template<class Scalar>
00260 void ReductTargetReductionOp<Scalar>::reduce(
00261   const Ordinal              count
00262   ,const ReductTarget*const  inBuffer[]
00263   ,ReductTarget*const        inoutBuffer[]
00264   ) const
00265 {
00266   for( int i = 0; i < count; ++i )
00267     op_->reduce_reduct_objs( *inBuffer[i], inoutBuffer[i] );
00268 }
00269 
00270 } // namespace RTOpPack
00271 
00272 template<class Scalar>
00273 void RTOpPack::SPMD_all_reduce(
00274   const Teuchos::Comm<index_type>     *comm
00275   ,const RTOpT<Scalar>                &op
00276   ,const int                          num_cols
00277   ,const ReductTarget*const           i_reduct_objs[]
00278   ,ReductTarget*const                 reduct_objs[]
00279   )
00280 {
00281   using Teuchos::Workspace;
00282   Teuchos::WorkspaceStore* wss = Teuchos::get_default_workspace_store().get();
00283   Workspace<Teuchos::RefCountPtr<ReductTarget> >
00284     i_i_reduct_objs( wss, num_cols );
00285   Workspace<ReductTarget*>
00286     _i_i_reduct_objs( wss, num_cols );
00287   for( int kc = 0; kc < num_cols; ++kc ) {
00288     i_i_reduct_objs[kc] = op.reduct_obj_create();
00289     _i_i_reduct_objs[kc] = &*i_i_reduct_objs[kc];
00290   }
00291   ReductTargetSerializer<Scalar>
00292     serializer(Teuchos::rcp(&op,false));
00293   ReductTargetReductionOp<Scalar>
00294     reductOp(Teuchos::rcp(&op,false));
00295   reduceAll(
00296     *comm,serializer,reductOp
00297     ,num_cols,&i_reduct_objs[0],&_i_i_reduct_objs[0]
00298     );
00299   for( int kc = 0; kc < num_cols; ++kc ) {
00300     op.reduce_reduct_objs(*_i_i_reduct_objs[kc],reduct_objs[kc]);
00301   }
00302 }
00303 
00304 template<class Scalar>
00305 void RTOpPack::SPMD_apply_op(
00306   const Teuchos::Comm<index_type>               *comm
00307   ,const RTOpT<Scalar>                          &op
00308   ,const int                                    num_vecs
00309   ,const RTOpPack::ConstSubVectorView<Scalar>   sub_vecs[]
00310   ,const int                                    num_targ_vecs
00311   ,const RTOpPack::SubVectorView<Scalar>        targ_sub_vecs[]
00312   ,ReductTarget                                 *reduct_obj
00313   )
00314 {
00315   ReductTarget* reduct_objs[] = { reduct_obj };
00316   SPMD_apply_op(
00317     comm,op,1,num_vecs,sub_vecs,num_targ_vecs,targ_sub_vecs
00318     ,reduct_obj ? reduct_objs : NULL
00319     );
00320 }
00321 
00323 template<class Scalar>
00324 void RTOpPack::SPMD_apply_op(
00325   const Teuchos::Comm<index_type>                    *comm
00326   ,const RTOpT<Scalar>                               &op
00327   ,const int                                         num_cols
00328   ,const int                                         num_multi_vecs
00329   ,const RTOpPack::ConstSubMultiVectorView<Scalar>   sub_multi_vecs[]
00330   ,const int                                         num_targ_multi_vecs
00331   ,const RTOpPack::SubMultiVectorView<Scalar>        targ_sub_multi_vecs[]
00332   ,RTOpPack::ReductTarget*const                      reduct_objs[]
00333   )
00334 {
00335   using Teuchos::Workspace;
00336   Teuchos::WorkspaceStore* wss = Teuchos::get_default_workspace_store().get();
00337   int k, j, off;
00338   Workspace<ConstSubVectorView<Scalar> > c_sub_vecs(wss,num_multi_vecs*num_cols);
00339   if(sub_multi_vecs) {
00340     for( off = 0, j = 0; j < num_cols; ++j ) {
00341       for( k = 0; k < num_multi_vecs; ++k ) {
00342         const ConstSubMultiVectorView<Scalar> &mv = sub_multi_vecs[k];
00343         c_sub_vecs[off++].initialize(mv.globalOffset(),mv.subDim(),&mv(0,j),1);
00344       }
00345     }
00346   }
00347   Workspace<SubVectorView<Scalar> > c_targ_sub_vecs(wss,num_targ_multi_vecs*num_cols);
00348   if(targ_sub_multi_vecs) {
00349     for( off = 0, j = 0; j < num_cols; ++j ) {
00350       for( k = 0; k < num_targ_multi_vecs; ++k ) {
00351         const SubMultiVectorView<Scalar> &mv = targ_sub_multi_vecs[k];
00352         c_targ_sub_vecs[off++].initialize(mv.globalOffset(),mv.subDim(),&mv(0,j),1);
00353       }
00354     }
00355   }
00356   SPMD_apply_op(
00357     comm,op,num_cols
00358     ,num_multi_vecs, num_multi_vecs && sub_multi_vecs ? &c_sub_vecs[0] : NULL
00359     ,num_targ_multi_vecs, num_targ_multi_vecs && targ_sub_multi_vecs ? &c_targ_sub_vecs[0] : NULL
00360     ,reduct_objs
00361     );
00362 }
00363 
00364 template<class Scalar>
00365 void RTOpPack::SPMD_apply_op(
00366   const Teuchos::Comm<index_type>           *comm
00367   ,const RTOpT<Scalar>                      &op
00368   ,const int                                num_cols
00369   ,const int                                num_vecs
00370   ,const ConstSubVectorView<Scalar>         sub_vecs[]
00371   ,const int                                num_targ_vecs
00372   ,const SubVectorView<Scalar>              sub_targ_vecs[]
00373   ,ReductTarget*const                       reduct_objs[]
00374   )
00375 {
00376 #ifdef RTOPPACK_SPMD_APPLY_OP_DUMP
00377   Teuchos::RefCountPtr<Teuchos::FancyOStream>
00378     out = Teuchos::VerboseObjectBase::getDefaultOStream();
00379   Teuchos::OSTab tab(out);
00380   if(show_spmd_apply_op_dump) {
00381     *out << "\nEntering RTOpPack::SPMD_apply_op(...) ...\n";
00382     *out
00383       << "\ncomm = " << (comm?comm->description():"NULL")
00384       << "\nop = " << op.description()
00385       << "\nnum_cols = " << num_cols
00386       << "\nnum_vecs = " << num_vecs
00387       << "\nnum_targ_vecs = " << num_targ_vecs
00388       << "\n";
00389     if( num_vecs && sub_vecs ) {
00390       *out << "\nInput vectors:\n";
00391       Teuchos::OSTab tab(out);
00392       for( int kc = 0; kc < num_cols; ++kc ) {
00393         for( int k = 0; k < num_vecs; ++k ) {
00394           *out << "\nvecs["<<kc<<","<<k<<"] =\n";
00395           print(sub_vecs[kc*num_vecs+k],*out);
00396         }
00397       }
00398     }
00399     if( num_targ_vecs && sub_targ_vecs ) {
00400       *out << "\nInput/output vectors *before* transforamtion:\n";
00401       Teuchos::OSTab tab(out);
00402       for( int kc = 0; kc < num_cols; ++kc ) {
00403         for( int k = 0; k < num_targ_vecs; ++k ) {
00404           *out << "\nvecs["<<kc<<","<<k<<"] =\n";
00405           print(sub_targ_vecs[kc*num_targ_vecs+k],*out);
00406         }
00407       }
00408     }
00409     if(reduct_objs) {
00410       *out << "\nInput/output reduction objects *before* reduction:\n";
00411       Teuchos::OSTab tab(out);
00412       for( int kc = 0; kc < num_cols; ++kc ) {
00413         *out
00414           << "\nreduct_objs["<<kc<<"] =\n"
00415           << describe(*reduct_objs[kc],Teuchos::VERB_EXTREME);
00416       }
00417     }
00418   }
00419 #endif // RTOPPACK_SPMD_APPLY_OP_DUMP
00420   using Teuchos::Workspace;
00421   Teuchos::WorkspaceStore* wss = Teuchos::get_default_workspace_store().get();
00422   if( reduct_objs == NULL && sub_vecs == NULL && sub_targ_vecs == NULL ) {
00423     // This is a transformation operation with no data on this processor.
00424     // Therefore, we can just exist!
00425   }
00426   else {
00427     const int localSubDim =
00428       ( num_vecs
00429         ? ( sub_vecs ? sub_vecs[0].subDim() : 0 )
00430         : ( sub_targ_vecs ? sub_targ_vecs[0].subDim() : 0 )
00431         );
00432     // See if we need to do any global communication at all?
00433     if( comm==NULL || reduct_objs == NULL ) {
00434       if( ( sub_vecs || sub_targ_vecs ) && localSubDim ) {
00435         for( int kc = 0; kc < num_cols; ++kc ) {
00436           op.apply_op(
00437             num_vecs,sub_vecs+kc*num_vecs,num_targ_vecs,sub_targ_vecs+kc*num_targ_vecs
00438             ,reduct_objs ? reduct_objs[kc] : NULL
00439             );
00440         }
00441       }
00442     }
00443     else {
00444       // Check the preconditions for excluding empty target vectors.
00445       TEST_FOR_EXCEPTION(
00446         ( ( num_vecs && !sub_vecs) || ( num_targ_vecs && !sub_targ_vecs) ) && !( !sub_vecs && !sub_targ_vecs )
00447         ,std::logic_error
00448         ,"SPMD_apply_op(...): Error, invalid arguments num_vecs = " << num_vecs
00449         << ", sub_vecs = " << sub_vecs << ", num_targ_vecs = " << num_targ_vecs
00450         << ", sub_targ_vecs = " << sub_targ_vecs
00451         );
00452       //
00453       // There is a non-null reduction target object and we are using
00454       // SPMD so we need to reduce it across processors
00455       //
00456       // Allocate the intermediate target object and perform the
00457       // reduction for the vector elements on this processor.
00458       //
00459       Workspace<Teuchos::RefCountPtr<ReductTarget> >
00460         i_reduct_objs( wss, num_cols );
00461       for( int kc = 0; kc < num_cols; ++kc ) {
00462         i_reduct_objs[kc] = op.reduct_obj_create();
00463         if( ( sub_vecs || sub_targ_vecs ) && localSubDim ) {
00464           op.apply_op(
00465             num_vecs, sub_vecs+kc*num_vecs, num_targ_vecs, sub_targ_vecs+kc*num_targ_vecs
00466             ,&*i_reduct_objs[kc]
00467             );
00468         }
00469       }
00470 #ifdef RTOPPACK_SPMD_APPLY_OP_DUMP
00471       if(show_spmd_apply_op_dump) {
00472         if(reduct_objs) {
00473           *out << "\nIntermediate reduction objects in this process before global reduction:\n";
00474           Teuchos::OSTab tab(out);
00475           for( int kc = 0; kc < num_cols; ++kc ) {
00476             *out
00477               << "\ni_reduct_objs["<<kc<<"] =\n"
00478               << describe(*i_reduct_objs[kc],Teuchos::VERB_EXTREME);
00479           }
00480         }
00481       }
00482 #endif // RTOPPACK_SPMD_APPLY_OP_DUMP
00483       //
00484       // Reduce the local intermediate reduction objects into the global reduction objects
00485       //
00486       Workspace<const ReductTarget*>
00487         _i_reduct_objs( wss, num_cols );
00488       for( int kc = 0; kc < num_cols; ++kc ) {
00489         _i_reduct_objs[kc] = &*i_reduct_objs[kc];
00490       }
00491 #ifdef RTOPPACK_SPMD_APPLY_OP_DUMP
00492       if(show_spmd_apply_op_dump) {
00493         if(reduct_objs) {
00494           *out << "\nPerforming global reduction ...\n";
00495         }
00496       }
00497 #endif // RTOPPACK_SPMD_APPLY_OP_DUMP
00498       SPMD_all_reduce(comm,op,num_cols,&_i_reduct_objs[0],reduct_objs);
00499     }
00500   }
00501 #ifdef RTOPPACK_SPMD_APPLY_OP_DUMP
00502   if(show_spmd_apply_op_dump) {
00503     if( num_targ_vecs && sub_targ_vecs ) {
00504       *out << "\nInput/output vectors *after* transforamtion:\n";
00505       Teuchos::OSTab tab(out);
00506       for( int kc = 0; kc < num_cols; ++kc ) {
00507         for( int k = 0; k < num_targ_vecs; ++k ) {
00508           *out << "\nvecs["<<kc<<","<<k<<"] =\n";
00509           print(sub_targ_vecs[kc*num_targ_vecs+k],*out);
00510         }
00511       }
00512     }
00513     if(reduct_objs) {
00514       *out << "\nInput/output reduction objects *after* reduction:\n";
00515       Teuchos::OSTab tab(out);
00516       for( int kc = 0; kc < num_cols; ++kc ) {
00517         *out
00518           << "\nreduct_objs["<<kc<<"] =\n"
00519           << describe(*reduct_objs[kc],Teuchos::VERB_EXTREME);
00520       }
00521     }
00522     *out << "\nLeaving RTOpPack::SPMD_apply_op(...) ...\n";
00523   }
00524 #endif // RTOPPACK_SPMD_APPLY_OP_DUMP
00525 }
00526 
00527 #endif // RTOPPACK_SPMD_APPLY_OP_HPP

Generated on Thu Sep 18 12:30:43 2008 for RTOp Package Browser (Single Doxygen Collection) by doxygen 1.3.9.1