Tpetra Matrix/Vector Services Version of the Day
Tpetra_HybridPlatform.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 // 
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00038 // 
00039 // ************************************************************************
00040 // @HEADER
00041 
00042 #ifndef TPETRA_HYBRIDPLATFORM_HPP
00043 #define TPETRA_HYBRIDPLATFORM_HPP
00044 
00045 #include <Teuchos_Describable.hpp>
00046 #include <Teuchos_ParameterList.hpp>
00047 #include <string>
00048 #include <cstdio> // for std::sscanf
00049 
00050 #include <Kokkos_SerialNode.hpp>
00051 #ifdef HAVE_KOKKOS_TBB
00052 #include <Kokkos_TBBNode.hpp>
00053 #endif
00054 #ifdef HAVE_KOKKOS_THREADPOOL
00055 #include <Kokkos_TPINode.hpp>
00056 #endif
00057 #ifdef HAVE_KOKKOS_OPENMP
00058 #include <Kokkos_OpenMPNode.hpp>
00059 #endif
00060 #ifdef HAVE_KOKKOS_THRUST
00061 #include <Kokkos_ThrustGPUNode.hpp>
00062 #endif
00063 
00064 namespace Tpetra {
00065 
00067 
00072   class HybridPlatform : public Teuchos::Describable {
00073     public:
00075 
00076 
00078       HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl);
00079 
00081       ~HybridPlatform();
00082 
00084 
00086 
00087 
00089       Teuchos::RCP<const Teuchos::Comm<int> > getComm() const;
00090 
00091       void createNode();
00092 
00094       template <template <class Node> class UserCode> 
00095       void runUserCode();
00096 
00098       template <class UserCode> 
00099       void runUserCode(UserCode &code);
00100 
00102 
00103     private:
00104       HybridPlatform(const HybridPlatform &platform); // not supported
00105       const Teuchos::RCP<const Teuchos::Comm<int> > comm_;
00106       Teuchos::ParameterList instList_;
00107       Teuchos::RCP<Kokkos::SerialNode>    serialNode_;
00108       bool nodeCreated_;
00109 #ifdef HAVE_KOKKOS_TBB
00110       Teuchos::RCP<Kokkos::TBBNode>       tbbNode_;
00111 #endif
00112 #ifdef HAVE_KOKKOS_THREADPOOL
00113       Teuchos::RCP<Kokkos::TPINode>       tpiNode_;
00114 #endif
00115 #ifdef HAVE_KOKKOS_OPENMP
00116       Teuchos::RCP<Kokkos::OpenMPNode>    ompNode_;
00117 #endif
00118 #ifdef HAVE_KOKKOS_THRUST
00119       Teuchos::RCP<Kokkos::ThrustGPUNode> thrustNode_;
00120 #endif
00121 
00122       enum NodeType {
00123         SERIALNODE
00124 #ifdef HAVE_KOKKOS_TBB
00125         , TBBNODE
00126 #endif        
00127 #ifdef HAVE_KOKKOS_THREADPOOL
00128         , TPINODE
00129 #endif        
00130 #ifdef HAVE_KOKKOS_OPENMP
00131         , OMPNODE
00132 #endif        
00133 #ifdef HAVE_KOKKOS_THRUST
00134         , THRUSTGPUNODE
00135 #endif        
00136       } nodeType_;
00137   };
00138 
00139   HybridPlatform::HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl)
00140   : comm_(comm)
00141   , nodeCreated_(false)
00142   , nodeType_(SERIALNODE)
00143   {
00144     // ParameterList format:
00145     // 
00146     // Node designation sublists have a name beginning with one of the following: % = [
00147     // and satisfying the following format:
00148     //   %M=N    is satisfied if mod(myrank,M) == N
00149     //   =N      is satisfied if myrank == N
00150     //   [M,N]   is satisfied if myrank \in [M,N]
00151     // 
00152     // A node designation sublist must have a parameter entry of type std::string named "NodeType". The value indicates the type of the Node.
00153     // The activated node designation sublist will be passed to the Node constructor.
00154     // 
00155     // For example:
00156     // "%2=0"  ->  
00157     //    NodeType     = "Kokkos::ThrustGPUNode"
00158     //    DeviceNumber = 0
00159     //    Verbose      = 1
00160     // "%2=1"  ->
00161     //    NodeType     = "Kokkos::TPINode"
00162     //    NumThreads   = 8
00163     // 
00164     // In this scenario, nodes that are equivalent to zero module 2, i.e., even nodes, will be selected to use ThrustGPUNode objects
00165     // and initialized with the parameter list containing
00166     //    NodeType   = "Kokkos::ThrustGPUNode"
00167     //    DeviceNumber = 0
00168     //    Verbose      = 1
00169     // Nodes that are equivalent to one modulo 2, i.e., odd nodes, will be selected to use TPINode objects and initialized with the 
00170     // parameter list containing 
00171     //    NodeType   = "Kokkos::TPINode"
00172     //    NumThreads = 8
00173     // 
00174     // If multiple node designation sublists match the processor rank, then the first enounctered node designation will be used.
00175     // I don't know if ParameterList respects any ordering, therefore, multiple matching designations are to be avoided.
00176 
00177     const int myrank = comm_->getRank();
00178     std::string desigNode("");
00179     bool matchFound = false;
00180     for (Teuchos::ParameterList::ConstIterator it = pl.begin(); it != pl.end(); ++it) {
00181       if (it->second.isList()) {
00182         int parsedLen, M, N;
00183         const std::string &name = it->first;
00184         const Teuchos::ParameterList &sublist = Teuchos::getValue<Teuchos::ParameterList>(it->second);
00185         // select and assign instList_;
00186         parsedLen = 0;
00187         if (std::sscanf(name.c_str(),"%%%d=%d%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00188           if ((myrank % M) == N) {
00189             matchFound = true;
00190           }
00191         }
00192         parsedLen = 0;
00193         if (std::sscanf(name.c_str(),"=%d%n",&N,&parsedLen) == 1 && (size_t)parsedLen == name.length()) {
00194           if (myrank == N) {
00195             matchFound = true;
00196           }
00197         }
00198         parsedLen = 0;
00199         if (std::sscanf(name.c_str(),"[%d,%d]%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00200           if (M <= myrank && myrank <= N) {
00201             matchFound = true;
00202           }
00203         }
00204         if (name == "default") {
00205           matchFound = true;
00206         }
00207         if (matchFound) {
00208           try {
00209             desigNode = sublist.get<std::string>("NodeType");
00210           }
00211           catch (Teuchos::Exceptions::InvalidParameterName &e) {
00212             TEUCHOS_TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00213               std::endl << Teuchos::typeName(*this) << ": Invalid machine file." << std::endl 
00214               << "Missing parameter \"NodeType\" on Node " << myrank << " for Node designator " << "\"" << name << "\":" << std::endl 
00215               << sublist << std::endl);
00216           }
00217           if (desigNode == "Kokkos::SerialNode") {
00218             nodeType_ = SERIALNODE;
00219           }
00220 #ifdef HAVE_KOKKOS_THREADPOOL
00221           else if (desigNode == "Kokkos::TPINode") {
00222             nodeType_ = TPINODE;
00223           }
00224 #endif
00225 #ifdef HAVE_KOKKOS_TBB
00226           else if (desigNode == "Kokkos::TBBNode") {
00227             nodeType_ = TBBNODE;
00228           }
00229 #endif
00230 #ifdef HAVE_KOKKOS_OPENMP
00231           else if (desigNode == "Kokkos::OpenMPNode") {
00232             nodeType_ = OMPNODE;
00233           }
00234 #endif
00235 #ifdef HAVE_KOKKOS_THRUST
00236           else if (desigNode == "Kokkos::ThrustGPUNode") {
00237             nodeType_ = THRUSTGPUNODE;
00238           }
00239 #endif
00240           else {
00241             matchFound = false;
00242           }
00243           if (matchFound) {
00244             instList_ = sublist;
00245             break;
00246           }
00247         }
00248       }
00249     }
00250     if (!matchFound) {
00251       TEUCHOS_TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00252           Teuchos::typeName(*this) << ": No matching node type on rank " << myrank);
00253     }
00254   } 
00255 
00256   HybridPlatform::~HybridPlatform() 
00257   {}
00258 
00259   Teuchos::RCP<const Teuchos::Comm<int> > 
00260   HybridPlatform::getComm() const {
00261     return comm_;
00262   }
00263 
00264   void HybridPlatform::createNode() {
00265     using Teuchos::rcp;
00266     if (nodeCreated_) return;
00267     switch (nodeType_) {
00268       case SERIALNODE:
00269         serialNode_ = rcp(new Kokkos::SerialNode(instList_));
00270         break;
00271 #ifdef HAVE_KOKKOS_TBB
00272       case TBBNODE:
00273         tbbNode_ = rcp(new Kokkos::TBBNode(instList_));
00274         break;
00275 #endif        
00276 #ifdef HAVE_KOKKOS_OPENMP
00277       case OMPNODE:
00278         ompNode_ = rcp(new Kokkos::OpenMPNode(instList_));
00279         break;
00280 #endif        
00281 #ifdef HAVE_KOKKOS_THREADPOOL
00282       case TPINODE:
00283         tpiNode_  = rcp(new Kokkos::TPINode(instList_));
00284         break;
00285 #endif        
00286 #ifdef HAVE_KOKKOS_THRUST
00287       case THRUSTGPUNODE:
00288         thrustNode_ = rcp(new Kokkos::ThrustGPUNode(instList_));
00289         break;
00290 #endif        
00291       default:
00292         TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, 
00293             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00294     } // end of switch
00295     nodeCreated_ = true;
00296   }
00297 
00298   template <class UserCode>
00299   void HybridPlatform::runUserCode(UserCode &codeobj) {
00300     createNode();
00301     switch (nodeType_) {
00302       case SERIALNODE:
00303         codeobj.template run<Kokkos::SerialNode>(instList_,comm_, serialNode_);
00304         break;
00305 #ifdef HAVE_KOKKOS_TBB
00306       case TBBNODE:
00307         codeobj.template run<Kokkos::TBBNode>(instList_,comm_, tbbNode_);
00308         break;
00309 #endif        
00310 #ifdef HAVE_KOKKOS_OPENMP
00311       case OMPNODE:
00312         codeobj.template run<Kokkos::OpenMPNode>(instList_,comm_, ompNode_);
00313         break;
00314 #endif        
00315 #ifdef HAVE_KOKKOS_THREADPOOL
00316       case TPINODE:
00317         codeobj.template run<Kokkos::TPINode>(instList_,comm_, tpiNode_);
00318         break;
00319 #endif        
00320 #ifdef HAVE_KOKKOS_THRUST
00321       case THRUSTGPUNODE:
00322         codeobj.template run<Kokkos::ThrustGPUNode>(instList_,comm_, thrustNode_);
00323         break;
00324 #endif        
00325       default:
00326         TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, 
00327             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00328     } // end of switch
00329   }
00330 
00331   template <template<class Node> class UserCode>
00332   void HybridPlatform::runUserCode() {
00333     createNode();
00334     switch (nodeType_) {
00335       case SERIALNODE:
00336         UserCode<Kokkos::SerialNode>::run(instList_,comm_, serialNode_);
00337         break;
00338 #ifdef HAVE_KOKKOS_TBB
00339       case TBBNODE:
00340         UserCode<Kokkos::TBBNode>::run(instList_,comm_, tbbNode_);
00341         break;
00342 #endif        
00343 #ifdef HAVE_KOKKOS_OPENMP
00344       case OMPNODE:
00345         UserCode<Kokkos::OpenMPNode>::run(instList_,comm_, ompNode_);
00346         break;
00347 #endif        
00348 #ifdef HAVE_KOKKOS_THREADPOOL
00349       case TPINODE:
00350         UserCode<Kokkos::TPINode>::run(instList_,comm_, tpiNode_);
00351         break;
00352 #endif        
00353 #ifdef HAVE_KOKKOS_THRUST
00354       case THRUSTGPUNODE:
00355         UserCode<Kokkos::ThrustGPUNode>::run(instList_,comm_, thrustNode_);
00356         break;
00357 #endif        
00358       default:
00359         TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, 
00360             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00361     } // end of switch
00362   }
00363 
00364 } // namespace Tpetra
00365 
00366 #endif // TPETRA_HYBRIDPLATFORM_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines