Tpetra Matrix/Vector Services Version of the Day
Tpetra_HybridPlatform.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_HYBRIDPLATFORM_HPP
00030 #define TPETRA_HYBRIDPLATFORM_HPP
00031 
00032 #include <Teuchos_Describable.hpp>
00033 #include <Teuchos_ParameterList.hpp>
00034 #include <string>
00035 #include <cstdio> // for std::sscanf
00036 
00037 #include <Kokkos_SerialNode.hpp>
00038 #ifdef HAVE_KOKKOS_TBB
00039 #include <Kokkos_TBBNode.hpp>
00040 #endif
00041 #ifdef HAVE_KOKKOS_THREADPOOL
00042 #include <Kokkos_TPINode.hpp>
00043 #endif
00044 #ifdef HAVE_KOKKOS_THRUST
00045 #include <Kokkos_ThrustGPUNode.hpp>
00046 #endif
00047 
00048 namespace Tpetra {
00049 
00051 
00056   class HybridPlatform : public Teuchos::Describable {
00057     public:
00059 
00060 
00062       HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl);
00063 
00065       ~HybridPlatform();
00066 
00068 
00070 
00071 
00073       Teuchos::RCP<const Teuchos::Comm<int> > getComm() const;
00074 
00075       void createNode();
00076 
00078       template <template <class Node> class UserCode> 
00079       void runUserCode();
00080 
00082       template <class UserCode> 
00083       void runUserCode(UserCode &code);
00084 
00086 
00087     private:
00088       HybridPlatform(const HybridPlatform &platform); // not supported
00089       const Teuchos::RCP<const Teuchos::Comm<int> > comm_;
00090       Teuchos::ParameterList instList_;
00091       Teuchos::RCP<Kokkos::SerialNode>    serialNode_;
00092       bool nodeCreated_;
00093 #ifdef HAVE_KOKKOS_TBB
00094       Teuchos::RCP<Kokkos::TBBNode>       tbbNode_;
00095 #endif
00096 #ifdef HAVE_KOKKOS_THREADPOOL
00097       Teuchos::RCP<Kokkos::TPINode>       tpiNode_;
00098 #endif
00099 #ifdef HAVE_KOKKOS_THRUST
00100       Teuchos::RCP<Kokkos::ThrustGPUNode> thrustNode_;
00101 #endif
00102 
00103       enum NodeType {
00104         SERIALNODE
00105 #ifdef HAVE_KOKKOS_TBB
00106         , TBBNODE
00107 #endif        
00108 #ifdef HAVE_KOKKOS_THREADPOOL
00109         , TPINODE
00110 #endif        
00111 #ifdef HAVE_KOKKOS_THRUST
00112         , THRUSTGPUNODE
00113 #endif        
00114       } nodeType_;
00115   };
00116 
00117   HybridPlatform::HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl)
00118   : comm_(comm)
00119   , nodeCreated_(false)
00120   , nodeType_(SERIALNODE)
00121   {
00122     // ParameterList format:
00123     // 
00124     // Node designation sublists have a name beginning with one of the following: % = [
00125     // and satisfying the following format:
00126     //   %M=N    is satisfied if mod(myrank,M) == N
00127     //   =N      is satisfied if myrank == N
00128     //   [M,N]   is satisfied if myrank \in [M,N]
00129     // 
00130     // A node designation sublist must have a parameter entry of type std::string named "NodeType". The value indicates the type of the Node.
00131     // The activated node designation sublist will be passed to the Node constructor.
00132     // 
00133     // For example:
00134     // "%2=0"  ->  
00135     //    NodeType     = "Kokkos::ThrustGPUNode"
00136     //    DeviceNumber = 0
00137     //    Verbose      = 1
00138     // "%2=1"  ->
00139     //    NodeType     = "Kokkos::TPINode"
00140     //    NumThreads   = 8
00141     // 
00142     // In this scenario, nodes that are equivalent to zero module 2, i.e., even nodes, will be selected to use ThrustGPUNode objects
00143     // and initialized with the parameter list containing
00144     //    NodeType   = "Kokkos::ThrustGPUNode"
00145     //    DeviceNumber = 0
00146     //    Verbose      = 1
00147     // Nodes that are equivalent to one modulo 2, i.e., odd nodes, will be selected to use TPINode objects and initialized with the 
00148     // parameter list containing 
00149     //    NodeType   = "Kokkos::TPINode"
00150     //    NumThreads = 8
00151     // 
00152     // If multiple node designation sublists match the processor rank, then the first enounctered node designation will be used.
00153     // I don't know if ParameterList respects any ordering, therefore, multiple matching designations are to be avoided.
00154 
00155     const int myrank = comm_->getRank();
00156     std::string desigNode("");
00157     bool matchFound = false;
00158     for (Teuchos::ParameterList::ConstIterator it = pl.begin(); it != pl.end(); ++it) {
00159       if (it->second.isList()) {
00160         int parsedLen, M, N;
00161         const std::string &name = it->first;
00162         const Teuchos::ParameterList &sublist = Teuchos::getValue<Teuchos::ParameterList>(it->second);
00163         // select and assign instList_;
00164         parsedLen = 0;
00165         if (std::sscanf(name.c_str(),"%%%d=%d%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00166           if ((myrank % M) == N) {
00167             matchFound = true;
00168           }
00169         }
00170         parsedLen = 0;
00171         if (std::sscanf(name.c_str(),"=%d%n",&N,&parsedLen) == 1 && (size_t)parsedLen == name.length()) {
00172           if (myrank == N) {
00173             matchFound = true;
00174           }
00175         }
00176         parsedLen = 0;
00177         if (std::sscanf(name.c_str(),"[%d,%d]%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00178           if (M <= myrank && myrank <= N) {
00179             matchFound = true;
00180           }
00181         }
00182         if (name == "default") {
00183           matchFound = true;
00184         }
00185         if (matchFound) {
00186           try {
00187             desigNode = sublist.get<std::string>("NodeType");
00188           }
00189           catch (Teuchos::Exceptions::InvalidParameterName &e) {
00190             TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00191               std::endl << Teuchos::typeName(*this) << ": Invalid machine file." << std::endl 
00192               << "Missing parameter \"NodeType\" on Node " << myrank << " for Node designator " << "\"" << name << "\":" << std::endl 
00193               << sublist << std::endl);
00194           }
00195           if (desigNode == "Kokkos::SerialNode") {
00196             nodeType_ = SERIALNODE;
00197           }
00198 #ifdef HAVE_KOKKOS_THREADPOOL
00199           else if (desigNode == "Kokkos::TPINode") {
00200             nodeType_ = TPINODE;
00201           }
00202 #endif
00203 #ifdef HAVE_KOKKOS_TBB
00204           else if (desigNode == "Kokkos::TBBNode") {
00205             nodeType_ = TBBNODE;
00206           }
00207 #endif
00208 #ifdef HAVE_KOKKOS_THRUST
00209           else if (desigNode == "Kokkos::ThrustGPUNode") {
00210             nodeType_ = THRUSTGPUNODE;
00211           }
00212 #endif
00213           else {
00214             matchFound = false;
00215           }
00216           if (matchFound) {
00217             instList_ = sublist;
00218             break;
00219           }
00220         }
00221       }
00222     }
00223     if (!matchFound) {
00224       TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00225           Teuchos::typeName(*this) << ": No matching node type on rank " << myrank);
00226     }
00227   } 
00228 
00229   HybridPlatform::~HybridPlatform() 
00230   {}
00231 
00232   Teuchos::RCP<const Teuchos::Comm<int> > 
00233   HybridPlatform::getComm() const {
00234     return comm_;
00235   }
00236 
00237   void HybridPlatform::createNode() {
00238     using Teuchos::rcp;
00239     if (nodeCreated_) return;
00240     switch (nodeType_) {
00241       case SERIALNODE:
00242         serialNode_ = rcp(new Kokkos::SerialNode(instList_));
00243         break;
00244 #ifdef HAVE_KOKKOS_TBB
00245       case TBBNODE:
00246         tbbNode_ = rcp(new Kokkos::TBBNode(instList_));
00247         break;
00248 #endif        
00249 #ifdef HAVE_KOKKOS_THREADPOOL
00250       case TPINODE:
00251         tpiNode_  = rcp(new Kokkos::TPINode(instList_));
00252         break;
00253 #endif        
00254 #ifdef HAVE_KOKKOS_THRUST
00255       case THRUSTGPUNODE:
00256         thrustNode_ = rcp(new Kokkos::ThrustGPUNode(instList_));
00257         break;
00258 #endif        
00259       default:
00260         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00261             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00262     } // end of switch
00263     nodeCreated_ = true;
00264   }
00265 
00266   template <class UserCode>
00267   void HybridPlatform::runUserCode(UserCode &codeobj) {
00268     createNode();
00269     switch (nodeType_) {
00270       case SERIALNODE:
00271         codeobj.template run<Kokkos::SerialNode>(instList_,comm_, serialNode_);
00272         break;
00273 #ifdef HAVE_KOKKOS_TBB
00274       case TBBNODE:
00275         codeobj.template run<Kokkos::TBBNode>(instList_,comm_, tbbNode_);
00276         break;
00277 #endif        
00278 #ifdef HAVE_KOKKOS_THREADPOOL
00279       case TPINODE:
00280         codeobj.template run<Kokkos::TPINode>(instList_,comm_, tpiNode_);
00281         break;
00282 #endif        
00283 #ifdef HAVE_KOKKOS_THRUST
00284       case THRUSTGPUNODE:
00285         codeobj.template run<Kokkos::ThrustGPUNode>(instList_,comm_, thrustNode_);
00286         break;
00287 #endif        
00288       default:
00289         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00290             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00291     } // end of switch
00292   }
00293 
00294   template <template<class Node> class UserCode>
00295   void HybridPlatform::runUserCode() {
00296     createNode();
00297     switch (nodeType_) {
00298       case SERIALNODE:
00299         UserCode<Kokkos::SerialNode>::run(instList_,comm_, serialNode_);
00300         break;
00301 #ifdef HAVE_KOKKOS_TBB
00302       case TBBNODE:
00303         UserCode<Kokkos::TBBNode>::run(instList_,comm_, tbbNode_);
00304         break;
00305 #endif        
00306 #ifdef HAVE_KOKKOS_THREADPOOL
00307       case TPINODE:
00308         UserCode<Kokkos::TPINode>::run(instList_,comm_, tpiNode_);
00309         break;
00310 #endif        
00311 #ifdef HAVE_KOKKOS_THRUST
00312       case THRUSTGPUNODE:
00313         UserCode<Kokkos::ThrustGPUNode>::run(instList_,comm_, thrustNode_);
00314         break;
00315 #endif        
00316       default:
00317         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00318             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00319     } // end of switch
00320   }
00321 
00322 } // namespace Tpetra
00323 
00324 #endif // TPETRA_HYBRIDPLATFORM_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines