Tpetra Matrix/Vector Services Version of the Day
Tpetra_HybridPlatform.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_HYBRIDPLATFORM_HPP
00030 #define TPETRA_HYBRIDPLATFORM_HPP
00031 
00032 #include <Teuchos_Describable.hpp>
00033 #include <Teuchos_ParameterList.hpp>
00034 #include <string>
00035 #include <cstdio> // for std::sscanf
00036 
00037 #include <Kokkos_SerialNode.hpp>
00038 #ifdef HAVE_KOKKOS_TBB
00039 #include <Kokkos_TBBNode.hpp>
00040 #endif
00041 #ifdef HAVE_KOKKOS_THREADPOOL
00042 #include <Kokkos_TPINode.hpp>
00043 #endif
00044 #ifdef HAVE_KOKKOS_THRUST
00045 #include <Kokkos_ThrustGPUNode.hpp>
00046 #endif
00047 
00048 namespace Tpetra {
00049 
00051 
00056   class HybridPlatform : public Teuchos::Describable {
00057     public:
00059 
00060 
00062       HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl);
00063 
00065       ~HybridPlatform();
00066 
00068 
00070 
00071 
00073       Teuchos::RCP<const Teuchos::Comm<int> > getComm() const;
00074 
00075       void createNode();
00076 
00078       template <template <class Node> class UserCode> 
00079       void runUserCode();
00080 
00082 
00083     private:
00084       HybridPlatform(const HybridPlatform &platform); // not supported
00085       const Teuchos::RCP<const Teuchos::Comm<int> > comm_;
00086       Teuchos::ParameterList instList_;
00087       Teuchos::RCP<Kokkos::SerialNode>    serialNode_;
00088       bool nodeCreated_;
00089 #ifdef HAVE_KOKKOS_TBB
00090       Teuchos::RCP<Kokkos::TBBNode>       tbbNode_;
00091 #endif
00092 #ifdef HAVE_KOKKOS_THREADPOOL
00093       Teuchos::RCP<Kokkos::TPINode>       tpiNode_;
00094 #endif
00095 #ifdef HAVE_KOKKOS_THRUST
00096       Teuchos::RCP<Kokkos::ThrustGPUNode> thrustNode_;
00097 #endif
00098 
00099       enum NodeType {
00100         SERIALNODE
00101 #ifdef HAVE_KOKKOS_TBB
00102         , TBBNODE
00103 #endif        
00104 #ifdef HAVE_KOKKOS_THREADPOOL
00105         , TPINODE
00106 #endif        
00107 #ifdef HAVE_KOKKOS_THRUST
00108         , THRUSTGPUNODE
00109 #endif        
00110       } nodeType_;
00111   };
00112 
00113   HybridPlatform::HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl)
00114   : comm_(comm)
00115   , nodeCreated_(false)
00116   , nodeType_(SERIALNODE)
00117   {
00118     // ParameterList format:
00119     // 
00120     // Node designation sublists have a name beginning with one of the following: % = [
00121     // and satisfying the following format:
00122     //   %M=N    is satisfied if mod(myrank,M) == N
00123     //   =N      is satisfied if myrank == N
00124     //   [M,N]   is satisfied if myrank \in [M,N]
00125     // 
00126     // A node designation sublist must have a parameter entry of type std::string named "NodeType". The value indicates the type of the Node.
00127     // The activated node designation sublist will be passed to the Node constructor.
00128     // 
00129     // For example:
00130     // "%2=0"  ->  
00131     //    NodeType     = "Kokkos::ThrustGPUNode"
00132     //    DeviceNumber = 0
00133     //    Verbose      = 1
00134     // "%2=1"  ->
00135     //    NodeType     = "Kokkos::TPINode"
00136     //    NumThreads   = 8
00137     // 
00138     // In this scenario, nodes that are equivalent to zero module 2, i.e., even nodes, will be selected to use ThrustGPUNode objects
00139     // and initialized with the parameter list containing
00140     //    NodeType   = "Kokkos::ThrustGPUNode"
00141     //    DeviceNumber = 0
00142     //    Verbose      = 1
00143     // Nodes that are equivalent to one modulo 2, i.e., odd nodes, will be selected to use TPINode objects and initialized with the 
00144     // parameter list containing 
00145     //    NodeType   = "Kokkos::TPINode"
00146     //    NumThreads = 8
00147     // 
00148     // If multiple node designation sublists match the processor rank, then the first enounctered node designation will be used.
00149     // I don't know if ParameterList respects any ordering, therefore, multiple matching designations are to be avoided.
00150 
00151     const int myrank = comm_->getRank();
00152     std::string desigNode("");
00153     bool matchFound = false;
00154     for (Teuchos::ParameterList::ConstIterator it = pl.begin(); it != pl.end(); ++it) {
00155       if (it->second.isList()) {
00156         int parsedLen, M, N;
00157         const std::string &name = it->first;
00158         const Teuchos::ParameterList &sublist = Teuchos::getValue<Teuchos::ParameterList>(it->second);
00159         // select and assign instList_;
00160         parsedLen = 0;
00161         if (std::sscanf(name.c_str(),"%%%d=%d%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00162           if ((myrank % M) == N) {
00163             matchFound = true;
00164           }
00165         }
00166         parsedLen = 0;
00167         if (std::sscanf(name.c_str(),"=%d%n",&N,&parsedLen) == 1 && (size_t)parsedLen == name.length()) {
00168           if (myrank == N) {
00169             matchFound = true;
00170           }
00171         }
00172         parsedLen = 0;
00173         if (std::sscanf(name.c_str(),"[%d,%d]%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00174           if (M <= myrank && myrank <= N) {
00175             matchFound = true;
00176           }
00177         }
00178         if (name == "default") {
00179           matchFound = true;
00180         }
00181         if (matchFound) {
00182           try {
00183             desigNode = sublist.get<std::string>("NodeType");
00184           }
00185           catch (Teuchos::Exceptions::InvalidParameterName &e) {
00186             TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00187               std::endl << Teuchos::typeName(*this) << ": Invalid machine file." << std::endl 
00188               << "Missing parameter \"NodeType\" on Node " << myrank << " for Node designator " << "\"" << name << "\":" << std::endl 
00189               << sublist << std::endl);
00190           }
00191           if (desigNode == "Kokkos::SerialNode") {
00192             nodeType_ = SERIALNODE;
00193           }
00194 #ifdef HAVE_KOKKOS_THREADPOOL
00195           else if (desigNode == "Kokkos::TPINode") {
00196             nodeType_ = TPINODE;
00197           }
00198 #endif
00199 #ifdef HAVE_KOKKOS_TBB
00200           else if (desigNode == "Kokkos::TBBNode") {
00201             nodeType_ = TBBNODE;
00202           }
00203 #endif
00204 #ifdef HAVE_KOKKOS_THRUST
00205           else if (desigNode == "Kokkos::ThrustGPUNode") {
00206             nodeType_ = THRUSTGPUNODE;
00207           }
00208 #endif
00209           else {
00210             matchFound = false;
00211           }
00212           if (matchFound) {
00213             instList_ = sublist;
00214             break;
00215           }
00216         }
00217       }
00218     }
00219     if (!matchFound) {
00220       TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00221           Teuchos::typeName(*this) << ": No matching node type on rank " << myrank);
00222     }
00223   } 
00224 
00225   HybridPlatform::~HybridPlatform() 
00226   {}
00227 
00228   Teuchos::RCP<const Teuchos::Comm<int> > 
00229   HybridPlatform::getComm() const {
00230     return comm_;
00231   }
00232 
00233   void HybridPlatform::createNode() {
00234     using Teuchos::rcp;
00235     if (nodeCreated_) return;
00236     switch (nodeType_) {
00237       case SERIALNODE:
00238         serialNode_ = rcp(new Kokkos::SerialNode(instList_));
00239         break;
00240 #ifdef HAVE_KOKKOS_TBB
00241       case TBBNODE:
00242         tbbNode_ = rcp(new Kokkos::TBBNode(instList_));
00243         break;
00244 #endif        
00245 #ifdef HAVE_KOKKOS_THREADPOOL
00246       case TPINODE:
00247         tpiNode_  = rcp(new Kokkos::TPINode(instList_));
00248         break;
00249 #endif        
00250 #ifdef HAVE_KOKKOS_THRUST
00251       case THRUSTGPUNODE:
00252         thrustNode_ = rcp(new Kokkos::ThrustGPUNode(instList_));
00253         break;
00254 #endif        
00255       default:
00256         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00257             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00258     } // end of switch
00259     nodeCreated_ = true;
00260   }
00261 
00262   template<template<class Node> class UserCode>
00263   void HybridPlatform::runUserCode() {
00264     createNode();
00265     switch (nodeType_) {
00266       case SERIALNODE:
00267         UserCode<Kokkos::SerialNode>::run(instList_,comm_, serialNode_);
00268         break;
00269 #ifdef HAVE_KOKKOS_TBB
00270       case TBBNODE:
00271         UserCode<Kokkos::TBBNode>::run(instList_,comm_, tbbNode_);
00272         break;
00273 #endif        
00274 #ifdef HAVE_KOKKOS_THREADPOOL
00275       case TPINODE:
00276         UserCode<Kokkos::TPINode>::run(instList_,comm_, tpiNode_);
00277         break;
00278 #endif        
00279 #ifdef HAVE_KOKKOS_THRUST
00280       case THRUSTGPUNODE:
00281         UserCode<Kokkos::ThrustGPUNode>::run(instList_,comm_, thrustNode_);
00282         break;
00283 #endif        
00284       default:
00285         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00286             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00287     } // end of switch
00288   }
00289 
00290 } // namespace Tpetra
00291 
00292 #endif // TPETRA_HYBRIDPLATFORM_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines