Tpetra_HybridPlatform.hpp

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_HYBRIDPLATFORM_HPP
00030 #define TPETRA_HYBRIDPLATFORM_HPP
00031 
00032 #include <Teuchos_Describable.hpp>
00033 #include <Teuchos_RCP.hpp>
00034 #include <Teuchos_ParameterList.hpp>
00035 #include <Teuchos_Comm.hpp> 
00036 #include <Teuchos_TypeNameTraits.hpp>
00037 #include <string>
00038 #include <cstdio> // for std::sscanf
00039 
00040 #include <Kokkos_SerialNode.hpp>
00041 #ifdef HAVE_KOKKOS_TBB
00042 #include <Kokkos_TBBNode.hpp>
00043 #endif
00044 #ifdef HAVE_KOKKOS_THREADPOOL
00045 #include <Kokkos_TPINode.hpp>
00046 #endif
00047 #ifdef HAVE_KOKKOS_THRUST
00048 #include <Kokkos_ThrustGPUNode.hpp>
00049 #endif
00050 
00051 namespace Tpetra {
00052 
00054 
00059   class HybridPlatform : public Teuchos::Describable {
00060     public:
00062 
00063 
00065       HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl);
00066 
00068       ~HybridPlatform();
00069 
00071 
00073 
00074 
00076       Teuchos::RCP<const Teuchos::Comm<int> > getComm() const;
00077 
00078       void createNode();
00079 
00081       template <template <class Node> class UserCode> 
00082       void runUserCode();
00083 
00085 
00086     private:
00087       HybridPlatform(const HybridPlatform &platform); // not supported
00088       const Teuchos::RCP<const Teuchos::Comm<int> > comm_;
00089       Teuchos::ParameterList instList_;
00090       Teuchos::RCP<Kokkos::SerialNode>    serialNode_;
00091       bool nodeCreated_;
00092 #ifdef HAVE_KOKKOS_TBB
00093       Teuchos::RCP<Kokkos::TBBNode>       tbbNode_;
00094 #endif
00095 #ifdef HAVE_KOKKOS_THREADPOOL
00096       Teuchos::RCP<Kokkos::TPINode>       tpiNode_;
00097 #endif
00098 #ifdef HAVE_KOKKOS_THRUST
00099       Teuchos::RCP<Kokkos::ThrustGPUNode> thrustNode_;
00100 #endif
00101 
00102       enum NodeType {
00103         SERIALNODE
00104 #ifdef HAVE_KOKKOS_TBB
00105         , TBBNODE
00106 #endif        
00107 #ifdef HAVE_KOKKOS_THREADPOOL
00108         , TPINODE
00109 #endif        
00110 #ifdef HAVE_KOKKOS_THRUST
00111         , THRUSTGPUNODE
00112 #endif        
00113       } nodeType_;
00114   };
00115 
00116   HybridPlatform::HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl)
00117   : comm_(comm)
00118   , nodeCreated_(false)
00119   , nodeType_(SERIALNODE)
00120   {
00121     // ParameterList format:
00122     // 
00123     // Node designation sublists have a name beginning with one of the following: % = [
00124     // and satisfying the following format:
00125     //   %M=N    is satisfied if mod(myrank,M) == N
00126     //   =N      is satisfied if myrank == N
00127     //   [M,N]   is satisfied if myrank \in [M,N]
00128     // 
00129     // A node designation sublist must have a parameter entry of type std::string named "NodeType". The value indicates the type of the Node.
00130     // The activated node designation sublist will be passed to the Node constructor.
00131     // 
00132     // For example:
00133     // "%2=0"  ->  
00134     //    NodeType     = "Kokkos::ThrustGPUNode"
00135     //    DeviceNumber = 0
00136     //    Verbose      = 1
00137     // "%2=1"  ->
00138     //    NodeType     = "Kokkos::TPINode"
00139     //    NumThreads   = 8
00140     // 
00141     // In this scenario, nodes that are equivalent to zero module 2, i.e., even nodes, will be selected to use ThrustGPUNode objects
00142     // and initialized with the parameter list containing
00143     //    NodeType   = "Kokkos::ThrustGPUNode"
00144     //    DeviceNumber = 0
00145     //    Verbose      = 1
00146     // Nodes that are equivalent to one modulo 2, i.e., odd nodes, will be selected to use TPINode objects and initialized with the 
00147     // parameter list containing 
00148     //    NodeType   = "Kokkos::TPINode"
00149     //    NumThreads = 8
00150     // 
00151     // If multiple node designation sublists match the processor rank, then the first enounctered node designation will be used.
00152     // I don't know if ParameterList respects any ordering, therefore, multiple matching designations are to be avoided.
00153 
00154     const int myrank = comm_->getRank();
00155     std::string desigNode("");
00156     bool matchFound = false;
00157     for (Teuchos::ParameterList::ConstIterator it = pl.begin(); it != pl.end(); ++it) {
00158       if (it->second.isList()) {
00159         int parsedLen, M, N;
00160         const std::string &name = it->first;
00161         const Teuchos::ParameterList &sublist = Teuchos::getValue<Teuchos::ParameterList>(it->second);
00162         // select and assign instList_;
00163         parsedLen = 0;
00164         if (std::sscanf(name.c_str(),"%%%d=%d%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00165           if ((myrank % M) == N) {
00166             matchFound = true;
00167           }
00168         }
00169         parsedLen = 0;
00170         if (std::sscanf(name.c_str(),"=%d%n",&N,&parsedLen) == 1 && (size_t)parsedLen == name.length()) {
00171           if (myrank == N) {
00172             matchFound = true;
00173           }
00174         }
00175         parsedLen = 0;
00176         if (std::sscanf(name.c_str(),"[%d,%d]%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00177           if (M <= myrank && myrank <= N) {
00178             matchFound = true;
00179           }
00180         }
00181         if (name == "default") {
00182           matchFound = true;
00183         }
00184         if (matchFound) {
00185           try {
00186             desigNode = sublist.get<std::string>("NodeType");
00187           }
00188           catch (Teuchos::Exceptions::InvalidParameterName &e) {
00189             TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00190               std::endl << Teuchos::typeName(*this) << ": Invalid machine file." << std::endl 
00191               << "Missing parameter \"NodeType\" on Node " << myrank << " for Node designator " << "\"" << name << "\":" << std::endl 
00192               << sublist << std::endl);
00193           }
00194           if (desigNode == "Kokkos::SerialNode") {
00195             nodeType_ = SERIALNODE;
00196           }
00197 #ifdef HAVE_KOKKOS_THREADPOOL
00198           else if (desigNode == "Kokkos::TPINode") {
00199             nodeType_ = TPINODE;
00200           }
00201 #endif
00202 #ifdef HAVE_KOKKOS_TBB
00203           else if (desigNode == "Kokkos::TBBNode") {
00204             nodeType_ = TBBNODE;
00205           }
00206 #endif
00207 #ifdef HAVE_KOKKOS_THRUST
00208           else if (desigNode == "Kokkos::ThrustGPUNode") {
00209             nodeType_ = THRUSTGPUNODE;
00210           }
00211 #endif
00212           else {
00213             matchFound = false;
00214           }
00215           if (matchFound) {
00216             instList_ = sublist;
00217             break;
00218           }
00219         }
00220       }
00221     }
00222     if (!matchFound) {
00223       TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00224           Teuchos::typeName(*this) << ": No matching node type on rank " << myrank);
00225     }
00226   } 
00227 
00228   HybridPlatform::~HybridPlatform() 
00229   {}
00230 
00231   Teuchos::RCP<const Teuchos::Comm<int> > 
00232   HybridPlatform::getComm() const {
00233     return comm_;
00234   }
00235 
00236   void HybridPlatform::createNode() {
00237     using Teuchos::rcp;
00238     if (nodeCreated_) return;
00239     switch (nodeType_) {
00240       case SERIALNODE:
00241         serialNode_ = rcp(new Kokkos::SerialNode(instList_));
00242         break;
00243 #ifdef HAVE_KOKKOS_TBB
00244       case TBBNODE:
00245         tbbNode_ = rcp(new Kokkos::TBBNode(instList_));
00246         break;
00247 #endif        
00248 #ifdef HAVE_KOKKOS_THREADPOOL
00249       case TPINODE:
00250         tpiNode_  = rcp(new Kokkos::TPINode(instList_));
00251         break;
00252 #endif        
00253 #ifdef HAVE_KOKKOS_THRUST
00254       case THRUSTGPUNODE:
00255         thrustNode_ = rcp(new Kokkos::ThrustGPUNode(instList_));
00256         break;
00257 #endif        
00258       default:
00259         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00260             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00261     } // end of switch
00262     nodeCreated_ = true;
00263   }
00264 
00265   template<template<class Node> class UserCode>
00266   void HybridPlatform::runUserCode() {
00267     createNode();
00268     switch (nodeType_) {
00269       case SERIALNODE:
00270         UserCode<Kokkos::SerialNode>::run(instList_,comm_, serialNode_);
00271         break;
00272 #ifdef HAVE_KOKKOS_TBB
00273       case TBBNODE:
00274         UserCode<Kokkos::TBBNode>::run(instList_,comm_, tbbNode_);
00275         break;
00276 #endif        
00277 #ifdef HAVE_KOKKOS_THREADPOOL
00278       case TPINODE:
00279         UserCode<Kokkos::TPINode>::run(instList_,comm_, tpiNode_);
00280         break;
00281 #endif        
00282 #ifdef HAVE_KOKKOS_THRUST
00283       case THRUSTGPUNODE:
00284         UserCode<Kokkos::ThrustGPUNode>::run(instList_,comm_, thrustNode_);
00285         break;
00286 #endif        
00287       default:
00288         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00289             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00290     } // end of switch
00291   }
00292 
00293 } // namespace Tpetra
00294 
00295 #endif // TPETRA_HYBRIDPLATFORM_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines
Generated on Wed Apr 13 10:21:41 2011 for Tpetra Matrix/Vector Services by  doxygen 1.6.3