Tpetra_HybridPlatform.hpp

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_HYBRIDPLATFORM_HPP
00030 #define TPETRA_HYBRIDPLATFORM_HPP
00031 
00032 #include <Teuchos_Describable.hpp>
00033 #include <Teuchos_RCP.hpp>
00034 #include <Teuchos_ParameterList.hpp>
00035 #include <Teuchos_Comm.hpp> 
00036 #include <Teuchos_TypeNameTraits.hpp>
00037 #include <string>
00038 #include <cstdio> // for std::sscanf
00039 
00040 #include <Kokkos_SerialNode.hpp>
00041 #ifdef HAVE_KOKKOS_TBB
00042 #include <Kokkos_TBBNode.hpp>
00043 #endif
00044 #ifdef HAVE_KOKKOS_THREADPOOL
00045 #include <Kokkos_TPINode.hpp>
00046 #endif
00047 #ifdef HAVE_KOKKOS_THRUST
00048 #include <Kokkos_ThrustGPUNode.hpp>
00049 #endif
00050 
00051 namespace Tpetra {
00052 
00054 
00059   class HybridPlatform : public Teuchos::Describable {
00060     public:
00062 
00063 
00065       HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl);
00066 
00068       ~HybridPlatform();
00069 
00071 
00073 
00074 
00076       Teuchos::RCP<const Teuchos::Comm<int> > getComm() const;
00077 
00078       void createNode();
00079 
00081       template <template <class Node> class UserCode> 
00082       void runUserCode();
00083 
00085 
00086     private:
00087       HybridPlatform(const HybridPlatform &platform); // not supported
00088       const Teuchos::RCP<const Teuchos::Comm<int> > comm_;
00089       Teuchos::ParameterList instList_;
00090       Teuchos::RCP<Kokkos::SerialNode>    serialNode_;
00091       bool nodeCreated_;
00092 #ifdef HAVE_KOKKOS_TBB
00093       Teuchos::RCP<Kokkos::TBBNode>       tbbNode_;
00094 #endif
00095 #ifdef HAVE_KOKKOS_THREADPOOL
00096       Teuchos::RCP<Kokkos::TPINode>       tpiNode_;
00097 #endif
00098 #ifdef HAVE_KOKKOS_THRUST
00099       Teuchos::RCP<Kokkos::ThrustGPUNode> thrustNode_;
00100 #endif
00101 
00102       enum NodeType {
00103         SERIALNODE
00104 #ifdef HAVE_KOKKOS_TBB
00105         , TBBNODE
00106 #endif        
00107 #ifdef HAVE_KOKKOS_THREADPOOL
00108         , TPINODE
00109 #endif        
00110 #ifdef HAVE_KOKKOS_THRUST
00111         , THRUSTGPUNODE
00112 #endif        
00113       } nodeType_;
00114   };
00115 
00116   HybridPlatform::HybridPlatform(const Teuchos::RCP<const Teuchos::Comm<int> > &comm, Teuchos::ParameterList &pl)
00117   : comm_(comm)
00118   , nodeCreated_(false)
00119   , nodeType_(SERIALNODE)
00120   {
00121     // ParameterList format:
00122     // 
00123     // Node designation sublists have a name beginning with one of the following: % = [
00124     // and satisfying the following format:
00125     //   %M=N    is satisfied if mod(myrank,M) == N
00126     //   =N      is satisfied if myrank == N
00127     //   [M,N]   is satisfied if myrank \in [M,N]
00128     // 
00129     // A node designation sublist must have a parameter entry of type std::string named "NodeType". The value indicates the type of the Node.
00130     // The activated node designation sublist will be passed to the Node constructor.
00131     // 
00132     // For example:
00133     // "%2=0"  ->  
00134     //    NodeType     = "Kokkos::ThrustGPUNode"
00135     //    DeviceNumber = 0
00136     //    Verbose      = 1
00137     // "%2=1"  ->
00138     //    NodeType     = "Kokkos::TPINode"
00139     //    NumThreads   = 8
00140     // 
00141     // In this scenario, nodes that are equivalent to zero module 2, i.e., even nodes, will be selected to use ThrustGPUNode objects
00142     // and initialized with the parameter list containing
00143     //    NodeType   = "Kokkos::ThrustGPUNode"
00144     //    DeviceNumber = 0
00145     //    Verbose      = 1
00146     // Nodes that are equivalent to one modulo 2, i.e., odd nodes, will be selected to use TPINode objects and initialized with the 
00147     // parameter list containing 
00148     //    NodeType   = "Kokkos::TPINode"
00149     //    NumThreads = 8
00150     // 
00151     // If multiple node designation sublists match the processor rank, then the first enounctered node designation will be used.
00152     // I don't know if ParameterList respects any ordering, therefore, multiple matching designations are to be avoided.
00153 
00154     const int myrank = comm_->getRank();
00155     std::string desigNode("");
00156     bool matchFound = false;
00157     for (Teuchos::ParameterList::ConstIterator it = pl.begin(); it != pl.end(); ++it) {
00158       if (it->second.isList()) {
00159         int parsedLen, M, N;
00160         const std::string &name = it->first;
00161         const Teuchos::ParameterList &sublist = Teuchos::getValue<Teuchos::ParameterList>(it->second);
00162         // select and assign instList_;
00163         parsedLen = 0;
00164         if (std::sscanf(name.c_str(),"%%%d=%d%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00165           if ((myrank % M) == N) {
00166             matchFound = true;
00167           }
00168         }
00169         parsedLen = 0;
00170         if (std::sscanf(name.c_str(),"=%d%n",&N,&parsedLen) == 1 && (size_t)parsedLen == name.length()) {
00171           if (myrank == N) {
00172             matchFound = true;
00173           }
00174         }
00175         parsedLen = 0;
00176         if (std::sscanf(name.c_str(),"[%d,%d]%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00177           if (M <= myrank && myrank <= N) {
00178             matchFound = true;
00179           }
00180         }
00181         if (matchFound) {
00182           try {
00183             desigNode = sublist.get<std::string>("NodeType");
00184           }
00185           catch (Teuchos::Exceptions::InvalidParameterName &e) {
00186             TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00187               std::endl << Teuchos::typeName(*this) << ": Invalid machine file." << std::endl 
00188               << "Missing parameter \"NodeType\" on Node " << myrank << " for Node designator " << "\"" << name << "\":" << std::endl 
00189               << sublist << std::endl);
00190           }
00191           if (desigNode == "Kokkos::SerialNode") {
00192             nodeType_ = SERIALNODE;
00193           }
00194 #ifdef HAVE_KOKKOS_THREADPOOL
00195           else if (desigNode == "Kokkos::TPINode") {
00196             nodeType_ = TPINODE;
00197           }
00198 #endif
00199 #ifdef HAVE_KOKKOS_TBB
00200           else if (desigNode == "Kokkos::TBBNode") {
00201             nodeType_ = TBBNODE;
00202           }
00203 #endif
00204 #ifdef HAVE_KOKKOS_THRUST
00205           else if (desigNode == "Kokkos::ThrustGPUNode") {
00206             nodeType_ = THRUSTGPUNODE;
00207           }
00208 #endif
00209           else {
00210             matchFound = false;
00211           }
00212           if (matchFound) {
00213             instList_ = sublist;
00214             break;
00215           }
00216         }
00217       }
00218     }
00219     if (!matchFound) {
00220       TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00221           Teuchos::typeName(*this) << ": No matching node type on rank " << myrank);
00222     }
00223   } 
00224 
00225   HybridPlatform::~HybridPlatform() 
00226   {}
00227 
00228   Teuchos::RCP<const Teuchos::Comm<int> > 
00229   HybridPlatform::getComm() const {
00230     return comm_;
00231   }
00232 
00233   void HybridPlatform::createNode() {
00234     using Teuchos::rcp;
00235     if (nodeCreated_) return;
00236     switch (nodeType_) {
00237       case SERIALNODE:
00238         serialNode_ = rcp(new Kokkos::SerialNode(instList_));
00239         break;
00240 #ifdef HAVE_KOKKOS_TBB
00241       case TBBNODE:
00242         tbbNode_ = rcp(new Kokkos::TBBNode(instList_));
00243         break;
00244 #endif        
00245 #ifdef HAVE_KOKKOS_THREADPOOL
00246       case TPINODE:
00247         tpiNode_  = rcp(new Kokkos::TPINode(instList_));
00248         break;
00249 #endif        
00250 #ifdef HAVE_KOKKOS_THRUST
00251       case THRUSTGPUNODE:
00252         thrustNode_ = rcp(new Kokkos::ThrustGPUNode(instList_));
00253         break;
00254 #endif        
00255       default:
00256         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00257             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00258     } // end of switch
00259     nodeCreated_ = true;
00260   }
00261 
00262   template<template<class Node> class UserCode>
00263   void HybridPlatform::runUserCode() {
00264     createNode();
00265     switch (nodeType_) {
00266       case SERIALNODE:
00267         UserCode<Kokkos::SerialNode>::run(comm_, serialNode_);
00268         break;
00269 #ifdef HAVE_KOKKOS_TBB
00270       case TBBNODE:
00271         UserCode<Kokkos::TBBNode>::run(comm_, tbbNode_);
00272         break;
00273 #endif        
00274 #ifdef HAVE_KOKKOS_THREADPOOL
00275       case TPINODE:
00276         UserCode<Kokkos::TPINode>::run(comm_, tpiNode_);
00277         break;
00278 #endif        
00279 #ifdef HAVE_KOKKOS_THRUST
00280       case THRUSTGPUNODE:
00281         UserCode<Kokkos::ThrustGPUNode>::run(comm_, thrustNode_);
00282         break;
00283 #endif        
00284       default:
00285         TEST_FOR_EXCEPTION(true, std::runtime_error, 
00286             Teuchos::typeName(*this) << "::runUserCode(): Invalid node type." << std::endl);
00287     } // end of switch
00288   }
00289 
00290 } // namespace Tpetra
00291 
00292 #endif // TPETRA_HYBRIDPLATFORM_HPP

Generated on Tue Jul 13 09:39:06 2010 for Tpetra Matrix/Vector Services by  doxygen 1.4.7