Tpetra Matrix/Vector Services Version of the Day
Tpetra_HybridPlatform.cpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 // 
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00038 // 
00039 // ************************************************************************
00040 // @HEADER
00041 
00042 #include <Tpetra_HybridPlatform.hpp>
00043 #include <cstdio> // for std::sscanf
00044 
00045 // This macro is only for use by Tpetra developers.
00046 // It should only be invoked in the Tpetra namespace.
00047 #define TPETRA_HYBRIDPLATFORM_ADD_NODE_SUPPORT_DEF(N) \
00048   template <> bool HybridPlatform::isNodeSupported<N>() {return true;}
00049 
00050 namespace Tpetra {
00051   TPETRA_HYBRIDPLATFORM_ADD_NODE_SUPPORT_DEF(KokkosClassic::SerialNode)
00052 #ifdef HAVE_KOKKOSCLASSIC_TBB
00053   TPETRA_HYBRIDPLATFORM_ADD_NODE_SUPPORT_DEF(KokkosClassic::TBBNode)
00054 #endif        
00055 #ifdef HAVE_KOKKOSCLASSIC_OPENMP
00056   TPETRA_HYBRIDPLATFORM_ADD_NODE_SUPPORT_DEF(KokkosClassic::OpenMPNode)
00057 #endif        
00058 #ifdef HAVE_KOKKOSCLASSIC_THREADPOOL
00059   TPETRA_HYBRIDPLATFORM_ADD_NODE_SUPPORT_DEF(KokkosClassic::TPINode)
00060 #endif        
00061 #ifdef HAVE_KOKKOSCLASSIC_THRUST
00062   TPETRA_HYBRIDPLATFORM_ADD_NODE_SUPPORT_DEF(KokkosClassic::ThrustGPUNode)
00063 #endif
00064 
00065   HybridPlatform::
00066   HybridPlatform (const Teuchos::RCP<const Teuchos::Comm<int> >& comm, 
00067       Teuchos::ParameterList& pl)
00068     : comm_ (comm)
00069     , nodeCreated_ (false)
00070     , nodeType_ (SERIALNODE)
00071   {
00072     // ParameterList format:
00073     // 
00074     // Node designation sublists have a name beginning with one of the
00075     // following characters: % = [ and satisfying the following
00076     // format:
00077 
00078     //   %M=N    is satisfied if mod(myrank,M) == N
00079     //   =N      is satisfied if myrank == N
00080     //   [M,N]   is satisfied if myrank \in [M,N]
00081     // 
00082     // A node designation sublist must have a parameter entry of type
00083     // std::string named "NodeType". The value indicates the type of
00084     // the Node.  The activated node designation sublist will be
00085     // passed to the Node constructor.
00086     // 
00087     // For example:
00088     // "%2=0"  ->  
00089     //    NodeType     = "KokkosClassic::ThrustGPUNode"
00090     //    DeviceNumber = 0
00091     //    Verbose      = 1
00092     // "%2=1"  ->
00093     //    NodeType     = "KokkosClassic::TPINode"
00094     //    NumThreads   = 8
00095     // 
00096     // In this scenario, nodes that are equivalent to zero module 2,
00097     // that is, even nodes, will be selected to use ThrustGPUNode
00098     // objects and initialized with the parameter list containing
00099     //    NodeType   = "KokkosClassic::ThrustGPUNode"
00100     //    DeviceNumber = 0
00101     //    Verbose      = 1
00102     //
00103     // Nodes that are equivalent to one modulo 2, i.e., odd nodes,
00104     // will be selected to use TPINode objects and initialized with
00105     // the parameter list containing
00106     //    NodeType   = "KokkosClassic::TPINode"
00107     //    NumThreads = 8
00108     // 
00109     // If multiple node designation sublists match the process rank,
00110     // then the first encountered node designation will be used.  I
00111     // don't know if ParameterList respects any ordering, therefore,
00112     // multiple matching designations are to be avoided.
00113 
00114     const int myrank = comm_->getRank();
00115     std::string desigNode("");
00116     bool matchFound = false;
00117     for (Teuchos::ParameterList::ConstIterator it = pl.begin(); it != pl.end(); ++it) {
00118       if (it->second.isList()) {
00119         int parsedLen, M, N;
00120         const std::string &name = it->first;
00121         const Teuchos::ParameterList &sublist = Teuchos::getValue<Teuchos::ParameterList>(it->second);
00122         // select and assign instList_;
00123         parsedLen = 0;
00124         if (std::sscanf(name.c_str(),"%%%d=%d%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00125           if ((myrank % M) == N) {
00126             matchFound = true;
00127           }
00128         }
00129         parsedLen = 0;
00130         if (std::sscanf(name.c_str(),"=%d%n",&N,&parsedLen) == 1 && (size_t)parsedLen == name.length()) {
00131           if (myrank == N) {
00132             matchFound = true;
00133           }
00134         }
00135         parsedLen = 0;
00136         if (std::sscanf(name.c_str(),"[%d,%d]%n",&M,&N,&parsedLen) == 2 && (size_t)parsedLen == name.length()) {
00137           if (M <= myrank && myrank <= N) {
00138             matchFound = true;
00139           }
00140         }
00141         if (name == "default") {
00142           matchFound = true;
00143         }
00144         if (matchFound) {
00145           try {
00146             desigNode = sublist.get<std::string>("NodeType");
00147           }
00148           catch (Teuchos::Exceptions::InvalidParameterName &e) {
00149             TEUCHOS_TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00150               std::endl << Teuchos::typeName(*this) << ": Invalid machine file." << std::endl 
00151               << "Missing parameter \"NodeType\" on Node " << myrank << " for Node designator " << "\"" << name << "\":" << std::endl 
00152               << sublist << std::endl);
00153           }
00154           if (desigNode == "KokkosClassic::SerialNode") {
00155             nodeType_ = SERIALNODE;
00156           }
00157 #ifdef HAVE_KOKKOSCLASSIC_THREADPOOL
00158           else if (desigNode == "KokkosClassic::TPINode") {
00159             nodeType_ = TPINODE;
00160           }
00161 #endif
00162 #ifdef HAVE_KOKKOSCLASSIC_TBB
00163           else if (desigNode == "KokkosClassic::TBBNode") {
00164             nodeType_ = TBBNODE;
00165           }
00166 #endif
00167 #ifdef HAVE_KOKKOSCLASSIC_OPENMP
00168           else if (desigNode == "KokkosClassic::OpenMPNode") {
00169             nodeType_ = OMPNODE;
00170           }
00171 #endif
00172 #ifdef HAVE_KOKKOSCLASSIC_THRUST
00173           else if (desigNode == "KokkosClassic::ThrustGPUNode") {
00174             nodeType_ = THRUSTGPUNODE;
00175           }
00176 #endif
00177           else {
00178             matchFound = false;
00179           }
00180           if (matchFound) {
00181             instList_ = sublist;
00182             break;
00183           }
00184         }
00185       }
00186     }
00187     if (!matchFound) {
00188       TEUCHOS_TEST_FOR_EXCEPTION_PURE_MSG(true, std::runtime_error, 
00189           Teuchos::typeName(*this) << ": No matching node type on rank " << myrank);
00190     }
00191   } 
00192 
00193   HybridPlatform::~HybridPlatform() 
00194   {}
00195 
00196   RCP<ParameterList> HybridPlatform::listSupportedNodes()
00197   {
00198     RCP<ParameterList> list = Teuchos::parameterList();
00199     {
00200       ParameterList subpl;
00201       subpl.set("NodeType","KokkosClassic::SerialNode");
00202       subpl.setParameters( KokkosClassic::SerialNode::getDefaultParameters() );
00203       list->set("=-1",subpl);
00204     }
00205 #ifdef HAVE_KOKKOSCLASSIC_TBB
00206     {
00207       ParameterList subpl;
00208       subpl.set("NodeType","KokkosClassic::TBBNode");
00209       subpl.setParameters( KokkosClassic::TBBNode::getDefaultParameters() );
00210       list->set("=-2",subpl);
00211     }
00212 #endif        
00213 #ifdef HAVE_KOKKOSCLASSIC_OPENMP
00214     {
00215       ParameterList subpl;
00216       subpl.set("NodeType","KokkosClassic::OpenMPNode");
00217       subpl.setParameters( KokkosClassic::OpenMPNode::getDefaultParameters() );
00218       list->set("=-3",subpl);
00219     }
00220 #endif        
00221 #ifdef HAVE_KOKKOSCLASSIC_THREADPOOL
00222     {
00223       ParameterList subpl;
00224       subpl.set("NodeType","KokkosClassic::TPINode");
00225       subpl.setParameters( KokkosClassic::TPINode::getDefaultParameters() );
00226       list->set("=-4",subpl);
00227     }
00228 #endif        
00229 #ifdef HAVE_KOKKOSCLASSIC_THRUST
00230     {
00231       ParameterList subpl;
00232       subpl.set("NodeType","KokkosClassic::ThrustGPUNode");
00233       subpl.setParameters( KokkosClassic::ThrustGPUNode::getDefaultParameters() );
00234       list->set("=-5",subpl);
00235     }
00236 #endif        
00237     return list;
00238   }
00239 
00240   Teuchos::RCP<const Teuchos::Comm<int> > 
00241   HybridPlatform::getComm () const {
00242     return comm_;
00243   }
00244 
00245   void HybridPlatform::createNode () {
00246     using Teuchos::rcp;
00247 
00248     if (nodeCreated_) {
00249       return;
00250     }
00251     switch (nodeType_) {
00252     case SERIALNODE:
00253       serialNode_ = rcp (new KokkosClassic::SerialNode (instList_));
00254       break;
00255 #ifdef HAVE_KOKKOSCLASSIC_TBB
00256     case TBBNODE:
00257       tbbNode_ = rcp (new KokkosClassic::TBBNode (instList_));
00258       break;
00259 #endif        
00260 #ifdef HAVE_KOKKOSCLASSIC_OPENMP
00261     case OMPNODE:
00262       ompNode_ = rcp (new KokkosClassic::OpenMPNode (instList_));
00263       break;
00264 #endif        
00265 #ifdef HAVE_KOKKOSCLASSIC_THREADPOOL
00266     case TPINODE:
00267       tpiNode_  = rcp (new KokkosClassic::TPINode (instList_));
00268       break;
00269 #endif        
00270 #ifdef HAVE_KOKKOSCLASSIC_THRUST
00271     case THRUSTGPUNODE:
00272       thrustNode_ = rcp (new KokkosClassic::ThrustGPUNode (instList_));
00273       break;
00274 #endif        
00275     default:
00276       TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, 
00277         Teuchos::typeName(*this) << "::runUserCode(): Invalid Node type." 
00278   << std::endl);
00279     } // end of switch
00280     nodeCreated_ = true;
00281   }
00282 } // namespace Tpetra
00283 
00284 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines