Tpetra Matrix/Vector Services Version of the Day
Tpetra_MpiPlatform.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 //
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ************************************************************************
00040 // @HEADER
00041 
00042 #ifndef TPETRA_MPIPLATFORM_HPP
00043 #define TPETRA_MPIPLATFORM_HPP
00044 
00045 #include <Tpetra_ConfigDefs.hpp>
00046 #include <Tpetra_Core.hpp>
00047 #include <Kokkos_DefaultNode.hpp>
00048 #include <Teuchos_DefaultMpiComm.hpp>
00049 #include <Teuchos_Describable.hpp>
00050 
00051 namespace Tpetra {
00052 
00083   template <class Node>
00084   class MpiPlatform : public Teuchos::Describable {
00085   public:
00087 
00088 
00090     typedef Node NodeType;
00091 
00093 
00094 
00095 
00106     explicit MpiPlatform (const Teuchos::RCP<NodeType>& node) :
00107       comm_ (Teuchos::createMpiComm<int> (Teuchos::opaqueWrapper<MPI_Comm> (MPI_COMM_WORLD))),
00108       node_ (node)
00109     {
00110       // mfh 29 Jun 2014: Don't initialize the Node yet.  This ensures
00111       // that (new) Kokkos won't get initialized with the wrong
00112       // command-line arguments, at least not until getNode() is
00113       // called.  Initializing Kokkos with the wrong command-line
00114       // arguments may result in poor performance due to the wrong
00115       // assignment of software threads to hardware execution units.
00116       //
00117       // if (node_.is_null ()) {
00118       //   node_ = KokkosClassic::Details::getNode<NodeType> ();
00119       // }
00120     }
00121 
00133     MpiPlatform (int* argc,
00134                  char*** argv,
00135                  const Teuchos::RCP<NodeType>& node) :
00136       comm_ (Teuchos::null),
00137       node_ (node)
00138     {
00139       initialize (argc, argv);
00140       comm_ = getDefaultComm ();
00141 
00142       // mfh 29 Jun 2014: Don't initialize the Node yet.  See above note.
00143       //
00144       // if (node_.is_null ()) {
00145       //   node_ = KokkosClassic::Details::getNode<Node> ();
00146       // }
00147     }
00148 
00169     MpiPlatform (const Teuchos::RCP<NodeType>& node,
00170                  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm)
00171       : comm_ (Teuchos::null),
00172         node_ (node)
00173     {
00174       TEUCHOS_TEST_FOR_EXCEPTION(
00175         rawMpiComm.is_null (), std::invalid_argument, "Tpetra::MpiPlatform "
00176         "constructor: The input RCP<OpaqueWrapper<MPI_Comm> > is null.  That "
00177         "means something different than MPI_COMM_NULL.  If you want to give "
00178         "MPI_COMM_NULL to this constructor, please wrap MPI_COMM_NULL in a "
00179         "nonnull Teuchos::OpaqueWrapper by using the "
00180         "Teuchos::opaqueWrapper<MPI_Comm>() nonmember constructor.");
00181       comm_ = Teuchos::createMpiComm<int> (rawMpiComm);
00182 
00183       // mfh 29 Jun 2014: Don't initialize the Node yet.  See above note.
00184       //
00185       // if (node_.is_null ()) {
00186       //   node_ = KokkosClassic::Details::getNode<NodeType> ();
00187       // }
00188     }
00189 
00213     MpiPlatform (int* argc,
00214                  char*** argv,
00215                  const Teuchos::RCP<NodeType>& node,
00216                  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm)
00217       : comm_ (Teuchos::null),
00218         node_ (node)
00219     {
00220       TEUCHOS_TEST_FOR_EXCEPTION(
00221         rawMpiComm.is_null (), std::invalid_argument, "Tpetra::MpiPlatform "
00222         "constructor: The input RCP<OpaqueWrapper<MPI_Comm> > is null.  That "
00223         "means something different than MPI_COMM_NULL.  If you want to give "
00224         "MPI_COMM_NULL to this constructor, please wrap MPI_COMM_NULL in a "
00225         "nonnull Teuchos::OpaqueWrapper by using the "
00226         "Teuchos::opaqueWrapper<MPI_Comm>() nonmember constructor.");
00227       comm_ = Teuchos::createMpiComm<int> (rawMpiComm);
00228 
00229       // NOTE (mfh 29 Jun 2014): The OpaqueWrapper might wrap the
00230       // MPI_Comm in something that calls MPI_Comm_free.  Thus, we
00231       // can't just ignore it; we have to give it to the comm_ so that
00232       // its destructor (which might call MPI_Comm_free) will be
00233       // called at the right time.  This is why we don't set comm
00234       // using getDefaultComm().  This is also why we pass comm_
00235       // directly to initialize(): that way there aren't two
00236       // references to the raw MPI_Comm floating around, and comm_'s
00237       // destructor will get to do the right thing.
00238       initialize (argc, argv, comm_);
00239 
00240       // mfh 29 Jun 2014: Don't initialize the Node yet.  See above note.
00241       //
00242       // if (node_.is_null ()) {
00243       //   node_ = KokkosClassic::Details::getNode<NodeType> ();
00244       // }
00245     }
00246 
00259     MpiPlatform (const Teuchos::RCP<NodeType>& node, MPI_Comm rawMpiComm)
00260       : comm_ (Teuchos::createMpiComm<int> (Teuchos::opaqueWrapper<MPI_Comm> (rawMpiComm))),
00261         node_ (node)
00262     {
00263       // mfh 29 Jun 2014: Don't initialize the Node yet.  See above note.
00264       //
00265       // if (node_.is_null ()) {
00266       //   node_ = KokkosClassic::Details::getNode<NodeType> ();
00267       // }
00268     }
00269 
00284     MpiPlatform (int* argc,
00285                  char*** argv,
00286                  const Teuchos::RCP<NodeType>& node,
00287                  MPI_Comm rawMpiComm)
00288       : comm_ (Teuchos::null),
00289         node_ (node)
00290     {
00291       initialize (argc, argv, rawMpiComm);
00292       comm_ = getDefaultComm ();
00293 
00294       // mfh 29 Jun 2014: Don't initialize the Node yet.  See above note.
00295       //
00296       // if (node_.is_null ()) {
00297       //   node_ = KokkosClassic::Details::getNode<NodeType> ();
00298       // }
00299     }
00300 
00302     virtual ~MpiPlatform () {}
00303 
00305 
00306 
00307 
00309     Teuchos::RCP<const Teuchos::Comm<int> > getComm () const {
00310       TEUCHOS_TEST_FOR_EXCEPTION(
00311         comm_.is_null (), std::logic_error, "Tpetra::MpiPlatform::getComm: "
00312         "The default communicator is null.  This should never happen.  "
00313         "Please report this bug to the Tpetra developers.");
00314       return comm_;
00315     }
00316 
00324     Teuchos::RCP<NodeType> getNode () const {
00325       typedef MpiPlatform<NodeType> this_type;
00326       if (node_.is_null ()) {
00327         // NOTE (mfh 29 Jun 2014): Creating an instance of one of the
00328         // new Kokkos wrapper Nodes _must_ call Kokkos::initialize.
00329         // If Kokkos has not been initialized yet, this may result in
00330         // Kokkos being initialized correctly, since we have no way to
00331         // pass it the command-line arguments at this point.  This is
00332         // why we should prefer the *Platform constructors that take
00333         // argc and argv, since they can (and do) call
00334         // Kokkos::initialize (by calling Tpetra::initialize).
00335         //
00336         // mfh 29 Jun 2014: We're only keeping the *Platform classes
00337         // for backwards compatibility anyway, so I don't feel bad
00338         // about the const_cast here.
00339         const_cast<this_type*> (this)->node_ =
00340           KokkosClassic::Details::getNode<NodeType> ();
00341         TEUCHOS_TEST_FOR_EXCEPTION(
00342           node_.is_null (), std::logic_error, "Tpetra::MpiPlatform::getNode: "
00343           "KokkosClassic::Details::getNode<NodeType>() returned null.  "
00344           "This should never happen.  "
00345           "Please report this bug to the Tpetra developers.");
00346       }
00347       return node_;
00348     }
00349 
00351   protected:
00353     Teuchos::RCP<const Teuchos::Comm<int> > comm_;
00355     Teuchos::RCP<NodeType> node_;
00356 
00357   private:
00359     MpiPlatform (const MpiPlatform<NodeType>& platform);
00361     MpiPlatform& operator= (const MpiPlatform<NodeType>& platform);
00362   };
00363 
00372   template <>
00373   class MpiPlatform<KokkosClassic::DefaultNode::DefaultNodeType> :
00374     public Teuchos::Describable {
00375   public:
00377 
00378 
00380     typedef KokkosClassic::DefaultNode::DefaultNodeType NodeType;
00381 
00383 
00384 
00385 
00387     MpiPlatform ();
00388 
00394     MpiPlatform (int* argc, char*** argv);
00395 
00412     explicit MpiPlatform (const Teuchos::RCP<NodeType>& node);
00413 
00425     MpiPlatform (int* argc, char*** argv, const Teuchos::RCP<NodeType>& node);
00426 
00447     MpiPlatform (const Teuchos::RCP<NodeType>& node,
00448                  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm);
00449 
00473     MpiPlatform (int* argc,
00474                  char*** argv,
00475                  const Teuchos::RCP<NodeType>& node,
00476                  const Teuchos::RCP<const Teuchos::OpaqueWrapper<MPI_Comm> >& rawMpiComm);
00477 
00490     MpiPlatform (const Teuchos::RCP<NodeType>& node, MPI_Comm rawMpiComm);
00491 
00506     MpiPlatform (int* argc,
00507                  char*** argv,
00508                  const Teuchos::RCP<NodeType>& node,
00509                  MPI_Comm rawMpiComm);
00510 
00512     virtual ~MpiPlatform ();
00513 
00515 
00516 
00517 
00519     Teuchos::RCP<const Teuchos::Comm<int> > getComm () const;
00520 
00528     Teuchos::RCP<NodeType> getNode () const;
00529 
00531   private:
00533     MpiPlatform (const MpiPlatform<NodeType>& platform);
00534 
00536     MpiPlatform& operator= (const MpiPlatform<NodeType>& platform);
00537 
00538   protected:
00540     RCP<const Teuchos::Comm<int> > comm_;
00541 
00543     RCP<NodeType> node_;
00544   };
00545 
00546 } // namespace Tpetra
00547 
00548 #endif // TPETRA_MPIPLATFORM_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines