Tpetra_DistObject.hpp

// @HEADER
// ***********************************************************************
// 
//          Tpetra: Templated Linear Algebra Services Package
//                 Copyright (2004) Sandia Corporation
// 
// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
// license for use of this work by or on behalf of the U.S. Government.
// 
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//  
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//  
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
// USA
// Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
// 
// ***********************************************************************
// @HEADER

#ifndef TPETRA_DISTOBJECT_HPP
#define TPETRA_DISTOBJECT_HPP

#include "Tpetra_ConfigDefs.hpp"
#include "Tpetra_Map.hpp"
#include "Tpetra_CombineMode.hpp"
#include "Tpetra_Import.hpp"
#include "Tpetra_Export.hpp"
#include "Tpetra_Distributor.hpp"

#include <Teuchos_OrdinalTraits.hpp>
#include <Teuchos_RCP.hpp>
#include <Teuchos_Comm.hpp>

namespace Tpetra {


  template<typename Ordinal, typename Scalar>
  class DistObject : public Teuchos::Object {

  public:



    DistObject(const Map<Ordinal>& map, Teuchos::RCP<const Teuchos::Comm<Ordinal> > comm);

    DistObject(const Map<Ordinal>& map, 
           Teuchos::RCP<const Teuchos::Comm<Ordinal> > comm,
           const std::string & Label);

    DistObject(const DistObject<Ordinal, Scalar>& source);

    virtual ~DistObject();




    void doImport(const DistObject<Ordinal, Scalar> & source, 
                  const Import<Ordinal> & importer, CombineMode CM);

    void doExport(const DistObject<Ordinal, Scalar> & dest, 
                  const Export<Ordinal> & exporter, CombineMode CM);

    void doImport(const DistObject<Ordinal, Scalar> & source,
                  const Export<Ordinal> & exporter, CombineMode CM);

    void doExport(const DistObject<Ordinal, Scalar> & dest,
                  const Import<Ordinal> & importer, CombineMode CM);




    inline bool isDistributed() const;

    inline const Map<Ordinal> & getMap() const;





    void print(std::ostream &os) const;


  protected:

    virtual void doTransfer(const DistObject<Ordinal,Scalar> &source,
                            CombineMode CM,
                            Ordinal numSameIDs,
                            const Teuchos::ArrayView<const Ordinal> &permuteToLIDs,
                            const Teuchos::ArrayView<const Ordinal> &permuteFromLIDs,
                            const Teuchos::ArrayView<const Ordinal> &remoteLIDs,
                            const Teuchos::ArrayView<const Ordinal> &exportLIDs,
                            Distributor<Ordinal> &distor,
                            bool doReverse);

    // The following four methods must be implemented by the derived class


    virtual bool checkSizes(const DistObject<Ordinal, Scalar> & source, Ordinal &packetSize) = 0;


    virtual void copyAndPermute(const DistObject<Ordinal, Scalar> & source,
                                      Ordinal numSameIDs,
                                const Teuchos::ArrayView<const Ordinal> &permuteToLIDs,
                                const Teuchos::ArrayView<const Ordinal> &permuteFromLIDs) = 0;


    virtual void packAndPrepare(const DistObject<Ordinal,Scalar> & source,
                                const Teuchos::ArrayView<const Ordinal> &exportLIDs,
                                const Teuchos::ArrayView<Scalar> &exports,
                                Distributor<Ordinal> &distor) = 0;


    virtual void unpackAndCombine(const Teuchos::ArrayView<const Ordinal> &importLIDs,
                                  const Teuchos::ArrayView<const Scalar> &imports,
                                  Distributor<Ordinal> &distor,
                                  CombineMode CM) = 0;

  private:

    const Map<Ordinal> map_;
    Teuchos::RCP<const Teuchos::Comm<Ordinal> > Comm_;
    // buffers into which packed data is imported
    Teuchos::Array<Scalar> imports_;
    // buffers from which packed data is exported
    Teuchos::Array<Scalar> exports_;

  }; // class DistObject

  template <typename Ordinal, typename Scalar>
  DistObject<Ordinal,Scalar>::DistObject(const Map<Ordinal>& map, Teuchos::RCP<const Teuchos::Comm<Ordinal> > comm)
  : Teuchos::Object("Tpetra::DistObject")
  , map_(map)
  , Comm_(comm)
  {}

  template <typename Ordinal, typename Scalar>
  DistObject<Ordinal,Scalar>::DistObject(const Map<Ordinal>& map, 
      Teuchos::RCP<const Teuchos::Comm<Ordinal> > comm, const std::string & Label)
  : Teuchos::Object(Label.c_str())
  , map_(map)
  , Comm_(comm)
  {}

  template <typename Ordinal, typename Scalar>
  DistObject<Ordinal,Scalar>::DistObject(const DistObject<Ordinal, Scalar>& source)
  : Teuchos::Object(source.label())
  , map_(source.map_)
  , Comm_(source.Comm_)
  {}

  template <typename Ordinal, typename Scalar>
  DistObject<Ordinal,Scalar>::~DistObject() 
  {}

  template <typename Ordinal, typename Scalar>
  void DistObject<Ordinal,Scalar>::doImport(const DistObject<Ordinal,Scalar> & source, 
                                            const Import<Ordinal> & importer, CombineMode CM) 
  {
    TEST_FOR_EXCEPTION( getMap() != importer.getTargetMap(), std::runtime_error, "Target Maps don't match.");
    TEST_FOR_EXCEPTION( source.getMap() != importer.getSourceMap(), std::runtime_error, "Source Maps don't match.");
    Ordinal numSameIDs = importer.getNumSameIDs();
    const Teuchos::ArrayView<const Ordinal> exportLIDs      = importer.getExportLIDs();
    const Teuchos::ArrayView<const Ordinal> remoteLIDs      = importer.getRemoteLIDs();
    const Teuchos::ArrayView<const Ordinal> permuteToLIDs   = importer.getPermuteToLIDs();
    const Teuchos::ArrayView<const Ordinal> permuteFromLIDs = importer.getPermuteFromLIDs();
    this->doTransfer(source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
                     importer.getDistributor(), false);
  }

  template <typename Ordinal, typename Scalar>
  void DistObject<Ordinal,Scalar>::doExport(const DistObject<Ordinal,Scalar> & dest, 
                                            const Export<Ordinal> & exporter, CombineMode CM) 
  {
    TEST_FOR_EXCEPTION( getMap() != exporter.getTargetMap(), std::runtime_error, "Target Maps don't match.");
    TEST_FOR_EXCEPTION( dest.getMap() != exporter.getSourceMap(), std::runtime_error, "Source Maps don't match.");
    Ordinal numSameIDs = exporter.getNumSameIDs();
    Teuchos::ArrayView<const Ordinal> exportLIDs      = exporter.getExportLIDs();
    Teuchos::ArrayView<const Ordinal> remoteLIDs      = exporter.getRemoteLIDs();
    Teuchos::ArrayView<const Ordinal> permuteToLIDs   = exporter.getPermuteToLIDs();
    Teuchos::ArrayView<const Ordinal> permuteFromLIDs = exporter.getPermuteFromLIDs();
    doTransfer(dest, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
               exporter.getDistributor(), false);
  }

  template <typename Ordinal, typename Scalar>
  void DistObject<Ordinal,Scalar>::doImport(const DistObject<Ordinal,Scalar> & source,
                                            const Export<Ordinal> & exporter, CombineMode CM)
  {
    TEST_FOR_EXCEPTION( getMap() != exporter.getTargetMap(), std::runtime_error, "Target Maps don't match.");
    TEST_FOR_EXCEPTION( source.getMap() != exporter.getSourceMap(), std::runtime_error, "Source Maps don't match.");
    Ordinal numSameIDs = exporter.getNumSameIDs();
    Teuchos::ArrayView<const Ordinal> exportLIDs      = exporter.getRemoteLIDs();
    Teuchos::ArrayView<const Ordinal> remoteLIDs      = exporter.getExportLIDs();
    Teuchos::ArrayView<const Ordinal> permuteToLIDs   = exporter.getPermuteFromLIDs();
    Teuchos::ArrayView<const Ordinal> permuteFromLIDs = exporter.getPermuteToLIDs();
    doTransfer(source, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
               exporter.getDistributor(), true);
  }

  template <typename Ordinal, typename Scalar>
  void DistObject<Ordinal,Scalar>::doExport(const DistObject<Ordinal, Scalar> & dest,
                                            const Import<Ordinal> & importer, CombineMode CM)
  {
    TEST_FOR_EXCEPTION( getMap() != importer.getTargetMap(), std::runtime_error, "Target Maps don't match.");
    TEST_FOR_EXCEPTION( dest.getMap() != importer.getSourceMap(), std::runtime_error, "Source Maps don't match.");
    Ordinal numSameIDs = importer.getNumSameIDs();
    Teuchos::ArrayView<const Ordinal> exportLIDs      = importer.getRemoteLIDs();
    Teuchos::ArrayView<const Ordinal> remoteLIDs      = importer.getExportLIDs();
    Teuchos::ArrayView<const Ordinal> permuteToLIDs   = importer.getPermuteFromLIDs();
    Teuchos::ArrayView<const Ordinal> permuteFromLIDs = importer.getPermuteToLIDs();
    doTransfer(dest, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
               importer.getDistributor(), true);
  }

  template <typename Ordinal, typename Scalar>
  bool DistObject<Ordinal,Scalar>::isDistributed() const 
  {
    return map_.isDistributed();
  }

  template <typename Ordinal, typename Scalar>
  const Map<Ordinal> & DistObject<Ordinal,Scalar>::getMap() const 
  {
    return map_;
  }

  template <typename Ordinal, typename Scalar>
  void DistObject<Ordinal,Scalar>::doTransfer(
      const DistObject<Ordinal, Scalar> & source,
      CombineMode CM,
      Ordinal numSameIDs, 
      const Teuchos::ArrayView<const Ordinal> &permuteToLIDs, 
      const Teuchos::ArrayView<const Ordinal> &permuteFromLIDs,
      const Teuchos::ArrayView<const Ordinal> &remoteLIDs,    
      const Teuchos::ArrayView<const Ordinal> &exportLIDs,
      Distributor<Ordinal> &distor, bool doReverse) 
  {
    Ordinal packetSize;
    TEST_FOR_EXCEPTION( checkSizes(source,packetSize) == false, std::runtime_error, 
        "Tpetra::DistObject::doTransfer(): checkSizes() indicates that DistOjbects are not size-compatible.");
    Ordinal sbufLen = exportLIDs.size()*packetSize;
    Ordinal rbufLen = remoteLIDs.size()*packetSize;
    exports_.resize(sbufLen);
    imports_.resize(rbufLen);
    if (numSameIDs + permuteToLIDs.size()) {
      copyAndPermute(source,numSameIDs,permuteToLIDs,permuteFromLIDs);
    }
    packAndPrepare(source,exportLIDs,exports_(),distor);
    if ((isDistributed() && doReverse) || (source.isDistributed() && !doReverse)) 
    {
      // call one of the doPostsAndWaits functions
      if (doReverse) {
        distor.doReversePostsAndWaits(exports_().getConst(),packetSize,imports_());
      }
      else {
        distor.doPostsAndWaits(exports_().getConst(),packetSize,imports_());
      }
      unpackAndCombine(remoteLIDs,imports_(),distor,CM);
    }
  }

  template <typename Ordinal, typename Scalar>
  void DistObject<Ordinal,Scalar>::print(std::ostream &os) const
  {
    using std::endl;
    os << "Tpetra::DistObject" << endl
       << " export buffer size: " << exports_.size() << endl
       << " import buffer size: " << imports_.size() << endl
       << "Map:" << endl
       << map_;
  }

} // namespace Tpetra

#endif /* TPETRA_DISTOBJECT_HPP */

---