Tpetra Matrix/Vector Services Version of the Day
Tpetra_DistObject.hpp
00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
00008 // the U.S. Government retains certain rights in this software.
00009 // 
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00038 // 
00039 // ************************************************************************
00040 // @HEADER
00041 
00042 #ifndef TPETRA_DISTOBJECT_HPP
00043 #define TPETRA_DISTOBJECT_HPP
00044 
00045 #include "Tpetra_ConfigDefs.hpp"
00046 #include "Tpetra_Map.hpp"
00047 #include "Tpetra_Import.hpp"
00048 #include "Tpetra_Export.hpp"
00049 #include "Tpetra_Distributor.hpp"
00050 
00051 #include <Kokkos_DefaultNode.hpp>
00052 #include <Teuchos_Describable.hpp>
00053 
00054 namespace Tpetra {
00055 
00113   template <class Packet, 
00114             class LocalOrdinal = int, 
00115             class GlobalOrdinal = LocalOrdinal, 
00116             class Node = Kokkos::DefaultNode::DefaultNodeType>
00117   class DistObject : virtual public Teuchos::Describable {
00118   public:
00120 
00121 
00123     explicit DistObject (const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& map);
00124 
00126     DistObject (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source);
00127 
00129     virtual ~DistObject ();
00130 
00132 
00133 
00134 
00136     void 
00137     doImport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source, 
00138               const Import<LocalOrdinal,GlobalOrdinal,Node>& importer, 
00139               CombineMode CM);
00140 
00142     void 
00143     doExport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest, 
00144               const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter, 
00145               CombineMode CM);
00146 
00148     void 
00149     doImport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source,
00150               const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter, 
00151               CombineMode CM);
00152 
00154     void 
00155     doExport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& dest,
00156               const Import<LocalOrdinal,GlobalOrdinal,Node>& importer, 
00157               CombineMode CM);
00158 
00160 
00161 
00162 
00168     inline bool isDistributed () const;
00169 
00171     inline const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& 
00172     getMap() const { return map_; }
00173 
00175 
00176 
00177 
00182     void print (std::ostream &os) const;
00183 
00185 
00186 
00188 
00189 
00190 
00195     virtual std::string description () const;
00196 
00197 
00202     virtual void 
00203     describe (Teuchos::FancyOStream &out, 
00204               const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const;
00206 
00207   protected:
00208 
00216     enum ReverseOption {
00217       DoForward, //*!< Perform the transfer in forward mode.
00218       DoReverse  //*!< Perform the transfer in reverse mode.
00219     };
00220 
00243     virtual void 
00244     doTransfer (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00245                 CombineMode CM,
00246                 size_t numSameIDs,
00247                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00248                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00249                 const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00250                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00251                 Distributor &distor,
00252                 ReverseOption revOp);
00253 
00264 
00265 
00269     virtual bool 
00270     checkSizes (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source) = 0;
00271 
00289     virtual void 
00290     copyAndPermute (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source,
00291                     size_t numSameIDs,
00292                     const Teuchos::ArrayView<const LocalOrdinal>& permuteToLIDs,
00293                     const Teuchos::ArrayView<const LocalOrdinal>& permuteFromLIDs) = 0;
00294 
00316     virtual void 
00317     packAndPrepare (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source,
00318                     const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
00319                     Teuchos::Array<Packet>& exports,
00320                     const Teuchos::ArrayView<size_t>& numPacketsPerLID,
00321                     size_t& constantNumPackets,
00322                     Distributor &distor) = 0;
00323 
00343     virtual void 
00344     unpackAndCombine (const Teuchos::ArrayView<const LocalOrdinal> &importLIDs,
00345                       const Teuchos::ArrayView<const Packet> &imports,
00346                       const Teuchos::ArrayView<size_t> &numPacketsPerLID,
00347                       size_t constantNumPackets,
00348                       Distributor &distor,
00349                       CombineMode CM) = 0;
00351     
00358     virtual void createViews () const {}
00359 
00375     virtual void createViewsNonConst (Kokkos::ReadWriteOption rwo) {}
00376 
00387     virtual void releaseViews () const {}
00388 
00390     Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > map_;
00391 
00392   private:
00394     Teuchos::Array<Packet> imports_;
00395 
00406     Teuchos::Array<size_t> numImportPacketsPerLID_;
00407 
00409     Teuchos::Array<Packet> exports_;
00410 
00421     Teuchos::Array<size_t> numExportPacketsPerLID_;
00422   }; // class DistObject
00423 
00424   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00425   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00426   DistObject (const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> >& map)
00427     : map_ (map)
00428   {}
00429 
00430   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00431   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00432   DistObject (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source)
00433     : map_ (source.map_)
00434   {}
00435 
00436   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00437   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::~DistObject() 
00438   {}
00439   
00440   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00441   std::string
00442   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::description () const
00443   {
00444     using Teuchos::TypeNameTraits;
00445 
00446     std::ostringstream os;
00447     os << "Tpetra::DistObject<"
00448        << TypeNameTraits<Packet>::name ()
00449        << ", " << TypeNameTraits<LocalOrdinal>::name ()
00450        << ", " << TypeNameTraits<GlobalOrdinal>::name ()
00451        << ", " << TypeNameTraits<Node>::name ()
00452        << ">";
00453     return os.str ();
00454   }
00455 
00456   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00457   void 
00458   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00459   describe (Teuchos::FancyOStream &out, 
00460             const Teuchos::EVerbosityLevel verbLevel) const 
00461   {
00462     using Teuchos::rcpFromRef;
00463     using std::endl;
00464 
00465     const Teuchos::EVerbosityLevel vl = (verbLevel == Teuchos::VERB_DEFAULT) ? 
00466       Teuchos::VERB_LOW : verbLevel;
00467 
00468     if (vl != Teuchos::VERB_NONE) {
00469       out << this->description () << endl;
00470       Teuchos::OSTab tab (rcpFromRef (out));
00471       out << "Export buffer size (in packets): " << exports_.size() << endl
00472           << "Import buffer size (in packets): " << imports_.size() << endl
00473           << "Map over which this object is distributed:" << endl;
00474       map_->describe (out, vl);
00475     }
00476   }
00477 
00478   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00479   void 
00480   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00481   doImport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00482             const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, 
00483             CombineMode CM) 
00484   {
00485     TEUCHOS_TEST_FOR_EXCEPTION(*getMap() != *importer.getTargetMap(), 
00486       std::invalid_argument, "doImport: The target DistObject's Map is not "
00487       "identical to the Import's target Map.");
00488     TEUCHOS_TEST_FOR_EXCEPTION(*A.getMap() != *importer.getSourceMap(), 
00489       std::invalid_argument, "doImport: The source DistObject's Map is not "
00490       "identical to the Import's source Map.");
00491     size_t numSameIDs = importer.getNumSameIDs();
00492 
00493     typedef ArrayView<const LocalOrdinal> view_type;
00494     const view_type exportLIDs      = importer.getExportLIDs();
00495     const view_type remoteLIDs      = importer.getRemoteLIDs();
00496     const view_type permuteToLIDs   = importer.getPermuteToLIDs();
00497     const view_type permuteFromLIDs = importer.getPermuteFromLIDs();
00498     this->doTransfer (A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, 
00499                       remoteLIDs, exportLIDs, importer.getDistributor (), 
00500                       DoForward);
00501   }
00502 
00503   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00504   void 
00505   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00506   doExport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00507             const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, 
00508             CombineMode CM) 
00509   {
00510     TEUCHOS_TEST_FOR_EXCEPTION(   *getMap() != *exporter.getTargetMap(), std::invalid_argument, 
00511       "doExport: The target DistObject's Map is not identical to the Export's target Map.");
00512     TEUCHOS_TEST_FOR_EXCEPTION( *A.getMap() != *exporter.getSourceMap(), std::invalid_argument, 
00513       "doExport: The source DistObject's Map is not identical to the Export's source Map.");
00514     size_t numSameIDs = exporter.getNumSameIDs();
00515 
00516     typedef ArrayView<const LocalOrdinal> view_type;
00517     view_type exportLIDs      = exporter.getExportLIDs();
00518     view_type remoteLIDs      = exporter.getRemoteLIDs();
00519     view_type permuteToLIDs   = exporter.getPermuteToLIDs();
00520     view_type permuteFromLIDs = exporter.getPermuteFromLIDs();
00521     doTransfer (A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, 
00522                 exportLIDs, exporter.getDistributor (), DoForward);
00523   }
00524 
00525   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00526   void 
00527   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00528   doImport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00529             const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, 
00530             CombineMode CM) 
00531   {
00532     TEUCHOS_TEST_FOR_EXCEPTION(  * getMap() != *exporter.getSourceMap(), std::invalid_argument,
00533       "doImport (with Export): The target DistObject's Map is not identical to the Export's source Map.");
00534     TEUCHOS_TEST_FOR_EXCEPTION( *A.getMap() != *exporter.getTargetMap(), std::invalid_argument,
00535       "doImport (with Export): The source DistObject's Map is not identical to the Export's target Map.");
00536     size_t numSameIDs = exporter.getNumSameIDs();
00537 
00538     typedef ArrayView<const LocalOrdinal> view_type;
00539     view_type exportLIDs      = exporter.getRemoteLIDs();
00540     view_type remoteLIDs      = exporter.getExportLIDs();
00541     view_type permuteToLIDs   = exporter.getPermuteFromLIDs();
00542     view_type permuteFromLIDs = exporter.getPermuteToLIDs();
00543     doTransfer (A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, 
00544                 exportLIDs, exporter.getDistributor (), DoReverse);
00545   }
00546 
00547   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00548   void 
00549   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00550   doExport (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00551             const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, 
00552             CombineMode CM) 
00553   {
00554     TEUCHOS_TEST_FOR_EXCEPTION( *getMap() != *importer.getSourceMap(), 
00555       std::invalid_argument, "doExport (with Import): The target object's Map "
00556       "is not identical to the Import's source Map.");
00557     TEUCHOS_TEST_FOR_EXCEPTION( *A.getMap() != *importer.getTargetMap(), 
00558       std::invalid_argument, "doExport (with Import): The source object's Map "
00559       "is not identical to the Import's target Map.");
00560     size_t numSameIDs = importer.getNumSameIDs();
00561 
00562     typedef ArrayView<const LocalOrdinal> view_type;
00563     view_type exportLIDs      = importer.getRemoteLIDs();
00564     view_type remoteLIDs      = importer.getExportLIDs();
00565     view_type permuteToLIDs   = importer.getPermuteFromLIDs();
00566     view_type permuteFromLIDs = importer.getPermuteToLIDs();
00567     doTransfer (A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, 
00568                 exportLIDs, importer.getDistributor (), DoReverse);
00569   }
00570 
00571   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00572   bool 
00573   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::isDistributed() const {
00574     return map_->isDistributed ();
00575   }
00576 
00577   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00578   void 
00579   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::
00580   doTransfer (const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>& source,
00581               CombineMode CM,
00582               size_t numSameIDs, 
00583               const Teuchos::ArrayView<const LocalOrdinal>& permuteToLIDs, 
00584               const Teuchos::ArrayView<const LocalOrdinal>& permuteFromLIDs,
00585               const Teuchos::ArrayView<const LocalOrdinal>& remoteLIDs,    
00586               const Teuchos::ArrayView<const LocalOrdinal>& exportLIDs,
00587               Distributor &distor, 
00588               ReverseOption revOp) 
00589   {
00590     using Teuchos::as;
00591 
00592     TEUCHOS_TEST_FOR_EXCEPTION( ! checkSizes(source), std::invalid_argument, 
00593       "Tpetra::DistObject::doTransfer(): checkSizes() indicates that the "
00594       "destination object is not a legal target for redistribution from the "
00595       "source object.  This probably means that they do not have the same "
00596       "dimensions.  For example, MultiVectors must have the same number of "
00597       "rows and columns.");
00598     Kokkos::ReadWriteOption rwo = Kokkos::ReadWrite;
00599     if (CM == INSERT || CM == REPLACE) {
00600       const size_t numIDsToWrite = 
00601         numSameIDs + permuteToLIDs.size() + remoteLIDs.size();
00602       if (numIDsToWrite == this->getMap()->getNodeNumElements()) {
00603         // We're overwriting all of our local data in the destination
00604         // object, so a write-only view suffices.
00605         //
00606         // FIXME (mfh 10 Apr 2012) This doesn't make sense for a
00607         // CrsMatrix with a dynamic graph.  INSERT mode could mean
00608         // that we're adding new entries to the object, but we don't
00609         // want to get rid of the old ones.
00610         rwo = Kokkos::WriteOnly;
00611       }
00612     }
00613     // Tell the source to create a read-only view of its data.  On a
00614     // discrete accelerator such as a GPU, this brings EVERYTHING from
00615     // device memory to host memory.
00616     //
00617     // FIXME (mfh 23 Mar 2012) By passing in the list of GIDs (or
00618     // rather, local LIDs to send) and packet counts, createViews()
00619     // could create a "sparse view" that only brings in the necessary
00620     // data from device to host memory.
00621     source.createViews();
00622 
00623     // Tell the target to create a view of its data.  Depending on
00624     // rwo, this could be a write-only view or a read-and-write view.
00625     // On a discrete accelerator such as a GPU, a write-only view only
00626     // requires a transfer from host to device memory.  A
00627     // read-and-write view requires a two-way transfer.  This has the
00628     // same problem as createViews(): it transfers EVERYTHING, not
00629     // just the necessary data.
00630     //
00631     // FIXME (mfh 23 Mar 2012) By passing in the list of GIDs (or
00632     // rather, local LIDs into which to receive) and packet counts,
00633     // createViewsNonConst() could create a "sparse view" that only
00634     // transfers the necessary data.
00635     this->createViewsNonConst(rwo); 
00636 
00637     if (numSameIDs + permuteToLIDs.size()) {
00638       // There is at least one GID to copy or permute.
00639       copyAndPermute (source, numSameIDs, permuteToLIDs, permuteFromLIDs);
00640     }
00641     size_t constantNumPackets = 0;
00642     numExportPacketsPerLID_.resize(exportLIDs.size());
00643     numImportPacketsPerLID_.resize(remoteLIDs.size());
00644 
00645     // Ask the source to pack data.  Also ask it whether there are a
00646     // constant number of packets per element (constantNumPackets is
00647     // an output argument).  If there are, constantNumPackets will
00648     // come back nonzero.  Otherwise, the source will fill the
00649     // numExportPacketsPerLID_ array.
00650     packAndPrepare (source, exportLIDs, exports_, numExportPacketsPerLID_(), 
00651                     constantNumPackets, distor);
00652 
00653     // We don't need the source's data anymore, so it can let go of
00654     // its views.  On a discrete accelerator, this frees host memory,
00655     // since device memory has the "master" version of the data.
00656     source.releaseViews();
00657 
00658     if (constantNumPackets != 0) {
00659       // There are a constant number of packets per element.  We
00660       // already know (from the number of "remote" (incoming)
00661       // elements) how many incoming elements we expect, so we can
00662       // resize the buffer accordingly.
00663       const size_t rbufLen = remoteLIDs.size() * constantNumPackets;
00664       if (as<size_t> (imports_.size()) != rbufLen) {
00665         imports_.resize (rbufLen);
00666       }
00667     }
00668     if ((isDistributed() && revOp == DoReverse) || 
00669         (source.isDistributed() && revOp == DoForward)) {
00670       // call one of the doPostsAndWaits functions
00671       if (revOp == DoReverse) {
00672         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00673           distor.doReversePostsAndWaits (numExportPacketsPerLID_().getConst(), 1,
00674                                          numImportPacketsPerLID_());
00675           size_t totalImportPackets = 0;
00676           for (Array_size_type i = 0; i < numImportPacketsPerLID_.size(); ++i) {
00677             totalImportPackets += numImportPacketsPerLID_[i];
00678           }
00679           imports_.resize(totalImportPackets);
00680           distor.doReversePostsAndWaits (exports_().getConst(),
00681                                          numExportPacketsPerLID_(),
00682                                          imports_(), 
00683                                          numImportPacketsPerLID_());
00684         }
00685         else {
00686           distor.doReversePostsAndWaits (exports_().getConst(),
00687                                          constantNumPackets,
00688                                          imports_());
00689         }
00690       }
00691       else { // revOp == DoForward
00692         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00693           distor.doPostsAndWaits (numExportPacketsPerLID_().getConst(), 1,
00694                                   numImportPacketsPerLID_());
00695           size_t totalImportPackets = 0;
00696           for (Array_size_type i = 0; i < numImportPacketsPerLID_.size(); ++i) {
00697             totalImportPackets += numImportPacketsPerLID_[i];
00698           }
00699           imports_.resize(totalImportPackets);
00700           distor.doPostsAndWaits (exports_().getConst(), 
00701                                   numExportPacketsPerLID_(),
00702                                   imports_(), 
00703                                   numImportPacketsPerLID_());
00704         }
00705         else {
00706           distor.doPostsAndWaits (exports_().getConst(), 
00707                                   constantNumPackets, 
00708                                   imports_());
00709         }
00710       }
00711       unpackAndCombine (remoteLIDs, imports_(), numImportPacketsPerLID_(), 
00712                         constantNumPackets, distor, CM);
00713     }
00714     this->releaseViews();
00715   }
00716 
00717   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00718   void 
00719   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::print (std::ostream &os) const
00720   {
00721     using Teuchos::FancyOStream;
00722     using Teuchos::getFancyOStream;
00723     using Teuchos::RCP;
00724     using Teuchos::rcpFromRef;
00725     using std::endl;
00726 
00727     RCP<FancyOStream> out = getFancyOStream (rcpFromRef (os));
00728     this->describe (*out, Teuchos::VERB_DEFAULT);
00729   }
00730 
00731 } // namespace Tpetra
00732 
00733 #endif /* TPETRA_DISTOBJECT_HPP */
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines