Amesos Package Browser (Single Doxygen Collection) Development
Amesos_Scalapack.h
Go to the documentation of this file.
00001 /*
00002   TODO:
00003   1)  Eliminate mb_ and grid_mb_  DONE 
00004 
00005   1)  Allow user level control over whether to use a 1D or 2D data 
00006       distribution.
00007   2)  Figure out how to redistribute the vectors.  (Allow no more than 
00008       nb vectors at a time for now).  Done - I think 
00009   3)  Move the code from software:MyExamples/TwodMap/TwoDMap.cpp in to
00010       this routine (for the 2D case) Done - I think
00011   4)  Create the ScaLAPACK 2D dense matrix.  Done - I think 
00012  */
00013 
00014 // @HEADER
00015 // ***********************************************************************
00016 // 
00017 //                Amesos: Direct Sparse Solver Package
00018 //                 Copyright (2004) Sandia Corporation
00019 // 
00020 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00021 // license for use of this work by or on behalf of the U.S. Government.
00022 // 
00023 // This library is free software; you can redistribute it and/or modify
00024 // it under the terms of the GNU Lesser General Public License as
00025 // published by the Free Software Foundation; either version 2.1 of the
00026 // License, or (at your option) any later version.
00027 //  
00028 // This library is distributed in the hope that it will be useful, but
00029 // WITHOUT ANY WARRANTY; without even the implied warranty of
00030 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00031 // Lesser General Public License for more details.
00032 //  
00033 // You should have received a copy of the GNU Lesser General Public
00034 // License along with this library; if not, write to the Free Software
00035 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00036 // USA
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00038 // 
00039 // ***********************************************************************
00040 // @HEADER
00041 
00042 #ifndef _AMESOS_SCALAPACK_H_
00043 #define _AMESOS_SCALAPACK_H_
00044 
00045 #include "Amesos_ConfigDefs.h"
00046 #include "Amesos_BaseSolver.h"
00047 #include "Amesos_NoCopiable.h"
00048 #include "Amesos_Utils.h"
00049 #include "Amesos_Time.h"
00050 #include "Amesos_Status.h"
00051 #include "Amesos_Control.h"
00052 #include "Epetra_LinearProblem.h"
00053 #include "Epetra_Time.h"
00054 #ifdef EPETRA_MPI
00055 #include "Epetra_MpiComm.h"
00056 #else
00057 #include "Epetra_Comm.h"
00058 #endif
00059 #include "Epetra_CrsGraph.h"
00060 
00062 
00146 class Amesos_Scalapack: public Amesos_BaseSolver,
00147   private Amesos_Time,
00148   private Amesos_NoCopiable,
00149   private Amesos_Utils,
00150   private Amesos_Control, 
00151   private Amesos_Status 
00152  { 
00153 
00154 public: 
00155 
00157 
00158 
00165   Amesos_Scalapack( const Epetra_LinearProblem& LinearProblem );
00166 
00168 
00170   ~Amesos_Scalapack(void);
00172 
00174 
00176 
00183     int SymbolicFactorization() ;
00184 
00186 
00211     int NumericFactorization() ;
00212 
00214 
00234     int Solve();
00235 
00237   
00239 
00240 #if 0
00241 
00242   char * Label() const {return(Epetra_Object::Label());};
00243 #endif
00244     
00246   const Epetra_LinearProblem *GetProblem() const { return(Problem_); };
00247 
00249 
00252   bool MatrixShapeOK() const ;
00253 
00255 
00267   int SetUseTranspose(bool UseTranspose) {UseTranspose_ = UseTranspose; return(0);};
00268 
00270   bool UseTranspose() const {return(UseTranspose_);};
00271 
00273   const Epetra_Comm & Comm() const {return(GetProblem()->GetOperator()->Comm());};
00274 
00276 
00292   int SetParameters( Teuchos::ParameterList &ParameterList ) ;
00293 
00295   int NumSymbolicFact() const { return( Amesos_Status::NumSymbolicFact_ ); }
00296 
00298   int NumNumericFact() const { return( Amesos_Status::NumNumericFact_ ); }
00299 
00301   int NumSolve() const { return( Amesos_Status::NumSolve_ ); }
00302 
00304   void PrintTiming() const;
00305   
00307   void PrintStatus() const;
00308  
00310   void GetTiming( Teuchos::ParameterList &TimingParameterList ) const { Amesos_Time::GetTiming(TimingParameterList); }
00311  
00313 
00314  private:  
00315   /*
00316   RedistributeA - Convert matrix to a dense ScaLAPACK matrix
00317     Preconditions:
00318       Problem_ must be set 
00319       SetParameters() 
00320       ScaLAPACK1DMap and ScaLAPACK1DMatrix must either be 0 or be pointers to 
00321         appropriatly allocate objects.  If they are non-zero, those objects
00322   will be deleted and recreated.  
00323   
00324     Postconditions:
00325       nprow_, npcol_, DescA_ 
00326       RowMatrixA_ 
00327       ScaLAPACK1DMap_ 
00328       ScaLAPACK1DMatrix_ 
00329       ImportToScaLAPACK1D_
00330       ImportBackToOriginal_
00331 
00332    */
00333   int RedistributeA();
00334 
00335   /*
00336     ConvertToScalapack - Convert matirx to form expected by Scalapack: Ai, Ap, Aval
00337     Preconditions:
00338       Problem_ 
00339     Postconditions:
00340       nprow_, npcol_, 
00341   */
00342   int ConvertToScalapack();     
00343 
00344   /*
00345     PerformNumericFactorization - Call Scalapack to perform numeric factorization
00346     Preconditions:
00347 
00348     Postconditions:
00349       DenseA_, DescA_
00350   */
00351   int PerformNumericFactorization(); 
00352 
00353  protected:
00354 
00355   int iam_;                              //  Process number (i.e. Comm().MyPID() 
00356   
00357   int NumGlobalElements_;                //  Number of rows and columns in the Problem_->GetOperator()
00358   int NumGlobalNonzeros_; 
00359 
00360   //
00361   //  The following variables are required for the ScaLAPACK interface:
00362   //
00363   int nprow_ ;                           //  number of process rows: 1 for now
00364   int npcol_ ;                           //  number of process columns
00365   int ictxt_ ;                           //  BLACS context
00366   int m_per_p_;                          //  Number of columns per process
00367   int DescA_[10];                        //  ScaLAPACK array descriptor 
00368 
00369   Epetra_Map *ScaLAPACK1DMap_ ;          //  Points to a 1D Map which matches a ScaLAPACK 1D
00370                                          //  blocked (not block cyclic) distribution
00371   Epetra_CrsMatrix *ScaLAPACK1DMatrix_ ; //  Points to a  ScaLAPACK 1D
00372                                          //  blocked (not block cyclic) distribution
00373   Epetra_Map *VectorMap_ ;               //  Points to a Map for vectors X and B
00374   std::vector<double> DenseA_;                //  The data in a ScaLAPACK 1D blocked format
00375   std::vector<int> Ipiv_ ;                    //  ScaLAPACK pivot information
00376   int NumOurRows_ ;
00377   int NumOurColumns_ ;
00378  
00379 
00380   bool UseTranspose_;     
00381   const Epetra_LinearProblem * Problem_;
00382   
00383   
00384   // some timing internal to ScaLAPACK
00385   double ConTime_;                        // time to convert to ScaLAPACKformat
00386   double SymTime_;                        // time for symbolic factorization
00387   double NumTime_;                        // time for numeric factorization
00388   double SolTime_;                        // time for solution
00389   double VecTime_;                        // time to redistribute vectors
00390   double MatTime_;                        // time to redistribute matrix
00391   
00392   //
00393   //  Control of the data distribution
00394   //
00395   bool TwoD_distribution_;  // True if 2D data distribution is used
00396   int grid_nb_;             // Row and Column blocking factor (only used in 2D distribution)  
00397   int mypcol_;              // Process column in the ScaLAPACK2D grid
00398   int myprow_;              // Process row in the ScaLAPACK2D grid
00399   Epetra_CrsMatrix* FatOut_;//
00400 
00401   //
00402   //  Blocking factors (For both 1D and 2D data distributions)
00403   //
00404   int nb_;
00405   int lda_;
00406 
00407   Epetra_Time * Time_;
00408   
00409 };  // End of  class Amesos_Scalapack  
00410 #endif /* _AMESOS_SCALAPACK_H_ */
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines