FEI Version of the Day
fei_Graph_Impl.cpp
00001 /*--------------------------------------------------------------------*/
00002 /*    Copyright 2005 Sandia Corporation.                              */
00003 /*    Under the terms of Contract DE-AC04-94AL85000, there is a       */
00004 /*    non-exclusive license for use of this work by or on behalf      */
00005 /*    of the U.S. Government.  Export of this program may require     */
00006 /*    a license from the United States Government.                    */
00007 /*--------------------------------------------------------------------*/
00008 
00009 #include <fei_macros.hpp>
00010 
00011 #include <fei_Graph_Impl.hpp>
00012 #include <fei_EqnComm.hpp>
00013 #include <fei_CommUtils.hpp>
00014 #include <fei_TemplateUtils.hpp>
00015 #include <fei_VectorSpace.hpp>
00016 
00017 #undef fei_file
00018 #define fei_file "fei_Graph_Impl.cpp"
00019 #include <fei_ErrMacros.hpp>
00020 
00021 //----------------------------------------------------------------------------
00022 fei::Graph_Impl::Graph_Impl(MPI_Comm comm, int firstLocalRow, int lastLocalRow)
00023   : localGraphData_(NULL),
00024     remoteGraphData_(),
00025     eqnComm_(),
00026     firstLocalRow_(firstLocalRow),
00027     lastLocalRow_(lastLocalRow),
00028     localProc_(0),
00029     numProcs_(1),
00030     comm_(comm)
00031 {
00032   localProc_ = fei::localProc(comm_);
00033   numProcs_  = fei::numProcs(comm_);
00034   //for remoteGraphData_, we don't know what the range of row-numbers will
00035   //be, so we'll just construct it with -1,-1
00036   remoteGraphData_.resize(numProcs_);
00037   for(int p=0; p<numProcs_; ++p) {
00038     remoteGraphData_[p] = new remote_table_type(-1, -1);
00039   }
00040   eqnComm_.reset(new fei::EqnComm(comm_, lastLocalRow-firstLocalRow+1));
00041   localGraphData_       = new table_type(firstLocalRow_, lastLocalRow_);
00042 }
00043 
00044 //----------------------------------------------------------------------------
00045 fei::Graph_Impl::~Graph_Impl()
00046 {
00047   delete localGraphData_;
00048   for(unsigned i=0; i<remoteGraphData_.size(); ++i) {
00049     delete remoteGraphData_[i];
00050   }
00051 }
00052 
00053 //----------------------------------------------------------------------------
00054 int fei::Graph_Impl::addIndices(int row, int len, const int* indices)
00055 {
00056   if (row < 0) {
00057     return(-1);
00058   }
00059 
00060   if (row < firstLocalRow_ || row > lastLocalRow_) {
00061     int p = eqnComm_->getOwnerProc(row);
00062     remoteGraphData_[p]->addIndices(row, len, indices);
00063   }
00064   else localGraphData_->addIndices(row, len, indices);
00065 
00066   return(0);
00067 }
00068 
00069 //----------------------------------------------------------------------------
00070 int fei::Graph_Impl::addSymmetricIndices(int numIndices, int* indices,
00071           bool diagonal)
00072 {
00073   if (diagonal) {
00074     addDiagonals(numIndices, indices);
00075     return(0);
00076   }
00077 
00078   bool all_local = true;
00079   if (numProcs_ > 1) {
00080     for(int i=0; i<numIndices; ++i) {
00081       if (indices[i] < 0) {
00082   return(-1);
00083       }
00084 
00085       bool local = true;
00086       if (indices[i] < firstLocalRow_ || indices[i] > lastLocalRow_) {
00087     all_local = false;
00088     local = false;
00089       }
00090 
00091       if (!local) {
00092         int p = eqnComm_->getOwnerProc(indices[i]);
00093   remoteGraphData_[p]->addIndices(indices[i], numIndices, indices);
00094       }
00095     }
00096   }
00097 
00098   if (all_local) {
00099     localGraphData_->addIndices(numIndices, indices,
00100           numIndices, indices);
00101   }
00102   else {
00103     for(int i=0; i<numIndices; ++i) {
00104       if (indices[i] >= firstLocalRow_ && indices[i] <= lastLocalRow_) {
00105     localGraphData_->addIndices(indices[i], numIndices, indices);
00106       }
00107     }
00108   }
00109 
00110   return(0);
00111 }
00112 
00113 //----------------------------------------------------------------------------
00114 void fei::Graph_Impl::addDiagonals(int numIndices, int* indices)
00115 {
00116   bool all_local = true;
00117   int i;
00118   if (numProcs_ > 1) {
00119     for(i=0; i<numIndices; ++i) {
00120       int ind = indices[i];
00121       if (ind < 0) {
00122         throw std::runtime_error("fei::Graph_Impl::addDiagonals given negative index");
00123       }
00124 
00125       bool local = true;
00126       if (ind < firstLocalRow_ || ind > lastLocalRow_) {
00127     all_local = false;
00128     local = false;
00129       }
00130 
00131       if (!local) {
00132         int p=eqnComm_->getOwnerProc(ind);
00133   remoteGraphData_[p]->addIndices(ind, 1, &ind);
00134       }
00135     }
00136   }
00137 
00138   if (all_local) {
00139     localGraphData_->addDiagonals(numIndices, indices);
00140   }
00141   else {
00142     for(i=0; i<numIndices; ++i) {
00143       int ind = indices[i];
00144       if (ind >= firstLocalRow_ && ind <= lastLocalRow_) {
00145     localGraphData_->addIndices(ind, 1, &ind);
00146       }
00147     }
00148   }
00149 }
00150 
00151 //----------------------------------------------------------------------------
00152 int fei::Graph_Impl::writeLocalGraph(FEI_OSTREAM& os, bool debug,
00153             bool prefixLinesWithPoundSign)
00154 {
00155   if (localGraphData_ == NULL) {
00156     if (prefixLinesWithPoundSign) {
00157       os << "# fei::Graph_Impl::writeLocalGraph numRows: 0"<<FEI_ENDL;
00158     }
00159     else {
00160       os << "0" << FEI_ENDL;
00161     }
00162     return(0);
00163   }
00164 
00165   if (prefixLinesWithPoundSign) {
00166     os << "# fei::Graph_Impl::writeLocalGraph numRows: ";
00167   }
00168   os << localGraphData_->getMap().size() <<FEI_ENDL;
00169 
00170   if (prefixLinesWithPoundSign) {
00171     writeToStream(*localGraphData_, os, "# ");
00172   }
00173   else {
00174     writeToStream(*localGraphData_, os, NULL);
00175   }
00176 
00177   return( 0 );
00178 }
00179 
00180 //----------------------------------------------------------------------------
00181 int fei::Graph_Impl::writeRemoteGraph(FEI_OSTREAM& os)
00182 {
00183   for(unsigned p=0; p<remoteGraphData_.size(); ++p) {
00184     writeToStream(*remoteGraphData_[p], os, "# ");
00185   }
00186 
00187   return( 0 );
00188 }
00189 
00190 //----------------------------------------------------------------------------
00191 int fei::Graph_Impl::gatherFromOverlap()
00192 {
00193   if (numProcs_ == 1) return(0);
00194 
00195 #ifndef FEI_SER
00196   //this function gathers shared-but-not-owned data onto the
00197   //owning processors.
00198 
00199   //iterate the remoteGraphData_ array and create a list of processors
00200   //that we will be sending data to. (processors which own graph rows
00201   //that we share.)
00202   std::vector<int> sendProcs;
00203   for(unsigned i=0; i<remoteGraphData_.size(); ++i) {
00204     if ((int)i == localProc_) continue;
00205     if (remoteGraphData_[i] != NULL) {
00206       if (remoteGraphData_[i]->getMap().size() == 0) {
00207         continue;
00208       }
00209 
00210       sendProcs.push_back((int)i);
00211     }
00212   }
00213 
00214   //now we can find out which procs we'll be receiving from.
00215   std::vector<int> recvProcs;
00216   fei::mirrorProcs(comm_, sendProcs, recvProcs);
00217 
00218   //next we'll declare arrays to receive into.
00219   std::vector<std::vector<int> > recv_ints(recvProcs.size());
00220 
00221   //...and an array for the sizes of the recv buffers:
00222   std::vector<int> recv_sizes(recvProcs.size());
00223   std::vector<MPI_Request> mpiReqs(recvProcs.size());
00224   std::vector<MPI_Status> mpiStatuses(recvProcs.size());
00225 
00226   int tag1 = 11113;
00227 
00228   unsigned offset = 0;
00229   for(unsigned i=0; i<recvProcs.size(); ++i) {
00230     MPI_Irecv(&recv_sizes[i], 1, MPI_INT, recvProcs[i],
00231               tag1, comm_, &mpiReqs[i]);
00232   }
00233 
00234   //now we'll pack our to-be-sent data into buffers, and send the
00235   //sizes to the receiving procs:
00236   std::vector<std::vector<int> > send_ints(sendProcs.size());
00237 
00238   for(unsigned i=0; i<sendProcs.size(); ++i) {
00239     int proc = sendProcs[i];
00240 
00241     fei::packRaggedTable(*(remoteGraphData_[proc]), send_ints[i]);
00242 
00243     int isize = send_ints[i].size();
00244 
00245     MPI_Send(&isize, 1, MPI_INT, proc, tag1, comm_);
00246   }
00247 
00248   if (mpiReqs.size() > 0) {
00249     MPI_Waitall(mpiReqs.size(), &mpiReqs[0], &mpiStatuses[0]);
00250   }
00251 
00252   //now resize our recv buffers, and post the recvs.
00253   for(size_t i=0; i<recvProcs.size(); ++i) {
00254     int intsize = recv_sizes[i];
00255 
00256     recv_ints[i].resize(intsize);
00257 
00258     MPI_Irecv(&(recv_ints[i][0]), intsize, MPI_INT, recvProcs[i],
00259               tag1, comm_, &mpiReqs[i]);
00260   }
00261 
00262   //now send our packed buffers.
00263   for(size_t i=0; i<sendProcs.size(); ++i) {
00264     int proc = sendProcs[i];
00265 
00266     MPI_Send(&(send_ints[i][0]), send_ints[i].size(), MPI_INT,
00267              proc, tag1, comm_);
00268   }
00269 
00270   if (mpiReqs.size() > 0) {
00271     MPI_Waitall(mpiReqs.size(), &mpiReqs[0], &mpiStatuses[0]);
00272   }
00273 
00274   for(unsigned i=0; i<recvProcs.size(); ++i) {
00275     std::vector<int> recvdata = recv_ints[i];
00276     int numRows = recvdata[0];
00277     int* rowNumbers = &recvdata[1];
00278     int* rowLengths = rowNumbers+numRows;
00279     int* packedCols = rowLengths+numRows;
00280     offset = 0;
00281     for(int r=0; r<numRows; ++r) {
00282       addIndices(rowNumbers[r], rowLengths[r], &packedCols[offset]);
00283       offset += rowLengths[r];
00284     }
00285   }
00286 
00287 #endif
00288   return(0);
00289 }
00290 
00291 //----------------------------------------------------------------------------
00292 int fei::Graph_Impl::getLocalRowLength(int row)
00293 {
00294   table_row_type* colIndices = localGraphData_->getRow(row);
00295   if (colIndices == NULL) {
00296     return(-1);
00297   }
00298 
00299   return(colIndices->size());
00300 }
00301 
00302 //----------------------------------------------------------------------------
00303 int fei::Graph_Impl::getNumLocalRows()
00304 {
00305   int numLocalRows = localGraphData_->getMap().size();
00306 
00307   return(numLocalRows);
00308 }
00309 
00310 //----------------------------------------------------------------------------
00311 int fei::Graph_Impl::getNumLocalNonzeros() const
00312 {
00313   int numNonzeros = fei::countNonzeros(*localGraphData_);
00314 
00315   return(numNonzeros);
00316 }
00317 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Friends