FEI Version of the Day
fei_Vector_core.cpp
00001 /*--------------------------------------------------------------------*/
00002 /*    Copyright 2005 Sandia Corporation.                              */
00003 /*    Under the terms of Contract DE-AC04-94AL85000, there is a       */
00004 /*    non-exclusive license for use of this work by or on behalf      */
00005 /*    of the U.S. Government.  Export of this program may require     */
00006 /*    a license from the United States Government.                    */
00007 /*--------------------------------------------------------------------*/
00008 
00009 #include "fei_fstream.hpp"
00010 
00011 #include "fei_Vector_core.hpp"
00012 #include "fei_VectorSpace.hpp"
00013 #include "fei_CSVec.hpp"
00014 #include "snl_fei_RecordCollection.hpp"
00015 #include "fei_TemplateUtils.hpp"
00016 #include "fei_impl_utils.hpp"
00017 
00018 #include <fstream>
00019 #include <sstream>
00020 #undef fei_file
00021 #define fei_file "fei_Vector_core.cpp"
00022 
00023 #include "fei_ErrMacros.hpp"
00024 
00025 fei::Vector_core::Vector_core(fei::SharedPtr<fei::VectorSpace> vecSpace,
00026                               int numLocalEqns)
00027   : eqnComm_(),
00028     vecSpace_(vecSpace),
00029     comm_(vecSpace->getCommunicator()),
00030     firstLocalOffset_(0),
00031     lastLocalOffset_(0),
00032     numLocal_(0),
00033     work_indices_(),
00034     work_indices2_(),
00035     haveFEVector_(false),
00036     remotelyOwned_(),
00037     sendProcs_(),
00038     recvProcs_(),
00039     recv_sizes_(),
00040     sendRecvProcsNeedUpdated_(true),
00041     overlapAlreadySet_(false),
00042     dbgprefix_("Vcore: ")
00043 {
00044   eqnComm_.reset(new fei::EqnComm(comm_,numLocalEqns));
00045   remotelyOwned_.resize(fei::numProcs(comm_));
00046   for(unsigned i=0; i<remotelyOwned_.size(); ++i) {
00047     remotelyOwned_[i] = new CSVec;
00048   }
00049 
00050   const std::vector<int>& offsets = eqnComm_->getGlobalOffsets();
00051   firstLocalOffset_ = offsets[fei::localProc(comm_)];
00052   lastLocalOffset_ = offsets[fei::localProc(comm_)+1] - 1;
00053   numLocal_ = lastLocalOffset_ - firstLocalOffset_ + 1;
00054 
00055   if (output_level_ >= fei::BRIEF_LOGS && output_stream_ != NULL) {
00056     FEI_OSTREAM& os = *output_stream_;
00057     os<<dbgprefix_<<" ctor firstLocal="<<firstLocalOffset_<<", lastLocal="
00058      <<lastLocalOffset_<<FEI_ENDL;
00059   }
00060 }
00061 
00062 fei::Vector_core::~Vector_core()
00063 {
00064   for(unsigned i=0; i<remotelyOwned_.size(); ++i) {
00065     delete remotelyOwned_[i];
00066   }
00067 }
00068 
00069 void fei::Vector_core::setOverlap(int numRemoteEqns,
00070                                   const int* remoteEqns)
00071 {
00072   if (numRemoteEqns == 0 && remoteEqns == NULL) {
00073     if (overlapAlreadySet_) return;
00074   }
00075 
00076   if (output_level_ >= fei::BRIEF_LOGS && output_stream_ != NULL) {
00077     FEI_OSTREAM& os = *output_stream_;
00078     os << dbgprefix_<<"setOverlap"<<FEI_ENDL;
00079   }
00080 
00081   int local_proc = fei::localProc(comm_);
00082 
00083   if (numRemoteEqns != 0 && remoteEqns != NULL) {
00084     for(int i=0; i<numRemoteEqns; ++i) {
00085       int proc = eqnComm_->getOwnerProc(remoteEqns[i]);
00086       if (proc == local_proc) continue;
00087 
00088       fei::add_entry(*remotelyOwned_[proc], remoteEqns[i], 0.0);
00089     }
00090   }
00091   else {
00092     std::vector<int> eqns;
00093     vecSpace_->getIndices_SharedAndOwned(eqns);
00094 
00095     for(size_t i=0; i<eqns.size(); ++i) {
00096       int proc = eqnComm_->getOwnerProc(eqns[i]);
00097       if (proc == local_proc) continue;
00098 
00099       fei::add_entry(*remotelyOwned_[proc], eqns[i], 0.0);
00100     }
00101   }
00102 
00103   overlapAlreadySet_ = true;
00104 }
00105 
00106 int fei::Vector_core::scatterToOverlap()
00107 {
00108   if (fei::numProcs(comm_) == 1 || haveFEVector()) {
00109     return(0);
00110   }
00111 
00112 #ifndef FEI_SER
00113   if (!overlapAlreadySet_) {
00114     setOverlap();
00115   }
00116 
00117   //...and now the overlap is whatever is in our remotelyOwned_ vectors.
00118 
00119 //  if (sendRecvProcsNeedUpdated_) {
00120     //first find out which procs we'll be receiving from.
00121     std::vector<int> recvProcs;
00122     for(unsigned i=0; i<remotelyOwned_.size(); ++i) {
00123       if ((int)i == fei::localProc(comm_)) continue;
00124       if (remotelyOwned_[i]->size() == 0) continue;
00125   
00126       recvProcs.push_back((int)i);
00127     }
00128   
00129     //find out the send-procs.
00130     std::vector<int> sendProcs;
00131     fei::mirrorProcs(comm_, recvProcs, sendProcs);
00132     sendRecvProcsNeedUpdated_ = false;
00133 //  }
00134 
00135   //declare arrays to send from, and corresponding sizes
00136   std::vector<std::vector<int> > send_ints(sendProcs.size());
00137   std::vector<std::vector<double> > send_doubles(sendProcs.size());
00138   std::vector<int> send_sizes(sendProcs.size());
00139 
00140   std::vector<MPI_Request> mpiReqs(sendProcs.size()+recvProcs.size());
00141   std::vector<MPI_Status> mpiStatuses(sendProcs.size()+recvProcs.size());
00142   int tag1 = 11111;
00143   int tag2 = 11112;
00144 
00145   //first, the procs we're going to send to, have to let us know
00146   //how much data we're supposed to send. So we have to receive
00147   //sizes and then indices from the "send"-procs.
00148   for(unsigned i=0; i<sendProcs.size(); ++i) {
00149     MPI_Irecv(&send_sizes[i], 1, MPI_INT, sendProcs[i],
00150               tag1, comm_, &mpiReqs[i]);
00151   }
00152 
00153   //now we'll send the sizes of our remotely-owned data to the
00154   //procs that we will be receiving the data from, and also the
00155   //indices that we want to receive.
00156   for(unsigned i=0; i<recvProcs.size(); ++i) {
00157     int proc = recvProcs[i];
00158 
00159     int size = remotelyOwned_[proc]->size();
00160     MPI_Send(&size, 1, MPI_INT, proc, tag1, comm_);
00161   }
00162  
00163   MPI_Waitall(sendProcs.size(), &mpiReqs[0], &mpiStatuses[0]);
00164 
00165   //now resize our send_ints and send_doubles arrays, and post the recvs
00166   //for indices that we're supposed to pack.
00167   for(unsigned i=0; i<sendProcs.size(); ++i) {
00168     int proc = sendProcs[i];
00169     int size = send_sizes[i];
00170     send_ints[i].resize(size);
00171     MPI_Irecv(&(send_ints[i][0]), size, MPI_INT, proc, tag1,
00172               comm_, &mpiReqs[i]);
00173     send_doubles[i].resize(size);
00174   }
00175 
00176   //now send the indices that we want to receive data for.
00177   for(unsigned i=0; i<recvProcs.size(); ++i) {
00178     int proc = recvProcs[i];
00179     int size = remotelyOwned_[proc]->size();
00180     int* indices = &(remotelyOwned_[proc]->indices())[0];
00181     MPI_Send(indices, size, MPI_INT, proc, tag1, comm_);
00182   }
00183 
00184   MPI_Waitall(sendProcs.size(), &mpiReqs[0], &mpiStatuses[0]);
00185 
00186   //now post our recvs.
00187   for(unsigned i=0; i<recvProcs.size(); ++i) {
00188     int proc = recvProcs[i];
00189     int size = remotelyOwned_[proc]->size();
00190     double* coefs = &(remotelyOwned_[proc]->coefs())[0];
00191     MPI_Irecv(coefs, size, MPI_DOUBLE, proc, tag2, comm_, &mpiReqs[i]);
00192   }
00193 
00194   //now pack and send the coefs that the other procs need from us.
00195   for(unsigned i=0; i<sendProcs.size(); ++i) {
00196     int proc = sendProcs[i];
00197 
00198     int num = send_sizes[i];
00199     int err = copyOutOfUnderlyingVector(num, &(send_ints[i][0]),
00200                                         &(send_doubles[i][0]), 0);
00201     if (err != 0) {
00202       FEI_COUT << "fei::Vector_core::scatterToOverlap ERROR getting data to send."<<FEI_ENDL;
00203       return(err);
00204     }
00205 
00206     MPI_Send(&(send_doubles[i][0]), num, MPI_DOUBLE, proc,
00207              tag2, comm_);
00208   }
00209 
00210   MPI_Waitall(recvProcs.size(), &mpiReqs[0], &mpiStatuses[0]);
00211 
00212 #endif  //#ifndef FEI_SER
00213 
00214   return(0);
00215 }
00216 
00217 int fei::Vector_core::copyOut(int numValues,
00218           const int* indices,
00219           double* values,
00220           int vectorIndex) const
00221 {
00222   const std::vector<CSVec*>& remote = remotelyOwned();
00223 
00224   for(int i=0; i<numValues; ++i) {
00225     int ind = indices[i];
00226 
00227     int local = ind - firstLocalOffset_;
00228     if (local < 0 || local >= numLocal_) {
00229       if (ind < 0) {
00230   continue;
00231       }
00232 
00233       int proc = eqnComm_->getOwnerProc(ind);
00234 
00235       int insertPoint = -1;
00236       int idx = fei::binarySearch(ind, remote[proc]->indices(), insertPoint);
00237       if (idx < 0) {
00238   fei::console_out() << "fei::Vector_core::copyOut: proc " << fei::localProc(comm_)
00239        << ", index " << ind << " not in remotelyOwned_ vec object for proc "
00240             <<proc<<FEI_ENDL;
00241   ERReturn(-1);
00242       }
00243       else {
00244   values[i] = remote[proc]->coefs()[idx];
00245       }
00246     }
00247     else {
00248       CHK_ERR( copyOutOfUnderlyingVector(1, &ind, &(values[i]), vectorIndex) );
00249     }
00250   }
00251 
00252   return(0);
00253 }
00254 
00255 int fei::Vector_core::giveToVector(int numValues,
00256                const int* indices,
00257                const double* values,
00258                bool sumInto,
00259                int vectorIndex)
00260 {
00261   std::vector<CSVec*>& remote = remotelyOwned();
00262 
00263   for(int i=0; i<numValues; ++i) {
00264     int ind = indices[i];
00265     double val = values[i];
00266 
00267     if (ind < 0) {
00268 //      throw std::runtime_error("negative index not allowed");
00269       //preservation of existing behavior: certain Sierra scenarios
00270       //involve passing negative indices for positions that should be
00271       //ignored... so we'll continue rather than throwing.
00272       continue;
00273     }
00274     int local = ind - firstLocalOffset_;
00275     if (local < 0 || local >= numLocal_) {
00276       int proc = eqnComm_->getOwnerProc(ind);
00277       if (output_level_ >= fei::BRIEF_LOGS && output_stream_ != NULL) {
00278         FEI_OSTREAM& os = *output_stream_;
00279         os << dbgprefix_<<"giveToVector remote["<<proc<<"]("
00280          <<ind<<","<<val<<")"<<FEI_ENDL;
00281       }
00282 
00283       if (sumInto) {
00284   fei::add_entry( *remote[proc], ind, val);
00285       }
00286       else {
00287   fei::put_entry( *remote[proc], ind, val);
00288       }
00289     }
00290     else {
00291       int err = giveToUnderlyingVector(1, &ind, &val, sumInto, vectorIndex);
00292       if (err != 0) {
00293   fei::console_out() << "giveToVector sumIn ERROR, ind: " << ind
00294        << ", val: " << val << FEI_ENDL;
00295   ERReturn(-1);
00296       }
00297     }
00298   }
00299 
00300   return(0);
00301 }
00302 
00303 int fei::Vector_core::assembleFieldData(int fieldID,
00304               int idType,
00305               int numIDs,
00306               const int* IDs,
00307               const double* data,
00308               bool sumInto,
00309               int vectorIndex)
00310 {
00311   if (vecSpace_.get() == NULL) ERReturn(-1);
00312 
00313   int fieldSize = vecSpace_->getFieldSize(fieldID);
00314 
00315   work_indices_.resize(numIDs*fieldSize);
00316   int* indicesPtr = &work_indices_[0];
00317 
00318   CHK_ERR( vecSpace_->getGlobalIndices(numIDs, IDs, idType, fieldID,
00319           indicesPtr) );
00320 
00321   CHK_ERR( giveToVector(numIDs*fieldSize, indicesPtr, data, sumInto, vectorIndex) );
00322 
00323   return(0);
00324 }
00325 
00326 void pack_send_buffers(const std::vector<int>& sendProcs,
00327                        const std::vector<fei::CSVec*>& remotelyOwned,
00328                        std::vector<std::vector<char> >& send_chars)
00329 {
00330   for(size_t i=0; i<sendProcs.size(); ++i) {
00331     int proc = sendProcs[i];
00332     fei::impl_utils::pack_indices_coefs(remotelyOwned[proc]->indices(),
00333                        remotelyOwned[proc]->coefs(), send_chars[i]);
00334 
00335     fei::set_values(*remotelyOwned[proc], 0.0);
00336   }
00337 }
00338 
00339 int fei::Vector_core::gatherFromOverlap(bool accumulate)
00340 {
00341   if (fei::numProcs(comm_) == 1 || haveFEVector()) {
00342     return(0);
00343   }
00344 
00345 //int localproc = fei::localProc(comm_);
00346 //std::ostringstream osstr;
00347 //osstr << "proc."<<localproc;
00348 //std::string str=osstr.str();
00349 //static std::ofstream ofs(str.c_str());
00350 
00351 #ifndef FEI_SER
00352   //declare arrays to hold the buffers we'll be receiving.
00353   std::vector<std::vector<char> > recv_chars;
00354   std::vector<std::vector<char> > send_chars(sendProcs_.size());
00355 
00356   std::vector<MPI_Request> mpiReqs;
00357   std::vector<MPI_Request> mpiReqs2;
00358   std::vector<MPI_Status> mpiStatuses;
00359   int tag1 = 11111;
00360 
00361   if (sendRecvProcsNeedUpdated_) {
00362     //first create the list of procs we'll be sending to.
00363     for(unsigned i=0; i<remotelyOwned_.size(); ++i) {
00364       if ((int)i == fei::localProc(comm_)) continue;
00365       if (remotelyOwned_[i]->size() == 0) continue;
00366   
00367       sendProcs_.push_back(i);
00368     }
00369   
00370     std::vector<int> tmpSendProcs;
00371     vecSpace_->getSendProcs(tmpSendProcs);
00372     for(size_t i=0; i<tmpSendProcs.size(); ++i) {
00373       bool found = false;
00374       for(size_t j=0; j<sendProcs_.size(); ++j) {
00375         if (sendProcs_[j] == tmpSendProcs[i]) {
00376           found = true;
00377           break;
00378         }
00379         if (sendProcs_[j] > tmpSendProcs[i]) {
00380           sendProcs_.insert(sendProcs_.begin()+j, tmpSendProcs[i]);
00381           found = true;
00382           break;
00383         }
00384       }
00385       if (!found) sendProcs_.push_back(tmpSendProcs[i]);
00386     }
00387 
00388     recvProcs_.clear();
00389     fei::mirrorProcs(comm_, sendProcs_, recvProcs_);
00390     sendRecvProcsNeedUpdated_ = false;
00391   }
00392   
00393 //  int num_remote_sends_needed = 0;
00394 //  for(unsigned i=0; i<remotelyOwned_.size(); ++i) {
00395 //    if ((int)i==fei::localProc(comm_)) continue;
00396 //    if (remotelyOwned_[i]->size() > 0) num_remote_sends_needed++;
00397 //  }
00398 //  if (num_remote_sends_needed != (int)sendProcs_.size()) {
00399 //    std::cout << "proc " << fei::localProc(comm_)<<", num_remote_sends_needed="<<num_remote_sends_needed<<", sendProcs_.size()="<<sendProcs_.size()<<std::endl;
00400 //  }
00401 
00402   send_chars.resize(sendProcs_.size());
00403 
00404   pack_send_buffers(sendProcs_, remotelyOwned_, send_chars);
00405 
00406   recv_sizes_.resize(recvProcs_.size());
00407   mpiReqs.resize(recvProcs_.size());
00408   mpiStatuses.resize(recvProcs_.size());
00409   mpiReqs2.resize(recvProcs_.size());
00410   mpiStatuses.resize(recvProcs_.size());
00411   recv_chars.resize(recvProcs_.size());
00412 
00413 
00414   //post the recvs for the sizes.
00415   for(size_t i=0; i<recvProcs_.size(); ++i) {
00416     int proc = recvProcs_[i];
00417     MPI_Irecv(&recv_sizes_[i], 1, MPI_INT, proc,
00418               tag1, comm_, &mpiReqs[i]);
00419   }
00420 
00421   //send the sizes of data we'll be sending.
00422   for(unsigned i=0; i<sendProcs_.size(); ++i) {
00423     int proc = sendProcs_[i];
00424     int size = send_chars[i].size();
00425 //std::cout<<"proc "<<fei::localProc(comm_)<<" sending "<<size<<" chars ("<<remotelyOwned_[proc]->indices().size()<<" indices/coefs) to proc "<<proc<<" accum="<<accumulate<<std::endl;
00426     MPI_Send(&size, 1, MPI_INT, proc, tag1, comm_);
00427   }
00428 
00429   //now post the recvs for the data.
00430   for(size_t i=0; i<recvProcs_.size(); ++i) {
00431     int index;
00432     MPI_Status status;
00433     MPI_Waitany(mpiReqs.size(), &mpiReqs[0], &index, &status);
00434 
00435     recv_chars[index].resize(recv_sizes_[index]);
00436     MPI_Irecv(&(recv_chars[index][0]), recv_sizes_[index], MPI_CHAR, recvProcs_[index],
00437               tag1, comm_, &mpiReqs2[index]);
00438   }
00439 
00440   //now send the outgoing data.
00441   for(size_t i=0; i<sendProcs_.size(); ++i) {
00442     int proc = sendProcs_[i];
00443     int size = send_chars[i].size();
00444     MPI_Send(&(send_chars[i][0]), size, MPI_CHAR, proc, tag1, comm_);
00445   }
00446 
00447   std::vector<int> indices;
00448   std::vector<double> coefs;
00449   int numRecvProcs = recvProcs_.size();
00450   for(size_t i=0; i<recvProcs_.size(); ++i) {
00451     int index;
00452     MPI_Status status;
00453     MPI_Waitany(numRecvProcs, &mpiReqs2[0], &index, &status);
00454   }
00455 
00456   //now store the data we've received.
00457   for(size_t i=0; i<recvProcs_.size(); ++i) {
00458     fei::impl_utils::unpack_indices_coefs(recv_chars[i], indices, coefs);
00459     int num = indices.size();
00460     if (num == 0) continue;
00461 //ofs<<"proc "<<fei::localProc(comm_)<<" recvd "<<num<<"inds/coefs from proc "<<recvProcs_[i]<<" accum="<<accumulate<<std::endl;
00462 //for(int ii=0; ii<num;++ii)ofs<<"   "<<indices[ii]<<" "<<coefs[ii]<<std::endl;
00463     int err = giveToUnderlyingVector(num, &(indices[0]),
00464                                      &(coefs[0]), accumulate, 0);
00465     if (err != 0) {
00466       FEI_COUT << "fei::Vector_core::gatherFromOverlap ERROR storing recvd data" << FEI_ENDL;
00467       return(err);
00468     }
00469   }
00470 
00471 #endif  //#ifndef FEI_SER
00472 
00473   return(0);
00474 }
00475 
00476 int fei::Vector_core::copyOutFieldData(int fieldID,
00477              int idType,
00478              int numIDs,
00479              const int* IDs,
00480              double* data,
00481              int vectorIndex)
00482 {
00483   if (vecSpace_.get() == NULL) ERReturn(-1);
00484 
00485   int fieldSize = vecSpace_->getFieldSize(fieldID);
00486 
00487   if (haveFEVector_) {
00488     snl_fei::RecordCollection* collection = NULL;
00489     CHK_ERR( vecSpace_->getRecordCollection(idType, collection) );
00490     int nodeNumber;
00491     int dofOffset;
00492     int foffset;
00493     std::vector<int>& eqnNums = vecSpace_->getEqnNumbers();
00494     int* vspcEqnPtr = eqnNums.size() > 0 ? &eqnNums[0] : NULL;
00495 
00496     int offset = 0;
00497     for(int i=0; i<numIDs; ++i) {
00498       fei::Record<int>* node = collection->getRecordWithID(IDs[i]);
00499       if (node == NULL) {
00500         ERReturn(-1);
00501       }
00502 
00503       nodeNumber = node->getNumber();
00504       int* eqnNumbers = vspcEqnPtr+node->getOffsetIntoEqnNumbers();
00505       node->getFieldMask()->getFieldEqnOffset(fieldID, foffset);
00506       dofOffset = eqnNumbers[foffset] - eqnNumbers[0];
00507       for(int j=0; j<fieldSize; ++j) {
00508   CHK_ERR( copyOut_FE(nodeNumber, dofOffset+j, data[offset++]));
00509       }
00510     }
00511   }
00512   else {
00513     work_indices_.resize(numIDs*fieldSize*2);
00514     int* indicesPtr = &work_indices_[0];
00515 
00516     CHK_ERR( vecSpace_->getGlobalIndices(numIDs, IDs, idType,
00517            fieldID, indicesPtr) );
00518 
00519     CHK_ERR( copyOut(numIDs*fieldSize, indicesPtr, data) );
00520   }
00521 
00522   return(0);
00523 }
00524 
00525 int fei::Vector_core::writeToFile(const char* filename,
00526                                     bool matrixMarketFormat)
00527 {
00528   int numProcs = fei::numProcs(comm_);
00529   int localProc =fei::localProc(comm_);
00530 
00531   double coef;
00532 
00533   static char mmbanner[] = "%%MatrixMarket matrix array real general";
00534 
00535   for(int p=0; p<numProcs; ++p) {
00536     fei::Barrier(comm_);
00537     if (p != localProc) continue;
00538 
00539     FEI_OFSTREAM* outFile = NULL;
00540     if (p==0) {
00541       outFile = new FEI_OFSTREAM(filename, IOS_OUT);
00542       FEI_OFSTREAM& ofref = *outFile;
00543       if (matrixMarketFormat) {
00544         ofref << mmbanner << FEI_ENDL;
00545         ofref << eqnComm_->getGlobalOffsets()[numProcs] << " 1" << FEI_ENDL;
00546       }
00547       else {
00548         ofref << eqnComm_->getGlobalOffsets()[numProcs] << FEI_ENDL;
00549       }
00550     }
00551     else outFile = new FEI_OFSTREAM(filename, IOS_APP);
00552     FEI_OFSTREAM& ofref = *outFile;
00553     ofref.setf(IOS_SCIENTIFIC, IOS_FLOATFIELD);
00554     ofref.precision(13);
00555 
00556     for(int i=firstLocalOffset_; i<=lastLocalOffset_; ++i) {
00557       CHK_ERR( copyOut(1, &i, &coef) );
00558       if (matrixMarketFormat) {
00559         ofref << " " << coef << FEI_ENDL;
00560       }
00561       else {
00562         ofref << i << " " << coef << FEI_ENDL;
00563       }
00564     }
00565     
00566     delete outFile;
00567   } 
00568     
00569   return(0);
00570 }
00571 
00572 int fei::Vector_core::writeToStream(FEI_OSTREAM& ostrm,
00573           bool matrixMarketFormat)
00574 {
00575   int numProcs = fei::numProcs(comm_);
00576   int local_proc =fei::localProc(comm_);
00577 
00578   double coef;
00579 
00580   static char mmbanner[] = "%%MatrixMarket matrix array real general";
00581 
00582   IOS_FMTFLAGS oldf = ostrm.setf(IOS_SCIENTIFIC, IOS_FLOATFIELD);
00583   ostrm.precision(13);
00584 
00585   for(int proc=0; proc<numProcs; ++proc) {
00586     fei::Barrier(comm_);
00587     if (proc != local_proc) continue;
00588 
00589     if (proc==0) {
00590       if (matrixMarketFormat) {
00591   ostrm << mmbanner << FEI_ENDL;
00592   ostrm << eqnComm_->getGlobalOffsets()[numProcs] << " 1" << FEI_ENDL;
00593       }
00594       else {
00595   ostrm << eqnComm_->getGlobalOffsets()[numProcs] << FEI_ENDL;
00596       }
00597     }
00598 
00599     for(int p=0; p<local_proc; ++p) {
00600       for(size_t ii=0; ii<remotelyOwned_[p]->size(); ++ii) {
00601   if (matrixMarketFormat) {
00602     ostrm << " " << remotelyOwned_[p]->coefs()[ii] << FEI_ENDL;
00603   }
00604   else {
00605     ostrm << " " << remotelyOwned_[p]->indices()[ii] << " "
00606     << remotelyOwned_[p]->coefs()[ii] << FEI_ENDL;
00607   }
00608       }
00609     }
00610 
00611     for(int i=firstLocalOffset_; i<=lastLocalOffset_; ++i) {
00612       CHK_ERR( copyOut(1, &i, &coef) );
00613       if (matrixMarketFormat) {
00614   ostrm << " " << coef << FEI_ENDL;
00615       }
00616       else {
00617   ostrm << " " << i << " " << coef << FEI_ENDL;
00618       }
00619     }
00620 
00621     for(int p=local_proc+1; p<numProcs; ++p) {
00622       for(size_t ii=0; ii<remotelyOwned_[p]->size(); ++ii) {
00623   if (matrixMarketFormat) {
00624     ostrm << " " << remotelyOwned_[p]->coefs()[ii] << FEI_ENDL;
00625   }
00626   else {
00627     ostrm << " " << remotelyOwned_[p]->indices()[ii] << " "
00628     << remotelyOwned_[p]->coefs()[ii] << FEI_ENDL;
00629   }
00630       }
00631     }
00632   }
00633 
00634   ostrm.setf(oldf, IOS_FLOATFIELD);
00635 
00636   return(0);
00637 }
00638 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Friends