ExternalRows_dh.c

00001 /*@HEADER
00002 // ***********************************************************************
00003 //
00004 //       Ifpack: Object-Oriented Algebraic Preconditioner Package
00005 //                 Copyright (2009) Sandia Corporation
00006 //
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 //
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00025 //
00026 // ***********************************************************************
00027 //@HEADER
00028 */
00029 
00030 #include "ExternalRows_dh.h"
00031 #include "Factor_dh.h"
00032 #include "Euclid_dh.h"
00033 #include "SubdomainGraph_dh.h"
00034 #include "Mem_dh.h"
00035 #include "Parser_dh.h"
00036 #include "Hash_dh.h"
00037 
00038  /* tags for MPI comms */
00039 enum
00040 { ROW_CT_TAG, NZ_CT_TAG, ROW_LENGTH_TAG, ROW_NUMBER_TAG,
00041   CVAL_TAG, FILL_TAG, AVAL_TAG
00042 };
00043 
00044 #undef __FUNC__
00045 #define __FUNC__ "ExternalRows_dhCreate"
00046 void
00047 ExternalRows_dhCreate (ExternalRows_dh * er)
00048 {
00049   START_FUNC_DH
00050     struct _extrows_dh *tmp =
00051     (struct _extrows_dh *) MALLOC_DH (sizeof (struct _extrows_dh));
00052   CHECK_V_ERROR;
00053   *er = tmp;
00054 
00055   if (MAX_MPI_TASKS < np_dh)
00056     {
00057       SET_V_ERROR ("MAX_MPI_TASKS is too small; change, then recompile!");
00058     }
00059 
00060   {
00061     int i;
00062     for (i = 0; i < MAX_MPI_TASKS; ++i)
00063       {
00064     tmp->rcv_row_lengths[i] = NULL;
00065     tmp->rcv_row_numbers[i] = NULL;
00066       }
00067   }
00068 
00069   tmp->cvalExt = NULL;
00070   tmp->fillExt = NULL;
00071   tmp->avalExt = NULL;
00072   tmp->my_row_counts = NULL;
00073   tmp->my_row_numbers = NULL;
00074   tmp->cvalSend = NULL;
00075   tmp->fillSend = NULL;
00076   tmp->avalSend = NULL;
00077   tmp->rowLookup = NULL;
00078   tmp->sg = NULL;
00079   tmp->F = NULL;
00080   tmp->debug = Parser_dhHasSwitch (parser_dh, "-debug_ExtRows");
00081 END_FUNC_DH}
00082 
00083 #undef __FUNC__
00084 #define __FUNC__ "ExternalRows_dhDestroy"
00085 void
00086 ExternalRows_dhDestroy (ExternalRows_dh er)
00087 {
00088   START_FUNC_DH int i;
00089 
00090   for (i = 0; i < MAX_MPI_TASKS; ++i)
00091     {
00092       if (er->rcv_row_lengths[i] != NULL)
00093     {
00094       FREE_DH (er->rcv_row_lengths[i]);
00095       CHECK_V_ERROR;
00096     }
00097       if (er->rcv_row_numbers[i] != NULL)
00098     {
00099       FREE_DH (er->rcv_row_numbers[i]);
00100       CHECK_V_ERROR;
00101     }
00102     }
00103 
00104   if (er->cvalExt != NULL)
00105     {
00106       FREE_DH (er->cvalExt);
00107       CHECK_V_ERROR;
00108     }
00109   if (er->fillExt != NULL)
00110     {
00111       FREE_DH (er->fillExt);
00112       CHECK_V_ERROR;
00113     }
00114   if (er->avalExt != NULL)
00115     {
00116       FREE_DH (er->avalExt);
00117       CHECK_V_ERROR;
00118     }
00119 
00120   if (er->my_row_counts != NULL)
00121     {
00122       FREE_DH (er->my_row_counts);
00123       CHECK_V_ERROR;
00124     }
00125   if (er->my_row_numbers != NULL)
00126     {
00127       FREE_DH (er->my_row_numbers);
00128       CHECK_V_ERROR;
00129     }
00130 
00131   if (er->cvalSend != NULL)
00132     {
00133       FREE_DH (er->cvalSend);
00134       CHECK_V_ERROR;
00135     }
00136   if (er->fillSend != NULL)
00137     {
00138       FREE_DH (er->fillSend);
00139       CHECK_V_ERROR;
00140     }
00141   if (er->avalSend != NULL)
00142     {
00143       FREE_DH (er->avalSend);
00144       CHECK_V_ERROR;
00145     }
00146 
00147   if (er->rowLookup != NULL)
00148     {
00149       Hash_dhDestroy (er->rowLookup);
00150       CHECK_V_ERROR;
00151     }
00152   FREE_DH (er);
00153   CHECK_V_ERROR;
00154 END_FUNC_DH}
00155 
00156 #undef __FUNC__
00157 #define __FUNC__ "ExternalRows_dhInit"
00158 void
00159 ExternalRows_dhInit (ExternalRows_dh er, Euclid_dh ctx)
00160 {
00161   START_FUNC_DH er->sg = ctx->sg;
00162   er->F = ctx->F;
00163 END_FUNC_DH}
00164 
00165 /*=====================================================================
00166  * method for accessing external rows
00167  *=====================================================================*/
00168 
00169 #undef __FUNC__
00170 #define __FUNC__ "ExternalRows_dhGetRow"
00171 void
00172 ExternalRows_dhGetRow (ExternalRows_dh er, int globalRow,
00173                int *len, int **cval, int **fill, REAL_DH ** aval)
00174 {
00175   START_FUNC_DH if (er->rowLookup == NULL)
00176     {
00177       *len = 0;
00178     }
00179 
00180   else
00181     {
00182       HashData *r = NULL;
00183       r = Hash_dhLookup (er->rowLookup, globalRow);
00184       CHECK_V_ERROR;
00185       if (r != NULL)
00186     {
00187       *len = r->iData;
00188       if (cval != NULL)
00189         *cval = r->iDataPtr;
00190       if (fill != NULL)
00191         *fill = r->iDataPtr2;
00192       if (aval != NULL)
00193         *aval = r->fDataPtr;
00194     }
00195       else
00196     {
00197       *len = 0;
00198     }
00199     }
00200 END_FUNC_DH}
00201 
00202 /*=====================================================================
00203  * methods for receiving  external rows from lower-ordered subdomains
00204  *=====================================================================*/
00205 static void rcv_ext_storage_private (ExternalRows_dh extRows);
00206 static void build_hash_table_private (ExternalRows_dh er);
00207 static void rcv_external_rows_private (ExternalRows_dh er);
00208 static void allocate_ext_row_storage_private (ExternalRows_dh er);
00209 static void print_received_rows_private (ExternalRows_dh er);
00210 
00211 #undef __FUNC__
00212 #define __FUNC__ "ExternalRows_dhRecvRows"
00213 void
00214 ExternalRows_dhRecvRows (ExternalRows_dh er)
00215 {
00216   START_FUNC_DH bool debug = false;
00217   if (logFile != NULL && er->debug)
00218     debug = true;
00219 
00220   if (er->sg->loCount > 0)
00221     {
00222       /* get number of rows and length of each row to be received
00223          from each lower ordered nabor.
00224          (allocates: *rcv_row_lengths[], *rcv_row_numbers[])
00225        */
00226       rcv_ext_storage_private (er);
00227       CHECK_V_ERROR;
00228 
00229 
00230       /* allocate data structures for receiving the rows (no comms) 
00231          (allocates: cvalExt, fillExt, avalExt)
00232          (no communications)
00233        */
00234       allocate_ext_row_storage_private (er);
00235       CHECK_V_ERROR;
00236 
00237 
00238       /* construct hash table for  external row lookup (no comms) 
00239          (Creates/allocates: rowLookup)
00240          (no communications)
00241        */
00242       build_hash_table_private (er);
00243       CHECK_V_ERROR;
00244 
00245       /* receive the actual row structures and values 
00246          from lower ordered neighbors 
00247        */
00248       rcv_external_rows_private (er);
00249       CHECK_V_ERROR;
00250 
00251       if (debug)
00252     {
00253       print_received_rows_private (er);
00254       CHECK_V_ERROR;
00255     }
00256     }
00257 END_FUNC_DH}
00258 
00259 #undef __FUNC__
00260 #define __FUNC__ "rcv_ext_storage_private"
00261 void
00262 rcv_ext_storage_private (ExternalRows_dh er)
00263 {
00264   START_FUNC_DH int i;
00265   int loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
00266   int *rcv_row_counts = er->rcv_row_counts;
00267   int *rcv_nz_counts = er->rcv_nz_counts;
00268   int **lengths = er->rcv_row_lengths, **numbers = er->rcv_row_numbers;
00269   bool debug = false;
00270 
00271   if (logFile != NULL && er->debug)
00272     debug = true;
00273 
00274   /* get number of rows, and total nonzeros, that each lo-nabor will send */
00275   for (i = 0; i < loCount; ++i)
00276     {
00277       int nabor = loNabors[i];
00278       MPI_Irecv (rcv_row_counts + i, 1, MPI_INT, nabor, ROW_CT_TAG, comm_dh,
00279          er->req1 + i);
00280       MPI_Irecv (rcv_nz_counts + i, 1, MPI_INT, nabor, NZ_CT_TAG, comm_dh,
00281          er->req2 + i);
00282     }
00283   MPI_Waitall (loCount, er->req1, er->status);
00284   MPI_Waitall (loCount, er->req2, er->status);
00285 
00286   if (debug)
00287     {
00288       fprintf (logFile,
00289            "\nEXR rcv_ext_storage_private:: <nabor,rowCount,nzCount>\nEXR ");
00290       for (i = 0; i < loCount; ++i)
00291     {
00292       fprintf (logFile, "<%i,%i,%i> ", loNabors[i], rcv_row_counts[i],
00293            rcv_nz_counts[i]);
00294     }
00295     }
00296 
00297   /* get lengths and global number of each row to be received */
00298   for (i = 0; i < loCount; ++i)
00299     {
00300       int nz = rcv_nz_counts[i];
00301       int nabor = loNabors[i];
00302       lengths[i] = (int *) MALLOC_DH (nz * sizeof (int));
00303       CHECK_V_ERROR;
00304       numbers[i] = (int *) MALLOC_DH (nz * sizeof (int));
00305       CHECK_V_ERROR;
00306       MPI_Irecv (lengths[i], nz, MPI_INT, nabor, ROW_LENGTH_TAG, comm_dh,
00307          er->req1 + i);
00308       MPI_Irecv (numbers[i], nz, MPI_INT, nabor, ROW_NUMBER_TAG, comm_dh,
00309          er->req2 + i);
00310     }
00311   MPI_Waitall (loCount, er->req1, er->status);
00312   MPI_Waitall (loCount, er->req2, er->status);
00313 
00314   if (debug)
00315     {
00316       int j, nz;
00317       for (i = 0; i < loCount; ++i)
00318     {
00319       fprintf (logFile,
00320            "\nEXR rows <number,length> to be received from P_%i\nEXR ",
00321            loNabors[i]);
00322       nz = rcv_row_counts[i];
00323       for (j = 0; j < nz; ++j)
00324         fprintf (logFile, "<%i,%i> ", numbers[i][j], lengths[i][j]);
00325       fprintf (logFile, "\n");
00326     }
00327     }
00328 
00329 END_FUNC_DH}
00330 
00331 /* allocates: cvalExt, fillExt, avalExt */
00332 #undef __FUNC__
00333 #define __FUNC__ "allocate_ext_row_storage_private"
00334 void
00335 allocate_ext_row_storage_private (ExternalRows_dh er)
00336 {
00337   START_FUNC_DH int i, nz = 0;
00338   int loCount = er->sg->loCount;
00339   int *rcv_nz_counts = er->rcv_nz_counts;
00340 
00341   /* count total number of nonzeros to be received */
00342   for (i = 0; i < loCount; ++i)
00343     nz += rcv_nz_counts[i];
00344 
00345   /* allocate buffers */
00346   er->cvalExt = (int *) MALLOC_DH (nz * sizeof (int));
00347   CHECK_V_ERROR;
00348   er->fillExt = (int *) MALLOC_DH (nz * sizeof (int));
00349   CHECK_V_ERROR;
00350   er->avalExt = (REAL_DH *) MALLOC_DH (nz * sizeof (REAL_DH));
00351   CHECK_V_ERROR;
00352 END_FUNC_DH}
00353 
00354 #undef __FUNC__
00355 #define __FUNC__ "build_hash_table_private"
00356 void
00357 build_hash_table_private (ExternalRows_dh er)
00358 {
00359   START_FUNC_DH int loCount = er->sg->loCount;
00360   int i, j, offset, rowCt = 0;
00361   Hash_dh table;
00362   HashData record;
00363   int *extRowCval = er->cvalExt, *extRowFill = er->fillExt;
00364   REAL_DH *extRowAval = er->avalExt;
00365   int *rcv_row_counts = er->rcv_row_counts;
00366   int **rcv_row_numbers = er->rcv_row_numbers;
00367   int **rcv_row_lengths = er->rcv_row_lengths;
00368 
00369   /* count total number of rows to be received */
00370   for (i = 0; i < loCount; ++i)
00371     rowCt += rcv_row_counts[i];
00372 
00373   /* build table for looking up external rows */
00374   Hash_dhCreate (&table, rowCt);
00375   CHECK_V_ERROR;
00376   er->rowLookup = table;
00377   offset = 0;
00378 
00379   /* loop over lower ordered nabors in subdomain graph */
00380   for (i = 0; i < loCount; ++i)
00381     {
00382 
00383       /* number of rows to be received from nabor(i) */
00384       int rowCount = rcv_row_counts[i];
00385 
00386       /* loop over rows to be received from nabor(i) */
00387       for (j = 0; j < rowCount; ++j)
00388     {
00389 
00390       /* insert a record to locate row(j) in the hash table */
00391       int row = rcv_row_numbers[i][j];
00392       int rowLength = rcv_row_lengths[i][j];
00393       record.iData = rowLength;
00394       record.iDataPtr = extRowCval + offset;
00395       record.iDataPtr2 = extRowFill + offset;
00396       record.fDataPtr = extRowAval + offset;
00397       Hash_dhInsert (table, row, &record);
00398       CHECK_V_ERROR;
00399       offset += rowLength;
00400     }
00401     }
00402 END_FUNC_DH}
00403 
00404 #undef __FUNC__
00405 #define __FUNC__ "rcv_external_rows_private"
00406 void
00407 rcv_external_rows_private (ExternalRows_dh er)
00408 {
00409   START_FUNC_DH int *rcv_nz_counts = er->rcv_nz_counts;
00410   int i, loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
00411   int nabor, nz = 0, offset = 0;
00412   int *extRowCval = er->cvalExt, *extRowFill = er->fillExt;
00413   double *extRowAval = er->avalExt;
00414 
00415   /* start receives of external rows */
00416   nz = 0;
00417   for (i = 0; i < loCount; ++i)
00418     {
00419       nabor = loNabors[i];
00420       nz = rcv_nz_counts[i];
00421       MPI_Irecv (extRowCval + offset, nz, MPI_INT, nabor, CVAL_TAG, comm_dh,
00422          er->req1 + i);
00423       MPI_Irecv (extRowFill + offset, nz, MPI_INT, nabor, FILL_TAG, comm_dh,
00424          er->req2 + i);
00425       MPI_Irecv (extRowAval + offset, nz, MPI_DOUBLE, nabor, AVAL_TAG,
00426          comm_dh, er->req3 + i);
00427       offset += nz;
00428     }
00429 
00430   /* wait for external rows to arrive */
00431   MPI_Waitall (loCount, er->req1, er->status);
00432   MPI_Waitall (loCount, er->req2, er->status);
00433   MPI_Waitall (loCount, er->req3, er->status);
00434 END_FUNC_DH}
00435 
00436 
00437 #undef __FUNC__
00438 #define __FUNC__ "print_received_rows_private"
00439 void
00440 print_received_rows_private (ExternalRows_dh er)
00441 {
00442   START_FUNC_DH bool noValues = (Parser_dhHasSwitch (parser_dh, "-noValues"));
00443   int i, j, k, rwCt, idx = 0, nabor;
00444   int loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
00445   int n = er->F->n;
00446 
00447   fprintf (logFile,
00448        "\nEXR ================= received rows, printed from buffers ==============\n");
00449 
00450   /* loop over nabors from whom we received rows */
00451   for (i = 0; i < loCount; ++i)
00452     {
00453       rwCt = er->rcv_row_counts[i];
00454       nabor = loNabors[i];
00455       fprintf (logFile, "\nEXR Rows received from P_%i:\n", nabor);
00456 
00457       /* loop over each row to be received from this nabor */
00458       for (j = 0; j < rwCt; ++j)
00459     {
00460       int rowNum = er->rcv_row_numbers[i][j];
00461       int rowLen = er->rcv_row_lengths[i][j];
00462       fprintf (logFile, "EXR %i :: ", 1 + rowNum);
00463       for (k = 0; k < rowLen; ++k)
00464         {
00465           if (noValues)
00466         {
00467           fprintf (logFile, "%i,%i ; ", er->cvalExt[idx],
00468                er->fillExt[idx]);
00469         }
00470           else
00471         {
00472           fprintf (logFile, "%i,%i,%g ; ", er->cvalExt[idx],
00473                er->fillExt[idx], er->avalExt[idx]);
00474         }
00475           ++idx;
00476         }
00477       fprintf (logFile, "\n");
00478     }
00479     }
00480 
00481   fprintf (logFile,
00482        "\nEXR =============== received rows, printed from hash table =============\n");
00483   for (i = 0; i < n; ++i)
00484     {
00485       int len, *cval, *fill;
00486       REAL_DH *aval;
00487       ExternalRows_dhGetRow (er, i, &len, &cval, &fill, &aval);
00488       CHECK_V_ERROR;
00489       if (len > 0)
00490     {
00491       fprintf (logFile, "EXR %i :: ", i + 1);
00492       for (j = 0; j < len; ++j)
00493         {
00494           if (noValues)
00495         {
00496           fprintf (logFile, "%i,%i ; ", cval[j], fill[j]);
00497         }
00498           else
00499         {
00500           fprintf (logFile, "%i,%i,%g ; ", cval[j], fill[j], aval[j]);
00501         }
00502         }
00503       fprintf (logFile, "\n");
00504     }
00505     }
00506 
00507 END_FUNC_DH}
00508 
00509 /*=====================================================================
00510  * methods for sending rows to higher ordered nabors in subdomain graph
00511  *=====================================================================*/
00512 
00513 static void send_ext_storage_private (ExternalRows_dh er);
00514 static void send_external_rows_private (ExternalRows_dh er);
00515 static void waitfor_sends_private (ExternalRows_dh er);
00516 
00517 #undef __FUNC__
00518 #define __FUNC__ "ExternalRows_dhSendRows"
00519 void
00520 ExternalRows_dhSendRows (ExternalRows_dh er)
00521 {
00522   START_FUNC_DH if (er->sg->hiCount > 0)
00523     {
00524       /* send number of rows and length of each row to be sent
00525          to each higher ordered nabor.
00526        */
00527       send_ext_storage_private (er);
00528       CHECK_V_ERROR;
00529 
00530       /* send the row's colum indices, fill levels, and values */
00531       send_external_rows_private (er);
00532       CHECK_V_ERROR;
00533 
00534       waitfor_sends_private (er);
00535       CHECK_V_ERROR;
00536     }
00537 END_FUNC_DH}
00538 
00539 
00540 #undef __FUNC__
00541 #define __FUNC__ "send_ext_storage_private"
00542 void
00543 send_ext_storage_private (ExternalRows_dh er)
00544 {
00545   START_FUNC_DH int nz, i, j;
00546   int *nzCounts, *nzNumbers;
00547   int hiCount = er->sg->hiCount, *hiNabors = er->sg->hiNabors;
00548   int *rp = er->F->rp, *diag = er->F->diag;
00549   int m = er->F->m;
00550   int beg_row = er->F->beg_row;
00551   int rowCount = er->F->bdry_count; /* number of boundary rows */
00552   int first_bdry = er->F->first_bdry;
00553   bool debug = false;
00554 
00555   if (logFile != NULL && er->debug)
00556     debug = true;
00557 
00558   /* allocate storage to hold nz counts for each row */
00559   nzCounts = er->my_row_counts = (int *) MALLOC_DH (rowCount * sizeof (int));
00560   CHECK_V_ERROR;
00561   nzNumbers = er->my_row_numbers =
00562     (int *) MALLOC_DH (rowCount * sizeof (int));
00563   CHECK_V_ERROR;
00564 
00565   /* count nonzeros in upper triangular portion of each boundary row */
00566   nz = 0;
00567   for (i = first_bdry, j = 0; i < m; ++i, ++j)
00568     {
00569       int tmp = (rp[i + 1] - diag[i]);
00570       nz += tmp;
00571       nzCounts[j] = tmp;
00572     }
00573   er->nzSend = nz;
00574 
00575   if (debug)
00576     {
00577       fprintf (logFile, "EXR send_ext_storage_private:: rowCount = %i\n",
00578            rowCount);
00579       fprintf (logFile, "EXR send_ext_storage_private:: nz Count = %i\n", nz);
00580     }
00581 
00582   /* send  number of rows, and total nonzeros, to higher ordered nabors */
00583   for (i = 0; i < hiCount; ++i)
00584     {
00585       int nabor = hiNabors[i];
00586       MPI_Isend (&rowCount, 1, MPI_INT, nabor, ROW_CT_TAG, comm_dh,
00587          er->req1 + i);
00588       MPI_Isend (&nz, 1, MPI_INT, nabor, NZ_CT_TAG, comm_dh, er->req2 + i);
00589     }
00590 
00591   /* set up array for global row numbers */
00592   for (i = 0, j = first_bdry; j < m; ++i, ++j)
00593     {
00594       nzNumbers[i] = j + beg_row;
00595     }
00596 
00597   /* start sends of length and global number of each of this processor's
00598      boundary row to higher ordered nabors; the receiving processor will
00599      use this information to allocate storage buffers for the actual
00600      row structures and values.
00601    */
00602   for (i = 0; i < hiCount; ++i)
00603     {
00604       int nabor = hiNabors[i];
00605       MPI_Isend (nzNumbers, rowCount, MPI_INT, nabor, ROW_NUMBER_TAG, comm_dh,
00606          er->req3 + i);
00607       MPI_Isend (nzCounts, rowCount, MPI_INT, nabor, ROW_LENGTH_TAG, comm_dh,
00608          er->req4 + i);
00609     }
00610 
00611 END_FUNC_DH}
00612 
00613 
00614 #undef __FUNC__
00615 #define __FUNC__ "send_external_rows_private"
00616 void
00617 send_external_rows_private (ExternalRows_dh er)
00618 {
00619   START_FUNC_DH
00620     int i, j, hiCount = er->sg->hiCount, *hiNabors = er->sg->hiNabors;
00621   int offset, nz = er->nzSend;
00622   int *cvalSend, *fillSend;
00623   REAL_DH *avalSend;
00624   int *cval = er->F->cval, *fill = er->F->fill;
00625   int m = er->F->m;
00626   int *rp = er->F->rp, *diag = er->F->diag;
00627   int first_bdry = er->F->first_bdry;
00628   REAL_DH *aval = er->F->aval;
00629   bool debug = false;
00630 
00631   if (logFile != NULL && er->debug)
00632     debug = true;
00633 
00634   /* allocate buffers to hold upper triangular portion of boundary rows */
00635   cvalSend = er->cvalSend = (int *) MALLOC_DH (nz * sizeof (int));
00636   CHECK_V_ERROR;
00637   fillSend = er->fillSend = (int *) MALLOC_DH (nz * sizeof (int));
00638   CHECK_V_ERROR;
00639   avalSend = er->avalSend = (double *) MALLOC_DH (nz * sizeof (double));
00640   CHECK_V_ERROR;
00641 
00642   /* copy upper triangular portion of boundary rows int send buffers */
00643   offset = 0;
00644   for (i = first_bdry, j = 0; i < m; ++i, ++j)
00645     {
00646       int tmp = (rp[i + 1] - diag[i]);
00647 
00648       memcpy (cvalSend + offset, cval + diag[i], tmp * sizeof (int));
00649       memcpy (fillSend + offset, fill + diag[i], tmp * sizeof (int));
00650       memcpy (avalSend + offset, aval + diag[i], tmp * sizeof (double));
00651       offset += tmp;
00652     }
00653 
00654   if (debug)
00655     {
00656       int beg_row = er->F->beg_row;
00657       int idx = 0;
00658       bool noValues = (Parser_dhHasSwitch (parser_dh, "-noValues"));
00659 
00660       fprintf (logFile,
00661            "\nEXR ======================= send buffers ======================\n");
00662 
00663       for (i = first_bdry, j = 0; i < m; ++i, ++j)
00664     {
00665       int tmp = (rp[i + 1] - diag[i]);
00666       fprintf (logFile, "EXR %i :: ", i + beg_row);
00667 
00668       for (j = 0; j < tmp; ++j)
00669         {
00670           if (noValues)
00671         {
00672           fprintf (logFile, "%i,%i ; ", cvalSend[idx], fillSend[idx]);
00673         }
00674           else
00675         {
00676           fprintf (logFile, "%i,%i,%g ; ", cvalSend[idx],
00677                fillSend[idx], avalSend[idx]);
00678         }
00679           ++idx;
00680         }
00681       fprintf (logFile, "\n");
00682     }
00683     }
00684 
00685   /* start sends to higher-ordred nabors */
00686   for (i = 0; i < hiCount; ++i)
00687     {
00688       int nabor = hiNabors[i];
00689       MPI_Isend (cvalSend, nz, MPI_INT, nabor, CVAL_TAG, comm_dh,
00690          er->cval_req + i);
00691       MPI_Isend (fillSend, nz, MPI_INT, nabor, FILL_TAG, comm_dh,
00692          er->fill_req + i);
00693       MPI_Isend (avalSend, nz, MPI_DOUBLE, nabor, AVAL_TAG, comm_dh,
00694          er->aval_req + i);
00695     }
00696 END_FUNC_DH}
00697 
00698 
00699 #undef __FUNC__
00700 #define __FUNC__ "waitfor_sends_private"
00701 void
00702 waitfor_sends_private (ExternalRows_dh er)
00703 {
00704   START_FUNC_DH MPI_Status * status = er->status;
00705   int hiCount = er->sg->hiCount;
00706 
00707   if (hiCount)
00708     {
00709       MPI_Waitall (hiCount, er->req1, status);
00710       MPI_Waitall (hiCount, er->req2, status);
00711       MPI_Waitall (hiCount, er->req3, status);
00712       MPI_Waitall (hiCount, er->req4, status);
00713       MPI_Waitall (hiCount, er->cval_req, status);
00714       MPI_Waitall (hiCount, er->fill_req, status);
00715       MPI_Waitall (hiCount, er->aval_req, status);
00716     }
00717 END_FUNC_DH}

Generated on Wed May 12 21:30:17 2010 for IFPACK by  doxygen 1.4.7