void PrintMatUsingGetRow(void* A, HYPRE_Int beg_row, HYPRE_Int m,
HYPRE_Int *n2o_row, HYPRE_Int *n2o_col, char *filename)
{
START_FUNC_DH
FILE *fp;
HYPRE_Int *o2n_col = NULL, pe, i, j, *cval, len;
HYPRE_Int newCol, newRow;
double *aval;
/* form inverse column permutation */
if (n2o_col != NULL) {
o2n_col = (HYPRE_Int*)MALLOC_DH(m*sizeof(HYPRE_Int)); CHECK_V_ERROR;
for (i=0; i<m; ++i) o2n_col[n2o_col[i]] = i;
}
for (pe=0; pe<np_dh; ++pe) {
hypre_MPI_Barrier(comm_dh);
if (myid_dh == pe) {
if (pe == 0) {
fp=fopen(filename, "w");
} else {
fp=fopen(filename, "a");
}
if (fp == NULL) {
hypre_sprintf(msgBuf_dh, "can't open %s for writing\n", filename);
SET_V_ERROR(msgBuf_dh);
}
for (i=0; i<m; ++i) {
if (n2o_row == NULL) {
EuclidGetRow(A, i+beg_row, &len, &cval, &aval); CHECK_V_ERROR;
for (j=0; j<len; ++j) {
hypre_fprintf(fp, "%i %i %g\n", i+1, cval[j], aval[j]);
}
EuclidRestoreRow(A, i, &len, &cval, &aval); CHECK_V_ERROR;
} else {
newRow = n2o_row[i] + beg_row;
EuclidGetRow(A, newRow, &len, &cval, &aval); CHECK_V_ERROR;
for (j=0; j<len; ++j) {
newCol = o2n_col[cval[j]-beg_row] + beg_row;
hypre_fprintf(fp, "%i %i %g\n", i+1, newCol, aval[j]);
}
EuclidRestoreRow(A, i, &len, &cval, &aval); CHECK_V_ERROR;
}
}
fclose(fp);
}
}
if (n2o_col != NULL) {
FREE_DH(o2n_col); CHECK_V_ERROR;
}
END_FUNC_DH
}
/*************************************************************************
* This function prints a vector on each processor
**************************************************************************/
void hypre_PrintVector(HYPRE_Int *v, HYPRE_Int n, char *msg,
hypre_PilutSolverGlobals *globals)
{
HYPRE_Int i, penum;
for (penum=0; penum<npes; penum++) {
if (mype == penum) {
hypre_printf("PE %d %s: ", mype, msg);
for (i=0; i<n; i++)
hypre_printf("%d ", v[i]);
hypre_printf("\n");
}
hypre_MPI_Barrier( pilut_comm );
}
}
HYPRE_Int
main( HYPRE_Int argc,
char *argv[] )
{
HYPRE_Int num_procs, myid;
HYPRE_Int verbose = 0, build_matrix_type = 1;
HYPRE_Int index, matrix_arg_index, commpkg_flag=3;
HYPRE_Int i, k, ierr=0;
HYPRE_Int row_start, row_end;
HYPRE_Int col_start, col_end, global_num_rows;
HYPRE_Int *row_part, *col_part;
char *csrfilename;
HYPRE_Int preload = 0, loop = 0, loop2 = LOOP2;
HYPRE_Int bcast_rows[2], *info;
hypre_ParCSRMatrix *parcsr_A, *small_A;
HYPRE_ParCSRMatrix A_temp, A_temp_small;
hypre_CSRMatrix *A_CSR;
hypre_ParCSRCommPkg *comm_pkg;
HYPRE_Int nx, ny, nz;
HYPRE_Int P, Q, R;
HYPRE_Int p, q, r;
HYPRE_Real values[4];
hypre_ParVector *x_new;
hypre_ParVector *y_new, *y;
HYPRE_Int *row_starts;
HYPRE_Real ans;
HYPRE_Real start_time, end_time, total_time, *loop_times;
HYPRE_Real T_avg, T_std;
HYPRE_Int noparmprint = 0;
#if mydebug
HYPRE_Int j, tmp_int;
#endif
/*-----------------------------------------------------------
* Initialize MPI
*-----------------------------------------------------------*/
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs );
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid );
/*-----------------------------------------------------------
* default - is 27pt laplace
*-----------------------------------------------------------*/
build_matrix_type = 2;
matrix_arg_index = argc;
/*-----------------------------------------------------------
* Parse command line
*-----------------------------------------------------------*/
index = 1;
while ( index < argc)
{
if ( strcmp(argv[index], "-verbose") == 0 )
{
index++;
verbose = 1;
}
else if ( strcmp(argv[index], "-fromonecsrfile") == 0 )
{
index++;
build_matrix_type = 1;
matrix_arg_index = index; /*this tells where the name is*/
}
else if ( strcmp(argv[index], "-commpkg") == 0 )
{
index++;
commpkg_flag = atoi(argv[index++]);
}
else if ( strcmp(argv[index], "-laplacian") == 0 )
{
index++;
build_matrix_type = 2;
matrix_arg_index = index;
}
else if ( strcmp(argv[index], "-27pt") == 0 )
{
index++;
build_matrix_type = 4;
matrix_arg_index = index;
}
/*
else if ( strcmp(argv[index], "-nopreload") == 0 )
{
index++;
preload = 0;
}
*/
else if ( strcmp(argv[index], "-loop") == 0 )
{
index++;
loop = atoi(argv[index++]);
}
else if ( strcmp(argv[index], "-noparmprint") == 0 )
{
index++;
noparmprint = 1;
}
else
{
index++;
/*hypre_printf("Warning: Unrecogized option '%s'\n",argv[index++] );*/
}
}
/*-----------------------------------------------------------
* Setup the Matrix problem
*-----------------------------------------------------------*/
/*-----------------------------------------------------------
* Get actual partitioning-
* read in an actual csr matrix.
*-----------------------------------------------------------*/
if (build_matrix_type ==1) /*read in a csr matrix from one file */
{
if (matrix_arg_index < argc)
{
csrfilename = argv[matrix_arg_index];
}
else
{
hypre_printf("Error: No filename specified \n");
exit(1);
}
if (myid == 0)
{
/*hypre_printf(" FromFile: %s\n", csrfilename);*/
A_CSR = hypre_CSRMatrixRead(csrfilename);
}
row_part = NULL;
col_part = NULL;
parcsr_A = hypre_CSRMatrixToParCSRMatrix(hypre_MPI_COMM_WORLD, A_CSR,
row_part, col_part);
if (myid == 0) hypre_CSRMatrixDestroy(A_CSR);
}
else if (build_matrix_type ==2)
{
myBuildParLaplacian(argc, argv, matrix_arg_index, &A_temp, !noparmprint);
parcsr_A = (hypre_ParCSRMatrix *) A_temp;
}
else if (build_matrix_type ==4)
{
myBuildParLaplacian27pt(argc, argv, matrix_arg_index, &A_temp, !noparmprint);
parcsr_A = (hypre_ParCSRMatrix *) A_temp;
}
/*-----------------------------------------------------------
* create a small problem so that timings are more accurate -
* code gets run twice (small laplace)
*-----------------------------------------------------------*/
/*this is no longer being used - preload = 0 is set at the beginning */
if (preload == 1)
{
/*hypre_printf("preload!\n");*/
values[1] = -1;
values[2] = -1;
values[3] = -1;
values[0] = - 6.0 ;
nx = 2;
ny = num_procs;
nz = 2;
P = 1;
Q = num_procs;
R = 1;
p = myid % P;
q = (( myid - p)/P) % Q;
r = ( myid - p - P*q)/( P*Q );
A_temp_small = (HYPRE_ParCSRMatrix) GenerateLaplacian(hypre_MPI_COMM_WORLD, nx, ny, nz,
P, Q, R, p, q, r, values);
small_A = (hypre_ParCSRMatrix *) A_temp_small;
/*do comm packages*/
hypre_NewCommPkgCreate(small_A);
hypre_NewCommPkgDestroy(small_A);
hypre_MatvecCommPkgCreate(small_A);
hypre_ParCSRMatrixDestroy(small_A);
}
/*-----------------------------------------------------------
* Prepare for timing
*-----------------------------------------------------------*/
/* instead of preloading, let's not time the first one if more than one*/
if (!loop)
{
loop = 1;
/* and don't do any timings */
}
else
{
loop +=1;
if (loop < 2) loop = 2;
}
loop_times = hypre_CTAlloc(HYPRE_Real, loop);
/******************************************************************************************/
hypre_MPI_Barrier(hypre_MPI_COMM_WORLD);
if (commpkg_flag == 1 || commpkg_flag ==3 )
{
/*-----------------------------------------------------------
* Create new comm package
*-----------------------------------------------------------*/
if (!myid) hypre_printf("********************************************************\n" );
/*do loop times*/
for (i=0; i< loop; i++)
{
loop_times[i] = 0.0;
for (k=0; k< loop2; k++)
{
hypre_MPI_Barrier(hypre_MPI_COMM_WORLD);
start_time = hypre_MPI_Wtime();
#if mpip_on
if (i==(loop-1)) hypre_MPI_Pcontrol(1);
#endif
hypre_NewCommPkgCreate(parcsr_A);
#if mpip_on
if (i==(loop-1)) hypre_MPI_Pcontrol(0);
#endif
end_time = hypre_MPI_Wtime();
end_time = end_time - start_time;
hypre_MPI_Allreduce(&end_time, &total_time, 1,
HYPRE_MPI_REAL, hypre_MPI_MAX, hypre_MPI_COMM_WORLD);
loop_times[i] += total_time;
if ( !((i+1)== loop && (k+1) == loop2)) hypre_NewCommPkgDestroy(parcsr_A);
}/*end of loop2 */
} /*end of loop*/
/* calculate the avg and std. */
if (loop > 1)
{
/* calculate the avg and std. */
stats_mo(loop_times, loop, &T_avg, &T_std);
if (!myid) hypre_printf(" NewCommPkgCreate: AVG. wall clock time = %f seconds\n", T_avg);
if (!myid) hypre_printf(" STD. for %d runs = %f\n", loop-1, T_std);
if (!myid) hypre_printf(" (Note: avg./std. timings exclude run 0.)\n");
if (!myid) hypre_printf("********************************************************\n" );
for (i=0; i< loop; i++)
{
if (!myid) hypre_printf(" run %d = %f sec.\n", i, loop_times[i]);
}
if (!myid) hypre_printf("********************************************************\n" );
}
else
{
if (!myid) hypre_printf("********************************************************\n" );
if (!myid) hypre_printf(" NewCommPkgCreate:\n");
if (!myid) hypre_printf(" run time = %f sec.\n", loop_times[0]);
if (!myid) hypre_printf("********************************************************\n" );
}
/*-----------------------------------------------------------
* Verbose printing
*-----------------------------------------------------------*/
/*some verification*/
global_num_rows = hypre_ParCSRMatrixGlobalNumRows(parcsr_A);
if (verbose)
{
ierr = hypre_ParCSRMatrixGetLocalRange( parcsr_A,
&row_start, &row_end ,
&col_start, &col_end );
comm_pkg = hypre_ParCSRMatrixCommPkg(parcsr_A);
hypre_printf("myid = %i, my ACTUAL local range: [%i, %i]\n", myid,
row_start, row_end);
ierr = hypre_GetAssumedPartitionRowRange( myid, global_num_rows, &row_start,
&row_end);
hypre_printf("myid = %i, my assumed local range: [%i, %i]\n", myid,
row_start, row_end);
hypre_printf("myid = %d, num_recvs = %d\n", myid,
hypre_ParCSRCommPkgNumRecvs(comm_pkg) );
#if mydebug
for (i=0; i < hypre_ParCSRCommPkgNumRecvs(comm_pkg); i++)
{
hypre_printf("myid = %d, recv proc = %d, vec_starts = [%d : %d]\n",
myid, hypre_ParCSRCommPkgRecvProcs(comm_pkg)[i],
hypre_ParCSRCommPkgRecvVecStarts(comm_pkg)[i],
hypre_ParCSRCommPkgRecvVecStarts(comm_pkg)[i+1]-1);
}
#endif
hypre_printf("myid = %d, num_sends = %d\n", myid,
hypre_ParCSRCommPkgNumSends(comm_pkg) );
#if mydebug
for (i=0; i <hypre_ParCSRCommPkgNumSends(comm_pkg) ; i++)
{
tmp_int = hypre_ParCSRCommPkgSendMapStarts(comm_pkg)[i+1] -
hypre_ParCSRCommPkgSendMapStarts(comm_pkg)[i];
index = hypre_ParCSRCommPkgSendMapStarts(comm_pkg)[i];
for (j=0; j< tmp_int; j++)
{
hypre_printf("myid = %d, send proc = %d, send element = %d\n",myid,
hypre_ParCSRCommPkgSendProcs(comm_pkg)[i],
hypre_ParCSRCommPkgSendMapElmts(comm_pkg)[index+j]);
}
}
#endif
}
/*-----------------------------------------------------------
* To verify correctness (if commpkg_flag = 3)
*-----------------------------------------------------------*/
if (commpkg_flag == 3 )
{
/*do a matvec - we are assuming a square matrix */
row_starts = hypre_ParCSRMatrixRowStarts(parcsr_A);
x_new = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD, global_num_rows, row_starts);
hypre_ParVectorSetPartitioningOwner(x_new, 0);
hypre_ParVectorInitialize(x_new);
hypre_ParVectorSetRandomValues(x_new, 1);
y_new = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD, global_num_rows, row_starts);
hypre_ParVectorSetPartitioningOwner(y_new, 0);
hypre_ParVectorInitialize(y_new);
hypre_ParVectorSetConstantValues(y_new, 0.0);
/*y = 1.0*A*x+1.0*y */
hypre_ParCSRMatrixMatvec (1.0, parcsr_A, x_new, 1.0, y_new);
}
/*-----------------------------------------------------------
* Clean up after MyComm
*-----------------------------------------------------------*/
hypre_NewCommPkgDestroy(parcsr_A);
}
/******************************************************************************************/
/******************************************************************************************/
hypre_MPI_Barrier(hypre_MPI_COMM_WORLD);
if (commpkg_flag > 1 )
{
/*-----------------------------------------------------------
* Set up standard comm package
*-----------------------------------------------------------*/
bcast_rows[0] = 23;
bcast_rows[1] = 1789;
if (!myid) hypre_printf("********************************************************\n" );
/*do loop times*/
for (i=0; i< loop; i++)
{
loop_times[i] = 0.0;
for (k=0; k< loop2; k++)
{
hypre_MPI_Barrier(hypre_MPI_COMM_WORLD);
start_time = hypre_MPI_Wtime();
#if time_gather
info = hypre_CTAlloc(HYPRE_Int, num_procs);
hypre_MPI_Allgather(bcast_rows, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, hypre_MPI_COMM_WORLD);
#endif
hypre_MatvecCommPkgCreate(parcsr_A);
end_time = hypre_MPI_Wtime();
end_time = end_time - start_time;
hypre_MPI_Allreduce(&end_time, &total_time, 1,
HYPRE_MPI_REAL, hypre_MPI_MAX, hypre_MPI_COMM_WORLD);
loop_times[i] += total_time;
if ( !((i+1)== loop && (k+1) == loop2)) hypre_MatvecCommPkgDestroy(hypre_ParCSRMatrixCommPkg(parcsr_A));
}/* end of loop 2*/
} /*end of loop*/
/* calculate the avg and std. */
if (loop > 1)
{
stats_mo(loop_times, loop, &T_avg, &T_std);
if (!myid) hypre_printf("Current CommPkgCreate: AVG. wall clock time = %f seconds\n", T_avg);
if (!myid) hypre_printf(" STD. for %d runs = %f\n", loop-1, T_std);
if (!myid) hypre_printf(" (Note: avg./std. timings exclude run 0.)\n");
if (!myid) hypre_printf("********************************************************\n" );
for (i=0; i< loop; i++)
{
if (!myid) hypre_printf(" run %d = %f sec.\n", i, loop_times[i]);
}
if (!myid) hypre_printf("********************************************************\n" );
}
else
{
if (!myid) hypre_printf("********************************************************\n" );
if (!myid) hypre_printf(" Current CommPkgCreate:\n");
if (!myid) hypre_printf(" run time = %f sec.\n", loop_times[0]);
if (!myid) hypre_printf("********************************************************\n" );
}
/*-----------------------------------------------------------
* Verbose printing
*-----------------------------------------------------------*/
/*some verification*/
if (verbose)
{
ierr = hypre_ParCSRMatrixGetLocalRange( parcsr_A,
&row_start, &row_end ,
&col_start, &col_end );
comm_pkg = hypre_ParCSRMatrixCommPkg(parcsr_A);
hypre_printf("myid = %i, std - my local range: [%i, %i]\n", myid,
row_start, row_end);
ierr = hypre_ParCSRMatrixGetLocalRange( parcsr_A,
&row_start, &row_end ,
&col_start, &col_end );
hypre_printf("myid = %d, std - num_recvs = %d\n", myid,
hypre_ParCSRCommPkgNumRecvs(comm_pkg) );
#if mydebug
for (i=0; i < hypre_ParCSRCommPkgNumRecvs(comm_pkg); i++)
{
hypre_printf("myid = %d, std - recv proc = %d, vec_starts = [%d : %d]\n",
myid, hypre_ParCSRCommPkgRecvProcs(comm_pkg)[i],
hypre_ParCSRCommPkgRecvVecStarts(comm_pkg)[i],
hypre_ParCSRCommPkgRecvVecStarts(comm_pkg)[i+1]-1);
}
#endif
hypre_printf("myid = %d, std - num_sends = %d\n", myid,
hypre_ParCSRCommPkgNumSends(comm_pkg));
#if mydebug
for (i=0; i <hypre_ParCSRCommPkgNumSends(comm_pkg) ; i++)
{
tmp_int = hypre_ParCSRCommPkgSendMapStarts(comm_pkg)[i+1] -
hypre_ParCSRCommPkgSendMapStarts(comm_pkg)[i];
index = hypre_ParCSRCommPkgSendMapStarts(comm_pkg)[i];
for (j=0; j< tmp_int; j++)
{
hypre_printf("myid = %d, std - send proc = %d, send element = %d\n",myid,
hypre_ParCSRCommPkgSendProcs(comm_pkg)[i],
hypre_ParCSRCommPkgSendMapElmts(comm_pkg)[index+j]);
}
}
#endif
}
/*-----------------------------------------------------------
* Verify correctness
*-----------------------------------------------------------*/
if (commpkg_flag == 3 )
{
global_num_rows = hypre_ParCSRMatrixGlobalNumRows(parcsr_A);
row_starts = hypre_ParCSRMatrixRowStarts(parcsr_A);
y = hypre_ParVectorCreate(hypre_MPI_COMM_WORLD, global_num_rows,row_starts);
hypre_ParVectorSetPartitioningOwner(y, 0);
hypre_ParVectorInitialize(y);
hypre_ParVectorSetConstantValues(y, 0.0);
hypre_ParCSRMatrixMatvec (1.0, parcsr_A, x_new, 1.0, y);
}
}
/*-----------------------------------------------------------
* Compare matvecs for both comm packages (3)
*-----------------------------------------------------------*/
if (commpkg_flag == 3 )
{
/*make sure that y and y_new are the same - now y_new should=0*/
hypre_ParVectorAxpy( -1.0, y, y_new );
hypre_ParVectorSetRandomValues(y, 1);
ans = hypre_ParVectorInnerProd( y, y_new );
if (!myid)
{
if ( fabs(ans) > 1e-8 )
{
hypre_printf("!!!!! WARNING !!!!! should be zero if correct = %6.10f\n",
ans);
}
else
{
hypre_printf("Matvecs match ( should be zero = %6.10f )\n",
ans);
}
}
}
/*-----------------------------------------------------------
* Clean up
*-----------------------------------------------------------*/
hypre_ParCSRMatrixDestroy(parcsr_A); /*this calls the standard comm
package destroy - but we'll destroy
ours separately until it is
incorporated */
if (commpkg_flag == 3 )
{
hypre_ParVectorDestroy(x_new);
hypre_ParVectorDestroy(y);
hypre_ParVectorDestroy(y_new);
}
hypre_MPI_Finalize();
return(ierr);
}
HYPRE_Int main(HYPRE_Int argc, char *argv[])
{
HYPRE_Int mype, npes;
HYPRE_Int symmetric;
HYPRE_Int num_runs;
Matrix *A;
ParaSails *ps;
FILE *file;
HYPRE_Int n, beg_row, end_row;
HYPRE_Real time0, time1;
HYPRE_Real setup_time, solve_time;
HYPRE_Real max_setup_time, max_solve_time;
HYPRE_Real cost;
HYPRE_Real *x, *b;
HYPRE_Int i, niter;
HYPRE_Real thresh;
HYPRE_Real threshg;
HYPRE_Int nlevels;
HYPRE_Real filter;
HYPRE_Real loadbal;
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &mype);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &npes);
/* Read number of rows in matrix */
symmetric = atoi(argv[1]);
num_runs = atoi(argv[2]);
file = fopen(argv[3], "r");
assert(file != NULL);
#ifdef EMSOLVE
hypre_fscanf(file, "%*d %d\n", &n);
#else
hypre_fscanf(file, "%d\n", &n);
#endif
fclose(file);
assert(n >= npes);
beg_row = (HYPRE_Int) ((HYPRE_Real)(mype*n) / npes) + 1; /* assumes 1-based */
end_row = (HYPRE_Int) ((HYPRE_Real)((mype+1)* n) / npes);
if (mype == 0)
assert(beg_row == 1);
if (mype == npes-1)
assert(end_row == n);
#ifdef EMSOLVE
beg_row--;
end_row--;
#endif
x = (HYPRE_Real *) malloc((end_row-beg_row+1) * sizeof(HYPRE_Real));
b = (HYPRE_Real *) malloc((end_row-beg_row+1) * sizeof(HYPRE_Real));
A = MatrixCreate(hypre_MPI_COMM_WORLD, beg_row, end_row);
MatrixRead(A, argv[3]);
if (mype == 0)
hypre_printf("%s\n", argv[3]);
/* MatrixPrint(A, "A"); */
/* Right-hand side */
if (argc > 4)
{
RhsRead(b, A, argv[4]);
if (mype == 0)
hypre_printf("Using rhs from %s\n", argv[4]);
}
else
{
for (i=0; i<end_row-beg_row+1; i++)
b[i] = (HYPRE_Real) (2*rand()) / (HYPRE_Real) RAND_MAX - 1.0;
}
while (num_runs && num_runs >= -1)
{
/* Initial guess */
for (i=0; i<end_row-beg_row+1; i++)
x[i] = 0.0;
if (num_runs == -1)
{
thresh = 0.0;
nlevels = 0;
filter = 0.0;
loadbal = 0.0;
}
else
{
if (mype == 0)
{
#if PARASAILS_EXT_PATTERN
hypre_printf("Enter parameters threshg, thresh, nlevels, "
"filter, beta:\n");
fflush(stdout);
hypre_scanf("%lf %lf %d %lf %lf", &threshg, &thresh, &nlevels,
&filter, &loadbal);
#else
hypre_printf("Enter parameters thresh, nlevels, "
"filter, beta:\n");
fflush(stdout);
hypre_scanf("%lf %d %lf %lf", &thresh, &nlevels,
&filter, &loadbal);
#endif
}
hypre_MPI_Bcast(&threshg, 1, hypre_MPI_DOUBLE, 0, hypre_MPI_COMM_WORLD);
hypre_MPI_Bcast(&thresh, 1, hypre_MPI_DOUBLE, 0, hypre_MPI_COMM_WORLD);
hypre_MPI_Bcast(&nlevels, 1, HYPRE_MPI_INT, 0, hypre_MPI_COMM_WORLD);
hypre_MPI_Bcast(&filter, 1, hypre_MPI_DOUBLE, 0, hypre_MPI_COMM_WORLD);
hypre_MPI_Bcast(&loadbal, 1, hypre_MPI_DOUBLE, 0, hypre_MPI_COMM_WORLD);
if (nlevels < 0)
break;
}
/**************
* Setup phase
**************/
hypre_MPI_Barrier(hypre_MPI_COMM_WORLD);
time0 = hypre_MPI_Wtime();
ps = ParaSailsCreate(hypre_MPI_COMM_WORLD, beg_row, end_row, symmetric);
ps->loadbal_beta = loadbal;
#if PARASAILS_EXT_PATTERN
ParaSailsSetupPatternExt(ps, A, threshg, thresh, nlevels);
#else
ParaSailsSetupPattern(ps, A, thresh, nlevels);
#endif
time1 = hypre_MPI_Wtime();
setup_time = time1-time0;
cost = ParaSailsStatsPattern(ps, A);
if (cost > 5.e11)
{
hypre_printf("Aborting setup and solve due to high cost.\n");
goto cleanup;
}
hypre_MPI_Barrier(hypre_MPI_COMM_WORLD);
time0 = hypre_MPI_Wtime();
err = ParaSailsSetupValues(ps, A, filter);
if (err != 0)
{
hypre_printf("ParaSailsSetupValues returned error.\n");
goto cleanup;
}
time1 = hypre_MPI_Wtime();
setup_time += (time1-time0);
ParaSailsStatsValues(ps, A);
if (!strncmp(argv[3], "testpsmat", 8))
MatrixPrint(ps->M, "M");
#if 0
if (mype == 0)
hypre_printf("SETTING UP VALUES AGAIN WITH FILTERED PATTERN\n");
ps->loadbal_beta = 0;
ParaSailsSetupValues(ps, A, 0.0);
#endif
/*****************
* Solution phase
*****************/
niter = 3000;
if (MatrixNnz(ps->M) == n) /* if diagonal preconditioner */
niter = 5000;
hypre_MPI_Barrier(hypre_MPI_COMM_WORLD);
time0 = hypre_MPI_Wtime();
if (symmetric == 1)
PCG_ParaSails(A, ps, b, x, 1.e-8, niter);
else
FGMRES_ParaSails(A, ps, b, x, 50, 1.e-8, niter);
time1 = hypre_MPI_Wtime();
solve_time = time1-time0;
hypre_MPI_Reduce(&setup_time, &max_setup_time, 1, hypre_MPI_DOUBLE, hypre_MPI_MAX, 0,
hypre_MPI_COMM_WORLD);
hypre_MPI_Reduce(&solve_time, &max_solve_time, 1, hypre_MPI_DOUBLE, hypre_MPI_MAX, 0,
hypre_MPI_COMM_WORLD);
if (mype == 0)
{
hypre_printf("**********************************************\n");
hypre_printf("*** Setup Solve Total\n");
hypre_printf("III %8.1f %8.1f %8.1f\n", max_setup_time, max_solve_time,
max_setup_time+max_solve_time);
hypre_printf("**********************************************\n");
}
cleanup:
ParaSailsDestroy(ps);
num_runs--;
}
free(x);
free(b);
MatrixDestroy(A);
hypre_MPI_Finalize();
return 0;
}
static void matvec_timing(MPI_Comm comm, Matrix *mat)
{
HYPRE_Real time0, time1;
HYPRE_Real trial1, trial2, trial3, trial4, trial5, trial6;
HYPRE_Real *temp1, *temp2;
HYPRE_Int i, mype;
HYPRE_Int n = mat->end_row - mat->beg_row + 1;
temp1 = (HYPRE_Real *) calloc(n, sizeof(HYPRE_Real));
temp2 = (HYPRE_Real *) calloc(n, sizeof(HYPRE_Real));
/* warm-up */
hypre_MPI_Barrier(comm);
for (i=0; i<100; i++)
MatrixMatvec(mat, temp1, temp2);
hypre_MPI_Barrier(comm);
time0 = hypre_MPI_Wtime();
for (i=0; i<100; i++)
MatrixMatvec(mat, temp1, temp2);
hypre_MPI_Barrier(comm);
time1 = hypre_MPI_Wtime();
trial1 = time1-time0;
hypre_MPI_Barrier(comm);
time0 = hypre_MPI_Wtime();
for (i=0; i<100; i++)
MatrixMatvec(mat, temp1, temp2);
hypre_MPI_Barrier(comm);
time1 = hypre_MPI_Wtime();
trial2 = time1-time0;
hypre_MPI_Barrier(comm);
time0 = hypre_MPI_Wtime();
for (i=0; i<100; i++)
MatrixMatvec(mat, temp1, temp2);
hypre_MPI_Barrier(comm);
time1 = hypre_MPI_Wtime();
trial3 = time1-time0;
hypre_MPI_Barrier(comm);
time0 = hypre_MPI_Wtime();
for (i=0; i<100; i++)
MatrixMatvecSerial(mat, temp1, temp2);
hypre_MPI_Barrier(comm);
time1 = hypre_MPI_Wtime();
trial4 = time1-time0;
hypre_MPI_Barrier(comm);
time0 = hypre_MPI_Wtime();
for (i=0; i<100; i++)
MatrixMatvecSerial(mat, temp1, temp2);
hypre_MPI_Barrier(comm);
time1 = hypre_MPI_Wtime();
trial5 = time1-time0;
hypre_MPI_Barrier(comm);
time0 = hypre_MPI_Wtime();
for (i=0; i<100; i++)
MatrixMatvecSerial(mat, temp1, temp2);
hypre_MPI_Barrier(comm);
time1 = hypre_MPI_Wtime();
trial6 = time1-time0;
hypre_MPI_Comm_rank(comm, &mype);
if (mype == 0)
hypre_printf("Timings: %f %f %f Serial: %f %f %f\n",
trial1, trial2, trial3, trial4, trial5, trial6);
fflush(stdout);
/* this is all we wanted, so don't waste any more cycles */
exit(0);
}
