void HypreSolver2D::build_solver()
{
HYPRE_BoomerAMGCreate(&solver);
//HYPRE_BoomerAMGSetPrintLevel(solver, 3); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(solver, 3); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(solver, 1); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(solver, 20); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
}
int main (int argc, char *argv[])
{
int myid, num_procs;
int n;
double gamma, h;
int vis;
HYPRE_SStructGrid grid;
HYPRE_SStructGraph graph;
HYPRE_SStructMatrix A;
HYPRE_SStructVector b;
HYPRE_SStructVector x;
HYPRE_Solver solver;
/* Initialize MPI */
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
/* Set default parameters */
n = 10;
vis = 0;
/* Parse command line */
{
int arg_index = 0;
int print_usage = 0;
while (arg_index < argc)
{
if ( strcmp(argv[arg_index], "-n") == 0 )
{
arg_index++;
n = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-vis") == 0 )
{
arg_index++;
vis = 1;
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
break;
}
else
{
arg_index++;
}
}
if ((print_usage) && (myid == 0))
{
printf("\n");
printf("Usage: %s [<options>]\n", argv[0]);
printf("\n");
printf(" -n <n> : problem size per processor (default: 10)\n");
printf(" -vis : save the solution for GLVis visualization\n");
printf("\n");
}
if (print_usage)
{
MPI_Finalize();
return (0);
}
}
/* Set the rhombus angle, gamma, and the mesh size, h, depending on the
number of processors np and the given n */
if (num_procs < 3)
{
if (myid ==0) printf("Must run with at least 3 processors!\n");
MPI_Finalize();
exit(1);
}
gamma = 2*M_PI/num_procs;
h = 1.0/n;
/* 1. Set up the grid. We will set up the grid so that processor X owns
part X. Note that each part has its own index space numbering. Later
we relate the parts to each other. */
{
int ndim = 2;
int nparts = num_procs;
/* Create an empty 2D grid object */
HYPRE_SStructGridCreate(MPI_COMM_WORLD, ndim, nparts, &grid);
/* Set the extents of the grid - each processor sets its grid boxes. Each
part has its own relative index space numbering */
{
int part = myid;
int ilower[2] = {1,1}; /* lower-left cell touching the origin */
int iupper[2] = {n,n}; /* upper-right cell */
HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
}
/* Set the variable type and number of variables on each part. These need
to be set in each part which is neighboring or contains boxes owned by
the processor. */
{
int i;
int nvars = 1;
HYPRE_SStructVariable vartypes[1] = {HYPRE_SSTRUCT_VARIABLE_NODE};
for (i = 0; i < nparts; i++)
HYPRE_SStructGridSetVariables(grid, i, nvars, vartypes);
}
/* Set the ordering of the variables in the finite element problem. This
is done by listing the variable offset directions relative to the
element's center. See the Reference Manual for more details. */
{
int part = myid;
int ordering[12] = { 0, -1, -1, /* [3]------[2] */
0, +1, -1, /* / / */
0, +1, +1, /* / / */
0, -1, +1 }; /* [0]------[1] */
HYPRE_SStructGridSetFEMOrdering(grid, part, ordering);
}
/* Now we need to set the spatial relation between each of the parts.
Since we are using nodal variables, we have to use SetSharedPart to
establish the connection at the origin. */
{
/* Relation to the clockwise-previous neighbor part, e.g. 0 and 1 for
the case of 6 parts. Note that we could have used SetNeighborPart
here instead of SetSharedPart. */
{
int part = myid;
/* the box of cells intersecting the boundary in the current part */
int ilower[2] = {1,1}, iupper[2] = {1,n};
/* share all data on the left side of the box */
int offset[2] = {-1,0};
int shared_part = (myid+1) % num_procs;
/* the box of cells intersecting the boundary in the neighbor */
int shared_ilower[2] = {1,1}, shared_iupper[2] = {n,1};
/* share all data on the bottom of the box */
int shared_offset[2] = {0,-1};
/* x/y-direction on the current part is -y/x on the neighbor */
int index_map[2] = {1,0};
int index_dir[2] = {-1,1};
HYPRE_SStructGridSetSharedPart(grid, part, ilower, iupper, offset,
shared_part, shared_ilower,
shared_iupper, shared_offset,
index_map, index_dir);
}
/* Relation to the clockwise-following neighbor part, e.g. 0 and 5 for
the case of 6 parts. Note that we could have used SetNeighborPart
here instead of SetSharedPart. */
{
int part = myid;
/* the box of cells intersecting the boundary in the current part */
int ilower[2] = {1,1}, iupper[2] = {n,1};
/* share all data on the bottom of the box */
int offset[2] = {0,-1};
int shared_part = (myid+num_procs-1) % num_procs;
/* the box of cells intersecting the boundary in the neighbor */
int shared_ilower[2] = {1,1}, shared_iupper[2] = {1,n};
/* share all data on the left side of the box */
int shared_offset[2] = {-1,0};
/* x/y-direction on the current part is y/-x on the neighbor */
int index_map[2] = {1,0};
int index_dir[2] = {1,-1};
HYPRE_SStructGridSetSharedPart(grid, part, ilower, iupper, offset,
shared_part, shared_ilower,
shared_iupper, shared_offset,
index_map, index_dir);
}
/* Relation to all other parts, e.g. 0 and 2,3,4. This can be
described only by SetSharedPart. */
{
int part = myid;
/* the (one cell) box that touches the origin */
int ilower[2] = {1,1}, iupper[2] = {1,1};
/* share all data in the bottom left corner (i.e. the origin) */
int offset[2] = {-1,-1};
int shared_part;
/* the box of one cell that touches the origin */
int shared_ilower[2] = {1,1}, shared_iupper[2] = {1,1};
/* share all data in the bottom left corner (i.e. the origin) */
int shared_offset[2] = {-1,-1};
/* x/y-direction on the current part is -x/-y on the neighbor, but
in this case the arguments are not really important since we are
only sharing a point */
int index_map[2] = {0,1};
int index_dir[2] = {-1,-1};
for (shared_part = 0; shared_part < myid-1; shared_part++)
HYPRE_SStructGridSetSharedPart(grid, part, ilower, iupper, offset,
shared_part, shared_ilower,
shared_iupper, shared_offset,
index_map, index_dir);
for (shared_part = myid+2; shared_part < num_procs; shared_part++)
HYPRE_SStructGridSetSharedPart(grid, part, ilower, iupper, offset,
shared_part, shared_ilower,
shared_iupper, shared_offset,
index_map, index_dir);
}
}
/* Now the grid is ready to be used */
HYPRE_SStructGridAssemble(grid);
}
/* 2. Set up the Graph - this determines the non-zero structure of the
matrix. */
{
int part;
/* Create the graph object */
HYPRE_SStructGraphCreate(MPI_COMM_WORLD, grid, &graph);
/* See MatrixSetObjectType below */
HYPRE_SStructGraphSetObjectType(graph, HYPRE_PARCSR);
/* Indicate that this problem uses finite element stiffness matrices and
load vectors, instead of stencils. */
for (part = 0; part < num_procs; part++)
HYPRE_SStructGraphSetFEM(graph, part);
/* The local stiffness matrix is full, so there is no need to call
HYPRE_SStructGraphSetFEMSparsity to set its sparsity pattern. */
/* Assemble the graph */
HYPRE_SStructGraphAssemble(graph);
}
/* 3. Set up the SStruct Matrix and right-hand side vector */
{
int part = myid;
/* Create the matrix object */
HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, graph, &A);
/* Use a ParCSR storage */
HYPRE_SStructMatrixSetObjectType(A, HYPRE_PARCSR);
/* Indicate that the matrix coefficients are ready to be set */
HYPRE_SStructMatrixInitialize(A);
/* Create an empty vector object */
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &b);
/* Use a ParCSR storage */
HYPRE_SStructVectorSetObjectType(b, HYPRE_PARCSR);
/* Indicate that the vector coefficients are ready to be set */
HYPRE_SStructVectorInitialize(b);
/* Set the matrix and vector entries by finite element assembly */
{
/* local stifness matrix and load vector */
double S[4][4], F[4];
int i, j, k;
int index[2];
/* set the values in the interior cells */
{
ComputeFEMRhombus(S, F, gamma, h);
for (i = 1; i <= n; i++)
for (j = 1; j <= n; j++)
{
index[0] = i;
index[1] = j;
HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
HYPRE_SStructVectorAddFEMValues(b, part, index, F);
}
}
/* cells having nodes 1,2 on the domain boundary */
{
ComputeFEMRhombus(S, F, gamma, h);
/* eliminate nodes 1,2 from S and F */
for (k = 0; k < 4; k++)
{
S[1][k] = S[k][1] = 0.0;
S[2][k] = S[k][2] = 0.0;
}
S[1][1] = 1.0;
S[2][2] = 1.0;
F[1] = 0.0;
F[2] = 0.0;
for (i = n; i <= n; i++)
for (j = 1; j <= n; j++)
{
index[0] = i;
index[1] = j;
HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
HYPRE_SStructVectorAddFEMValues(b, part, index, F);
}
}
/* cells having nodes 2,3 on the domain boundary */
{
ComputeFEMRhombus(S, F, gamma, h);
/* eliminate nodes 2,3 from S and F */
for (k = 0; k < 4; k++)
{
S[2][k] = S[k][2] = 0.0;
S[3][k] = S[k][3] = 0.0;
}
S[2][2] = 1.0;
S[3][3] = 1.0;
F[2] = 0.0;
F[3] = 0.0;
for (i = 1; i <= n; i++)
for (j = n; j <= n; j++)
{
index[0] = i;
index[1] = j;
HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
HYPRE_SStructVectorAddFEMValues(b, part, index, F);
}
}
/* cells having nodes 1,2,3 on the domain boundary */
{
ComputeFEMRhombus(S, F, gamma, h);
/* eliminate nodes 2,3 from S and F */
for (k = 0; k < 4; k++)
{
S[1][k] = S[k][1] = 0.0;
S[2][k] = S[k][2] = 0.0;
S[3][k] = S[k][3] = 0.0;
}
S[1][1] = 1.0;
S[2][2] = 1.0;
S[3][3] = 1.0;
F[1] = 0.0;
F[2] = 0.0;
F[3] = 0.0;
for (i = n; i <= n; i++)
for (j = n; j <= n; j++)
{
index[0] = i;
index[1] = j;
HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
HYPRE_SStructVectorAddFEMValues(b, part, index, F);
}
}
}
}
/* Collective calls finalizing the matrix and vector assembly */
HYPRE_SStructMatrixAssemble(A);
HYPRE_SStructVectorAssemble(b);
/* 4. Set up SStruct Vector for the solution vector x */
{
int part = myid;
int var = 0;
int nvalues = (n+1)*(n+1);
double *values;
/* Since the SetBoxValues() calls below set the values of the nodes in
the upper-right corners of the cells, the nodal box should start
from (0,0) instead of (1,1). */
int ilower[2] = {0,0};
int iupper[2] = {n,n};
values = calloc(nvalues, sizeof(double));
/* Create an empty vector object */
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &x);
/* Set the object type to ParCSR */
HYPRE_SStructVectorSetObjectType(x, HYPRE_PARCSR);
/* Indicate that the vector coefficients are ready to be set */
HYPRE_SStructVectorInitialize(x);
/* Set the values for the initial guess */
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, values);
free(values);
/* Finalize the vector assembly */
HYPRE_SStructVectorAssemble(x);
}
/* 5. Set up and call the solver (Solver options can be found in the
Reference Manual.) */
{
double final_res_norm;
int its;
HYPRE_ParCSRMatrix par_A;
HYPRE_ParVector par_b;
HYPRE_ParVector par_x;
/* Extract the ParCSR objects needed in the solver */
HYPRE_SStructMatrixGetObject(A, (void **) &par_A);
HYPRE_SStructVectorGetObject(b, (void **) &par_b);
HYPRE_SStructVectorGetObject(x, (void **) &par_x);
/* Here we construct a BoomerAMG solver. See the other SStruct examples
as well as the Reference manual for additional solver choices. */
HYPRE_BoomerAMGCreate(&solver);
HYPRE_BoomerAMGSetCoarsenType(solver, 6);
HYPRE_BoomerAMGSetStrongThreshold(solver, 0.25);
HYPRE_BoomerAMGSetTol(solver, 1e-6);
HYPRE_BoomerAMGSetPrintLevel(solver, 2);
HYPRE_BoomerAMGSetMaxIter(solver, 50);
/* call the setup */
HYPRE_BoomerAMGSetup(solver, par_A, par_b, par_x);
/* call the solve */
HYPRE_BoomerAMGSolve(solver, par_A, par_b, par_x);
/* get some info */
HYPRE_BoomerAMGGetNumIterations(solver, &its);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver,
&final_res_norm);
/* clean up */
HYPRE_BoomerAMGDestroy(solver);
/* Gather the solution vector */
HYPRE_SStructVectorGather(x);
/* Save the solution for GLVis visualization, see vis/glvis-ex13.sh */
if (vis)
{
FILE *file;
char filename[255];
int i, part = myid, var = 0;
int nvalues = (n+1)*(n+1);
double *values = calloc(nvalues, sizeof(double));
int ilower[2] = {0,0};
int iupper[2] = {n,n};
/* get all local data (including a local copy of the shared values) */
HYPRE_SStructVectorGetBoxValues(x, part, ilower, iupper,
var, values);
sprintf(filename, "%s.%06d", "vis/ex14.sol", myid);
if ((file = fopen(filename, "w")) == NULL)
{
printf("Error: can't open output file %s\n", filename);
MPI_Finalize();
exit(1);
}
/* finite element space header */
fprintf(file, "FiniteElementSpace\n");
fprintf(file, "FiniteElementCollection: H1_2D_P1\n");
fprintf(file, "VDim: 1\n");
fprintf(file, "Ordering: 0\n\n");
/* save solution */
for (i = 0; i < nvalues; i++)
fprintf(file, "%.14e\n", values[i]);
fflush(file);
fclose(file);
free(values);
/* save local finite element mesh */
GLVis_PrintLocalRhombusMesh("vis/ex14.mesh", n, myid, gamma);
/* additional visualization data */
if (myid == 0)
{
sprintf(filename, "%s", "vis/ex14.data");
file = fopen(filename, "w");
fprintf(file, "np %d\n", num_procs);
fflush(file);
fclose(file);
}
}
if (myid == 0)
{
printf("\n");
printf("Iterations = %d\n", its);
printf("Final Relative Residual Norm = %g\n", final_res_norm);
printf("\n");
}
}
/* Free memory */
HYPRE_SStructGridDestroy(grid);
HYPRE_SStructGraphDestroy(graph);
HYPRE_SStructMatrixDestroy(A);
HYPRE_SStructVectorDestroy(b);
HYPRE_SStructVectorDestroy(x);
/* Finalize MPI */
MPI_Finalize();
return 0;
}
HYPRE_Int hypre_seqAMGSetup( hypre_ParAMGData *amg_data,
HYPRE_Int p_level,
HYPRE_Int coarse_threshold)
{
/* Par Data Structure variables */
hypre_ParCSRMatrix **Par_A_array = hypre_ParAMGDataAArray(amg_data);
MPI_Comm comm = hypre_ParCSRMatrixComm(Par_A_array[0]);
MPI_Comm new_comm, seq_comm;
hypre_ParCSRMatrix *A_seq = NULL;
hypre_CSRMatrix *A_seq_diag;
hypre_CSRMatrix *A_seq_offd;
hypre_ParVector *F_seq = NULL;
hypre_ParVector *U_seq = NULL;
hypre_ParCSRMatrix *A;
HYPRE_Int **dof_func_array;
HYPRE_Int num_procs, my_id;
HYPRE_Int not_finished_coarsening;
HYPRE_Int level;
HYPRE_Solver coarse_solver;
/* misc */
dof_func_array = hypre_ParAMGDataDofFuncArray(amg_data);
/*MPI Stuff */
hypre_MPI_Comm_size(comm, &num_procs);
hypre_MPI_Comm_rank(comm,&my_id);
/*initial */
level = p_level;
not_finished_coarsening = 1;
/* convert A at this level to sequential */
A = Par_A_array[level];
{
double *A_seq_data = NULL;
HYPRE_Int *A_seq_i = NULL;
HYPRE_Int *A_seq_offd_i = NULL;
HYPRE_Int *A_seq_j = NULL;
double *A_tmp_data = NULL;
HYPRE_Int *A_tmp_i = NULL;
HYPRE_Int *A_tmp_j = NULL;
HYPRE_Int *info, *displs, *displs2;
HYPRE_Int i, j, size, num_nonzeros, total_nnz, cnt;
hypre_CSRMatrix *A_diag = hypre_ParCSRMatrixDiag(A);
hypre_CSRMatrix *A_offd = hypre_ParCSRMatrixOffd(A);
HYPRE_Int *col_map_offd = hypre_ParCSRMatrixColMapOffd(A);
HYPRE_Int *A_diag_i = hypre_CSRMatrixI(A_diag);
HYPRE_Int *A_offd_i = hypre_CSRMatrixI(A_offd);
HYPRE_Int *A_diag_j = hypre_CSRMatrixJ(A_diag);
HYPRE_Int *A_offd_j = hypre_CSRMatrixJ(A_offd);
double *A_diag_data = hypre_CSRMatrixData(A_diag);
double *A_offd_data = hypre_CSRMatrixData(A_offd);
HYPRE_Int num_rows = hypre_CSRMatrixNumRows(A_diag);
HYPRE_Int first_row_index = hypre_ParCSRMatrixFirstRowIndex(A);
hypre_MPI_Group orig_group, new_group;
HYPRE_Int *ranks, new_num_procs, *row_starts;
info = hypre_CTAlloc(HYPRE_Int, num_procs);
hypre_MPI_Allgather(&num_rows, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, comm);
ranks = hypre_CTAlloc(HYPRE_Int, num_procs);
new_num_procs = 0;
for (i=0; i < num_procs; i++)
if (info[i])
{
ranks[new_num_procs] = i;
info[new_num_procs++] = info[i];
}
MPI_Comm_group(comm, &orig_group);
hypre_MPI_Group_incl(orig_group, new_num_procs, ranks, &new_group);
MPI_Comm_create(comm, new_group, &new_comm);
hypre_MPI_Group_free(&new_group);
hypre_MPI_Group_free(&orig_group);
if (num_rows)
{
/* alloc space in seq data structure only for participating procs*/
HYPRE_BoomerAMGCreate(&coarse_solver);
HYPRE_BoomerAMGSetMaxRowSum(coarse_solver,
hypre_ParAMGDataMaxRowSum(amg_data));
HYPRE_BoomerAMGSetStrongThreshold(coarse_solver,
hypre_ParAMGDataStrongThreshold(amg_data));
HYPRE_BoomerAMGSetCoarsenType(coarse_solver,
hypre_ParAMGDataCoarsenType(amg_data));
HYPRE_BoomerAMGSetInterpType(coarse_solver,
hypre_ParAMGDataInterpType(amg_data));
HYPRE_BoomerAMGSetTruncFactor(coarse_solver,
hypre_ParAMGDataTruncFactor(amg_data));
HYPRE_BoomerAMGSetPMaxElmts(coarse_solver,
hypre_ParAMGDataPMaxElmts(amg_data));
if (hypre_ParAMGDataUserRelaxType(amg_data) > -1)
HYPRE_BoomerAMGSetRelaxType(coarse_solver,
hypre_ParAMGDataUserRelaxType(amg_data));
HYPRE_BoomerAMGSetRelaxOrder(coarse_solver,
hypre_ParAMGDataRelaxOrder(amg_data));
HYPRE_BoomerAMGSetRelaxWt(coarse_solver,
hypre_ParAMGDataUserRelaxWeight(amg_data));
if (hypre_ParAMGDataUserNumSweeps(amg_data) > -1)
HYPRE_BoomerAMGSetNumSweeps(coarse_solver,
hypre_ParAMGDataUserNumSweeps(amg_data));
HYPRE_BoomerAMGSetNumFunctions(coarse_solver,
hypre_ParAMGDataNumFunctions(amg_data));
HYPRE_BoomerAMGSetMaxIter(coarse_solver, 1);
HYPRE_BoomerAMGSetTol(coarse_solver, 0);
/* Create CSR Matrix, will be Diag part of new matrix */
A_tmp_i = hypre_CTAlloc(HYPRE_Int, num_rows+1);
A_tmp_i[0] = 0;
for (i=1; i < num_rows+1; i++)
A_tmp_i[i] = A_diag_i[i]-A_diag_i[i-1]+A_offd_i[i]-A_offd_i[i-1];
num_nonzeros = A_offd_i[num_rows]+A_diag_i[num_rows];
A_tmp_j = hypre_CTAlloc(HYPRE_Int, num_nonzeros);
A_tmp_data = hypre_CTAlloc(double, num_nonzeros);
cnt = 0;
for (i=0; i < num_rows; i++)
{
for (j=A_diag_i[i]; j < A_diag_i[i+1]; j++)
{
A_tmp_j[cnt] = A_diag_j[j]+first_row_index;
A_tmp_data[cnt++] = A_diag_data[j];
}
for (j=A_offd_i[i]; j < A_offd_i[i+1]; j++)
{
A_tmp_j[cnt] = col_map_offd[A_offd_j[j]];
A_tmp_data[cnt++] = A_offd_data[j];
}
}
displs = hypre_CTAlloc(HYPRE_Int, new_num_procs+1);
displs[0] = 0;
for (i=1; i < new_num_procs+1; i++)
displs[i] = displs[i-1]+info[i-1];
size = displs[new_num_procs];
A_seq_i = hypre_CTAlloc(HYPRE_Int, size+1);
A_seq_offd_i = hypre_CTAlloc(HYPRE_Int, size+1);
hypre_MPI_Allgatherv ( &A_tmp_i[1], num_rows, HYPRE_MPI_INT, &A_seq_i[1], info,
displs, HYPRE_MPI_INT, new_comm );
displs2 = hypre_CTAlloc(HYPRE_Int, new_num_procs+1);
A_seq_i[0] = 0;
displs2[0] = 0;
for (j=1; j < displs[1]; j++)
A_seq_i[j] = A_seq_i[j]+A_seq_i[j-1];
for (i=1; i < new_num_procs; i++)
{
for (j=displs[i]; j < displs[i+1]; j++)
{
A_seq_i[j] = A_seq_i[j]+A_seq_i[j-1];
}
}
A_seq_i[size] = A_seq_i[size]+A_seq_i[size-1];
displs2[new_num_procs] = A_seq_i[size];
for (i=1; i < new_num_procs+1; i++)
{
displs2[i] = A_seq_i[displs[i]];
info[i-1] = displs2[i] - displs2[i-1];
}
total_nnz = displs2[new_num_procs];
A_seq_j = hypre_CTAlloc(HYPRE_Int, total_nnz);
A_seq_data = hypre_CTAlloc(double, total_nnz);
hypre_MPI_Allgatherv ( A_tmp_j, num_nonzeros, HYPRE_MPI_INT,
A_seq_j, info, displs2,
HYPRE_MPI_INT, new_comm );
hypre_MPI_Allgatherv ( A_tmp_data, num_nonzeros, hypre_MPI_DOUBLE,
A_seq_data, info, displs2,
hypre_MPI_DOUBLE, new_comm );
hypre_TFree(displs);
hypre_TFree(displs2);
hypre_TFree(A_tmp_i);
hypre_TFree(A_tmp_j);
hypre_TFree(A_tmp_data);
row_starts = hypre_CTAlloc(HYPRE_Int,2);
row_starts[0] = 0;
row_starts[1] = size;
/* Create 1 proc communicator */
seq_comm = hypre_MPI_COMM_SELF;
A_seq = hypre_ParCSRMatrixCreate(seq_comm,size,size,
row_starts, row_starts,
0,total_nnz,0);
A_seq_diag = hypre_ParCSRMatrixDiag(A_seq);
A_seq_offd = hypre_ParCSRMatrixOffd(A_seq);
hypre_CSRMatrixData(A_seq_diag) = A_seq_data;
hypre_CSRMatrixI(A_seq_diag) = A_seq_i;
hypre_CSRMatrixJ(A_seq_diag) = A_seq_j;
hypre_CSRMatrixI(A_seq_offd) = A_seq_offd_i;
F_seq = hypre_ParVectorCreate(seq_comm, size, row_starts);
U_seq = hypre_ParVectorCreate(seq_comm, size, row_starts);
hypre_ParVectorOwnsPartitioning(F_seq) = 0;
hypre_ParVectorOwnsPartitioning(U_seq) = 0;
hypre_ParVectorInitialize(F_seq);
hypre_ParVectorInitialize(U_seq);
hypre_BoomerAMGSetup(coarse_solver,A_seq,F_seq,U_seq);
hypre_ParAMGDataCoarseSolver(amg_data) = coarse_solver;
hypre_ParAMGDataACoarse(amg_data) = A_seq;
hypre_ParAMGDataFCoarse(amg_data) = F_seq;
hypre_ParAMGDataUCoarse(amg_data) = U_seq;
hypre_ParAMGDataNewComm(amg_data) = new_comm;
}
hypre_TFree(info);
hypre_TFree(ranks);
}
return 0;
}
inline void numfact(unsigned int ncol, int* I, int* loc2glob, int* J, K* C) {
static_assert(std::is_same<double, K>::value, "Hypre only supports double-precision floating-point real numbers");
static_assert(S == 'G', "Hypre only supports nonsymmetric matrices");
HYPRE_IJMatrixCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], loc2glob[0], loc2glob[1], &_A);
HYPRE_IJMatrixSetObjectType(_A, HYPRE_PARCSR);
HYPRE_IJMatrixSetRowSizes(_A, I + 1);
_local = ncol;
int* rows = new int[3 * _local]();
int* diag_sizes = rows + _local;
int* offdiag_sizes = diag_sizes + _local;
rows[0] = I[0];
for(unsigned int i = 0; i < _local; ++i) {
std::for_each(J + rows[0], J + rows[0] + I[i + 1], [&](int& j) { (j < loc2glob[0] || loc2glob[1] < j) ? ++offdiag_sizes[i] : ++diag_sizes[i]; });
rows[0] += I[i + 1];
}
HYPRE_IJMatrixSetDiagOffdSizes(_A, diag_sizes, offdiag_sizes);
HYPRE_IJMatrixSetMaxOffProcElmts(_A, 0);
HYPRE_IJMatrixInitialize(_A);
std::iota(rows, rows + _local, loc2glob[0]);
HYPRE_IJMatrixSetValues(_A, _local, I + 1, rows, J, C);
HYPRE_IJMatrixAssemble(_A);
HYPRE_IJVectorCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], &_b);
HYPRE_IJVectorSetObjectType(_b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(_b);
HYPRE_IJVectorCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], &_x);
HYPRE_IJVectorSetObjectType(_x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(_x);
delete [] rows;
delete [] I;
delete [] loc2glob;
HYPRE_BoomerAMGCreate(_strategy == 1 ? &_solver : &_precond);
HYPRE_BoomerAMGSetCoarsenType(_strategy == 1 ? _solver : _precond, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(_strategy == 1 ? _solver : _precond, 6); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(_strategy == 1 ? _solver : _precond, 1); /* sweeps on each level */
HYPRE_BoomerAMGSetMaxLevels(_strategy == 1 ? _solver : _precond, 10); /* maximum number of levels */
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJMatrixGetObject(_A, reinterpret_cast<void**>(&parcsr_A));
HYPRE_ParVector par_b;
HYPRE_IJVectorGetObject(_b, reinterpret_cast<void**>(&par_b));
HYPRE_ParVector par_x;
HYPRE_IJVectorGetObject(_x, reinterpret_cast<void**>(&par_x));
if(_strategy == 1) {
HYPRE_BoomerAMGSetTol(_solver, 1.0e-8);
HYPRE_BoomerAMGSetMaxIter(_solver, 1000);
HYPRE_BoomerAMGSetPrintLevel(_solver, 1);
HYPRE_BoomerAMGSetup(_solver, parcsr_A, nullptr, nullptr);
}
else {
HYPRE_BoomerAMGSetTol(_precond, 0.0);
HYPRE_BoomerAMGSetMaxIter(_precond, 1);
HYPRE_BoomerAMGSetPrintLevel(_precond, 1);
if(_strategy == 2) {
HYPRE_ParCSRPCGCreate(DMatrix::_communicator, &_solver);
HYPRE_PCGSetMaxIter(_solver, 500);
HYPRE_PCGSetTol(_solver, 1.0e-8);
HYPRE_PCGSetTwoNorm(_solver, 1);
HYPRE_PCGSetPrintLevel(_solver, 1);
HYPRE_PCGSetLogging(_solver, 1);
HYPRE_PCGSetPrecond(_solver, reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSolve), reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSetup), _precond);
HYPRE_ParCSRPCGSetup(_solver, parcsr_A, par_b, par_x);
}
else {
HYPRE_ParCSRFlexGMRESCreate(DMatrix::_communicator, &_solver);
HYPRE_FlexGMRESSetKDim(_solver, 50);
HYPRE_FlexGMRESSetMaxIter(_solver, 500);
HYPRE_FlexGMRESSetTol(_solver, 1.0e-8);
HYPRE_FlexGMRESSetPrintLevel(_solver, 1);
HYPRE_FlexGMRESSetLogging(_solver, 1);
HYPRE_FlexGMRESSetPrecond(_solver, reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSolve), reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSetup), _precond);
HYPRE_ParCSRFlexGMRESSetup(_solver, parcsr_A, par_b, par_x);
}
}
}
int main (int argc, char *argv[])
{
HYPRE_Int i;
int myid, num_procs;
int N, n;
HYPRE_Int ilower, iupper;
HYPRE_Int local_size, extra;
int solver_id;
int print_solution, print_system;
double h, h2;
HYPRE_IJMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJVector b;
HYPRE_ParVector par_b;
HYPRE_IJVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver, precond;
/* Initialize MPI */
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
/* Default problem parameters */
n = 33;
solver_id = 0;
print_solution = 0;
print_system = 0;
/* Parse command line */
{
int arg_index = 0;
int print_usage = 0;
while (arg_index < argc)
{
if ( strcmp(argv[arg_index], "-n") == 0 )
{
arg_index++;
n = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-solver") == 0 )
{
arg_index++;
solver_id = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-print_solution") == 0 )
{
arg_index++;
print_solution = 1;
}
else if ( strcmp(argv[arg_index], "-print_system") == 0 )
{
arg_index++;
print_system = 1;
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
break;
}
else
{
arg_index++;
}
}
if ((print_usage) && (myid == 0))
{
printf("\n");
printf("Usage: %s [<options>]\n", argv[0]);
printf("\n");
printf(" -n <n> : problem size in each direction (default: 33)\n");
printf(" -solver <ID> : solver ID\n");
printf(" 0 - AMG (default) \n");
printf(" 1 - AMG-PCG\n");
printf(" 8 - ParaSails-PCG\n");
printf(" 50 - PCG\n");
printf(" 61 - AMG-FlexGMRES\n");
printf(" -print_solution : print the solution vector\n");
printf(" -print_system : print the matrix and rhs\n");
printf("\n");
}
if (print_usage)
{
MPI_Finalize();
return (0);
}
}
/* Preliminaries: want at least one processor per row */
if (n*n < num_procs) n = sqrt(num_procs) + 1;
N = n*n; /* global number of rows */
h = 1.0/(n+1); /* mesh size*/
h2 = h*h;
/* Each processor knows only of its own rows - the range is denoted by ilower
and upper. Here we partition the rows. We account for the fact that
N may not divide evenly by the number of processors. */
local_size = N/num_procs;
extra = N - local_size*num_procs;
ilower = local_size*myid;
ilower += hypre_min(myid, extra);
iupper = local_size*(myid+1);
iupper += hypre_min(myid+1, extra);
iupper = iupper - 1;
/* How many rows do I have? */
local_size = iupper - ilower + 1;
/* Create the matrix.
Note that this is a square matrix, so we indicate the row partition
size twice (since number of rows = number of cols) */
HYPRE_IJMatrixCreate(MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
/* Choose a parallel csr format storage (see the User's Manual) */
HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
/* Initialize before setting coefficients */
HYPRE_IJMatrixInitialize(A);
/* Now go through my local rows and set the matrix entries.
Each row has at most 5 entries. For example, if n=3:
A = [M -I 0; -I M -I; 0 -I M]
M = [4 -1 0; -1 4 -1; 0 -1 4]
Note that here we are setting one row at a time, though
one could set all the rows together (see the User's Manual).
*/
{
HYPRE_Int nnz;
double values[5];
HYPRE_Int cols[5];
for (i = ilower; i <= iupper; i++)
{
nnz = 0;
/* The left identity block:position i-n */
if ((i-n)>=0)
{
cols[nnz] = i-n;
values[nnz] = -1.0;
nnz++;
}
/* The left -1: position i-1 */
if (i%n)
{
cols[nnz] = i-1;
values[nnz] = -1.0;
nnz++;
}
/* Set the diagonal: position i */
cols[nnz] = i;
values[nnz] = 4.0;
nnz++;
/* The right -1: position i+1 */
if ((i+1)%n)
{
cols[nnz] = i+1;
values[nnz] = -1.0;
nnz++;
}
/* The right identity block:position i+n */
if ((i+n)< N)
{
cols[nnz] = i+n;
values[nnz] = -1.0;
nnz++;
}
/* Set the values for row i */
HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
}
}
/* Assemble after setting the coefficients */
HYPRE_IJMatrixAssemble(A);
/* Note: for the testing of small problems, one may wish to read
in a matrix in IJ format (for the format, see the output files
from the -print_system option).
In this case, one would use the following routine:
HYPRE_IJMatrixRead( <filename>, MPI_COMM_WORLD,
HYPRE_PARCSR, &A );
<filename> = IJ.A.out to read in what has been printed out
by -print_system (processor numbers are omitted).
A call to HYPRE_IJMatrixRead is an *alternative* to the
following sequence of HYPRE_IJMatrix calls:
Create, SetObjectType, Initialize, SetValues, and Assemble
*/
/* Get the parcsr matrix object to use */
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
/* Create the rhs and solution */
HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&b);
HYPRE_IJVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(b);
HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&x);
HYPRE_IJVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(x);
/* Set the rhs values to h^2 and the solution to zero */
{
double *rhs_values, *x_values;
HYPRE_Int *rows;
rhs_values = calloc(local_size, sizeof(double));
x_values = calloc(local_size, sizeof(double));
rows = calloc(local_size, sizeof(HYPRE_Int));
for (i=0; i<local_size; i++)
{
rhs_values[i] = h2;
x_values[i] = 0.0;
rows[i] = ilower + i;
}
HYPRE_IJVectorSetValues(b, local_size, rows, rhs_values);
HYPRE_IJVectorSetValues(x, local_size, rows, x_values);
free(x_values);
free(rhs_values);
free(rows);
}
HYPRE_IJVectorAssemble(b);
/* As with the matrix, for testing purposes, one may wish to read in a rhs:
HYPRE_IJVectorRead( <filename>, MPI_COMM_WORLD,
HYPRE_PARCSR, &b );
as an alternative to the
following sequence of HYPRE_IJVectors calls:
Create, SetObjectType, Initialize, SetValues, and Assemble
*/
HYPRE_IJVectorGetObject(b, (void **) &par_b);
HYPRE_IJVectorAssemble(x);
HYPRE_IJVectorGetObject(x, (void **) &par_x);
/* Print out the system - files names will be IJ.out.A.XXXXX
and IJ.out.b.XXXXX, where XXXXX = processor id */
if (print_system)
{
HYPRE_IJMatrixPrint(A, "IJ.out.A");
HYPRE_IJVectorPrint(b, "IJ.out.b");
}
/* Choose a solver and solve the system */
/* AMG */
if (solver_id == 0)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_BoomerAMGCreate(&solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_BoomerAMGSetPrintLevel(solver, 3); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(solver, 3); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(solver, 1); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(solver, 20); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(solver, 1e-7); /* conv. tolerance */
/* Now setup and solve! */
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver */
HYPRE_BoomerAMGDestroy(solver);
}
/* PCG */
else if (solver_id == 50)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* prints out the iteration info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver */
HYPRE_ParCSRPCGDestroy(solver);
}
/* PCG with AMG preconditioner */
else if (solver_id == 1)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the AMG preconditioner and specify any parameters */
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
/* Set the PCG preconditioner */
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destroy solver and preconditioner */
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
}
/* PCG with Parasails Preconditioner */
else if (solver_id == 8)
{
HYPRE_Int num_iterations;
double final_res_norm;
int sai_max_levels = 1;
double sai_threshold = 0.1;
double sai_filter = 0.05;
int sai_sym = 1;
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the ParaSails preconditioner and specify any parameters */
HYPRE_ParaSailsCreate(MPI_COMM_WORLD, &precond);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_ParaSailsSetParams(precond, sai_threshold, sai_max_levels);
HYPRE_ParaSailsSetFilter(precond, sai_filter);
HYPRE_ParaSailsSetSym(precond, sai_sym);
HYPRE_ParaSailsSetLogging(precond, 3);
/* Set the PCG preconditioner */
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
(HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup, precond);
/* Now setup and solve! */
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destory solver and preconditioner */
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_ParaSailsDestroy(precond);
}
/* Flexible GMRES with AMG Preconditioner */
else if (solver_id == 61)
{
HYPRE_Int num_iterations;
double final_res_norm;
int restart = 30;
int modify = 1;
/* Create solver */
HYPRE_ParCSRFlexGMRESCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_FlexGMRESSetKDim(solver, restart);
HYPRE_FlexGMRESSetMaxIter(solver, 1000); /* max iterations */
HYPRE_FlexGMRESSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_FlexGMRESSetPrintLevel(solver, 2); /* print solve info */
HYPRE_FlexGMRESSetLogging(solver, 1); /* needed to get run info later */
/* Now set up the AMG preconditioner and specify any parameters */
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */
/* Set the FlexGMRES preconditioner */
HYPRE_FlexGMRESSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
if (modify)
/* this is an optional call - if you don't call it, hypre_FlexGMRESModifyPCDefault
is used - which does nothing. Otherwise, you can define your own, similar to
the one used here */
HYPRE_FlexGMRESSetModifyPC( solver,
(HYPRE_PtrToModifyPCFcn) hypre_FlexGMRESModifyPCAMGExample);
/* Now setup and solve! */
HYPRE_ParCSRFlexGMRESSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRFlexGMRESSolve(solver, parcsr_A, par_b, par_x);
/* Run info - needed logging turned on */
HYPRE_FlexGMRESGetNumIterations(solver, &num_iterations);
HYPRE_FlexGMRESGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
/* Destory solver and preconditioner */
HYPRE_ParCSRFlexGMRESDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
}
else
{
if (myid ==0) printf("Invalid solver id specified.\n");
}
/* Print the solution */
if (print_solution)
HYPRE_IJVectorPrint(x, "ij.out.x");
/* Clean up */
HYPRE_IJMatrixDestroy(A);
HYPRE_IJVectorDestroy(b);
HYPRE_IJVectorDestroy(x);
/* Finalize MPI*/
MPI_Finalize();
return(0);
}
HYPRE_Int hypre_AMESetup(void *esolver)
{
HYPRE_Int ne, *edge_bc;
hypre_AMEData *ame_data = esolver;
hypre_AMSData *ams_data = ame_data -> precond;
if (ams_data -> beta_is_zero)
{
ame_data -> t1 = hypre_ParVectorInDomainOf(ams_data -> G);
ame_data -> t2 = hypre_ParVectorInDomainOf(ams_data -> G);
}
else
{
ame_data -> t1 = ams_data -> r1;
ame_data -> t2 = ams_data -> g1;
}
ame_data -> t3 = ams_data -> r0;
/* Eliminate boundary conditions in G = [Gii, Gib; 0, Gbb], i.e.,
compute [Gii, 0; 0, 0] */
{
HYPRE_Int i, j, k, nv;
HYPRE_Int *offd_edge_bc;
hypre_ParCSRMatrix *Gt;
nv = hypre_ParCSRMatrixNumCols(ams_data -> G);
ne = hypre_ParCSRMatrixNumRows(ams_data -> G);
edge_bc = hypre_TAlloc(HYPRE_Int, ne);
for (i = 0; i < ne; i++)
edge_bc[i] = 0;
/* Find boundary (eliminated) edges */
{
hypre_CSRMatrix *Ad = hypre_ParCSRMatrixDiag(ams_data -> A);
HYPRE_Int *AdI = hypre_CSRMatrixI(Ad);
HYPRE_Int *AdJ = hypre_CSRMatrixJ(Ad);
HYPRE_Real *AdA = hypre_CSRMatrixData(Ad);
hypre_CSRMatrix *Ao = hypre_ParCSRMatrixOffd(ams_data -> A);
HYPRE_Int *AoI = hypre_CSRMatrixI(Ao);
HYPRE_Real *AoA = hypre_CSRMatrixData(Ao);
HYPRE_Real l1_norm;
/* A row (edge) is boundary if its off-diag l1 norm is less than eps */
HYPRE_Real eps = DBL_EPSILON * 1e+4;
for (i = 0; i < ne; i++)
{
l1_norm = 0.0;
for (j = AdI[i]; j < AdI[i+1]; j++)
if (AdJ[j] != i)
l1_norm += fabs(AdA[j]);
if (AoI)
for (j = AoI[i]; j < AoI[i+1]; j++)
l1_norm += fabs(AoA[j]);
if (l1_norm < eps)
edge_bc[i] = 1;
}
}
hypre_ParCSRMatrixTranspose(ams_data -> G, &Gt, 1);
/* Use a Matvec communication to find which of the edges
connected to local vertices are on the boundary */
{
hypre_ParCSRCommHandle *comm_handle;
hypre_ParCSRCommPkg *comm_pkg;
HYPRE_Int num_sends, *int_buf_data;
HYPRE_Int index, start;
offd_edge_bc = hypre_CTAlloc(HYPRE_Int, hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(Gt)));
hypre_MatvecCommPkgCreate(Gt);
comm_pkg = hypre_ParCSRMatrixCommPkg(Gt);
num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
int_buf_data = hypre_CTAlloc(HYPRE_Int,
hypre_ParCSRCommPkgSendMapStart(comm_pkg,
num_sends));
index = 0;
for (i = 0; i < num_sends; i++)
{
start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
{
k = hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j);
int_buf_data[index++] = edge_bc[k];
}
}
comm_handle = hypre_ParCSRCommHandleCreate(11, comm_pkg,
int_buf_data, offd_edge_bc);
hypre_ParCSRCommHandleDestroy(comm_handle);
hypre_TFree(int_buf_data);
}
/* Eliminate boundary vertex entries in G^t */
{
hypre_CSRMatrix *Gtd = hypre_ParCSRMatrixDiag(Gt);
HYPRE_Int *GtdI = hypre_CSRMatrixI(Gtd);
HYPRE_Int *GtdJ = hypre_CSRMatrixJ(Gtd);
HYPRE_Real *GtdA = hypre_CSRMatrixData(Gtd);
hypre_CSRMatrix *Gto = hypre_ParCSRMatrixOffd(Gt);
HYPRE_Int *GtoI = hypre_CSRMatrixI(Gto);
HYPRE_Int *GtoJ = hypre_CSRMatrixJ(Gto);
HYPRE_Real *GtoA = hypre_CSRMatrixData(Gto);
HYPRE_Int bdr;
for (i = 0; i < nv; i++)
{
bdr = 0;
/* A vertex is boundary if it belongs to a boundary edge */
for (j = GtdI[i]; j < GtdI[i+1]; j++)
if (edge_bc[GtdJ[j]]) { bdr = 1; break; }
if (!bdr && GtoI)
for (j = GtoI[i]; j < GtoI[i+1]; j++)
if (offd_edge_bc[GtoJ[j]]) { bdr = 1; break; }
if (bdr)
{
for (j = GtdI[i]; j < GtdI[i+1]; j++)
/* if (!edge_bc[GtdJ[j]]) */
GtdA[j] = 0.0;
if (GtoI)
for (j = GtoI[i]; j < GtoI[i+1]; j++)
/* if (!offd_edge_bc[GtoJ[j]]) */
GtoA[j] = 0.0;
}
}
}
hypre_ParCSRMatrixTranspose(Gt, &ame_data -> G, 1);
hypre_ParCSRMatrixDestroy(Gt);
hypre_TFree(offd_edge_bc);
}
/* Compute G^t M G */
{
if (!hypre_ParCSRMatrixCommPkg(ame_data -> G))
hypre_MatvecCommPkgCreate(ame_data -> G);
if (!hypre_ParCSRMatrixCommPkg(ame_data -> M))
hypre_MatvecCommPkgCreate(ame_data -> M);
hypre_BoomerAMGBuildCoarseOperator(ame_data -> G,
ame_data -> M,
ame_data -> G,
&ame_data -> A_G);
hypre_ParCSRMatrixFixZeroRows(ame_data -> A_G);
}
/* Create AMG preconditioner and PCG-AMG solver for G^tMG */
{
HYPRE_BoomerAMGCreate(&ame_data -> B1_G);
HYPRE_BoomerAMGSetCoarsenType(ame_data -> B1_G, ams_data -> B_G_coarsen_type);
HYPRE_BoomerAMGSetAggNumLevels(ame_data -> B1_G, ams_data -> B_G_agg_levels);
HYPRE_BoomerAMGSetRelaxType(ame_data -> B1_G, ams_data -> B_G_relax_type);
HYPRE_BoomerAMGSetNumSweeps(ame_data -> B1_G, 1);
HYPRE_BoomerAMGSetMaxLevels(ame_data -> B1_G, 25);
HYPRE_BoomerAMGSetTol(ame_data -> B1_G, 0.0);
HYPRE_BoomerAMGSetMaxIter(ame_data -> B1_G, 1);
HYPRE_BoomerAMGSetStrongThreshold(ame_data -> B1_G, ams_data -> B_G_theta);
/* don't use exact solve on the coarsest level (matrix may be singular) */
HYPRE_BoomerAMGSetCycleRelaxType(ame_data -> B1_G,
ams_data -> B_G_relax_type,
3);
HYPRE_ParCSRPCGCreate(hypre_ParCSRMatrixComm(ame_data->A_G),
&ame_data -> B2_G);
HYPRE_PCGSetPrintLevel(ame_data -> B2_G, 0);
HYPRE_PCGSetTol(ame_data -> B2_G, 1e-12);
HYPRE_PCGSetMaxIter(ame_data -> B2_G, 20);
HYPRE_PCGSetPrecond(ame_data -> B2_G,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
ame_data -> B1_G);
HYPRE_ParCSRPCGSetup(ame_data -> B2_G,
(HYPRE_ParCSRMatrix)ame_data->A_G,
(HYPRE_ParVector)ame_data->t1,
(HYPRE_ParVector)ame_data->t2);
}
/* Setup LOBPCG */
{
HYPRE_Int seed = 75;
mv_InterfaceInterpreter* interpreter;
mv_MultiVectorPtr eigenvectors;
ame_data -> interpreter = hypre_CTAlloc(mv_InterfaceInterpreter,1);
interpreter = (mv_InterfaceInterpreter*) ame_data -> interpreter;
HYPRE_ParCSRSetupInterpreter(interpreter);
ame_data -> eigenvalues = hypre_CTAlloc(HYPRE_Real, ame_data -> block_size);
ame_data -> eigenvectors =
mv_MultiVectorCreateFromSampleVector(interpreter,
ame_data -> block_size,
ame_data -> t3);
eigenvectors = (mv_MultiVectorPtr) ame_data -> eigenvectors;
mv_MultiVectorSetRandom (eigenvectors, seed);
/* Make the initial vectors discretely divergence free */
{
HYPRE_Int i, j;
HYPRE_Real *data;
mv_TempMultiVector* tmp = mv_MultiVectorGetData(eigenvectors);
HYPRE_ParVector *v = (HYPRE_ParVector*)(tmp -> vector);
hypre_ParVector *vi;
for (i = 0; i < ame_data -> block_size; i++)
{
vi = (hypre_ParVector*) v[i];
data = hypre_VectorData(hypre_ParVectorLocalVector(vi));
for (j = 0; j < ne; j++)
if (edge_bc[j])
data[j] = 0.0;
hypre_AMEDiscrDivFreeComponent(esolver, vi);
}
}
}
hypre_TFree(edge_bc);
return hypre_error_flag;
}
HYPRE_Int main (HYPRE_Int argc, char *argv[])
{
HYPRE_Int i;
HYPRE_Int myid, num_procs;
HYPRE_Int N, n;
HYPRE_Int ilower, iupper;
HYPRE_Int local_size, extra;
HYPRE_Int solver_id;
HYPRE_Int print_solution;
double h, h2;
#ifdef HYPRE_FORTRAN
hypre_F90_Obj A;
hypre_F90_Obj parcsr_A;
hypre_F90_Obj b;
hypre_F90_Obj par_b;
hypre_F90_Obj x;
hypre_F90_Obj par_x;
hypre_F90_Obj solver, precond;
hypre_F90_Obj long_temp_COMM;
HYPRE_Int temp_COMM;
HYPRE_Int precond_id;
HYPRE_Int one = 1;
HYPRE_Int two = 2;
HYPRE_Int three = 3;
HYPRE_Int six = 6;
HYPRE_Int twenty = 20;
HYPRE_Int thousand = 1000;
HYPRE_Int hypre_type = HYPRE_PARCSR;
double oo1 = 1.e-3;
double tol = 1.e-7;
#else
HYPRE_IJMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_IJVector b;
HYPRE_ParVector par_b;
HYPRE_IJVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver, precond;
#endif
/* Initialize MPI */
hypre_MPI_Init(&argc, &argv);
hypre_MPI_Comm_rank(hypre_MPI_COMM_WORLD, &myid);
hypre_MPI_Comm_size(hypre_MPI_COMM_WORLD, &num_procs);
/* Default problem parameters */
n = 33;
solver_id = 0;
print_solution = 0;
/* Parse command line */
{
HYPRE_Int arg_index = 0;
HYPRE_Int print_usage = 0;
while (arg_index < argc)
{
if ( strcmp(argv[arg_index], "-n") == 0 )
{
arg_index++;
n = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-solver") == 0 )
{
arg_index++;
solver_id = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-print_solution") == 0 )
{
arg_index++;
print_solution = 1;
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
break;
}
else
{
arg_index++;
}
}
if ((print_usage) && (myid == 0))
{
hypre_printf("\n");
hypre_printf("Usage: %s [<options>]\n", argv[0]);
hypre_printf("\n");
hypre_printf(" -n <n> : problem size in each direction (default: 33)\n");
hypre_printf(" -solver <ID> : solver ID\n");
hypre_printf(" 0 - AMG (default) \n");
hypre_printf(" 1 - AMG-PCG\n");
hypre_printf(" 8 - ParaSails-PCG\n");
hypre_printf(" 50 - PCG\n");
hypre_printf(" -print_solution : print the solution vector\n");
hypre_printf("\n");
}
if (print_usage)
{
hypre_MPI_Finalize();
return (0);
}
}
/* Preliminaries: want at least one processor per row */
if (n*n < num_procs) n = sqrt(num_procs) + 1;
N = n*n; /* global number of rows */
h = 1.0/(n+1); /* mesh size*/
h2 = h*h;
/* Each processor knows only of its own rows - the range is denoted by ilower
and upper. Here we partition the rows. We account for the fact that
N may not divide evenly by the number of processors. */
local_size = N/num_procs;
extra = N - local_size*num_procs;
ilower = local_size*myid;
ilower += hypre_min(myid, extra);
iupper = local_size*(myid+1);
iupper += hypre_min(myid+1, extra);
iupper = iupper - 1;
/* How many rows do I have? */
local_size = iupper - ilower + 1;
/* Create the matrix.
Note that this is a square matrix, so we indicate the row partition
size twice (since number of rows = number of cols) */
#ifdef HYPRE_FORTRAN
long_temp_COMM = (hypre_F90_Obj) hypre_MPI_COMM_WORLD;
temp_COMM = (HYPRE_Int) hypre_MPI_COMM_WORLD;
HYPRE_IJMatrixCreate(&long_temp_COMM, &ilower, &iupper, &ilower, &iupper, &A);
#else
HYPRE_IJMatrixCreate(hypre_MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);
#endif
/* Choose a parallel csr format storage (see the User's Manual) */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixSetObjectType(&A, &hypre_type);
#else
HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);
#endif
/* Initialize before setting coefficients */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixInitialize(&A);
#else
HYPRE_IJMatrixInitialize(A);
#endif
/* Now go through my local rows and set the matrix entries.
Each row has at most 5 entries. For example, if n=3:
A = [M -I 0; -I M -I; 0 -I M]
M = [4 -1 0; -1 4 -1; 0 -1 4]
Note that here we are setting one row at a time, though
one could set all the rows together (see the User's Manual).
*/
{
HYPRE_Int nnz;
double values[5];
HYPRE_Int cols[5];
for (i = ilower; i <= iupper; i++)
{
nnz = 0;
/* The left identity block:position i-n */
if ((i-n)>=0)
{
cols[nnz] = i-n;
values[nnz] = -1.0;
nnz++;
}
/* The left -1: position i-1 */
if (i%n)
{
cols[nnz] = i-1;
values[nnz] = -1.0;
nnz++;
}
/* Set the diagonal: position i */
cols[nnz] = i;
values[nnz] = 4.0;
nnz++;
/* The right -1: position i+1 */
if ((i+1)%n)
{
cols[nnz] = i+1;
values[nnz] = -1.0;
nnz++;
}
/* The right identity block:position i+n */
if ((i+n)< N)
{
cols[nnz] = i+n;
values[nnz] = -1.0;
nnz++;
}
/* Set the values for row i */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixSetValues(&A, &one, &nnz, &i, &cols[0], &values[0]);
#else
HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
#endif
}
}
/* Assemble after setting the coefficients */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixAssemble(&A);
#else
HYPRE_IJMatrixAssemble(A);
#endif
/* Get the parcsr matrix object to use */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixGetObject(&A, &parcsr_A);
HYPRE_IJMatrixGetObject(&A, &parcsr_A);
#else
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);
#endif
/* Create the rhs and solution */
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorCreate(&temp_COMM, &ilower, &iupper, &b);
HYPRE_IJVectorSetObjectType(&b, &hypre_type);
HYPRE_IJVectorInitialize(&b);
#else
HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, ilower, iupper,&b);
HYPRE_IJVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(b);
#endif
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorCreate(&temp_COMM, &ilower, &iupper, &x);
HYPRE_IJVectorSetObjectType(&x, &hypre_type);
HYPRE_IJVectorInitialize(&x);
#else
HYPRE_IJVectorCreate(hypre_MPI_COMM_WORLD, ilower, iupper,&x);
HYPRE_IJVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_IJVectorInitialize(x);
#endif
/* Set the rhs values to h^2 and the solution to zero */
{
double *rhs_values, *x_values;
HYPRE_Int *rows;
rhs_values = calloc(local_size, sizeof(double));
x_values = calloc(local_size, sizeof(double));
rows = calloc(local_size, sizeof(HYPRE_Int));
for (i=0; i<local_size; i++)
{
rhs_values[i] = h2;
x_values[i] = 0.0;
rows[i] = ilower + i;
}
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorSetValues(&b, &local_size, &rows[0], &rhs_values[0]);
HYPRE_IJVectorSetValues(&x, &local_size, &rows[0], &x_values[0]);
#else
HYPRE_IJVectorSetValues(b, local_size, rows, rhs_values);
HYPRE_IJVectorSetValues(x, local_size, rows, x_values);
#endif
free(x_values);
free(rhs_values);
free(rows);
}
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorAssemble(&b);
HYPRE_IJVectorGetObject(&b, &par_b);
#else
HYPRE_IJVectorAssemble(b);
HYPRE_IJVectorGetObject(b, (void **) &par_b);
#endif
#ifdef HYPRE_FORTRAN
HYPRE_IJVectorAssemble(&x);
HYPRE_IJVectorGetObject(&x, &par_x);
#else
HYPRE_IJVectorAssemble(x);
HYPRE_IJVectorGetObject(x, (void **) &par_x);
#endif
/* Choose a solver and solve the system */
/* AMG */
if (solver_id == 0)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGCreate(&solver);
#else
HYPRE_BoomerAMGCreate(&solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGSetPrintLevel(&solver, &three); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(&solver, &six); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(&solver, &three); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(&solver, &one); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(&solver, &twenty); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(&solver, &tol); /* conv. tolerance */
#else
HYPRE_BoomerAMGSetPrintLevel(solver, 3); /* print solve info + parameters */
HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
HYPRE_BoomerAMGSetRelaxType(solver, 3); /* G-S/Jacobi hybrid relaxation */
HYPRE_BoomerAMGSetNumSweeps(solver, 1); /* Sweeeps on each level */
HYPRE_BoomerAMGSetMaxLevels(solver, 20); /* maximum number of levels */
HYPRE_BoomerAMGSetTol(solver, 1e-7); /* conv. tolerance */
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_BoomerAMGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGGetNumIterations(&solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGDestroy(&solver);
#else
HYPRE_BoomerAMGDestroy(solver);
#endif
}
/* PCG */
else if (solver_id == 50)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* prints out the iteration info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* prints out the iteration info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
#else
HYPRE_ParCSRPCGDestroy(solver);
#endif
}
/* PCG with AMG preconditioner */
else if (solver_id == 1)
{
HYPRE_Int num_iterations;
double final_res_norm;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* print solve info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now set up the AMG preconditioner and specify any parameters */
#ifdef HYPRE_FORTRAN
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(&precond, &one); /* print amg solution info*/
HYPRE_BoomerAMGSetCoarsenType(&precond, &six);
HYPRE_BoomerAMGSetRelaxType(&precond, &three);
HYPRE_BoomerAMGSetNumSweeps(&precond, &one);
HYPRE_BoomerAMGSetTol(&precond, &oo1);
#else
HYPRE_BoomerAMGCreate(&precond);
HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info*/
HYPRE_BoomerAMGSetCoarsenType(precond, 6);
HYPRE_BoomerAMGSetRelaxType(precond, 3);
HYPRE_BoomerAMGSetNumSweeps(precond, 1);
HYPRE_BoomerAMGSetTol(precond, 1e-3);
#endif
/* Set the PCG preconditioner */
#ifdef HYPRE_FORTRAN
precond_id = 2;
HYPRE_ParCSRPCGSetPrecond(&solver, &precond_id, &precond);
#else
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
(HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destroy solver and preconditioner */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
HYPRE_BoomerAMGDestroy(&precond);
#else
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_BoomerAMGDestroy(precond);
#endif
}
/* PCG with Parasails Preconditioner */
else if (solver_id == 8)
{
HYPRE_Int num_iterations;
double final_res_norm;
HYPRE_Int sai_max_levels = 1;
double sai_threshold = 0.1;
double sai_filter = 0.05;
HYPRE_Int sai_sym = 1;
/* Create solver */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGCreate(&temp_COMM, &solver);
#else
HYPRE_ParCSRPCGCreate(hypre_MPI_COMM_WORLD, &solver);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetMaxIter(&solver, &thousand); /* max iterations */
HYPRE_ParCSRPCGSetTol(&solver, &tol); /* conv. tolerance */
HYPRE_ParCSRPCGSetTwoNorm(&solver, &one); /* use the two norm as the stopping criteria */
HYPRE_ParCSRPCGSetPrintLevel(&solver, &two); /* print solve info */
#else
HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
#endif
/* Now set up the ParaSails preconditioner and specify any parameters */
#ifdef HYPRE_FORTRAN
HYPRE_ParaSailsCreate(&temp_COMM, &precond);
#else
HYPRE_ParaSailsCreate(hypre_MPI_COMM_WORLD, &precond);
#endif
/* Set some parameters (See Reference Manual for more parameters) */
#ifdef HYPRE_FORTRAN
HYPRE_ParaSailsSetParams(&precond, &sai_threshold, &sai_max_levels);
HYPRE_ParaSailsSetFilter(&precond, &sai_filter);
HYPRE_ParaSailsSetSym(&precond, &sai_sym);
HYPRE_ParaSailsSetLogging(&precond, &three);
#else
HYPRE_ParaSailsSetParams(precond, sai_threshold, sai_max_levels);
HYPRE_ParaSailsSetFilter(precond, sai_filter);
HYPRE_ParaSailsSetSym(precond, sai_sym);
HYPRE_ParaSailsSetLogging(precond, 3);
#endif
/* Set the PCG preconditioner */
#ifdef HYPRE_FORTRAN
precond_id = 4;
HYPRE_ParCSRPCGSetPrecond(&solver, &precond_id, &precond);
#else
HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
(HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup, precond);
#endif
/* Now setup and solve! */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGSetup(&solver, &parcsr_A, &par_b, &par_x);
HYPRE_ParCSRPCGSolve(&solver, &parcsr_A, &par_b, &par_x);
#else
HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);
#endif
/* Run info - needed logging turned on */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGGetNumIterations(&solver, &num_iterations);
HYPRE_ParCSRPCGGetFinalRelativeResidualNorm(&solver, &final_res_norm);
#else
HYPRE_PCGGetNumIterations(solver, &num_iterations);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
#endif
if (myid == 0)
{
hypre_printf("\n");
hypre_printf("Iterations = %d\n", num_iterations);
hypre_printf("Final Relative Residual Norm = %e\n", final_res_norm);
hypre_printf("\n");
}
/* Destory solver and preconditioner */
#ifdef HYPRE_FORTRAN
HYPRE_ParCSRPCGDestroy(&solver);
HYPRE_ParaSailsDestroy(&precond);
#else
HYPRE_ParCSRPCGDestroy(solver);
HYPRE_ParaSailsDestroy(precond);
#endif
}
else
{
if (myid ==0) hypre_printf("Invalid solver id specified.\n");
}
/* Print the solution */
#ifdef HYPRE_FORTRAN
if (print_solution)
HYPRE_IJVectorPrint(&x, "ij.out.x");
#else
if (print_solution)
HYPRE_IJVectorPrint(x, "ij.out.x");
#endif
/* Clean up */
#ifdef HYPRE_FORTRAN
HYPRE_IJMatrixDestroy(&A);
HYPRE_IJVectorDestroy(&b);
HYPRE_IJVectorDestroy(&x);
#else
HYPRE_IJMatrixDestroy(A);
HYPRE_IJVectorDestroy(b);
HYPRE_IJVectorDestroy(x);
#endif
/* Finalize MPI*/
hypre_MPI_Finalize();
return(0);
}
int main(int argc, char *argv[]) {
// printf("start\n");
int i, j, pi, pj;
int myid, num_procs;
int Nx, Ny, nx, ny;
//nx and ny are number of mesh cells
int solver_id;
int vis;
int ilower[2], iupper[2];
int nparts = 1;
int part = 0;
double hx, hy;
HYPRE_SStructGraph graph;
HYPRE_SStructGrid grid;
HYPRE_SStructStencil stencil;
HYPRE_SStructMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_SStructVector b;
HYPRE_ParVector par_b;
HYPRE_SStructVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver;
// printf("variables initialized\n");
//Initialize MPI
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
// printf("mpi initialized\n");
// Set default problem parameters
nx = 100;
ny = 100;
Nx = 1;
solver_id = 0;
vis = 0;
// printf("defaults set\n");
// Parse command line
int arg_index = 0;
while (arg_index < argc) {
if (strcmp(argv[arg_index], "-nx") == 0) {
arg_index++;
nx = atoi(argv[arg_index++]);
// printf("nx set\n");
} else if (strcmp(argv[arg_index], "-ny") == 0) {
arg_index++;
ny = atoi(argv[arg_index++]);
// printf("ny set\n");
} else if (strcmp(argv[arg_index], "-Nx") == 0) {
arg_index++;
Nx = atoi(argv[arg_index++]);
// printf("Nx set\n");
} else if (strcmp(argv[arg_index], "-solver") == 0) {
arg_index++;
solver_id = atoi(argv[arg_index++]);
// printf("solver set\n");
} else if (strcmp(argv[arg_index], "-vis") == 0) {
arg_index++;
vis = 1;
// printf("print solution\n");
} else {
arg_index++;
// printf("plus plus\n");
}
}
// printf("command line parsed\n");
// Set up processors in grid
Ny = num_procs / Nx;
hx = 1.0 / (Nx * nx);
hy = 1.0 / (Ny * ny);
if (Ny <= Nx) {
pj = myid / Nx;
pi = myid - pj * Ny;
} else {
pi = myid / Ny;
pj = myid - pi * Nx;
}
// printf("processors assigned\n");
// Determine each processor's piece of the grid
ilower[0] = pi * nx;
ilower[1] = pj * ny;
iupper[0] = ilower[0] + nx - 1;
iupper[1] = ilower[1] + ny - 1;
// printf("upper and lowers assigned\n");
HYPRE_SStructGridCreate(MPI_COMM_WORLD, 2, nparts, &grid);
// printf("grid created\n");
HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
// printf("grid set\n");
int nvars = 1;
HYPRE_SStructVariable vartypes[1] = { HYPRE_SSTRUCT_VARIABLE_CELL };
// printf("variable type set\n");
for (i = 0; i < nparts; i++) {
HYPRE_SStructGridSetVariables(grid, i, nvars, vartypes);
}
// printf("variable type set\n");
HYPRE_SStructGridAssemble(grid);
// printf("grid assembled\n");
// Set up the stencil
HYPRE_SStructStencilCreate(2, 5, &stencil);
int entry;
int offsets[5][2] = { { 0, 0 }, { -1, 0 }, { 1, 0 }, { 0, -1 }, { 0, 1 } };
int var = 0;
for (entry = 0; entry < 5; entry++) {
HYPRE_SStructStencilSetEntry(stencil, entry, offsets[entry], var);
}
// Create the graph
HYPRE_SStructGraphCreate(MPI_COMM_WORLD, grid, &graph);
int object_type = HYPRE_PARCSR;
HYPRE_SStructGraphSetObjectType(graph, object_type);
HYPRE_SStructGraphSetStencil(graph, part, var, stencil);
HYPRE_SStructGraphAssemble(graph);
int nentries = 5;
int nvalues = nentries * nx * ny;
double *values;
int stencil_indices[5];
// Setup the matrix
HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, graph, &A);
HYPRE_SStructMatrixSetObjectType(A, HYPRE_PARCSR);
HYPRE_SStructMatrixInitialize(A);
// Set the matrix coefficients
values = calloc(nvalues, sizeof(double));
for (j = 0; j < nentries; j++) {
stencil_indices[j] = j;
}
// Account for differences in hx and hy
double hx2 = hx * hx;
double hy2 = hy * hy;
double hx2inv = 1 / hx2;
double hy2inv = 1 / hy2;
for (i = 0; i < nvalues; i += nentries) {
values[i] = -2.0 * hx2inv - 2.0 * hy2inv;
values[i + 1] = hx2inv;
values[i + 2] = hx2inv;
values[i + 3] = hy2inv;
values[i + 4] = hy2inv;
}
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper, var, nentries,
stencil_indices, values);
// Set the dirchelet boundary condition
{
int bc_ilower[2];
int bc_iupper[2];
int nentries = 1;
int nvaluesx = nentries * nx; /* number of stencil entries times the length
of one side of my grid box */
int nvaluesy = nentries * ny;
double *valuesx, *valuesy, *center_valuesx, *center_valuesy;
int stencil_indices[1];
int center_index[1];
center_index[0] = 0;
valuesx = calloc(nvaluesx, sizeof(double));
center_valuesx = calloc(nvaluesx, sizeof(double));
for (j = 0; j < nvaluesx; j++){
valuesx[j] = 0.0;
center_valuesx[j] = -hx2inv;
}
valuesy = calloc(nvaluesy, sizeof(double));
center_valuesy = calloc(nvaluesy, sizeof(double));
for (j = 0; j < nvaluesy; j++){
valuesy[j] = 0.0;
center_valuesy[j] = -hy2inv;
}
// Recall: pi and pj describe position in the processor grid
if (pj == 0) {
// Bottom row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0] + nx - 1;
bc_iupper[1] = bc_ilower[1];
stencil_indices[0] = 3;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesy);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesy);
// printf("Bottom row zeroed\n");
}
if (pj == Ny - 1) {
// upper row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny + ny - 1;
bc_iupper[0] = bc_ilower[0] + ny - 1;
bc_iupper[1] = bc_ilower[1];
stencil_indices[0] = 4;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesy);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesy);
// printf("Upper row zeroed\n");
}
if (pi == 0) {
// Left row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0];
bc_iupper[1] = bc_ilower[1] + ny - 1;
stencil_indices[0] = 1;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesx);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesx);
// printf("left collumn zeroed\n");
}
if (pi == Nx - 1) {
// Right row of grid points
bc_ilower[0] = pi * nx + nx - 1;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0];
bc_iupper[1] = bc_ilower[1] + ny - 1;
stencil_indices[0] = 2;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesx);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesx);
// printf("right collumn zeroed\n");
}
free(valuesx);
free(center_valuesx);
free(valuesy);
free(center_valuesy);
}
HYPRE_SStructMatrixAssemble(A);
HYPRE_SStructMatrixGetObject(A, (void **)&parcsr_A);
// Create the right hand side and solution vectors
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &b);
HYPRE_SStructVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_SStructVectorInitialize(b);
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &x);
HYPRE_SStructVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_SStructVectorInitialize(x);
// Set the right hand side, exact solution, and initial guess values
double *rhs_values, *x_values, *exactsolution;
nvalues = nx * ny;
rhs_values = calloc(nvalues, sizeof(double));
x_values = calloc(nvalues, sizeof(double));
exactsolution = calloc(nvalues, sizeof(double));
// printf("RHS calculation begin\n");
for (j = 0; j < ny; j++) {
double y = (ilower[1] + j + .5) * hy;
for (i = 0; i < nx; i++) {
double x = (ilower[0] + i + .5) * hx;
rhs_values[i + j * nx] =
-8. * PI2 * sin(2. * M_PI * x) * sin(2. * M_PI * y);
exactsolution[i + j * nx] = 1.0 * sin(2. * M_PI * x) * sin(2. * M_PI * y);
x_values[i + j * nx] = 0.0;
}
}
// printf("RHS calculation ended\n");
HYPRE_SStructVectorSetBoxValues(b, part, ilower, iupper, var, rhs_values);
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, x_values);
//printf("%d %f\n", (ny*nx-1), exactsolution[ny*nx-1]);
free(x_values);
free(rhs_values);
HYPRE_SStructVectorAssemble(b);
// HYPRE_SStructVectorPrint("ss.initial.b", b, 0);
HYPRE_SStructVectorGetObject(b, (void **)&par_b);
HYPRE_SStructVectorAssemble(x);
HYPRE_SStructVectorGetObject(x, (void **)&par_x);
// Print out initial vectors
HYPRE_SStructMatrixPrint("SStructExact/ss.initial.A", A, 0);
HYPRE_SStructVectorPrint("SStructExact/ss.initial.b", b, 0);
// printf("About to solve\n");
// Select a solver
if (solver_id == 0) {
int num_iterations;
double final_res_norm;
// Create AMG solver
HYPRE_BoomerAMGCreate(&solver);
// Set solver parameters
HYPRE_BoomerAMGSetPrintLevel(solver, 3);
HYPRE_BoomerAMGSetCoarsenType(solver, 6);
HYPRE_BoomerAMGSetRelaxType(solver, 3);
HYPRE_BoomerAMGSetNumSweeps(solver, 1);
HYPRE_BoomerAMGSetMaxLevels(solver, 20);
HYPRE_BoomerAMGSetTol(solver, 1e-12);
HYPRE_BoomerAMGSetMaxIter(solver, 100);
// Set up and solve
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
//printf("solution obtained\n");
// Run information
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0) {
printf("\n");
printf("Iterations = %d\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
// Destroy solver
HYPRE_BoomerAMGDestroy(solver);
// HYPRE_ParVectorPrint(par_x, "SStructExact/ss.final.x");
char solutionfile[255];
sprintf(solutionfile, "%s.%06d", "ss.final.x", myid);
HYPRE_ParVectorPrint(par_x, solutionfile);
}
// Save the solution for visualization and L2 norm calculation
if (vis) {
FILE *file;
FILE *solution;
char filename[255];
char solutionfile[255];
sprintf(solutionfile, "%s.%06d.%d", "ss.final.x", myid, myid);
int nvalues = nx * ny;
double sum = 0;
double diff, diff2, L2;
// get the localally calculated and exact solutions)
double *values = calloc(nvalues+1, sizeof(double));
// Opens the local solution file, reads the solution to the array, values
// and closes the file.
// Opens the solution file
if ((solution = fopen(solutionfile, "rt")) == NULL) {
printf("Error: can't open output file %s\n", solutionfile);
MPI_Finalize();
exit(1);
}
// Reads the file to the array values
for (i = 0; i < nvalues + 1; i++) {
fscanf(solution, "%lf", &values[i]);
}
// Close the file
fflush(solution);
fclose(solution);
MPI_Barrier(MPI_COMM_WORLD);
// Calcluates the sum of the squared differences for the exact and numerical solutions on each processor
for (i = 1; i < nvalues + 1; i++) {
diff = values[i] - exactsolution[i-1];
diff2 = diff * diff;
if (diff2 > 10){
printf("%+6f %+6f %+6d\n", values[i], exactsolution[i-1],
i-1);
}
sum += diff2;
}
//printf("Myid is: %d and mysum is: %f\n", myid, sum);
// Ensure all processors are caught up
MPI_Barrier(MPI_COMM_WORLD);
// Passes the individual sums to processor 0 and sums them.
double *sendbuffer, *recvbuffer;
sendbuffer = calloc(1, sizeof(double));
sendbuffer[0] = sum;
recvbuffer = calloc(1, sizeof(double));
int root = 0;
int count = 1;
MPI_Reduce(sendbuffer, recvbuffer, count, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
// Calculates the L2 norm
if (myid == 0){
//printf("%f\n" , recvbuffer[myid]);
// nvalues = (nx-2)*(ny-2);
double totalvalues = nvalues * num_procs * 1.0;
L2 = sqrt(recvbuffer[0] / totalvalues);
printf("The L2 norm is %e\n", L2);
}
sprintf(filename, "%s.%06d", "vis/ss.sol", myid);
if ((file = fopen(filename, "w")) == NULL) {
printf("Error: can't open output file %s\n", filename);
MPI_Finalize();
exit(1);
}
// Save solution
for (i = 1; i < nvalues+1; i++) {
fprintf(file, "%.14e\n", values[i]);
}
// Vis Cleanup
fflush(file);
fclose(file);
free(values);
free(exactsolution);
free(sendbuffer);
free(recvbuffer);
// save global finite element mesh
if (myid == 0) {
GLVis_PrintGlobalMesh("vis/ss.mesh", Nx, Ny, nx, ny, hx, hy);
}
}
// Clean up
HYPRE_SStructMatrixDestroy(A);
HYPRE_SStructVectorDestroy(b);
HYPRE_SStructVectorDestroy(x);
// Finalize MPI*/
MPI_Finalize();
return (0);
}