void
hypre_F90_IFACE(hypre_sstructmatrixsetboxvalues, HYPRE_SSTRUCTMATRIXSETBOXVALUES)
(hypre_F90_Obj *matrix,
hypre_F90_Int *part,
hypre_F90_IntArray *ilower,
hypre_F90_IntArray *iupper,
hypre_F90_Int *var,
hypre_F90_Int *nentries,
hypre_F90_IntArray *entries,
hypre_F90_DblArray *values,
hypre_F90_Int *ierr)
{
*ierr = (hypre_F90_Int)
(HYPRE_SStructMatrixSetBoxValues(
hypre_F90_PassObj (HYPRE_SStructMatrix, matrix),
hypre_F90_PassInt (part),
hypre_F90_PassIntArray (ilower),
hypre_F90_PassIntArray (iupper),
hypre_F90_PassInt (var),
hypre_F90_PassInt (nentries),
hypre_F90_PassIntArray (entries),
hypre_F90_PassDblArray (values)));
}
HYPRE_Int
hypre_FacZeroCData( void *fac_vdata,
hypre_SStructMatrix *A )
{
hypre_FACData *fac_data = fac_vdata;
hypre_SStructGrid *grid;
hypre_SStructPGrid *p_cgrid;
hypre_StructGrid *cgrid;
hypre_BoxArray *cgrid_boxes;
hypre_Box *cgrid_box;
hypre_BoxManager *fboxman;
hypre_BoxManEntry **boxman_entries;
HYPRE_Int nboxman_entries;
hypre_Box scaled_box;
hypre_Box intersect_box;
hypre_SStructPMatrix *level_pmatrix;
hypre_StructStencil *stencils;
HYPRE_Int stencil_size;
hypre_Index *refine_factors;
hypre_Index temp_index;
hypre_Index ilower, iupper;
HYPRE_Int max_level = fac_data -> max_levels;
HYPRE_Int *level_to_part = fac_data -> level_to_part;
HYPRE_Int ndim = hypre_SStructMatrixNDim(A);
HYPRE_Int part_crse = 0;
HYPRE_Int part_fine = 1;
HYPRE_Int level;
HYPRE_Int nvars, var;
HYPRE_Int ci, i, j, rem, intersect_size;
double *values;
HYPRE_Int ierr = 0;
for (level= max_level; level> 0; level--)
{
level_pmatrix = hypre_SStructMatrixPMatrix(fac_data -> A_level[level], part_crse);
grid = (fac_data -> grid_level[level]);
refine_factors= &(fac_data -> refine_factors[level]);
p_cgrid= hypre_SStructGridPGrid(grid, part_crse);
nvars = hypre_SStructPGridNVars(p_cgrid);
for (var= 0; var< nvars; var++)
{
stencils = hypre_SStructPMatrixSStencil(level_pmatrix, var, var);
stencil_size= hypre_StructStencilSize(stencils);
/*---------------------------------------------------------------------
* For each variable, find the underlying boxes for each coarse box.
*---------------------------------------------------------------------*/
cgrid = hypre_SStructPGridSGrid(p_cgrid, var);
cgrid_boxes = hypre_StructGridBoxes(cgrid);
fboxman = hypre_SStructGridBoxManager(grid, part_fine, var);
hypre_ForBoxI(ci, cgrid_boxes)
{
cgrid_box= hypre_BoxArrayBox(cgrid_boxes, ci);
hypre_ClearIndex(temp_index);
hypre_StructMapCoarseToFine(hypre_BoxIMin(cgrid_box), temp_index,
*refine_factors, hypre_BoxIMin(&scaled_box));
for (i= 0; i< ndim; i++)
{
temp_index[i]= (*refine_factors)[i]-1;
}
hypre_StructMapCoarseToFine(hypre_BoxIMax(cgrid_box), temp_index,
*refine_factors, hypre_BoxIMax(&scaled_box));
hypre_BoxManIntersect(fboxman, hypre_BoxIMin(&scaled_box),
hypre_BoxIMax(&scaled_box), &boxman_entries,
&nboxman_entries);
for (i= 0; i< nboxman_entries; i++)
{
hypre_BoxManEntryGetExtents(boxman_entries[i], ilower, iupper);
hypre_BoxSetExtents(&intersect_box, ilower, iupper);
hypre_IntersectBoxes(&intersect_box, &scaled_box, &intersect_box);
/* adjust the box so that it is divisible by refine_factors */
for (j= 0; j< ndim; j++)
{
rem= hypre_BoxIMin(&intersect_box)[j]%(*refine_factors)[j];
if (rem)
{
hypre_BoxIMin(&intersect_box)[j]+=(*refine_factors)[j] - rem;
}
}
hypre_ClearIndex(temp_index);
hypre_StructMapFineToCoarse(hypre_BoxIMin(&intersect_box), temp_index,
*refine_factors, hypre_BoxIMin(&intersect_box));
hypre_StructMapFineToCoarse(hypre_BoxIMax(&intersect_box), temp_index,
*refine_factors, hypre_BoxIMax(&intersect_box));
intersect_size= hypre_BoxVolume(&intersect_box);
if (intersect_size > 0)
{
/*------------------------------------------------------------
* Coarse underlying box found. Now zero off.
*------------------------------------------------------------*/
values= hypre_CTAlloc(double, intersect_size);
for (j= 0; j< stencil_size; j++)
{
HYPRE_SStructMatrixSetBoxValues(fac_data -> A_level[level],
part_crse,
hypre_BoxIMin(&intersect_box),
hypre_BoxIMax(&intersect_box),
var, 1, &j, values);
HYPRE_SStructMatrixSetBoxValues(A,
level_to_part[level-1],
hypre_BoxIMin(&intersect_box),
hypre_BoxIMax(&intersect_box),
var, 1, &j, values);
}
hypre_TFree(values);
} /* if (intersect_size > 0) */
} /* for (i= 0; i< nboxman_entries; i++) */
hypre_TFree(boxman_entries);
} /* hypre_ForBoxI(ci, cgrid_boxes) */
} /* for (var= 0; var< nvars; var++) */
int main (int argc, char *argv[])
{
int myid, num_procs;
HYPRE_SStructGrid grid;
HYPRE_SStructGraph graph;
HYPRE_SStructStencil stencil;
HYPRE_SStructMatrix A;
HYPRE_SStructVector b;
HYPRE_SStructVector x;
/* We are using struct solvers for this example */
HYPRE_StructSolver solver;
HYPRE_StructSolver precond;
int object_type;
/* Initialize MPI */
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
if (num_procs != 2)
{
if (myid ==0) printf("Must run with 2 processors!\n");
MPI_Finalize();
return(0);
}
/* 1. Set up the 2D grid. This gives the index space in each part.
Here we only use one part and one variable. (So the part id is 0
and the variable id is 0) */
{
int ndim = 2;
int nparts = 1;
int part = 0;
/* Create an empty 2D grid object */
HYPRE_SStructGridCreate(MPI_COMM_WORLD, ndim, nparts, &grid);
/* Set the extents of the grid - each processor sets its grid
boxes. Each part has its own relative index space numbering,
but in this example all boxes belong to the same part. */
/* Processor 0 owns two boxes in the grid. */
if (myid == 0)
{
/* Add a new box to the grid */
{
int ilower[2] = {-3, 1};
int iupper[2] = {-1, 2};
HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
}
/* Add a new box to the grid */
{
int ilower[2] = {0, 1};
int iupper[2] = {2, 4};
HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
}
}
/* Processor 1 owns one box in the grid. */
else if (myid == 1)
{
/* Add a new box to the grid */
{
int ilower[2] = {3, 1};
int iupper[2] = {6, 4};
HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
}
}
/* Set the variable type and number of variables on each part. */
{
int i;
int nvars = 1;
HYPRE_SStructVariable vartypes[1] = {HYPRE_SSTRUCT_VARIABLE_CELL};
for (i = 0; i< nparts; i++)
HYPRE_SStructGridSetVariables(grid, i, nvars, vartypes);
}
/* Now the grid is ready to use */
HYPRE_SStructGridAssemble(grid);
}
/* 2. Define the discretization stencil(s) */
{
/* Create an empty 2D, 5-pt stencil object */
HYPRE_SStructStencilCreate(2, 5, &stencil);
/* Define the geometry of the stencil. Each represents a
relative offset (in the index space). */
{
int entry;
int offsets[5][2] = {{0,0}, {-1,0}, {1,0}, {0,-1}, {0,1}};
int var = 0;
/* Assign numerical values to the offsets so that we can
easily refer to them - the last argument indicates the
variable for which we are assigning this stencil - we are
just using one variable in this example so it is the first one (0) */
for (entry = 0; entry < 5; entry++)
HYPRE_SStructStencilSetEntry(stencil, entry, offsets[entry], var);
}
}
/* 3. Set up the Graph - this determines the non-zero structure
of the matrix and allows non-stencil relationships between the parts */
{
int var = 0;
int part = 0;
/* Create the graph object */
HYPRE_SStructGraphCreate(MPI_COMM_WORLD, grid, &graph);
/* See MatrixSetObjectType below */
object_type = HYPRE_STRUCT;
HYPRE_SStructGraphSetObjectType(graph, object_type);
/* Now we need to tell the graph which stencil to use for each
variable on each part (we only have one variable and one part) */
HYPRE_SStructGraphSetStencil(graph, part, var, stencil);
/* Here we could establish connections between parts if we
had more than one part using the graph. For example, we could
use HYPRE_GraphAddEntries() routine or HYPRE_GridSetNeighborBox() */
/* Assemble the graph */
HYPRE_SStructGraphAssemble(graph);
}
/* 4. Set up a SStruct Matrix */
{
int i,j;
int part = 0;
int var = 0;
/* Create the empty matrix object */
HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, graph, &A);
/* Set the object type (by default HYPRE_SSTRUCT). This determines the
data structure used to store the matrix. If you want to use unstructured
solvers, e.g. BoomerAMG, the object type should be HYPRE_PARCSR.
If the problem is purely structured (with one part), you may want to use
HYPRE_STRUCT to access the structured solvers. Here we have a purely
structured example. */
object_type = HYPRE_STRUCT;
HYPRE_SStructMatrixSetObjectType(A, object_type);
/* Get ready to set values */
HYPRE_SStructMatrixInitialize(A);
/* Each processor must set the stencil values for their boxes on each part.
In this example, we only set stencil entries and therefore use
HYPRE_SStructMatrixSetBoxValues. If we need to set non-stencil entries,
we have to use HYPRE_SStructMatrixSetValues (shown in a later example). */
if (myid == 0)
{
/* Set the matrix coefficients for some set of stencil entries
over all the gridpoints in my first box (account for boundary
grid points later) */
{
int ilower[2] = {-3, 1};
int iupper[2] = {-1, 2};
int nentries = 5;
int nvalues = 30; /* 6 grid points, each with 5 stencil entries */
double values[30];
int stencil_indices[5];
for (j = 0; j < nentries; j++) /* label the stencil indices -
these correspond to the offsets
defined above */
stencil_indices[j] = j;
for (i = 0; i < nvalues; i += nentries)
{
values[i] = 4.0;
for (j = 1; j < nentries; j++)
values[i+j] = -1.0;
}
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, nentries,
stencil_indices, values);
}
/* Set the matrix coefficients for some set of stencil entries
over the gridpoints in my second box */
{
int ilower[2] = {0, 1};
int iupper[2] = {2, 4};
int nentries = 5;
int nvalues = 60; /* 12 grid points, each with 5 stencil entries */
double values[60];
int stencil_indices[5];
for (j = 0; j < nentries; j++)
stencil_indices[j] = j;
for (i = 0; i < nvalues; i += nentries)
{
values[i] = 4.0;
for (j = 1; j < nentries; j++)
values[i+j] = -1.0;
}
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, nentries,
stencil_indices, values);
}
}
else if (myid == 1)
{
/* Set the matrix coefficients for some set of stencil entries
over the gridpoints in my box */
{
int ilower[2] = {3, 1};
int iupper[2] = {6, 4};
int nentries = 5;
int nvalues = 80; /* 16 grid points, each with 5 stencil entries */
double values[80];
int stencil_indices[5];
for (j = 0; j < nentries; j++)
stencil_indices[j] = j;
for (i = 0; i < nvalues; i += nentries)
{
values[i] = 4.0;
for (j = 1; j < nentries; j++)
values[i+j] = -1.0;
}
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, nentries,
stencil_indices, values);
}
}
/* For each box, set any coefficients that reach ouside of the
boundary to 0 */
if (myid == 0)
{
int maxnvalues = 6;
double values[6];
for (i = 0; i < maxnvalues; i++)
values[i] = 0.0;
{
/* Values below our first AND second box */
int ilower[2] = {-3, 1};
int iupper[2] = { 2, 1};
int stencil_indices[1] = {3};
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, 1,
stencil_indices, values);
}
{
/* Values to the left of our first box */
int ilower[2] = {-3, 1};
int iupper[2] = {-3, 2};
int stencil_indices[1] = {1};
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, 1,
stencil_indices, values);
}
{
/* Values above our first box */
int ilower[2] = {-3, 2};
int iupper[2] = {-1, 2};
int stencil_indices[1] = {4};
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, 1,
stencil_indices, values);
}
{
/* Values to the left of our second box (that do not border the
first box). */
int ilower[2] = { 0, 3};
int iupper[2] = { 0, 4};
int stencil_indices[1] = {1};
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, 1,
stencil_indices, values);
}
{
/* Values above our second box */
int ilower[2] = { 0, 4};
int iupper[2] = { 2, 4};
int stencil_indices[1] = {4};
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, 1,
stencil_indices, values);
}
}
else if (myid == 1)
{
int maxnvalues = 4;
double values[4];
for (i = 0; i < maxnvalues; i++)
values[i] = 0.0;
{
/* Values below our box */
int ilower[2] = { 3, 1};
int iupper[2] = { 6, 1};
int stencil_indices[1] = {3};
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, 1,
stencil_indices, values);
}
{
/* Values to the right of our box */
int ilower[2] = { 6, 1};
int iupper[2] = { 6, 4};
int stencil_indices[1] = {2};
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, 1,
stencil_indices, values);
}
{
/* Values above our box */
int ilower[2] = { 3, 4};
int iupper[2] = { 6, 4};
int stencil_indices[1] = {4};
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper,
var, 1,
stencil_indices, values);
}
}
/* This is a collective call finalizing the matrix assembly.
The matrix is now ``ready to be used'' */
HYPRE_SStructMatrixAssemble(A);
}
/* 5. Set up SStruct Vectors for b and x */
{
int i;
/* We have one part and one variable. */
int part = 0;
int var = 0;
/* Create an empty vector object */
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &b);
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &x);
/* As with the matrix, set the object type for the vectors
to be the struct type */
object_type = HYPRE_STRUCT;
HYPRE_SStructVectorSetObjectType(b, object_type);
HYPRE_SStructVectorSetObjectType(x, object_type);
/* Indicate that the vector coefficients are ready to be set */
HYPRE_SStructVectorInitialize(b);
HYPRE_SStructVectorInitialize(x);
if (myid == 0)
{
/* Set the vector coefficients over the gridpoints in my first box */
{
int ilower[2] = {-3, 1};
int iupper[2] = {-1, 2};
int nvalues = 6; /* 6 grid points */
double values[6];
for (i = 0; i < nvalues; i ++)
values[i] = 1.0;
HYPRE_SStructVectorSetBoxValues(b, part, ilower, iupper, var, values);
for (i = 0; i < nvalues; i ++)
values[i] = 0.0;
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, values);
}
/* Set the vector coefficients over the gridpoints in my second box */
{
int ilower[2] = { 0, 1};
int iupper[2] = { 2, 4};
int nvalues = 12; /* 12 grid points */
double values[12];
for (i = 0; i < nvalues; i ++)
values[i] = 1.0;
HYPRE_SStructVectorSetBoxValues(b, part, ilower, iupper, var, values);
for (i = 0; i < nvalues; i ++)
values[i] = 0.0;
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, values);
}
}
else if (myid == 1)
{
/* Set the vector coefficients over the gridpoints in my box */
{
int ilower[2] = { 3, 1};
int iupper[2] = { 6, 4};
int nvalues = 16; /* 16 grid points */
double values[16];
for (i = 0; i < nvalues; i ++)
values[i] = 1.0;
HYPRE_SStructVectorSetBoxValues(b, part, ilower, iupper, var, values);
for (i = 0; i < nvalues; i ++)
values[i] = 0.0;
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, values);
}
}
/* This is a collective call finalizing the vector assembly.
The vectors are now ``ready to be used'' */
HYPRE_SStructVectorAssemble(b);
HYPRE_SStructVectorAssemble(x);
}
/* 6. Set up and use a solver (See the Reference Manual for descriptions
of all of the options.) */
{
HYPRE_StructMatrix sA;
HYPRE_StructVector sb;
HYPRE_StructVector sx;
/* Because we are using a struct solver, we need to get the
object of the matrix and vectors to pass in to the struct solvers */
HYPRE_SStructMatrixGetObject(A, (void **) &sA);
HYPRE_SStructVectorGetObject(b, (void **) &sb);
HYPRE_SStructVectorGetObject(x, (void **) &sx);
/* Create an empty PCG Struct solver */
HYPRE_StructPCGCreate(MPI_COMM_WORLD, &solver);
/* Set PCG parameters */
HYPRE_StructPCGSetTol(solver, 1.0e-06);
HYPRE_StructPCGSetPrintLevel(solver, 2);
HYPRE_StructPCGSetMaxIter(solver, 50);
/* Create the Struct SMG solver for use as a preconditioner */
HYPRE_StructSMGCreate(MPI_COMM_WORLD, &precond);
/* Set SMG parameters */
HYPRE_StructSMGSetMaxIter(precond, 1);
HYPRE_StructSMGSetTol(precond, 0.0);
HYPRE_StructSMGSetZeroGuess(precond);
HYPRE_StructSMGSetNumPreRelax(precond, 1);
HYPRE_StructSMGSetNumPostRelax(precond, 1);
/* Set preconditioner and solve */
HYPRE_StructPCGSetPrecond(solver, HYPRE_StructSMGSolve,
HYPRE_StructSMGSetup, precond);
HYPRE_StructPCGSetup(solver, sA, sb, sx);
HYPRE_StructPCGSolve(solver, sA, sb, sx);
}
/* Free memory */
HYPRE_SStructGridDestroy(grid);
HYPRE_SStructStencilDestroy(stencil);
HYPRE_SStructGraphDestroy(graph);
HYPRE_SStructMatrixDestroy(A);
HYPRE_SStructVectorDestroy(b);
HYPRE_SStructVectorDestroy(x);
HYPRE_StructPCGDestroy(solver);
HYPRE_StructSMGDestroy(precond);
/* Finalize MPI */
MPI_Finalize();
return (0);
}
int main (int argc, char *argv[])
{
int myid, num_procs;
int n, N, pi, pj, pk;
double h;
double tol, theta;
int maxit, cycle_type;
int rlx_type, rlx_sweeps, rlx_weight, rlx_omega;
int amg_coarsen_type, amg_agg_levels, amg_rlx_type;
int amg_interp_type, amg_Pmax;
int singular_problem ;
HYPRE_Int time_index;
HYPRE_SStructGrid edge_grid;
HYPRE_SStructGraph A_graph;
HYPRE_SStructMatrix A;
HYPRE_SStructVector b;
HYPRE_SStructVector x;
HYPRE_SStructGrid node_grid;
HYPRE_SStructGraph G_graph;
HYPRE_SStructStencil G_stencil[3];
HYPRE_SStructMatrix G;
HYPRE_SStructVector xcoord, ycoord, zcoord;
HYPRE_Solver solver, precond;
/* Initialize MPI */
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
/* Set default parameters */
n = 10;
optionAlpha = 0;
optionBeta = 0;
maxit = 100;
tol = 1e-6;
cycle_type = 13;
rlx_type = 2;
rlx_sweeps = 1;
rlx_weight = 1.0;
rlx_omega = 1.0;
amg_coarsen_type = 10;
amg_agg_levels = 1;
amg_rlx_type = 6;
theta = 0.25;
amg_interp_type = 6;
amg_Pmax = 4;
singular_problem = 0;
/* Parse command line */
{
int arg_index = 0;
int print_usage = 0;
while (arg_index < argc)
{
if ( strcmp(argv[arg_index], "-n") == 0 )
{
arg_index++;
n = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-a") == 0 )
{
arg_index++;
optionAlpha = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-b") == 0 )
{
arg_index++;
optionBeta = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-maxit") == 0 )
{
arg_index++;
maxit = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-tol") == 0 )
{
arg_index++;
tol = atof(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-type") == 0 )
{
arg_index++;
cycle_type = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-rlx") == 0 )
{
arg_index++;
rlx_type = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-rlxn") == 0 )
{
arg_index++;
rlx_sweeps = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-rlxw") == 0 )
{
arg_index++;
rlx_weight = atof(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-rlxo") == 0 )
{
arg_index++;
rlx_omega = atof(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-ctype") == 0 )
{
arg_index++;
amg_coarsen_type = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-amgrlx") == 0 )
{
arg_index++;
amg_rlx_type = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-agg") == 0 )
{
arg_index++;
amg_agg_levels = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-itype") == 0 )
{
arg_index++;
amg_interp_type = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-pmax") == 0 )
{
arg_index++;
amg_Pmax = atoi(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-sing") == 0 )
{
arg_index++;
singular_problem = 1;
}
else if ( strcmp(argv[arg_index], "-theta") == 0 )
{
arg_index++;
theta = atof(argv[arg_index++]);
}
else if ( strcmp(argv[arg_index], "-help") == 0 )
{
print_usage = 1;
break;
}
else
{
arg_index++;
}
}
if ((print_usage) && (myid == 0))
{
printf("\n");
printf("Usage: %s [<options>]\n", argv[0]);
printf("\n");
printf(" -n <n> : problem size per processor (default: 10)\n");
printf(" -a <alpha_opt> : choice for the curl-curl coefficient (default: 1)\n");
printf(" -b <beta_opt> : choice for the mass coefficient (default: 1)\n");
printf("\n");
printf("PCG-AMS solver options: \n");
printf(" -maxit <num> : maximum number of iterations (100) \n");
printf(" -tol <num> : convergence tolerance (1e-6) \n");
printf(" -type <num> : 3-level cycle type (0-8, 11-14) \n");
printf(" -theta <num> : BoomerAMG threshold (0.25) \n");
printf(" -ctype <num> : BoomerAMG coarsening type \n");
printf(" -agg <num> : Levels of BoomerAMG agg. coarsening \n");
printf(" -amgrlx <num> : BoomerAMG relaxation type \n");
printf(" -itype <num> : BoomerAMG interpolation type \n");
printf(" -pmax <num> : BoomerAMG interpolation truncation \n");
printf(" -rlx <num> : relaxation type \n");
printf(" -rlxn <num> : number of relaxation sweeps \n");
printf(" -rlxw <num> : damping parameter (usually <=1) \n");
printf(" -rlxo <num> : SOR parameter (usually in (0,2)) \n");
printf(" -sing : curl-curl only (singular) problem \n");
printf("\n");
printf("\n");
}
if (print_usage)
{
MPI_Finalize();
return (0);
}
}
/* Figure out the processor grid (N x N x N). The local problem size is n^3,
while pi, pj and pk indicate the position in the processor grid. */
N = pow(num_procs,1.0/3.0) + 0.5;
if (num_procs != N*N*N)
{
if (myid == 0) printf("Can't run on %d processors, try %d.\n",
num_procs, N*N*N);
MPI_Finalize();
exit(1);
}
h = 1.0 / (N*n);
pk = myid / (N*N);
pj = myid/N - pk*N;
pi = myid - pj*N - pk*N*N;
/* Start timing */
time_index = hypre_InitializeTiming("SStruct Setup");
hypre_BeginTiming(time_index);
/* 1. Set up the edge and nodal grids. Note that we do this simultaneously
to make sure that they have the same extents. For simplicity we use
only one part to represent the unit cube. */
{
HYPRE_Int ndim = 3;
HYPRE_Int nparts = 1;
/* Create empty 2D grid objects */
HYPRE_SStructGridCreate(MPI_COMM_WORLD, ndim, nparts, &node_grid);
HYPRE_SStructGridCreate(MPI_COMM_WORLD, ndim, nparts, &edge_grid);
/* Set the extents of the grid - each processor sets its grid boxes. */
{
HYPRE_Int part = 0;
HYPRE_Int ilower[3] = {1 + pi*n, 1 + pj*n, 1 + pk*n};
HYPRE_Int iupper[3] = {n + pi*n, n + pj*n, n + pk*n};
HYPRE_SStructGridSetExtents(node_grid, part, ilower, iupper);
HYPRE_SStructGridSetExtents(edge_grid, part, ilower, iupper);
}
/* Set the variable type and number of variables on each grid. */
{
HYPRE_Int i;
HYPRE_Int nnodevars = 1;
HYPRE_Int nedgevars = 3;
HYPRE_SStructVariable nodevars[1] = {HYPRE_SSTRUCT_VARIABLE_NODE};
HYPRE_SStructVariable edgevars[3] = {HYPRE_SSTRUCT_VARIABLE_XEDGE,
HYPRE_SSTRUCT_VARIABLE_YEDGE,
HYPRE_SSTRUCT_VARIABLE_ZEDGE};
for (i = 0; i < nparts; i++)
{
HYPRE_SStructGridSetVariables(node_grid, i, nnodevars, nodevars);
HYPRE_SStructGridSetVariables(edge_grid, i, nedgevars, edgevars);
}
}
/* Since there is only one part, there is no need to call the
SetNeighborPart or SetSharedPart functions, which determine the spatial
relation between the parts. See Examples 12, 13 and 14 for
illustrations of these calls. */
/* Now the grids are ready to be used */
HYPRE_SStructGridAssemble(node_grid);
HYPRE_SStructGridAssemble(edge_grid);
}
/* 2. Create the finite element stiffness matrix A and load vector b. */
{
HYPRE_Int part = 0; /* this problem has only one part */
/* Set the ordering of the variables in the finite element problem. This
is done by listing the variable offset directions relative to the
element's center. See the Reference Manual for more details. */
{
HYPRE_Int ordering[48] = { 0, 0, -1, -1, /* x-edge [0]-[1] */
1, +1, 0, -1, /* y-edge [1]-[2] */
/* [7]------[6] */ 0, 0, +1, -1, /* x-edge [3]-[2] */
/* /| /| */ 1, -1, 0, -1, /* y-edge [0]-[3] */
/* / | / | */ 0, 0, -1, +1, /* x-edge [4]-[5] */
/* [4]------[5] | */ 1, +1, 0, +1, /* y-edge [5]-[6] */
/* | [3]----|-[2] */ 0, 0, +1, +1, /* x-edge [7]-[6] */
/* | / | / */ 1, -1, 0, +1, /* y-edge [4]-[7] */
/* |/ |/ */ 2, -1, -1, 0, /* z-edge [0]-[4] */
/* [0]------[1] */ 2, +1, -1, 0, /* z-edge [1]-[5] */
2, +1, +1, 0, /* z-edge [2]-[6] */
2, -1, +1, 0 }; /* z-edge [3]-[7] */
HYPRE_SStructGridSetFEMOrdering(edge_grid, part, ordering);
}
/* Set up the Graph - this determines the non-zero structure of the
matrix. */
{
HYPRE_Int part = 0;
/* Create the graph object */
HYPRE_SStructGraphCreate(MPI_COMM_WORLD, edge_grid, &A_graph);
/* See MatrixSetObjectType below */
HYPRE_SStructGraphSetObjectType(A_graph, HYPRE_PARCSR);
/* Indicate that this problem uses finite element stiffness matrices and
load vectors, instead of stencils. */
HYPRE_SStructGraphSetFEM(A_graph, part);
/* The edge finite element matrix is full, so there is no need to call the
HYPRE_SStructGraphSetFEMSparsity() function. */
/* Assemble the graph */
HYPRE_SStructGraphAssemble(A_graph);
}
/* Set up the SStruct Matrix and right-hand side vector */
{
/* Create the matrix object */
HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, A_graph, &A);
/* Use a ParCSR storage */
HYPRE_SStructMatrixSetObjectType(A, HYPRE_PARCSR);
/* Indicate that the matrix coefficients are ready to be set */
HYPRE_SStructMatrixInitialize(A);
/* Create an empty vector object */
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, edge_grid, &b);
/* Use a ParCSR storage */
HYPRE_SStructVectorSetObjectType(b, HYPRE_PARCSR);
/* Indicate that the vector coefficients are ready to be set */
HYPRE_SStructVectorInitialize(b);
}
/* Set the matrix and vector entries by finite element assembly */
{
/* local stiffness matrix and load vector */
double S[12][12], F[12];
int i, j, k;
HYPRE_Int index[3];
for (i = 1; i <= n; i++)
for (j = 1; j <= n; j++)
for (k = 1; k <= n; k++)
{
/* Compute the FEM matrix and r.h.s. for cell (i,j,k) with
coefficients evaluated at the cell center. */
index[0] = i + pi*n; index[1] = j + pj*n; index[2] = k + pk*n;
ComputeFEMND1(S,F,(pi*n+i)*h-h/2,(pj*n+j)*h-h/2,(pk*n+k)*h-h/2,h);
/* Eliminate boundary conditions on x = 0 */
if (index[0] == 1)
{
int ii, jj, bc_edges[4] = { 3, 11, 7, 8 };
for (ii = 0; ii < 4; ii++)
{
for (jj = 0; jj < 12; jj++)
S[bc_edges[ii]][jj] = S[jj][bc_edges[ii]] = 0.0;
S[bc_edges[ii]][bc_edges[ii]] = 1.0;
F[bc_edges[ii]] = 0.0;
}
}
/* Eliminate boundary conditions on y = 0 */
if (index[1] == 1)
{
int ii, jj, bc_edges[4] = { 0, 9, 4, 8 };
for (ii = 0; ii < 4; ii++)
{
for (jj = 0; jj < 12; jj++)
S[bc_edges[ii]][jj] = S[jj][bc_edges[ii]] = 0.0;
S[bc_edges[ii]][bc_edges[ii]] = 1.0;
F[bc_edges[ii]] = 0.0;
}
}
/* Eliminate boundary conditions on z = 0 */
if (index[2] == 1)
{
int ii, jj, bc_edges[4] = { 0, 1, 2, 3 };
for (ii = 0; ii < 4; ii++)
{
for (jj = 0; jj < 12; jj++)
S[bc_edges[ii]][jj] = S[jj][bc_edges[ii]] = 0.0;
S[bc_edges[ii]][bc_edges[ii]] = 1.0;
F[bc_edges[ii]] = 0.0;
}
}
/* Eliminate boundary conditions on x = 1 */
if (index[0] == N*n)
{
int ii, jj, bc_edges[4] = { 1, 10, 5, 9 };
for (ii = 0; ii < 4; ii++)
{
for (jj = 0; jj < 12; jj++)
S[bc_edges[ii]][jj] = S[jj][bc_edges[ii]] = 0.0;
S[bc_edges[ii]][bc_edges[ii]] = 1.0;
F[bc_edges[ii]] = 0.0;
}
}
/* Eliminate boundary conditions on y = 1 */
if (index[1] == N*n)
{
int ii, jj, bc_edges[4] = { 2, 10, 6, 11 };
for (ii = 0; ii < 4; ii++)
{
for (jj = 0; jj < 12; jj++)
S[bc_edges[ii]][jj] = S[jj][bc_edges[ii]] = 0.0;
S[bc_edges[ii]][bc_edges[ii]] = 1.0;
F[bc_edges[ii]] = 0.0;
}
}
/* Eliminate boundary conditions on z = 1 */
if (index[2] == N*n)
{
int ii, jj, bc_edges[4] = { 4, 5, 6, 7 };
for (ii = 0; ii < 4; ii++)
{
for (jj = 0; jj < 12; jj++)
S[bc_edges[ii]][jj] = S[jj][bc_edges[ii]] = 0.0;
S[bc_edges[ii]][bc_edges[ii]] = 1.0;
F[bc_edges[ii]] = 0.0;
}
}
/* Assemble the matrix */
HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
/* Assemble the vector */
HYPRE_SStructVectorAddFEMValues(b, part, index, F);
}
}
/* Collective calls finalizing the matrix and vector assembly */
HYPRE_SStructMatrixAssemble(A);
HYPRE_SStructVectorAssemble(b);
}
/* 3. Create the discrete gradient matrix G, which is needed in AMS. */
{
HYPRE_Int part = 0;
HYPRE_Int stencil_size = 2;
/* Define the discretization stencil relating the edges and nodes of the
grid. */
{
HYPRE_Int ndim = 3;
HYPRE_Int entry;
HYPRE_Int var = 0; /* the node variable */
/* The discrete gradient stencils connect edge to node variables. */
HYPRE_Int Gx_offsets[2][3] = {{-1,0,0},{0,0,0}}; /* x-edge [7]-[6] */
HYPRE_Int Gy_offsets[2][3] = {{0,-1,0},{0,0,0}}; /* y-edge [5]-[6] */
HYPRE_Int Gz_offsets[2][3] = {{0,0,-1},{0,0,0}}; /* z-edge [2]-[6] */
HYPRE_SStructStencilCreate(ndim, stencil_size, &G_stencil[0]);
HYPRE_SStructStencilCreate(ndim, stencil_size, &G_stencil[1]);
HYPRE_SStructStencilCreate(ndim, stencil_size, &G_stencil[2]);
for (entry = 0; entry < stencil_size; entry++)
{
HYPRE_SStructStencilSetEntry(G_stencil[0], entry, Gx_offsets[entry], var);
HYPRE_SStructStencilSetEntry(G_stencil[1], entry, Gy_offsets[entry], var);
HYPRE_SStructStencilSetEntry(G_stencil[2], entry, Gz_offsets[entry], var);
}
}
/* Set up the Graph - this determines the non-zero structure of the
matrix. */
{
HYPRE_Int nvars = 3;
HYPRE_Int var; /* the edge variables */
/* Create the discrete gradient graph object */
HYPRE_SStructGraphCreate(MPI_COMM_WORLD, edge_grid, &G_graph);
/* See MatrixSetObjectType below */
HYPRE_SStructGraphSetObjectType(G_graph, HYPRE_PARCSR);
/* Since the discrete gradient relates edge and nodal variables (it is a
rectangular matrix), we have to specify the domain (column) grid. */
HYPRE_SStructGraphSetDomainGrid(G_graph, node_grid);
/* Tell the graph which stencil to use for each edge variable on each
part (we only have one part). */
for (var = 0; var < nvars; var++)
HYPRE_SStructGraphSetStencil(G_graph, part, var, G_stencil[var]);
/* Assemble the graph */
HYPRE_SStructGraphAssemble(G_graph);
}
/* Set up the SStruct Matrix */
{
/* Create the matrix object */
HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, G_graph, &G);
/* Use a ParCSR storage */
HYPRE_SStructMatrixSetObjectType(G, HYPRE_PARCSR);
/* Indicate that the matrix coefficients are ready to be set */
HYPRE_SStructMatrixInitialize(G);
}
/* Set the discrete gradient values, assuming a "natural" orientation of
the edges (i.e. one in agreement with the coordinate directions). */
{
int i;
int nedges = n*(n+1)*(n+1);
double *values;
HYPRE_Int stencil_indices[2] = {0,1}; /* the nodes of each edge */
values = calloc(2*nedges, sizeof(double));
/* The edge orientation is fixed: from first to second node */
for (i = 0; i < nedges; i++)
{
values[2*i] = -1.0;
values[2*i+1] = 1.0;
}
/* Set the values in the discrete gradient x-edges */
{
HYPRE_Int var = 0;
HYPRE_Int ilower[3] = {1 + pi*n, 0 + pj*n, 0 + pk*n};
HYPRE_Int iupper[3] = {n + pi*n, n + pj*n, n + pk*n};
HYPRE_SStructMatrixSetBoxValues(G, part, ilower, iupper, var,
stencil_size, stencil_indices,
values);
}
/* Set the values in the discrete gradient y-edges */
{
HYPRE_Int var = 1;
HYPRE_Int ilower[3] = {0 + pi*n, 1 + pj*n, 0 + pk*n};
HYPRE_Int iupper[3] = {n + pi*n, n + pj*n, n + pk*n};
HYPRE_SStructMatrixSetBoxValues(G, part, ilower, iupper, var,
stencil_size, stencil_indices,
values);
}
/* Set the values in the discrete gradient z-edges */
{
HYPRE_Int var = 2;
HYPRE_Int ilower[3] = {0 + pi*n, 0 + pj*n, 1 + pk*n};
HYPRE_Int iupper[3] = {n + pi*n, n + pj*n, n + pk*n};
HYPRE_SStructMatrixSetBoxValues(G, part, ilower, iupper, var,
stencil_size, stencil_indices,
values);
}
free(values);
}
/* Finalize the matrix assembly */
HYPRE_SStructMatrixAssemble(G);
}
/* 4. Create the vectors of nodal coordinates xcoord, ycoord and zcoord,
which are needed in AMS. */
{
int i, j, k;
HYPRE_Int part = 0;
HYPRE_Int var = 0; /* the node variable */
HYPRE_Int index[3];
double xval, yval, zval;
/* Create empty vector objects */
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, node_grid, &xcoord);
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, node_grid, &ycoord);
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, node_grid, &zcoord);
/* Set the object type to ParCSR */
HYPRE_SStructVectorSetObjectType(xcoord, HYPRE_PARCSR);
HYPRE_SStructVectorSetObjectType(ycoord, HYPRE_PARCSR);
HYPRE_SStructVectorSetObjectType(zcoord, HYPRE_PARCSR);
/* Indicate that the vector coefficients are ready to be set */
HYPRE_SStructVectorInitialize(xcoord);
HYPRE_SStructVectorInitialize(ycoord);
HYPRE_SStructVectorInitialize(zcoord);
/* Compute and set the coordinates of the nodes */
for (i = 0; i <= n; i++)
for (j = 0; j <= n; j++)
for (k = 0; k <= n; k++)
{
index[0] = i + pi*n; index[1] = j + pj*n; index[2] = k + pk*n;
xval = index[0]*h;
yval = index[1]*h;
zval = index[2]*h;
HYPRE_SStructVectorSetValues(xcoord, part, index, var, &xval);
HYPRE_SStructVectorSetValues(ycoord, part, index, var, &yval);
HYPRE_SStructVectorSetValues(zcoord, part, index, var, &zval);
}
/* Finalize the vector assembly */
HYPRE_SStructVectorAssemble(xcoord);
HYPRE_SStructVectorAssemble(ycoord);
HYPRE_SStructVectorAssemble(zcoord);
}
/* 5. Set up a SStruct Vector for the solution vector x */
{
HYPRE_Int part = 0;
int nvalues = n*(n+1)*(n+1);
double *values;
values = calloc(nvalues, sizeof(double));
/* Create an empty vector object */
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, edge_grid, &x);
/* Set the object type to ParCSR */
HYPRE_SStructVectorSetObjectType(x, HYPRE_PARCSR);
/* Indicate that the vector coefficients are ready to be set */
HYPRE_SStructVectorInitialize(x);
/* Set the values for the initial guess x-edge */
{
HYPRE_Int var = 0;
HYPRE_Int ilower[3] = {1 + pi*n, 0 + pj*n, 0 + pk*n};
HYPRE_Int iupper[3] = {n + pi*n, n + pj*n, n + pk*n};
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, values);
}
/* Set the values for the initial guess y-edge */
{
HYPRE_Int var = 1;
HYPRE_Int ilower[3] = {0 + pi*n, 1 + pj*n, 0 + pk*n};
HYPRE_Int iupper[3] = {n + pi*n, n + pj*n, n + pk*n};
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, values);
}
/* Set the values for the initial guess z-edge */
{
HYPRE_Int var = 2;
HYPRE_Int ilower[3] = {0 + pi*n, 0 + pj*n, 1 + pk*n};
HYPRE_Int iupper[3] = {n + pi*n, n + pj*n, n + pk*n};
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, values);
}
free(values);
/* Finalize the vector assembly */
HYPRE_SStructVectorAssemble(x);
}
/* Finalize current timing */
hypre_EndTiming(time_index);
hypre_PrintTiming("SStruct phase times", MPI_COMM_WORLD);
hypre_FinalizeTiming(time_index);
hypre_ClearTiming();
/* 6. Set up and call the PCG-AMS solver (Solver options can be found in the
Reference Manual.) */
{
double final_res_norm;
HYPRE_Int its;
HYPRE_ParCSRMatrix par_A;
HYPRE_ParVector par_b;
HYPRE_ParVector par_x;
HYPRE_ParCSRMatrix par_G;
HYPRE_ParVector par_xcoord;
HYPRE_ParVector par_ycoord;
HYPRE_ParVector par_zcoord;
/* Extract the ParCSR objects needed in the solver */
HYPRE_SStructMatrixGetObject(A, (void **) &par_A);
HYPRE_SStructVectorGetObject(b, (void **) &par_b);
HYPRE_SStructVectorGetObject(x, (void **) &par_x);
HYPRE_SStructMatrixGetObject(G, (void **) &par_G);
HYPRE_SStructVectorGetObject(xcoord, (void **) &par_xcoord);
HYPRE_SStructVectorGetObject(ycoord, (void **) &par_ycoord);
HYPRE_SStructVectorGetObject(zcoord, (void **) &par_zcoord);
if (myid == 0)
printf("Problem size: %lld\n\n",
hypre_ParCSRMatrixGlobalNumRows((hypre_ParCSRMatrix*)par_A));
/* Start timing */
time_index = hypre_InitializeTiming("AMS Setup");
hypre_BeginTiming(time_index);
/* Create solver */
HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);
/* Set some parameters (See Reference Manual for more parameters) */
HYPRE_PCGSetMaxIter(solver, maxit); /* max iterations */
HYPRE_PCGSetTol(solver, tol); /* conv. tolerance */
HYPRE_PCGSetTwoNorm(solver, 0); /* use the two norm as the stopping criteria */
HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */
/* Create AMS preconditioner */
HYPRE_AMSCreate(&precond);
/* Set AMS parameters */
HYPRE_AMSSetMaxIter(precond, 1);
HYPRE_AMSSetTol(precond, 0.0);
HYPRE_AMSSetCycleType(precond, cycle_type);
HYPRE_AMSSetPrintLevel(precond, 1);
/* Set discrete gradient */
HYPRE_AMSSetDiscreteGradient(precond, par_G);
/* Set vertex coordinates */
HYPRE_AMSSetCoordinateVectors(precond,
par_xcoord, par_ycoord, par_zcoord);
if (singular_problem)
HYPRE_AMSSetBetaPoissonMatrix(precond, NULL);
/* Smoothing and AMG options */
HYPRE_AMSSetSmoothingOptions(precond,
rlx_type, rlx_sweeps,
rlx_weight, rlx_omega);
HYPRE_AMSSetAlphaAMGOptions(precond,
amg_coarsen_type, amg_agg_levels,
amg_rlx_type, theta, amg_interp_type,
amg_Pmax);
HYPRE_AMSSetBetaAMGOptions(precond,
amg_coarsen_type, amg_agg_levels,
amg_rlx_type, theta, amg_interp_type,
amg_Pmax);
/* Set the PCG preconditioner */
HYPRE_PCGSetPrecond(solver,
(HYPRE_PtrToSolverFcn) HYPRE_AMSSolve,
(HYPRE_PtrToSolverFcn) HYPRE_AMSSetup,
precond);
/* Call the setup */
HYPRE_ParCSRPCGSetup(solver, par_A, par_b, par_x);
/* Finalize current timing */
hypre_EndTiming(time_index);
hypre_PrintTiming("Setup phase times", MPI_COMM_WORLD);
hypre_FinalizeTiming(time_index);
hypre_ClearTiming();
/* Start timing again */
time_index = hypre_InitializeTiming("AMS Solve");
hypre_BeginTiming(time_index);
/* Call the solve */
HYPRE_ParCSRPCGSolve(solver, par_A, par_b, par_x);
/* Finalize current timing */
hypre_EndTiming(time_index);
hypre_PrintTiming("Solve phase times", MPI_COMM_WORLD);
hypre_FinalizeTiming(time_index);
hypre_ClearTiming();
/* Get some info */
HYPRE_PCGGetNumIterations(solver, &its);
HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
/* Clean up */
HYPRE_AMSDestroy(precond);
HYPRE_ParCSRPCGDestroy(solver);
/* Gather the solution vector */
HYPRE_SStructVectorGather(x);
if (myid == 0)
{
printf("\n");
printf("Iterations = %lld\n", its);
printf("Final Relative Residual Norm = %g\n", final_res_norm);
printf("\n");
}
}
/* Free memory */
HYPRE_SStructGridDestroy(edge_grid);
HYPRE_SStructGraphDestroy(A_graph);
HYPRE_SStructMatrixDestroy(A);
HYPRE_SStructVectorDestroy(b);
HYPRE_SStructVectorDestroy(x);
HYPRE_SStructGridDestroy(node_grid);
HYPRE_SStructGraphDestroy(G_graph);
HYPRE_SStructStencilDestroy(G_stencil[0]);
HYPRE_SStructStencilDestroy(G_stencil[1]);
HYPRE_SStructStencilDestroy(G_stencil[2]);
HYPRE_SStructMatrixDestroy(G);
HYPRE_SStructVectorDestroy(xcoord);
HYPRE_SStructVectorDestroy(ycoord);
HYPRE_SStructVectorDestroy(zcoord);
/* Finalize MPI */
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[]) {
// printf("start\n");
int i, j, pi, pj;
int myid, num_procs;
int Nx, Ny, nx, ny;
//nx and ny are number of mesh cells
int solver_id;
int vis;
int ilower[2], iupper[2];
int nparts = 1;
int part = 0;
double hx, hy;
HYPRE_SStructGraph graph;
HYPRE_SStructGrid grid;
HYPRE_SStructStencil stencil;
HYPRE_SStructMatrix A;
HYPRE_ParCSRMatrix parcsr_A;
HYPRE_SStructVector b;
HYPRE_ParVector par_b;
HYPRE_SStructVector x;
HYPRE_ParVector par_x;
HYPRE_Solver solver;
// printf("variables initialized\n");
//Initialize MPI
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
// printf("mpi initialized\n");
// Set default problem parameters
nx = 100;
ny = 100;
Nx = 1;
solver_id = 0;
vis = 0;
// printf("defaults set\n");
// Parse command line
int arg_index = 0;
while (arg_index < argc) {
if (strcmp(argv[arg_index], "-nx") == 0) {
arg_index++;
nx = atoi(argv[arg_index++]);
// printf("nx set\n");
} else if (strcmp(argv[arg_index], "-ny") == 0) {
arg_index++;
ny = atoi(argv[arg_index++]);
// printf("ny set\n");
} else if (strcmp(argv[arg_index], "-Nx") == 0) {
arg_index++;
Nx = atoi(argv[arg_index++]);
// printf("Nx set\n");
} else if (strcmp(argv[arg_index], "-solver") == 0) {
arg_index++;
solver_id = atoi(argv[arg_index++]);
// printf("solver set\n");
} else if (strcmp(argv[arg_index], "-vis") == 0) {
arg_index++;
vis = 1;
// printf("print solution\n");
} else {
arg_index++;
// printf("plus plus\n");
}
}
// printf("command line parsed\n");
// Set up processors in grid
Ny = num_procs / Nx;
hx = 1.0 / (Nx * nx);
hy = 1.0 / (Ny * ny);
if (Ny <= Nx) {
pj = myid / Nx;
pi = myid - pj * Ny;
} else {
pi = myid / Ny;
pj = myid - pi * Nx;
}
// printf("processors assigned\n");
// Determine each processor's piece of the grid
ilower[0] = pi * nx;
ilower[1] = pj * ny;
iupper[0] = ilower[0] + nx - 1;
iupper[1] = ilower[1] + ny - 1;
// printf("upper and lowers assigned\n");
HYPRE_SStructGridCreate(MPI_COMM_WORLD, 2, nparts, &grid);
// printf("grid created\n");
HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
// printf("grid set\n");
int nvars = 1;
HYPRE_SStructVariable vartypes[1] = { HYPRE_SSTRUCT_VARIABLE_CELL };
// printf("variable type set\n");
for (i = 0; i < nparts; i++) {
HYPRE_SStructGridSetVariables(grid, i, nvars, vartypes);
}
// printf("variable type set\n");
HYPRE_SStructGridAssemble(grid);
// printf("grid assembled\n");
// Set up the stencil
HYPRE_SStructStencilCreate(2, 5, &stencil);
int entry;
int offsets[5][2] = { { 0, 0 }, { -1, 0 }, { 1, 0 }, { 0, -1 }, { 0, 1 } };
int var = 0;
for (entry = 0; entry < 5; entry++) {
HYPRE_SStructStencilSetEntry(stencil, entry, offsets[entry], var);
}
// Create the graph
HYPRE_SStructGraphCreate(MPI_COMM_WORLD, grid, &graph);
int object_type = HYPRE_PARCSR;
HYPRE_SStructGraphSetObjectType(graph, object_type);
HYPRE_SStructGraphSetStencil(graph, part, var, stencil);
HYPRE_SStructGraphAssemble(graph);
int nentries = 5;
int nvalues = nentries * nx * ny;
double *values;
int stencil_indices[5];
// Setup the matrix
HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, graph, &A);
HYPRE_SStructMatrixSetObjectType(A, HYPRE_PARCSR);
HYPRE_SStructMatrixInitialize(A);
// Set the matrix coefficients
values = calloc(nvalues, sizeof(double));
for (j = 0; j < nentries; j++) {
stencil_indices[j] = j;
}
// Account for differences in hx and hy
double hx2 = hx * hx;
double hy2 = hy * hy;
double hx2inv = 1 / hx2;
double hy2inv = 1 / hy2;
for (i = 0; i < nvalues; i += nentries) {
values[i] = -2.0 * hx2inv - 2.0 * hy2inv;
values[i + 1] = hx2inv;
values[i + 2] = hx2inv;
values[i + 3] = hy2inv;
values[i + 4] = hy2inv;
}
HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper, var, nentries,
stencil_indices, values);
// Set the dirchelet boundary condition
{
int bc_ilower[2];
int bc_iupper[2];
int nentries = 1;
int nvaluesx = nentries * nx; /* number of stencil entries times the length
of one side of my grid box */
int nvaluesy = nentries * ny;
double *valuesx, *valuesy, *center_valuesx, *center_valuesy;
int stencil_indices[1];
int center_index[1];
center_index[0] = 0;
valuesx = calloc(nvaluesx, sizeof(double));
center_valuesx = calloc(nvaluesx, sizeof(double));
for (j = 0; j < nvaluesx; j++){
valuesx[j] = 0.0;
center_valuesx[j] = -hx2inv;
}
valuesy = calloc(nvaluesy, sizeof(double));
center_valuesy = calloc(nvaluesy, sizeof(double));
for (j = 0; j < nvaluesy; j++){
valuesy[j] = 0.0;
center_valuesy[j] = -hy2inv;
}
// Recall: pi and pj describe position in the processor grid
if (pj == 0) {
// Bottom row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0] + nx - 1;
bc_iupper[1] = bc_ilower[1];
stencil_indices[0] = 3;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesy);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesy);
// printf("Bottom row zeroed\n");
}
if (pj == Ny - 1) {
// upper row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny + ny - 1;
bc_iupper[0] = bc_ilower[0] + ny - 1;
bc_iupper[1] = bc_ilower[1];
stencil_indices[0] = 4;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesy);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesy);
// printf("Upper row zeroed\n");
}
if (pi == 0) {
// Left row of grid points
bc_ilower[0] = pi * nx;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0];
bc_iupper[1] = bc_ilower[1] + ny - 1;
stencil_indices[0] = 1;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesx);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesx);
// printf("left collumn zeroed\n");
}
if (pi == Nx - 1) {
// Right row of grid points
bc_ilower[0] = pi * nx + nx - 1;
bc_ilower[1] = pj * ny;
bc_iupper[0] = bc_ilower[0];
bc_iupper[1] = bc_ilower[1] + ny - 1;
stencil_indices[0] = 2;
HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
nentries, stencil_indices, valuesx);
HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesx);
// printf("right collumn zeroed\n");
}
free(valuesx);
free(center_valuesx);
free(valuesy);
free(center_valuesy);
}
HYPRE_SStructMatrixAssemble(A);
HYPRE_SStructMatrixGetObject(A, (void **)&parcsr_A);
// Create the right hand side and solution vectors
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &b);
HYPRE_SStructVectorSetObjectType(b, HYPRE_PARCSR);
HYPRE_SStructVectorInitialize(b);
HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &x);
HYPRE_SStructVectorSetObjectType(x, HYPRE_PARCSR);
HYPRE_SStructVectorInitialize(x);
// Set the right hand side, exact solution, and initial guess values
double *rhs_values, *x_values, *exactsolution;
nvalues = nx * ny;
rhs_values = calloc(nvalues, sizeof(double));
x_values = calloc(nvalues, sizeof(double));
exactsolution = calloc(nvalues, sizeof(double));
// printf("RHS calculation begin\n");
for (j = 0; j < ny; j++) {
double y = (ilower[1] + j + .5) * hy;
for (i = 0; i < nx; i++) {
double x = (ilower[0] + i + .5) * hx;
rhs_values[i + j * nx] =
-8. * PI2 * sin(2. * M_PI * x) * sin(2. * M_PI * y);
exactsolution[i + j * nx] = 1.0 * sin(2. * M_PI * x) * sin(2. * M_PI * y);
x_values[i + j * nx] = 0.0;
}
}
// printf("RHS calculation ended\n");
HYPRE_SStructVectorSetBoxValues(b, part, ilower, iupper, var, rhs_values);
HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, x_values);
//printf("%d %f\n", (ny*nx-1), exactsolution[ny*nx-1]);
free(x_values);
free(rhs_values);
HYPRE_SStructVectorAssemble(b);
// HYPRE_SStructVectorPrint("ss.initial.b", b, 0);
HYPRE_SStructVectorGetObject(b, (void **)&par_b);
HYPRE_SStructVectorAssemble(x);
HYPRE_SStructVectorGetObject(x, (void **)&par_x);
// Print out initial vectors
HYPRE_SStructMatrixPrint("SStructExact/ss.initial.A", A, 0);
HYPRE_SStructVectorPrint("SStructExact/ss.initial.b", b, 0);
// printf("About to solve\n");
// Select a solver
if (solver_id == 0) {
int num_iterations;
double final_res_norm;
// Create AMG solver
HYPRE_BoomerAMGCreate(&solver);
// Set solver parameters
HYPRE_BoomerAMGSetPrintLevel(solver, 3);
HYPRE_BoomerAMGSetCoarsenType(solver, 6);
HYPRE_BoomerAMGSetRelaxType(solver, 3);
HYPRE_BoomerAMGSetNumSweeps(solver, 1);
HYPRE_BoomerAMGSetMaxLevels(solver, 20);
HYPRE_BoomerAMGSetTol(solver, 1e-12);
HYPRE_BoomerAMGSetMaxIter(solver, 100);
// Set up and solve
HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
//printf("solution obtained\n");
// Run information
HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
if (myid == 0) {
printf("\n");
printf("Iterations = %d\n", num_iterations);
printf("Final Relative Residual Norm = %e\n", final_res_norm);
printf("\n");
}
// Destroy solver
HYPRE_BoomerAMGDestroy(solver);
// HYPRE_ParVectorPrint(par_x, "SStructExact/ss.final.x");
char solutionfile[255];
sprintf(solutionfile, "%s.%06d", "ss.final.x", myid);
HYPRE_ParVectorPrint(par_x, solutionfile);
}
// Save the solution for visualization and L2 norm calculation
if (vis) {
FILE *file;
FILE *solution;
char filename[255];
char solutionfile[255];
sprintf(solutionfile, "%s.%06d.%d", "ss.final.x", myid, myid);
int nvalues = nx * ny;
double sum = 0;
double diff, diff2, L2;
// get the localally calculated and exact solutions)
double *values = calloc(nvalues+1, sizeof(double));
// Opens the local solution file, reads the solution to the array, values
// and closes the file.
// Opens the solution file
if ((solution = fopen(solutionfile, "rt")) == NULL) {
printf("Error: can't open output file %s\n", solutionfile);
MPI_Finalize();
exit(1);
}
// Reads the file to the array values
for (i = 0; i < nvalues + 1; i++) {
fscanf(solution, "%lf", &values[i]);
}
// Close the file
fflush(solution);
fclose(solution);
MPI_Barrier(MPI_COMM_WORLD);
// Calcluates the sum of the squared differences for the exact and numerical solutions on each processor
for (i = 1; i < nvalues + 1; i++) {
diff = values[i] - exactsolution[i-1];
diff2 = diff * diff;
if (diff2 > 10){
printf("%+6f %+6f %+6d\n", values[i], exactsolution[i-1],
i-1);
}
sum += diff2;
}
//printf("Myid is: %d and mysum is: %f\n", myid, sum);
// Ensure all processors are caught up
MPI_Barrier(MPI_COMM_WORLD);
// Passes the individual sums to processor 0 and sums them.
double *sendbuffer, *recvbuffer;
sendbuffer = calloc(1, sizeof(double));
sendbuffer[0] = sum;
recvbuffer = calloc(1, sizeof(double));
int root = 0;
int count = 1;
MPI_Reduce(sendbuffer, recvbuffer, count, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
// Calculates the L2 norm
if (myid == 0){
//printf("%f\n" , recvbuffer[myid]);
// nvalues = (nx-2)*(ny-2);
double totalvalues = nvalues * num_procs * 1.0;
L2 = sqrt(recvbuffer[0] / totalvalues);
printf("The L2 norm is %e\n", L2);
}
sprintf(filename, "%s.%06d", "vis/ss.sol", myid);
if ((file = fopen(filename, "w")) == NULL) {
printf("Error: can't open output file %s\n", filename);
MPI_Finalize();
exit(1);
}
// Save solution
for (i = 1; i < nvalues+1; i++) {
fprintf(file, "%.14e\n", values[i]);
}
// Vis Cleanup
fflush(file);
fclose(file);
free(values);
free(exactsolution);
free(sendbuffer);
free(recvbuffer);
// save global finite element mesh
if (myid == 0) {
GLVis_PrintGlobalMesh("vis/ss.mesh", Nx, Ny, nx, ny, hx, hy);
}
}
// Clean up
HYPRE_SStructMatrixDestroy(A);
HYPRE_SStructVectorDestroy(b);
HYPRE_SStructVectorDestroy(x);
// Finalize MPI*/
MPI_Finalize();
return (0);
}
