HYPRE_Int
HYPRE_SStructGridCreate( MPI_Comm comm,
HYPRE_Int ndim,
HYPRE_Int nparts,
HYPRE_SStructGrid *grid_ptr )
{
hypre_SStructGrid *grid;
hypre_SStructPGrid **pgrids;
hypre_SStructPGrid *pgrid;
HYPRE_Int *nneighbors;
hypre_SStructNeighbor **neighbors;
hypre_Index **nbor_offsets;
HYPRE_Int *fem_nvars;
HYPRE_Int **fem_vars;
hypre_Index **fem_offsets;
HYPRE_Int i;
grid = hypre_TAlloc(hypre_SStructGrid, 1);
hypre_SStructGridComm(grid) = comm;
hypre_SStructGridNDim(grid) = ndim;
hypre_SStructGridNParts(grid) = nparts;
pgrids = hypre_TAlloc(hypre_SStructPGrid *, nparts);
nneighbors = hypre_TAlloc(HYPRE_Int, nparts);
neighbors = hypre_TAlloc(hypre_SStructNeighbor *, nparts);
nbor_offsets = hypre_TAlloc(hypre_Index *, nparts);
fem_nvars = hypre_TAlloc(HYPRE_Int, nparts);
fem_vars = hypre_TAlloc(HYPRE_Int *, nparts);
fem_offsets = hypre_TAlloc(hypre_Index *, nparts);
for (i = 0; i < nparts; i++)
{
hypre_SStructPGridCreate(comm, ndim, &pgrid);
pgrids[i] = pgrid;
nneighbors[i] = 0;
neighbors[i] = NULL;
nbor_offsets[i] = NULL;
fem_nvars[i] = 0;
fem_vars[i] = NULL;
fem_offsets[i] = NULL;
}
hypre_SStructGridPGrids(grid) = pgrids;
hypre_SStructGridNNeighbors(grid) = nneighbors;
hypre_SStructGridNeighbors(grid) = neighbors;
hypre_SStructGridNborOffsets(grid) = nbor_offsets;
hypre_SStructGridNUCVars(grid) = 0;
hypre_SStructGridUCVars(grid) = NULL;
hypre_SStructGridFEMNVars(grid) = fem_nvars;
hypre_SStructGridFEMVars(grid) = fem_vars;
hypre_SStructGridFEMOffsets(grid) = fem_offsets;
hypre_SStructGridBoxManagers(grid) = NULL;
hypre_SStructGridNborBoxManagers(grid) = NULL;
/* miscellaneous */
hypre_SStructGridLocalSize(grid) = 0;
hypre_SStructGridGlobalSize(grid) = 0;
hypre_SStructGridRefCount(grid) = 1;
/* GEC0902 ghost addition to the grid */
hypre_SStructGridGhlocalSize(grid) = 0;
*grid_ptr = grid;
return hypre_error_flag;
}
int
hypre_SStructUMatrixSetValues( hypre_SStructMatrix *matrix,
int part,
hypre_Index index,
int var,
int nentries,
int *entries,
double *values,
int add_to )
{
HYPRE_IJMatrix ijmatrix = hypre_SStructMatrixIJMatrix(matrix);
hypre_SStructGraph *graph = hypre_SStructMatrixGraph(matrix);
hypre_SStructGrid *grid = hypre_SStructGraphGrid(graph);
hypre_SStructStencil *stencil = hypre_SStructGraphStencil(graph, part, var);
int *vars = hypre_SStructStencilVars(stencil);
hypre_Index *shape = hypre_SStructStencilShape(stencil);
int size = hypre_SStructStencilSize(stencil);
hypre_IndexRef offset;
hypre_Index to_index;
hypre_SStructUVEntry *Uventry;
hypre_BoxMapEntry *map_entry;
hypre_SStructMapInfo *entry_info;
HYPRE_BigInt row_coord;
HYPRE_BigInt *col_coords;
int ncoeffs;
double *coeffs;
int i, entry;
int proc, myproc;
/* GEC1002 the matrix type */
int matrix_type = hypre_SStructMatrixObjectType(matrix);
hypre_SStructGridFindMapEntry(grid, part, index, var, &map_entry);
if (map_entry == NULL)
{
hypre_error_in_arg(1);
hypre_error_in_arg(2);
hypre_error_in_arg(3);
/* RDF: This printing shouldn't be on by default */
printf("Warning: Attempt to set coeffs for point not in grid\n");
printf("hypre_SStructUMatrixSetValues call aborted for grid point\n");
printf(" part=%d, var=%d, index=(%d, %d, %d)\n", part, var,
hypre_IndexD(index,0),
hypre_IndexD(index,1),
hypre_IndexD(index,2) );
return hypre_error_flag;
}
else
{
hypre_BoxMapEntryGetInfo(map_entry, (void **) &entry_info);
}
/* Only Set values if I am the owner process; off-process AddTo and Get
* values are done by IJ */
if (!add_to)
{
hypre_SStructMapEntryGetProcess(map_entry, &proc);
MPI_Comm_rank(hypre_SStructGridComm(grid), &myproc);
if (proc != myproc)
{
return hypre_error_flag;
}
}
/* GEC1002 get the rank using the function with the type=matrixtype*/
hypre_SStructMapEntryGetGlobalRank(map_entry, index, &row_coord, matrix_type);
col_coords = hypre_SStructMatrixTmpColCoords(matrix);
coeffs = hypre_SStructMatrixTmpCoeffs(matrix);
ncoeffs = 0;
for (i = 0; i < nentries; i++)
{
entry = entries[i];
if (entry < size)
{
/* stencil entries */
offset = shape[entry];
hypre_IndexX(to_index) = hypre_IndexX(index) + hypre_IndexX(offset);
hypre_IndexY(to_index) = hypre_IndexY(index) + hypre_IndexY(offset);
hypre_IndexZ(to_index) = hypre_IndexZ(index) + hypre_IndexZ(offset);
hypre_SStructGridFindMapEntry(grid, part, to_index, vars[entry],
&map_entry);
if (map_entry != NULL)
{
hypre_SStructMapEntryGetGlobalRank(map_entry, to_index,
&col_coords[ncoeffs],matrix_type);
coeffs[ncoeffs] = values[i];
ncoeffs++;
}
}
else
{
/* non-stencil entries */
entry -= size;
hypre_SStructGraphFindUVEntry(graph, part, index, var, &Uventry);
col_coords[ncoeffs] = hypre_SStructUVEntryRank(Uventry, entry);
coeffs[ncoeffs] = values[i];
ncoeffs++;
}
}
if (add_to > 0)
{
HYPRE_IJMatrixAddToValues(ijmatrix, 1, &ncoeffs, &row_coord,
(const HYPRE_BigInt *) col_coords,
(const double *) coeffs);
}
else if (add_to > -1)
{
HYPRE_IJMatrixSetValues(ijmatrix, 1, &ncoeffs, &row_coord,
(const HYPRE_BigInt *) col_coords,
(const double *) coeffs);
}
else
{
HYPRE_IJMatrixGetValues(ijmatrix, 1, &ncoeffs, &row_coord,
col_coords, values);
}
return hypre_error_flag;
}
int
hypre_SStructUMatrixSetBoxValues( hypre_SStructMatrix *matrix,
int part,
hypre_Index ilower,
hypre_Index iupper,
int var,
int nentries,
int *entries,
double *values,
int add_to )
{
HYPRE_IJMatrix ijmatrix = hypre_SStructMatrixIJMatrix(matrix);
hypre_SStructGraph *graph = hypre_SStructMatrixGraph(matrix);
hypre_SStructGrid *grid = hypre_SStructGraphGrid(graph);
hypre_SStructStencil *stencil = hypre_SStructGraphStencil(graph, part, var);
int *vars = hypre_SStructStencilVars(stencil);
hypre_Index *shape = hypre_SStructStencilShape(stencil);
int size = hypre_SStructStencilSize(stencil);
hypre_IndexRef offset;
hypre_BoxMap *map;
hypre_BoxMapEntry **map_entries;
int nmap_entries;
hypre_BoxMapEntry **map_to_entries;
int nmap_to_entries;
int nrows;
int *ncols;
HYPRE_BigInt *rows;
HYPRE_BigInt *cols;
double *ijvalues;
hypre_Box *box;
hypre_Box *to_box;
hypre_Box *map_box;
hypre_Box *int_box;
hypre_Index index;
hypre_Index rs, cs;
int sy, sz;
HYPRE_BigInt row_base, col_base;
int val_base;
int e, entry, ii, jj, i, j, k;
int proc, myproc;
/* GEC1002 the matrix type */
int matrix_type = hypre_SStructMatrixObjectType(matrix);
box = hypre_BoxCreate();
/*------------------------------------------
* all stencil entries
*------------------------------------------*/
if (entries[0] < size)
{
to_box = hypre_BoxCreate();
map_box = hypre_BoxCreate();
int_box = hypre_BoxCreate();
hypre_CopyIndex(ilower, hypre_BoxIMin(box));
hypre_CopyIndex(iupper, hypre_BoxIMax(box));
/* ZTODO: check that this change fixes multiple-entry problem */
nrows = hypre_BoxVolume(box)*nentries;
ncols = hypre_CTAlloc(int, nrows);
for (i = 0; i < nrows; i++)
{
ncols[i] = 1;
}
rows = hypre_CTAlloc(HYPRE_BigInt, nrows);
cols = hypre_CTAlloc(HYPRE_BigInt, nrows);
ijvalues = hypre_CTAlloc(double, nrows);
sy = (hypre_IndexX(iupper) - hypre_IndexX(ilower) + 1);
sz = (hypre_IndexY(iupper) - hypre_IndexY(ilower) + 1) * sy;
map = hypre_SStructGridMap(grid, part, var);
hypre_BoxMapIntersect(map, ilower, iupper, &map_entries, &nmap_entries);
for (ii = 0; ii < nmap_entries; ii++)
{
/* Only Set values if I am the owner process; off-process AddTo and Get
* values are done by IJ */
if (!add_to)
{
hypre_SStructMapEntryGetProcess(map_entries[ii], &proc);
MPI_Comm_rank(hypre_SStructGridComm(grid), &myproc);
if (proc != myproc)
{
continue;
}
}
/* GEC1002 introducing the strides based on the type of the matrix */
hypre_SStructMapEntryGetStrides(map_entries[ii], rs, matrix_type);
hypre_CopyIndex(ilower, hypre_BoxIMin(box));
hypre_CopyIndex(iupper, hypre_BoxIMax(box));
hypre_BoxMapEntryGetExtents(map_entries[ii],
hypre_BoxIMin(map_box),
hypre_BoxIMax(map_box));
hypre_IntersectBoxes(box, map_box, int_box);
hypre_CopyBox(int_box, box);
nrows = 0;
for (e = 0; e < nentries; e++)
{
entry = entries[e];
hypre_CopyBox(box, to_box);
offset = shape[entry];
hypre_BoxIMinX(to_box) += hypre_IndexX(offset);
hypre_BoxIMinY(to_box) += hypre_IndexY(offset);
hypre_BoxIMinZ(to_box) += hypre_IndexZ(offset);
hypre_BoxIMaxX(to_box) += hypre_IndexX(offset);
hypre_BoxIMaxY(to_box) += hypre_IndexY(offset);
hypre_BoxIMaxZ(to_box) += hypre_IndexZ(offset);
map = hypre_SStructGridMap(grid, part, vars[entry]);
hypre_BoxMapIntersect(map, hypre_BoxIMin(to_box),
hypre_BoxIMax(to_box),
&map_to_entries, &nmap_to_entries );
for (jj = 0; jj < nmap_to_entries; jj++)
{
/* GEC1002 introducing the strides based on the type of the matrix */
hypre_SStructMapEntryGetStrides(map_to_entries[jj], cs, matrix_type);
hypre_BoxMapEntryGetExtents(map_to_entries[jj],
hypre_BoxIMin(map_box),
hypre_BoxIMax(map_box));
hypre_IntersectBoxes(to_box, map_box, int_box);
hypre_CopyIndex(hypre_BoxIMin(int_box), index);
/* GEC1002 introducing the rank based on the type of the matrix */
hypre_SStructMapEntryGetGlobalRank(map_to_entries[jj],
index, &col_base,matrix_type);
hypre_IndexX(index) -= hypre_IndexX(offset);
hypre_IndexY(index) -= hypre_IndexY(offset);
hypre_IndexZ(index) -= hypre_IndexZ(offset);
/* GEC1002 introducing the rank based on the type of the matrix */
hypre_SStructMapEntryGetGlobalRank(map_entries[ii],
index, &row_base,matrix_type);
hypre_IndexX(index) -= hypre_IndexX(ilower);
hypre_IndexY(index) -= hypre_IndexY(ilower);
hypre_IndexZ(index) -= hypre_IndexZ(ilower);
val_base = e + (hypre_IndexX(index) +
hypre_IndexY(index)*sy +
hypre_IndexZ(index)*sz) * nentries;
for (k = 0; k < hypre_BoxSizeZ(int_box); k++)
{
for (j = 0; j < hypre_BoxSizeY(int_box); j++)
{
for (i = 0; i < hypre_BoxSizeX(int_box); i++)
{
rows[nrows] = row_base + (HYPRE_BigInt)(i*rs[0] + j*rs[1] + k*rs[2]);
cols[nrows] = col_base + (HYPRE_BigInt)(i*cs[0] + j*cs[1] + k*cs[2]);
ijvalues[nrows] =
values[val_base + (i + j*sy + k*sz)*nentries];
nrows++;
}
}
}
}
hypre_TFree(map_to_entries);
}
/*------------------------------------------
* set IJ values one stencil entry at a time
*------------------------------------------*/
if (add_to > 0)
{
HYPRE_IJMatrixAddToValues(ijmatrix, nrows, ncols,
(const HYPRE_BigInt *) rows,
(const HYPRE_BigInt *) cols,
(const double *) ijvalues);
}
else if (add_to > -1)
{
HYPRE_IJMatrixSetValues(ijmatrix, nrows, ncols,
(const HYPRE_BigInt *) rows,
(const HYPRE_BigInt *) cols,
(const double *) ijvalues);
}
else
{
HYPRE_IJMatrixGetValues(ijmatrix, nrows, ncols, rows, cols, values);
}
}
hypre_TFree(map_entries);
hypre_TFree(ncols);
hypre_TFree(rows);
hypre_TFree(cols);
hypre_TFree(ijvalues);
hypre_BoxDestroy(to_box);
hypre_BoxDestroy(map_box);
hypre_BoxDestroy(int_box);
}
/*--------------------------------------------------------------------------
* hypre_SStructSharedDOF_ParcsrMatRowsComm
* Given a sstruct_grid & parcsr matrix with rows corresponding to the
* sstruct_grid, determine and extract the rows that must be communicated.
* These rows are for shared dof that geometrically lie on processor
* boundaries but internally are stored on one processor.
* Algo:
* for each cellbox
* RECVs:
* i) stretch the cellbox to the variable box
* ii) in the appropriate (dof-dependent) direction, take the
* boundary and boxman_intersect to extract boxmanentries
* that contain these boundary edges.
* iii)loop over the boxmanentries and see if they belong
* on this proc or another proc
* a) if belong on another proc, these are the recvs:
* count and prepare the communication buffers and
* values.
*
* SENDs:
* i) form layer of cells that is one layer off cellbox
* (stretches in the appropriate direction)
* ii) boxman_intersect with the cellgrid boxman
* iii)loop over the boxmanentries and see if they belong
* on this proc or another proc
* a) if belong on another proc, these are the sends:
* count and prepare the communication buffers and
* values.
*
* Note: For the recv data, the dof can come from only one processor.
* For the send data, the dof can go to more than one processor
* (the same dof is on the boundary of several cells).
*--------------------------------------------------------------------------*/
HYPRE_Int
hypre_SStructSharedDOF_ParcsrMatRowsComm( hypre_SStructGrid *grid,
hypre_ParCSRMatrix *A,
HYPRE_Int *num_offprocrows_ptr,
hypre_MaxwellOffProcRow ***OffProcRows_ptr)
{
MPI_Comm A_comm= hypre_ParCSRMatrixComm(A);
MPI_Comm grid_comm= hypre_SStructGridComm(grid);
HYPRE_Int matrix_type= HYPRE_PARCSR;
HYPRE_Int nparts= hypre_SStructGridNParts(grid);
HYPRE_Int ndim = hypre_SStructGridNDim(grid);
hypre_SStructGrid *cell_ssgrid;
hypre_SStructPGrid *pgrid;
hypre_StructGrid *cellgrid;
hypre_BoxArray *cellboxes;
hypre_Box *box, *cellbox, vbox, boxman_entry_box;
hypre_Index loop_size, start;
HYPRE_Int loopi, loopj, loopk;
HYPRE_Int start_rank, end_rank, rank;
HYPRE_Int i, j, k, m, n, t, part, var, nvars;
HYPRE_SStructVariable *vartypes;
HYPRE_Int nbdry_slabs;
hypre_BoxArray *recv_slabs, *send_slabs;
hypre_Index varoffset;
hypre_BoxManager **boxmans, *cell_boxman;
hypre_BoxManEntry **boxman_entries, *entry;
HYPRE_Int nboxman_entries;
hypre_Index ishift, jshift, kshift, zero_index;
hypre_Index ilower, iupper, index;
HYPRE_Int proc, nprocs, myproc;
HYPRE_Int *SendToProcs, *RecvFromProcs;
HYPRE_Int **send_RowsNcols; /* buffer for rows & ncols */
HYPRE_Int *send_RowsNcols_alloc;
HYPRE_Int *send_ColsData_alloc;
HYPRE_Int *tot_nsendRowsNcols, *tot_sendColsData;
double **vals; /* buffer for cols & data */
HYPRE_Int *col_inds;
double *values;
hypre_MPI_Request *requests;
hypre_MPI_Status *status;
HYPRE_Int **rbuffer_RowsNcols;
double **rbuffer_ColsData;
HYPRE_Int num_sends, num_recvs;
hypre_MaxwellOffProcRow **OffProcRows;
HYPRE_Int *starts;
HYPRE_Int ierr= 0;
hypre_MPI_Comm_rank(A_comm, &myproc);
hypre_MPI_Comm_size(grid_comm, &nprocs);
start_rank= hypre_ParCSRMatrixFirstRowIndex(A);
end_rank = hypre_ParCSRMatrixLastRowIndex(A);
hypre_SetIndex(ishift, 1, 0, 0);
hypre_SetIndex(jshift, 0, 1, 0);
hypre_SetIndex(kshift, 0, 0, 1);
hypre_SetIndex(zero_index, 0, 0, 0);
/* need a cellgrid boxman to determine the send boxes -> only the cell dofs
are unique so a boxman intersect can be used to get the edges that
must be sent. */
HYPRE_SStructGridCreate(grid_comm, ndim, nparts, &cell_ssgrid);
vartypes= hypre_CTAlloc(HYPRE_SStructVariable, 1);
vartypes[0]= HYPRE_SSTRUCT_VARIABLE_CELL;
for (i= 0; i< nparts; i++)
{
pgrid= hypre_SStructGridPGrid(grid, i);
cellgrid= hypre_SStructPGridCellSGrid(pgrid);
cellboxes= hypre_StructGridBoxes(cellgrid);
hypre_ForBoxI(j, cellboxes)
{
box= hypre_BoxArrayBox(cellboxes, j);
HYPRE_SStructGridSetExtents(cell_ssgrid, i,
hypre_BoxIMin(box), hypre_BoxIMax(box));
}
HYPRE_SStructGridSetVariables(cell_ssgrid, i, 1, vartypes);
}