HYPRE_Int HYPRE_SStructGridCreate( MPI_Comm comm, HYPRE_Int ndim, HYPRE_Int nparts, HYPRE_SStructGrid *grid_ptr ) { hypre_SStructGrid *grid; hypre_SStructPGrid **pgrids; hypre_SStructPGrid *pgrid; HYPRE_Int *nneighbors; hypre_SStructNeighbor **neighbors; hypre_Index **nbor_offsets; HYPRE_Int *fem_nvars; HYPRE_Int **fem_vars; hypre_Index **fem_offsets; HYPRE_Int i; grid = hypre_TAlloc(hypre_SStructGrid, 1); hypre_SStructGridComm(grid) = comm; hypre_SStructGridNDim(grid) = ndim; hypre_SStructGridNParts(grid) = nparts; pgrids = hypre_TAlloc(hypre_SStructPGrid *, nparts); nneighbors = hypre_TAlloc(HYPRE_Int, nparts); neighbors = hypre_TAlloc(hypre_SStructNeighbor *, nparts); nbor_offsets = hypre_TAlloc(hypre_Index *, nparts); fem_nvars = hypre_TAlloc(HYPRE_Int, nparts); fem_vars = hypre_TAlloc(HYPRE_Int *, nparts); fem_offsets = hypre_TAlloc(hypre_Index *, nparts); for (i = 0; i < nparts; i++) { hypre_SStructPGridCreate(comm, ndim, &pgrid); pgrids[i] = pgrid; nneighbors[i] = 0; neighbors[i] = NULL; nbor_offsets[i] = NULL; fem_nvars[i] = 0; fem_vars[i] = NULL; fem_offsets[i] = NULL; } hypre_SStructGridPGrids(grid) = pgrids; hypre_SStructGridNNeighbors(grid) = nneighbors; hypre_SStructGridNeighbors(grid) = neighbors; hypre_SStructGridNborOffsets(grid) = nbor_offsets; hypre_SStructGridNUCVars(grid) = 0; hypre_SStructGridUCVars(grid) = NULL; hypre_SStructGridFEMNVars(grid) = fem_nvars; hypre_SStructGridFEMVars(grid) = fem_vars; hypre_SStructGridFEMOffsets(grid) = fem_offsets; hypre_SStructGridBoxManagers(grid) = NULL; hypre_SStructGridNborBoxManagers(grid) = NULL; /* miscellaneous */ hypre_SStructGridLocalSize(grid) = 0; hypre_SStructGridGlobalSize(grid) = 0; hypre_SStructGridRefCount(grid) = 1; /* GEC0902 ghost addition to the grid */ hypre_SStructGridGhlocalSize(grid) = 0; *grid_ptr = grid; return hypre_error_flag; }
int hypre_SStructUMatrixSetValues( hypre_SStructMatrix *matrix, int part, hypre_Index index, int var, int nentries, int *entries, double *values, int add_to ) { HYPRE_IJMatrix ijmatrix = hypre_SStructMatrixIJMatrix(matrix); hypre_SStructGraph *graph = hypre_SStructMatrixGraph(matrix); hypre_SStructGrid *grid = hypre_SStructGraphGrid(graph); hypre_SStructStencil *stencil = hypre_SStructGraphStencil(graph, part, var); int *vars = hypre_SStructStencilVars(stencil); hypre_Index *shape = hypre_SStructStencilShape(stencil); int size = hypre_SStructStencilSize(stencil); hypre_IndexRef offset; hypre_Index to_index; hypre_SStructUVEntry *Uventry; hypre_BoxMapEntry *map_entry; hypre_SStructMapInfo *entry_info; HYPRE_BigInt row_coord; HYPRE_BigInt *col_coords; int ncoeffs; double *coeffs; int i, entry; int proc, myproc; /* GEC1002 the matrix type */ int matrix_type = hypre_SStructMatrixObjectType(matrix); hypre_SStructGridFindMapEntry(grid, part, index, var, &map_entry); if (map_entry == NULL) { hypre_error_in_arg(1); hypre_error_in_arg(2); hypre_error_in_arg(3); /* RDF: This printing shouldn't be on by default */ printf("Warning: Attempt to set coeffs for point not in grid\n"); printf("hypre_SStructUMatrixSetValues call aborted for grid point\n"); printf(" part=%d, var=%d, index=(%d, %d, %d)\n", part, var, hypre_IndexD(index,0), hypre_IndexD(index,1), hypre_IndexD(index,2) ); return hypre_error_flag; } else { hypre_BoxMapEntryGetInfo(map_entry, (void **) &entry_info); } /* Only Set values if I am the owner process; off-process AddTo and Get * values are done by IJ */ if (!add_to) { hypre_SStructMapEntryGetProcess(map_entry, &proc); MPI_Comm_rank(hypre_SStructGridComm(grid), &myproc); if (proc != myproc) { return hypre_error_flag; } } /* GEC1002 get the rank using the function with the type=matrixtype*/ hypre_SStructMapEntryGetGlobalRank(map_entry, index, &row_coord, matrix_type); col_coords = hypre_SStructMatrixTmpColCoords(matrix); coeffs = hypre_SStructMatrixTmpCoeffs(matrix); ncoeffs = 0; for (i = 0; i < nentries; i++) { entry = entries[i]; if (entry < size) { /* stencil entries */ offset = shape[entry]; hypre_IndexX(to_index) = hypre_IndexX(index) + hypre_IndexX(offset); hypre_IndexY(to_index) = hypre_IndexY(index) + hypre_IndexY(offset); hypre_IndexZ(to_index) = hypre_IndexZ(index) + hypre_IndexZ(offset); hypre_SStructGridFindMapEntry(grid, part, to_index, vars[entry], &map_entry); if (map_entry != NULL) { hypre_SStructMapEntryGetGlobalRank(map_entry, to_index, &col_coords[ncoeffs],matrix_type); coeffs[ncoeffs] = values[i]; ncoeffs++; } } else { /* non-stencil entries */ entry -= size; hypre_SStructGraphFindUVEntry(graph, part, index, var, &Uventry); col_coords[ncoeffs] = hypre_SStructUVEntryRank(Uventry, entry); coeffs[ncoeffs] = values[i]; ncoeffs++; } } if (add_to > 0) { HYPRE_IJMatrixAddToValues(ijmatrix, 1, &ncoeffs, &row_coord, (const HYPRE_BigInt *) col_coords, (const double *) coeffs); } else if (add_to > -1) { HYPRE_IJMatrixSetValues(ijmatrix, 1, &ncoeffs, &row_coord, (const HYPRE_BigInt *) col_coords, (const double *) coeffs); } else { HYPRE_IJMatrixGetValues(ijmatrix, 1, &ncoeffs, &row_coord, col_coords, values); } return hypre_error_flag; }
int hypre_SStructUMatrixSetBoxValues( hypre_SStructMatrix *matrix, int part, hypre_Index ilower, hypre_Index iupper, int var, int nentries, int *entries, double *values, int add_to ) { HYPRE_IJMatrix ijmatrix = hypre_SStructMatrixIJMatrix(matrix); hypre_SStructGraph *graph = hypre_SStructMatrixGraph(matrix); hypre_SStructGrid *grid = hypre_SStructGraphGrid(graph); hypre_SStructStencil *stencil = hypre_SStructGraphStencil(graph, part, var); int *vars = hypre_SStructStencilVars(stencil); hypre_Index *shape = hypre_SStructStencilShape(stencil); int size = hypre_SStructStencilSize(stencil); hypre_IndexRef offset; hypre_BoxMap *map; hypre_BoxMapEntry **map_entries; int nmap_entries; hypre_BoxMapEntry **map_to_entries; int nmap_to_entries; int nrows; int *ncols; HYPRE_BigInt *rows; HYPRE_BigInt *cols; double *ijvalues; hypre_Box *box; hypre_Box *to_box; hypre_Box *map_box; hypre_Box *int_box; hypre_Index index; hypre_Index rs, cs; int sy, sz; HYPRE_BigInt row_base, col_base; int val_base; int e, entry, ii, jj, i, j, k; int proc, myproc; /* GEC1002 the matrix type */ int matrix_type = hypre_SStructMatrixObjectType(matrix); box = hypre_BoxCreate(); /*------------------------------------------ * all stencil entries *------------------------------------------*/ if (entries[0] < size) { to_box = hypre_BoxCreate(); map_box = hypre_BoxCreate(); int_box = hypre_BoxCreate(); hypre_CopyIndex(ilower, hypre_BoxIMin(box)); hypre_CopyIndex(iupper, hypre_BoxIMax(box)); /* ZTODO: check that this change fixes multiple-entry problem */ nrows = hypre_BoxVolume(box)*nentries; ncols = hypre_CTAlloc(int, nrows); for (i = 0; i < nrows; i++) { ncols[i] = 1; } rows = hypre_CTAlloc(HYPRE_BigInt, nrows); cols = hypre_CTAlloc(HYPRE_BigInt, nrows); ijvalues = hypre_CTAlloc(double, nrows); sy = (hypre_IndexX(iupper) - hypre_IndexX(ilower) + 1); sz = (hypre_IndexY(iupper) - hypre_IndexY(ilower) + 1) * sy; map = hypre_SStructGridMap(grid, part, var); hypre_BoxMapIntersect(map, ilower, iupper, &map_entries, &nmap_entries); for (ii = 0; ii < nmap_entries; ii++) { /* Only Set values if I am the owner process; off-process AddTo and Get * values are done by IJ */ if (!add_to) { hypre_SStructMapEntryGetProcess(map_entries[ii], &proc); MPI_Comm_rank(hypre_SStructGridComm(grid), &myproc); if (proc != myproc) { continue; } } /* GEC1002 introducing the strides based on the type of the matrix */ hypre_SStructMapEntryGetStrides(map_entries[ii], rs, matrix_type); hypre_CopyIndex(ilower, hypre_BoxIMin(box)); hypre_CopyIndex(iupper, hypre_BoxIMax(box)); hypre_BoxMapEntryGetExtents(map_entries[ii], hypre_BoxIMin(map_box), hypre_BoxIMax(map_box)); hypre_IntersectBoxes(box, map_box, int_box); hypre_CopyBox(int_box, box); nrows = 0; for (e = 0; e < nentries; e++) { entry = entries[e]; hypre_CopyBox(box, to_box); offset = shape[entry]; hypre_BoxIMinX(to_box) += hypre_IndexX(offset); hypre_BoxIMinY(to_box) += hypre_IndexY(offset); hypre_BoxIMinZ(to_box) += hypre_IndexZ(offset); hypre_BoxIMaxX(to_box) += hypre_IndexX(offset); hypre_BoxIMaxY(to_box) += hypre_IndexY(offset); hypre_BoxIMaxZ(to_box) += hypre_IndexZ(offset); map = hypre_SStructGridMap(grid, part, vars[entry]); hypre_BoxMapIntersect(map, hypre_BoxIMin(to_box), hypre_BoxIMax(to_box), &map_to_entries, &nmap_to_entries ); for (jj = 0; jj < nmap_to_entries; jj++) { /* GEC1002 introducing the strides based on the type of the matrix */ hypre_SStructMapEntryGetStrides(map_to_entries[jj], cs, matrix_type); hypre_BoxMapEntryGetExtents(map_to_entries[jj], hypre_BoxIMin(map_box), hypre_BoxIMax(map_box)); hypre_IntersectBoxes(to_box, map_box, int_box); hypre_CopyIndex(hypre_BoxIMin(int_box), index); /* GEC1002 introducing the rank based on the type of the matrix */ hypre_SStructMapEntryGetGlobalRank(map_to_entries[jj], index, &col_base,matrix_type); hypre_IndexX(index) -= hypre_IndexX(offset); hypre_IndexY(index) -= hypre_IndexY(offset); hypre_IndexZ(index) -= hypre_IndexZ(offset); /* GEC1002 introducing the rank based on the type of the matrix */ hypre_SStructMapEntryGetGlobalRank(map_entries[ii], index, &row_base,matrix_type); hypre_IndexX(index) -= hypre_IndexX(ilower); hypre_IndexY(index) -= hypre_IndexY(ilower); hypre_IndexZ(index) -= hypre_IndexZ(ilower); val_base = e + (hypre_IndexX(index) + hypre_IndexY(index)*sy + hypre_IndexZ(index)*sz) * nentries; for (k = 0; k < hypre_BoxSizeZ(int_box); k++) { for (j = 0; j < hypre_BoxSizeY(int_box); j++) { for (i = 0; i < hypre_BoxSizeX(int_box); i++) { rows[nrows] = row_base + (HYPRE_BigInt)(i*rs[0] + j*rs[1] + k*rs[2]); cols[nrows] = col_base + (HYPRE_BigInt)(i*cs[0] + j*cs[1] + k*cs[2]); ijvalues[nrows] = values[val_base + (i + j*sy + k*sz)*nentries]; nrows++; } } } } hypre_TFree(map_to_entries); } /*------------------------------------------ * set IJ values one stencil entry at a time *------------------------------------------*/ if (add_to > 0) { HYPRE_IJMatrixAddToValues(ijmatrix, nrows, ncols, (const HYPRE_BigInt *) rows, (const HYPRE_BigInt *) cols, (const double *) ijvalues); } else if (add_to > -1) { HYPRE_IJMatrixSetValues(ijmatrix, nrows, ncols, (const HYPRE_BigInt *) rows, (const HYPRE_BigInt *) cols, (const double *) ijvalues); } else { HYPRE_IJMatrixGetValues(ijmatrix, nrows, ncols, rows, cols, values); } } hypre_TFree(map_entries); hypre_TFree(ncols); hypre_TFree(rows); hypre_TFree(cols); hypre_TFree(ijvalues); hypre_BoxDestroy(to_box); hypre_BoxDestroy(map_box); hypre_BoxDestroy(int_box); }
/*-------------------------------------------------------------------------- * hypre_SStructSharedDOF_ParcsrMatRowsComm * Given a sstruct_grid & parcsr matrix with rows corresponding to the * sstruct_grid, determine and extract the rows that must be communicated. * These rows are for shared dof that geometrically lie on processor * boundaries but internally are stored on one processor. * Algo: * for each cellbox * RECVs: * i) stretch the cellbox to the variable box * ii) in the appropriate (dof-dependent) direction, take the * boundary and boxman_intersect to extract boxmanentries * that contain these boundary edges. * iii)loop over the boxmanentries and see if they belong * on this proc or another proc * a) if belong on another proc, these are the recvs: * count and prepare the communication buffers and * values. * * SENDs: * i) form layer of cells that is one layer off cellbox * (stretches in the appropriate direction) * ii) boxman_intersect with the cellgrid boxman * iii)loop over the boxmanentries and see if they belong * on this proc or another proc * a) if belong on another proc, these are the sends: * count and prepare the communication buffers and * values. * * Note: For the recv data, the dof can come from only one processor. * For the send data, the dof can go to more than one processor * (the same dof is on the boundary of several cells). *--------------------------------------------------------------------------*/ HYPRE_Int hypre_SStructSharedDOF_ParcsrMatRowsComm( hypre_SStructGrid *grid, hypre_ParCSRMatrix *A, HYPRE_Int *num_offprocrows_ptr, hypre_MaxwellOffProcRow ***OffProcRows_ptr) { MPI_Comm A_comm= hypre_ParCSRMatrixComm(A); MPI_Comm grid_comm= hypre_SStructGridComm(grid); HYPRE_Int matrix_type= HYPRE_PARCSR; HYPRE_Int nparts= hypre_SStructGridNParts(grid); HYPRE_Int ndim = hypre_SStructGridNDim(grid); hypre_SStructGrid *cell_ssgrid; hypre_SStructPGrid *pgrid; hypre_StructGrid *cellgrid; hypre_BoxArray *cellboxes; hypre_Box *box, *cellbox, vbox, boxman_entry_box; hypre_Index loop_size, start; HYPRE_Int loopi, loopj, loopk; HYPRE_Int start_rank, end_rank, rank; HYPRE_Int i, j, k, m, n, t, part, var, nvars; HYPRE_SStructVariable *vartypes; HYPRE_Int nbdry_slabs; hypre_BoxArray *recv_slabs, *send_slabs; hypre_Index varoffset; hypre_BoxManager **boxmans, *cell_boxman; hypre_BoxManEntry **boxman_entries, *entry; HYPRE_Int nboxman_entries; hypre_Index ishift, jshift, kshift, zero_index; hypre_Index ilower, iupper, index; HYPRE_Int proc, nprocs, myproc; HYPRE_Int *SendToProcs, *RecvFromProcs; HYPRE_Int **send_RowsNcols; /* buffer for rows & ncols */ HYPRE_Int *send_RowsNcols_alloc; HYPRE_Int *send_ColsData_alloc; HYPRE_Int *tot_nsendRowsNcols, *tot_sendColsData; double **vals; /* buffer for cols & data */ HYPRE_Int *col_inds; double *values; hypre_MPI_Request *requests; hypre_MPI_Status *status; HYPRE_Int **rbuffer_RowsNcols; double **rbuffer_ColsData; HYPRE_Int num_sends, num_recvs; hypre_MaxwellOffProcRow **OffProcRows; HYPRE_Int *starts; HYPRE_Int ierr= 0; hypre_MPI_Comm_rank(A_comm, &myproc); hypre_MPI_Comm_size(grid_comm, &nprocs); start_rank= hypre_ParCSRMatrixFirstRowIndex(A); end_rank = hypre_ParCSRMatrixLastRowIndex(A); hypre_SetIndex(ishift, 1, 0, 0); hypre_SetIndex(jshift, 0, 1, 0); hypre_SetIndex(kshift, 0, 0, 1); hypre_SetIndex(zero_index, 0, 0, 0); /* need a cellgrid boxman to determine the send boxes -> only the cell dofs are unique so a boxman intersect can be used to get the edges that must be sent. */ HYPRE_SStructGridCreate(grid_comm, ndim, nparts, &cell_ssgrid); vartypes= hypre_CTAlloc(HYPRE_SStructVariable, 1); vartypes[0]= HYPRE_SSTRUCT_VARIABLE_CELL; for (i= 0; i< nparts; i++) { pgrid= hypre_SStructGridPGrid(grid, i); cellgrid= hypre_SStructPGridCellSGrid(pgrid); cellboxes= hypre_StructGridBoxes(cellgrid); hypre_ForBoxI(j, cellboxes) { box= hypre_BoxArrayBox(cellboxes, j); HYPRE_SStructGridSetExtents(cell_ssgrid, i, hypre_BoxIMin(box), hypre_BoxIMax(box)); } HYPRE_SStructGridSetVariables(cell_ssgrid, i, 1, vartypes); }