/*@C
DMCompositeGetISLocalToGlobalMappings - gets an ISLocalToGlobalMapping for each DM in the DMComposite, maps to the composite global space
Collective on DM
Input Parameter:
. dm - the packer object
Output Parameters:
. ltogs - the individual mappings for each packed vector. Note that this includes
all the ghost points that individual ghosted DMDA's may have.
Level: advanced
Notes:
Each entry of ltogs should be destroyed with ISLocalToGlobalMappingDestroy(), the ltogs array should be freed with PetscFree().
.seealso DMDestroy(), DMCompositeAddDM(), DMCreateGlobalVector(),
DMCompositeGather(), DMCompositeCreate(), DMCompositeGetAccess(), DMCompositeScatter(),
DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),DMCompositeGetEntries()
@*/
PetscErrorCode DMCompositeGetISLocalToGlobalMappings(DM dm,ISLocalToGlobalMapping **ltogs)
{
PetscErrorCode ierr;
PetscInt i,*idx,n,cnt;
struct DMCompositeLink *next;
PetscMPIInt rank;
DM_Composite *com = (DM_Composite*)dm->data;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm,DM_CLASSID,1);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = PetscMalloc((com->nDM)*sizeof(ISLocalToGlobalMapping),ltogs);CHKERRQ(ierr);
next = com->next;
ierr = MPI_Comm_rank(((PetscObject)dm)->comm,&rank);CHKERRQ(ierr);
/* loop over packed objects, handling one at at time */
cnt = 0;
while (next) {
ISLocalToGlobalMapping ltog;
PetscMPIInt size;
const PetscInt *suboff,*indices;
Vec global;
/* Get sub-DM global indices for each local dof */
ierr = DMGetLocalToGlobalMapping(next->dm,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltog,&n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog,&indices);CHKERRQ(ierr);
ierr = PetscMalloc(n*sizeof(PetscInt),&idx);CHKERRQ(ierr);
/* Get the offsets for the sub-DM global vector */
ierr = DMGetGlobalVector(next->dm,&global);CHKERRQ(ierr);
ierr = VecGetOwnershipRanges(global,&suboff);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)global)->comm,&size);CHKERRQ(ierr);
/* Shift the sub-DM definition of the global space to the composite global space */
for (i=0; i<n; i++) {
PetscInt subi = indices[i],lo = 0,hi = size,t;
/* Binary search to find which rank owns subi */
while (hi-lo > 1) {
t = lo + (hi-lo)/2;
if (suboff[t] > subi) hi = t;
else lo = t;
}
idx[i] = subi - suboff[lo] + next->grstarts[lo];
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltog,&indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm,n,idx,PETSC_OWN_POINTER,&(*ltogs)[cnt]);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(next->dm,&global);CHKERRQ(ierr);
next = next->next;
cnt++;
}
PetscFunctionReturn(0);
}
int FormJacobian_Grid(GridCtx *grid,Mat *J)
{
Mat jac = *J;
PetscErrorCode ierr;
PetscInt i,j,row,mx,my,xs,ys,xm,ym,Xs,Ys,Xm,Ym,col[5];
PetscInt grow;
const PetscInt *ltog;
PetscScalar two = 2.0,one = 1.0,v[5],hx,hy,hxdhy,hydhx,value;
ISLocalToGlobalMapping ltogm;
mx = grid->mx; my = grid->my;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
hxdhy = hx/hy; hydhx = hy/hx;
/* Get ghost points */
ierr = DMDAGetCorners(grid->da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(grid->da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(grid->da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
/* Evaluate Jacobian of function */
for (j=ys; j<ys+ym; j++) {
row = (j - Ys)*Xm + xs - Xs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
if (i > 0 && i < mx-1 && j > 0 && j < my-1) {
v[0] = -hxdhy; col[0] = ltog[row - Xm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + Xm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else if ((i > 0 && i < mx-1) || (j > 0 && j < my-1)) {
value = .5*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
} else {
value = .25*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode DMCreateGlobalVector_Redundant(DM dm,Vec *gvec)
{
PetscErrorCode ierr;
DM_Redundant *red = (DM_Redundant*)dm->data;
ISLocalToGlobalMapping ltog,ltogb;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm,DM_CLASSID,1);
PetscValidPointer(gvec,2);
*gvec = 0;
ierr = VecCreate(PetscObjectComm((PetscObject)dm),gvec);CHKERRQ(ierr);
ierr = VecSetSizes(*gvec,red->n,red->N);CHKERRQ(ierr);
ierr = VecSetType(*gvec,dm->vectype);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(dm,<og);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMappingBlock(dm,<ogb);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(*gvec,ltog);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMappingBlock(*gvec,ltog);CHKERRQ(ierr);
ierr = VecSetDM(*gvec,dm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMCreateMatrix_Composite(DM dm,MatType mtype,Mat *J)
{
PetscErrorCode ierr;
PetscBool usenest;
ISLocalToGlobalMapping ltogmap,ltogmapb;
PetscFunctionBegin;
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype,MATNEST,&usenest);CHKERRQ(ierr);
if (usenest) {
ierr = DMCreateMatrix_Composite_Nest(dm,mtype,J);CHKERRQ(ierr);
} else {
ierr = DMCreateMatrix_Composite_AIJ(dm,mtype ? mtype : MATAIJ,J);CHKERRQ(ierr);
}
ierr = DMGetLocalToGlobalMapping(dm,<ogmap);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMappingBlock(dm,<ogmapb);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(*J,ltogmap,ltogmap);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMappingBlock(*J,ltogmapb,ltogmapb);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMCreateMatrix_Redundant(DM dm,Mat *J)
{
DM_Redundant *red = (DM_Redundant*)dm->data;
PetscErrorCode ierr;
ISLocalToGlobalMapping ltog,ltogb;
PetscInt i,rstart,rend,*cols;
PetscScalar *vals;
PetscFunctionBegin;
ierr = MatCreate(PetscObjectComm((PetscObject)dm),J);CHKERRQ(ierr);
ierr = MatSetSizes(*J,red->n,red->n,red->N,red->N);CHKERRQ(ierr);
ierr = MatSetType(*J,dm->mattype);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*J,red->n,NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(*J,1,red->n,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,red->n,NULL,red->N-red->n,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(*J,1,red->n,NULL,red->N-red->n,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(dm,<og);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMappingBlock(dm,<ogb);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMappingBlock(*J,ltogb,ltogb);CHKERRQ(ierr);
ierr = PetscMalloc2(red->N,&cols,red->N,&vals);CHKERRQ(ierr);
for (i=0; i<red->N; i++) {
cols[i] = i;
vals[i] = 0.0;
}
ierr = MatGetOwnershipRange(*J,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatSetValues(*J,1,&i,red->N,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree2(cols,vals);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
ComputeJacobian - Evaluates Jacobian matrix.
Input Parameters:
. x - input vector
. user - user-defined application context
Output Parameters:
. jac - Jacobian matrix
. flag - flag indicating matrix structure
Notes:
Due to grid point reordering with DMDAs, we must always work
with the local grid points, and then transform them to the new
global numbering with the "ltog" mapping
We cannot work directly with the global numbers for the original
uniprocessor grid!
*/
PetscErrorCode ComputeJacobian(AppCtx *user,Vec X,Mat jac)
{
PetscErrorCode ierr;
Vec localX = user->localX; /* local vector */
const PetscInt *ltog; /* local-to-global mapping */
PetscInt i,j,row,mx,my,col[5];
PetscInt xs,ys,xm,ym,gxs,gys,gxm,gym,grow;
PetscScalar two = 2.0,one = 1.0,lambda,v[5],hx,hy,hxdhy,hydhx,sc,*x;
ISLocalToGlobalMapping ltogm;
mx = user->mx; my = user->my; lambda = user->param;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
sc = hx*hy; hxdhy = hx/hy; hydhx = hy/hx;
/*
Scatter ghost points to local vector, using the 2-step process
DMGlobalToLocalBegin(), DMGlobalToLocalEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = DMGlobalToLocalBegin(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
/*
Get pointer to vector data
*/
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->da,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
/*
Get the global node numbers for all local nodes, including ghost points
*/
ierr = DMGetLocalToGlobalMapping(user->da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
/*
Compute entries for the locally owned part of the Jacobian.
- Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors. The "grow"
parameter computed below specifies the global row number
corresponding to each local grid point.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Always specify global row and columns of matrix entries.
- Here, we set all entries for a particular row at once.
*/
for (j=ys; j<ys+ym; j++) {
row = (j - gys)*gxm + xs - gxs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
/* boundary points */
if (i == 0 || j == 0 || i == mx-1 || j == my-1) {
ierr = MatSetValues(jac,1,&grow,1,&grow,&one,INSERT_VALUES);CHKERRQ(ierr);
continue;
}
/* interior grid points */
v[0] = -hxdhy; col[0] = ltog[row - gxm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy) - sc*lambda*PetscExpScalar(x[row]); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + gxm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscErrorCode ierr;
PetscInt M = 10,N = 8,m = PETSC_DECIDE;
PetscInt s =2,w=2,n = PETSC_DECIDE,nloc,l,i,j,kk;
PetscInt Xs,Xm,Ys,Ym,iloc,*iglobal;
const PetscInt *ltog;
PetscInt *lx = NULL,*ly = NULL;
PetscBool testorder = PETSC_FALSE,flg;
DMBoundaryType bx = DM_BOUNDARY_NONE,by= DM_BOUNDARY_NONE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDAStencilType st = DMDA_STENCIL_BOX;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,400,&viewer);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(NULL,NULL,"-NX",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-NY",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-w",&w,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xghosted",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yghosted",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-star",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-box",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-testorder",&testorder,NULL);CHKERRQ(ierr);
/*
Test putting two nodes in x and y on each processor, exact last processor
in x and y gets the rest.
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-distribute",&flg,NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,st,M,N,m,n,w,s,lx,ly,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!testorder) { /* turn off printing when testing ordering mappings */
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vectors:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n");CHKERRQ(ierr);
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (testorder) {
ISLocalToGlobalMapping ltogm;
ierr = DMGetLocalToGlobalMapping(da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltogm,&nloc);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,NULL,&Xm,&Ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = PetscMalloc1(nloc,&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscFPrintf(PETSC_COMM_SELF,stdout,"[%d] Problem with mapping: j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",rank,j,i,l,ltog[iloc+l],iglobal[kk]);CHKERRQ(ierr);
}
kk++;
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 13,s=1,dof=1;
DMDABoundaryType bx = DMDA_BOUNDARY_PERIODIC;
PetscErrorCode ierr;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
PetscDraw draw;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Create viewers */
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",280,480,600,200,&viewer);CHKERRQ(ierr);
ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr);
ierr = PetscDrawSetDoubleBuffer(draw);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetEnum(NULL,"-wrap",DMDABoundaryTypes,(PetscEnum*)&bx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-s",&s,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,dof,s,NULL,&da);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank+1;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = VecView(global,viewer);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ISLocalToGlobalMapping is;
ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal to global mapping: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(da,&is);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(is,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*
DMDAGetFaceInterpolation - Gets the interpolation for a face based coarse space
*/
PetscErrorCode DMDAGetFaceInterpolation(DM da,PC_Exotic *exotic,Mat Aglobal,MatReuse reuse,Mat *P)
{
PetscErrorCode ierr;
PetscInt dim,i,j,k,m,n,p,dof,Nint,Nface,Nwire,Nsurf,*Iint,*Isurf,cint = 0,csurf = 0,istart,jstart,kstart,*II,N,c = 0;
PetscInt mwidth,nwidth,pwidth,cnt,mp,np,pp,Ntotal,gl[6],*globals,Ng,*IIint,*IIsurf,Nt;
Mat Xint, Xsurf,Xint_tmp;
IS isint,issurf,is,row,col;
ISLocalToGlobalMapping ltg;
MPI_Comm comm;
Mat A,Aii,Ais,Asi,*Aholder,iAii;
MatFactorInfo info;
PetscScalar *xsurf,*xint;
#if defined(PETSC_USE_DEBUG_foo)
PetscScalar tmp;
#endif
PetscTable ht;
PetscFunctionBegin;
ierr = DMDAGetInfo(da,&dim,0,0,0,&mp,&np,&pp,&dof,0,0,0,0,0);
CHKERRQ(ierr);
if (dof != 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only for single field problems");
if (dim != 3) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only coded for 3d problems");
ierr = DMDAGetCorners(da,0,0,0,&m,&n,&p);
CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&istart,&jstart,&kstart,&mwidth,&nwidth,&pwidth);
CHKERRQ(ierr);
istart = istart ? -1 : 0;
jstart = jstart ? -1 : 0;
kstart = kstart ? -1 : 0;
/*
the columns of P are the interpolation of each coarse grid point (one for each vertex and edge)
to all the local degrees of freedom (this includes the vertices, edges and faces).
Xint are the subset of the interpolation into the interior
Xface are the interpolation onto faces but not into the interior
Xsurf are the interpolation onto the vertices and edges (the surfbasket)
Xint
Symbolically one could write P = (Xface) after interchanging the rows to match the natural ordering on the domain
Xsurf
*/
N = (m - istart)*(n - jstart)*(p - kstart);
Nint = (m-2-istart)*(n-2-jstart)*(p-2-kstart);
Nface = 2*((m-2-istart)*(n-2-jstart) + (m-2-istart)*(p-2-kstart) + (n-2-jstart)*(p-2-kstart));
Nwire = 4*((m-2-istart) + (n-2-jstart) + (p-2-kstart)) + 8;
Nsurf = Nface + Nwire;
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nint,6,NULL,&Xint);
CHKERRQ(ierr);
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nsurf,6,NULL,&Xsurf);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/*
Require that all 12 edges and 6 faces have at least one grid point. Otherwise some of the columns of
Xsurf will be all zero (thus making the coarse matrix singular).
*/
if (m-istart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in X direction must be at least 3");
if (n-jstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Y direction must be at least 3");
if (p-kstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Z direction must be at least 3");
cnt = 0;
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 0*Nsurf] = 1;
}
for (k=1; k<p-1-kstart; k++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 1*Nsurf] = 1;
for (j=1; j<n-1-jstart; j++) {
xsurf[cnt++ + 2*Nsurf] = 1;
/* these are the interior nodes */
xsurf[cnt++ + 3*Nsurf] = 1;
}
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 4*Nsurf] = 1;
}
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 5*Nsurf] = 1;
}
#if defined(PETSC_USE_DEBUG_foo)
for (i=0; i<Nsurf; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xsurf[i+j*Nsurf];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xsurf interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
#endif
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/* ierr = MatView(Xsurf,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
/*
I are the indices for all the needed vertices (in global numbering)
Iint are the indices for the interior values, I surf for the surface values
(This is just for the part of the global matrix obtained with MatGetSubMatrix(), it
is NOT the local DMDA ordering.)
IIint and IIsurf are the same as the Iint, Isurf except they are in the global numbering
*/
#define Endpoint(a,start,b) (a == 0 || a == (b-1-start))
ierr = PetscMalloc3(N,&II,Nint,&Iint,Nsurf,&Isurf);
CHKERRQ(ierr);
ierr = PetscMalloc2(Nint,&IIint,Nsurf,&IIsurf);
CHKERRQ(ierr);
for (k=0; k<p-kstart; k++) {
for (j=0; j<n-jstart; j++) {
for (i=0; i<m-istart; i++) {
II[c++] = i + j*mwidth + k*mwidth*nwidth;
if (!Endpoint(i,istart,m) && !Endpoint(j,jstart,n) && !Endpoint(k,kstart,p)) {
IIint[cint] = i + j*mwidth + k*mwidth*nwidth;
Iint[cint++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
} else {
IIsurf[csurf] = i + j*mwidth + k*mwidth*nwidth;
Isurf[csurf++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
}
}
}
}
if (c != N) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"c != N");
if (cint != Nint) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"cint != Nint");
if (csurf != Nsurf) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"csurf != Nsurf");
ierr = DMGetLocalToGlobalMapping(da,<g);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,N,II,II);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nint,IIint,IIint);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nsurf,IIsurf,IIsurf);
CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)da,&comm);
CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,N,II,PETSC_COPY_VALUES,&is);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nint,Iint,PETSC_COPY_VALUES,&isint);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nsurf,Isurf,PETSC_COPY_VALUES,&issurf);
CHKERRQ(ierr);
ierr = PetscFree3(II,Iint,Isurf);
CHKERRQ(ierr);
ierr = ISSort(is);
CHKERRQ(ierr);
ierr = MatGetSubMatrices(Aglobal,1,&is,&is,MAT_INITIAL_MATRIX,&Aholder);
CHKERRQ(ierr);
A = *Aholder;
ierr = PetscFree(Aholder);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,isint,MAT_INITIAL_MATRIX,&Aii);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,issurf,MAT_INITIAL_MATRIX,&Ais);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,issurf,isint,MAT_INITIAL_MATRIX,&Asi);
CHKERRQ(ierr);
/*
Solve for the interpolation onto the interior Xint
*/
ierr = MatMatMult(Ais,Xsurf,MAT_INITIAL_MATRIX,PETSC_DETERMINE,&Xint_tmp);
CHKERRQ(ierr);
ierr = MatScale(Xint_tmp,-1.0);
CHKERRQ(ierr);
if (exotic->directSolve) {
ierr = MatGetFactor(Aii,MATSOLVERPETSC,MAT_FACTOR_LU,&iAii);
CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&info);
CHKERRQ(ierr);
ierr = MatGetOrdering(Aii,MATORDERINGND,&row,&col);
CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(iAii,Aii,row,col,&info);
CHKERRQ(ierr);
ierr = ISDestroy(&row);
CHKERRQ(ierr);
ierr = ISDestroy(&col);
CHKERRQ(ierr);
ierr = MatLUFactorNumeric(iAii,Aii,&info);
CHKERRQ(ierr);
ierr = MatMatSolve(iAii,Xint_tmp,Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&iAii);
CHKERRQ(ierr);
} else {
Vec b,x;
PetscScalar *xint_tmp;
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&x);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&b);
CHKERRQ(ierr);
ierr = KSPSetOperators(exotic->ksp,Aii,Aii);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = VecPlaceArray(x,xint+i*Nint);
CHKERRQ(ierr);
ierr = VecPlaceArray(b,xint_tmp+i*Nint);
CHKERRQ(ierr);
ierr = KSPSolve(exotic->ksp,b,x);
CHKERRQ(ierr);
ierr = VecResetArray(x);
CHKERRQ(ierr);
ierr = VecResetArray(b);
CHKERRQ(ierr);
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&b);
CHKERRQ(ierr);
}
ierr = MatDestroy(&Xint_tmp);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
for (i=0; i<Nint; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xint[i+j*Nint];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xint interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
/* ierr =MatView(Xint,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
#endif
/* total faces */
Ntotal = mp*np*(pp+1) + mp*pp*(np+1) + np*pp*(mp+1);
/*
For each vertex, edge, face on process (in the same orderings as used above) determine its local number including ghost points
*/
cnt = 0;
{
gl[cnt++] = mwidth+1;
}
{
{
gl[cnt++] = mwidth*nwidth+1;
}
{
gl[cnt++] = mwidth*nwidth + mwidth; /* these are the interior nodes */ gl[cnt++] = mwidth*nwidth + mwidth+m-istart-1;
}
{
gl[cnt++] = mwidth*nwidth+mwidth*(n-jstart-1)+1;
}
}
{
gl[cnt++] = mwidth*nwidth*(p-kstart-1) + mwidth+1;
}
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* convert that to global numbering and get them on all processes */
ierr = ISLocalToGlobalMappingApply(ltg,6,gl,gl);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
ierr = PetscMalloc1(6*mp*np*pp,&globals);
CHKERRQ(ierr);
ierr = MPI_Allgather(gl,6,MPIU_INT,globals,6,MPIU_INT,PetscObjectComm((PetscObject)da));
CHKERRQ(ierr);
/* Number the coarse grid points from 0 to Ntotal */
ierr = MatGetSize(Aglobal,&Nt,NULL);
CHKERRQ(ierr);
ierr = PetscTableCreate(Ntotal/3,Nt+1,&ht);
CHKERRQ(ierr);
for (i=0; i<6*mp*np*pp; i++) {
ierr = PetscTableAddCount(ht,globals[i]+1);
CHKERRQ(ierr);
}
ierr = PetscTableGetCount(ht,&cnt);
CHKERRQ(ierr);
if (cnt != Ntotal) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Hash table size %D not equal to total number coarse grid points %D",cnt,Ntotal);
ierr = PetscFree(globals);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = PetscTableFind(ht,gl[i]+1,&gl[i]);
CHKERRQ(ierr);
gl[i]--;
}
ierr = PetscTableDestroy(&ht);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* construct global interpolation matrix */
ierr = MatGetLocalSize(Aglobal,&Ng,NULL);
CHKERRQ(ierr);
if (reuse == MAT_INITIAL_MATRIX) {
ierr = MatCreateAIJ(PetscObjectComm((PetscObject)da),Ng,PETSC_DECIDE,PETSC_DECIDE,Ntotal,Nint+Nsurf,NULL,Nint,NULL,P);
CHKERRQ(ierr);
} else {
ierr = MatZeroEntries(*P);
CHKERRQ(ierr);
}
ierr = MatSetOption(*P,MAT_ROW_ORIENTED,PETSC_FALSE);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nint,IIint,6,gl,xint,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nsurf,IIsurf,6,gl,xsurf,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = PetscFree2(IIint,IIsurf);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
{
Vec x,y;
PetscScalar *yy;
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),Ng,PETSC_DETERMINE,&y);
CHKERRQ(ierr);
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),PETSC_DETERMINE,Ntotal,&x);
CHKERRQ(ierr);
ierr = VecSet(x,1.0);
CHKERRQ(ierr);
ierr = MatMult(*P,x,y);
CHKERRQ(ierr);
ierr = VecGetArray(y,&yy);
CHKERRQ(ierr);
for (i=0; i<Ng; i++) {
if (PetscAbsScalar(yy[i]-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong p interpolation at i %D value %g",i,(double)PetscAbsScalar(yy[i]));
}
ierr = VecRestoreArray(y,&yy);
CHKERRQ(ierr);
ierr = VecDestroy(x);
CHKERRQ(ierr);
ierr = VecDestroy(y);
CHKERRQ(ierr);
}
#endif
ierr = MatDestroy(&Aii);
CHKERRQ(ierr);
ierr = MatDestroy(&Ais);
CHKERRQ(ierr);
ierr = MatDestroy(&Asi);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = ISDestroy(&is);
CHKERRQ(ierr);
ierr = ISDestroy(&isint);
CHKERRQ(ierr);
ierr = ISDestroy(&issurf);
CHKERRQ(ierr);
ierr = MatDestroy(&Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&Xsurf);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode MatSetupDM_HYPRESStruct(Mat mat,DM da)
{
PetscErrorCode ierr;
Mat_HYPRESStruct *ex = (Mat_HYPRESStruct*) mat->data;
PetscInt dim,dof,sw[3],nx,ny,nz;
PetscInt ilower[3],iupper[3],ssize,i;
DMBoundaryType px,py,pz;
DMDAStencilType st;
PetscInt nparts= 1; /* assuming only one part */
PetscInt part = 0;
ISLocalToGlobalMapping ltog;
PetscFunctionBegin;
ex->da = da;
ierr = PetscObjectReference((PetscObject)da);CHKERRQ(ierr);
ierr = DMDAGetInfo(ex->da,&dim,0,0,0,0,0,0,&dof,&sw[0],&px,&py,&pz,&st);CHKERRQ(ierr);
ierr = DMDAGetCorners(ex->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);CHKERRQ(ierr);
iupper[0] += ilower[0] - 1;
iupper[1] += ilower[1] - 1;
iupper[2] += ilower[2] - 1;
/* the hypre_Box is used to zero out the matrix entries in MatZeroValues() */
ex->hbox.imin[0] = ilower[0];
ex->hbox.imin[1] = ilower[1];
ex->hbox.imin[2] = ilower[2];
ex->hbox.imax[0] = iupper[0];
ex->hbox.imax[1] = iupper[1];
ex->hbox.imax[2] = iupper[2];
ex->dofs_order = 0;
/* assuming that the same number of dofs on each gridpoint. Also assume all cell-centred based */
ex->nvars= dof;
/* create the hypre grid object and set its information */
if (px || py || pz) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add periodic support by calling HYPRE_SStructGridSetPeriodic()");
PetscStackCallStandard(HYPRE_SStructGridCreate,(ex->hcomm,dim,nparts,&ex->ss_grid));
PetscStackCallStandard(HYPRE_SStructGridSetExtents,(ex->ss_grid,part,ex->hbox.imin,ex->hbox.imax));
{
HYPRE_SStructVariable *vartypes;
ierr = PetscMalloc1(ex->nvars,&vartypes);CHKERRQ(ierr);
for (i= 0; i< ex->nvars; i++) vartypes[i]= HYPRE_SSTRUCT_VARIABLE_CELL;
PetscStackCallStandard(HYPRE_SStructGridSetVariables,(ex->ss_grid, part, ex->nvars,vartypes));
ierr = PetscFree(vartypes);CHKERRQ(ierr);
}
PetscStackCallStandard(HYPRE_SStructGridAssemble,(ex->ss_grid));
sw[1] = sw[0];
sw[2] = sw[1];
/* PetscStackCallStandard(HYPRE_SStructGridSetNumGhost,(ex->ss_grid,sw)); */
/* create the hypre stencil object and set its information */
if (sw[0] > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for wider stencils");
if (st == DMDA_STENCIL_BOX) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for box stencils");
if (dim == 1) {
PetscInt offsets[3][1] = {{-1},{0},{1}};
PetscInt j, cnt;
ssize = 3*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i = 0; i < (ex->nvars); i++) {
for (j = 0; j < 3; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
} else if (dim == 2) {
PetscInt offsets[5][2] = {{0,-1},{-1,0},{0,0},{1,0},{0,1}};
PetscInt j, cnt;
ssize = 5*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i= 0; i< (ex->nvars); i++) {
for (j= 0; j< 5; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
} else if (dim == 3) {
PetscInt offsets[7][3] = {{0,0,-1},{0,-1,0},{-1,0,0},{0,0,0},{1,0,0},{0,1,0},{0,0,1}};
PetscInt j, cnt;
ssize = 7*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i= 0; i< (ex->nvars); i++) {
for (j= 0; j< 7; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
}
/* create the HYPRE graph */
PetscStackCallStandard(HYPRE_SStructGraphCreate,(ex->hcomm, ex->ss_grid, &(ex->ss_graph)));
/* set the stencil graph. Note that each variable has the same graph. This means that each
variable couples to all the other variable and with the same stencil pattern. */
for (i= 0; i< (ex->nvars); i++) {
PetscStackCallStandard(HYPRE_SStructGraphSetStencil,(ex->ss_graph,part,i,ex->ss_stencil));
}
PetscStackCallStandard(HYPRE_SStructGraphAssemble,(ex->ss_graph));
/* create the HYPRE sstruct vectors for rhs and solution */
PetscStackCallStandard(HYPRE_SStructVectorCreate,(ex->hcomm,ex->ss_grid,&ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorCreate,(ex->hcomm,ex->ss_grid,&ex->ss_x));
PetscStackCallStandard(HYPRE_SStructVectorInitialize,(ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorInitialize,(ex->ss_x));
PetscStackCallStandard(HYPRE_SStructVectorAssemble,(ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorAssemble,(ex->ss_x));
/* create the hypre matrix object and set its information */
PetscStackCallStandard(HYPRE_SStructMatrixCreate,(ex->hcomm,ex->ss_graph,&ex->ss_mat));
PetscStackCallStandard(HYPRE_SStructGridDestroy,(ex->ss_grid));
PetscStackCallStandard(HYPRE_SStructStencilDestroy,(ex->ss_stencil));
if (ex->needsinitialization) {
PetscStackCallStandard(HYPRE_SStructMatrixInitialize,(ex->ss_mat));
ex->needsinitialization = PETSC_FALSE;
}
/* set the global and local sizes of the matrix */
ierr = DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);CHKERRQ(ierr);
ierr = MatSetSizes(mat,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->rmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->cmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
if (dim == 3) {
mat->ops->setvalueslocal = MatSetValuesLocal_HYPRESStruct_3d;
mat->ops->zerorowslocal = MatZeroRowsLocal_HYPRESStruct_3d;
mat->ops->zeroentries = MatZeroEntries_HYPRESStruct_3d;
ierr = MatZeroEntries_HYPRESStruct_3d(mat);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only support for 3d DMDA currently");
/* get values that will be used repeatedly in MatSetValuesLocal() and MatZeroRowsLocal() repeatedly */
ierr = MatGetOwnershipRange(mat,&ex->rstart,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(ex->da,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog, (const PetscInt **) &ex->gindices);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(ex->da,0,0,0,&ex->gnx,&ex->gnxgny,&ex->gnxgnygnz);CHKERRQ(ierr);
ex->gnxgny *= ex->gnx;
ex->gnxgnygnz *= ex->gnxgny;
ierr = DMDAGetCorners(ex->da,&ex->xs,&ex->ys,&ex->zs,&ex->nx,&ex->ny,&ex->nz);CHKERRQ(ierr);
ex->nxny = ex->nx*ex->ny;
ex->nxnynz = ex->nz*ex->nxny;
PetscFunctionReturn(0);
}
static PetscErrorCode MatSetupDM_HYPREStruct(Mat mat,DM da)
{
PetscErrorCode ierr;
Mat_HYPREStruct *ex = (Mat_HYPREStruct*) mat->data;
PetscInt dim,dof,sw[3],nx,ny,nz,ilower[3],iupper[3],ssize,i;
DMBoundaryType px,py,pz;
DMDAStencilType st;
ISLocalToGlobalMapping ltog;
PetscFunctionBegin;
ex->da = da;
ierr = PetscObjectReference((PetscObject)da);CHKERRQ(ierr);
ierr = DMDAGetInfo(ex->da,&dim,0,0,0,0,0,0,&dof,&sw[0],&px,&py,&pz,&st);CHKERRQ(ierr);
ierr = DMDAGetCorners(ex->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);CHKERRQ(ierr);
iupper[0] += ilower[0] - 1;
iupper[1] += ilower[1] - 1;
iupper[2] += ilower[2] - 1;
/* the hypre_Box is used to zero out the matrix entries in MatZeroValues() */
ex->hbox.imin[0] = ilower[0];
ex->hbox.imin[1] = ilower[1];
ex->hbox.imin[2] = ilower[2];
ex->hbox.imax[0] = iupper[0];
ex->hbox.imax[1] = iupper[1];
ex->hbox.imax[2] = iupper[2];
/* create the hypre grid object and set its information */
if (dof > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Currently only support for scalar problems");
if (px || py || pz) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add periodic support by calling HYPRE_StructGridSetPeriodic()");
PetscStackCallStandard(HYPRE_StructGridCreate,(ex->hcomm,dim,&ex->hgrid));
PetscStackCallStandard(HYPRE_StructGridSetExtents,(ex->hgrid,(HYPRE_Int *)ilower,(HYPRE_Int *)iupper));
PetscStackCallStandard(HYPRE_StructGridAssemble,(ex->hgrid));
sw[1] = sw[0];
sw[2] = sw[1];
PetscStackCallStandard(HYPRE_StructGridSetNumGhost,(ex->hgrid,(HYPRE_Int *)sw));
/* create the hypre stencil object and set its information */
if (sw[0] > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for wider stencils");
if (st == DMDA_STENCIL_BOX) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for box stencils");
if (dim == 1) {
PetscInt offsets[3][1] = {{-1},{0},{1}};
ssize = 3;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
} else if (dim == 2) {
PetscInt offsets[5][2] = {{0,-1},{-1,0},{0,0},{1,0},{0,1}};
ssize = 5;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
} else if (dim == 3) {
PetscInt offsets[7][3] = {{0,0,-1},{0,-1,0},{-1,0,0},{0,0,0},{1,0,0},{0,1,0},{0,0,1}};
ssize = 7;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
}
/* create the HYPRE vector for rhs and solution */
PetscStackCallStandard(HYPRE_StructVectorCreate,(ex->hcomm,ex->hgrid,&ex->hb));
PetscStackCallStandard(HYPRE_StructVectorCreate,(ex->hcomm,ex->hgrid,&ex->hx));
PetscStackCallStandard(HYPRE_StructVectorInitialize,(ex->hb));
PetscStackCallStandard(HYPRE_StructVectorInitialize,(ex->hx));
PetscStackCallStandard(HYPRE_StructVectorAssemble,(ex->hb));
PetscStackCallStandard(HYPRE_StructVectorAssemble,(ex->hx));
/* create the hypre matrix object and set its information */
PetscStackCallStandard(HYPRE_StructMatrixCreate,(ex->hcomm,ex->hgrid,ex->hstencil,&ex->hmat));
PetscStackCallStandard(HYPRE_StructGridDestroy,(ex->hgrid));
PetscStackCallStandard(HYPRE_StructStencilDestroy,(ex->hstencil));
if (ex->needsinitialization) {
PetscStackCallStandard(HYPRE_StructMatrixInitialize,(ex->hmat));
ex->needsinitialization = PETSC_FALSE;
}
/* set the global and local sizes of the matrix */
ierr = DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);CHKERRQ(ierr);
ierr = MatSetSizes(mat,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->rmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->cmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
if (dim == 3) {
mat->ops->setvalueslocal = MatSetValuesLocal_HYPREStruct_3d;
mat->ops->zerorowslocal = MatZeroRowsLocal_HYPREStruct_3d;
mat->ops->zeroentries = MatZeroEntries_HYPREStruct_3d;
ierr = MatZeroEntries_HYPREStruct_3d(mat);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only support for 3d DMDA currently");
/* get values that will be used repeatedly in MatSetValuesLocal() and MatZeroRowsLocal() repeatedly */
ierr = MatGetOwnershipRange(mat,&ex->rstart,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(ex->da,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog, (const PetscInt **) &ex->gindices);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(ex->da,0,0,0,&ex->gnx,&ex->gnxgny,0);CHKERRQ(ierr);
ex->gnxgny *= ex->gnx;
ierr = DMDAGetCorners(ex->da,&ex->xs,&ex->ys,&ex->zs,&ex->nx,&ex->ny,0);CHKERRQ(ierr);
ex->nxny = ex->nx*ex->ny;
PetscFunctionReturn(0);
}
