PetscErrorCode FormJacobian_Grid(AppCtx *user,GridCtx *grid,Vec X, Mat *J,Mat *B)
{
Mat jac = *J;
PetscErrorCode ierr;
PetscInt i, j, row, mx, my, xs, ys, xm, ym, Xs, Ys, Xm, Ym, col[5], nloc, *ltog, grow;
PetscScalar two = 2.0, one = 1.0, lambda, v[5], hx, hy, hxdhy, hydhx, sc, *x, value;
Vec localX = grid->localX;
mx = grid->mx; my = grid->my; lambda = user->param;
hx = one/(double)(mx-1); hy = one/(double)(my-1);
sc = hx*hy; hxdhy = hx/hy; hydhx = hy/hx;
/* Get ghost points */
ierr = DMGlobalToLocalBegin(grid->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(grid->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMDAGetCorners(grid->da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(grid->da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(grid->da,&nloc,<og);CHKERRQ(ierr);
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/* Evaluate Jacobian of function */
for (j=ys; j<ys+ym; j++) {
row = (j - Ys)*Xm + xs - Xs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
if (i > 0 && i < mx-1 && j > 0 && j < my-1) {
v[0] = -hxdhy; col[0] = ltog[row - Xm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy) - sc*lambda*exp(x[row]); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + Xm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else if ((i > 0 && i < mx-1) || (j > 0 && j < my-1)){
value = .5*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
} else {
value = .25*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
int FormJacobian_Grid(GridCtx *grid,Mat *J)
{
Mat jac = *J;
PetscErrorCode ierr;
PetscInt i,j,row,mx,my,xs,ys,xm,ym,Xs,Ys,Xm,Ym,col[5];
PetscInt nloc,grow;
const PetscInt *ltog;
PetscScalar two = 2.0,one = 1.0,v[5],hx,hy,hxdhy,hydhx,value;
mx = grid->mx; my = grid->my;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
hxdhy = hx/hy; hydhx = hy/hx;
/* Get ghost points */
ierr = DMDAGetCorners(grid->da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(grid->da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(grid->da,&nloc,<og);CHKERRQ(ierr);
/* Evaluate Jacobian of function */
for (j=ys; j<ys+ym; j++) {
row = (j - Ys)*Xm + xs - Xs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
if (i > 0 && i < mx-1 && j > 0 && j < my-1) {
v[0] = -hxdhy; col[0] = ltog[row - Xm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + Xm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else if ((i > 0 && i < mx-1) || (j > 0 && j < my-1)) {
value = .5*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
} else {
value = .25*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = DMDARestoreGlobalIndices(grid->da,&nloc,<og);CHKERRQ(ierr);
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
/*
Forms the interpolation (and restriction) operator from
coarse grid to fine.
*/
PetscErrorCode FormInterpolation(AppCtx *user)
{
PetscErrorCode ierr;
PetscInt i,j,i_start,m_fine,j_start,m,n,*idx;
PetscInt m_ghost,n_ghost,*idx_c,m_ghost_c,n_ghost_c,m_coarse;
PetscInt row,i_start_ghost,j_start_ghost,cols[4], m_c;
PetscInt nc,ratio = user->ratio,m_c_local,m_fine_local;
PetscInt i_c,j_c,i_start_c,j_start_c,n_c,i_start_ghost_c,j_start_ghost_c,col;
PetscScalar v[4],x,y, one = 1.0;
Mat mat;
Vec Rscale;
ierr = DMDAGetCorners(user->fine.da,&i_start,&j_start,0,&m,&n,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->fine.da,&i_start_ghost,&j_start_ghost,0,&m_ghost,&n_ghost,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(user->fine.da,PETSC_NULL,&idx);CHKERRQ(ierr);
ierr = DMDAGetCorners(user->coarse.da,&i_start_c,&j_start_c,0,&m_c,&n_c,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->coarse.da,&i_start_ghost_c,&j_start_ghost_c,0,&m_ghost_c,&n_ghost_c,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(user->coarse.da,PETSC_NULL,&idx_c);CHKERRQ(ierr);
/* create interpolation matrix */
ierr = VecGetLocalSize(user->fine.x,&m_fine_local);CHKERRQ(ierr);
ierr = VecGetLocalSize(user->coarse.x,&m_c_local);CHKERRQ(ierr);
ierr = VecGetSize(user->fine.x,&m_fine);CHKERRQ(ierr);
ierr = VecGetSize(user->coarse.x,&m_coarse);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,m_fine_local,m_c_local,m_fine,m_coarse,
5,0,3,0,&mat);CHKERRQ(ierr);
/* loop over local fine grid nodes setting interpolation for those*/
for ( j=j_start; j<j_start+n; j++ ) {
for ( i=i_start; i<i_start+m; i++ ) {
/* convert to local "natural" numbering and then to PETSc global numbering */
row = idx[m_ghost*(j-j_start_ghost) + (i-i_start_ghost)];
i_c = (i/ratio); /* coarse grid node to left of fine grid node */
j_c = (j/ratio); /* coarse grid node below fine grid node */
/*
Only include those interpolation points that are truly
nonzero. Note this is very important for final grid lines
in x and y directions; since they have no right/top neighbors
*/
x = ((double)(i - i_c*ratio))/((double)ratio);
y = ((double)(j - j_c*ratio))/((double)ratio);
nc = 0;
/* one left and below; or we are right on it */
if (j_c < j_start_ghost_c || j_c > j_start_ghost_c+n_ghost_c) {
SETERRQ3(PETSC_COMM_SELF,1,"Sorry j %D %D %D",j_c,j_start_ghost_c,j_start_ghost_c+n_ghost_c);
}
if (i_c < i_start_ghost_c || i_c > i_start_ghost_c+m_ghost_c) {
SETERRQ3(PETSC_COMM_SELF,1,"Sorry i %D %D %D",i_c,i_start_ghost_c,i_start_ghost_c+m_ghost_c);
}
col = m_ghost_c*(j_c-j_start_ghost_c) + (i_c-i_start_ghost_c);
cols[nc] = idx_c[col];
v[nc++] = x*y - x - y + 1.0;
/* one right and below */
if (i_c*ratio != i) {
cols[nc] = idx_c[col+1];
v[nc++] = -x*y + x;
}
/* one left and above */
if (j_c*ratio != j) {
cols[nc] = idx_c[col+m_ghost_c];
v[nc++] = -x*y + y;
}
/* one right and above */
if (j_c*ratio != j && i_c*ratio != i) {
cols[nc] = idx_c[col+m_ghost_c+1];
v[nc++] = x*y;
}
ierr = MatSetValues(mat,1,&row,nc,cols,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecDuplicate(user->coarse.x,&Rscale);CHKERRQ(ierr);
ierr = VecSet(user->fine.x,one);CHKERRQ(ierr);
ierr = MatMultTranspose(mat,user->fine.x,Rscale);CHKERRQ(ierr);
ierr = VecReciprocal(Rscale);CHKERRQ(ierr);
user->Rscale = Rscale;
user->R = mat;
return 0;
}
/*
FormJacobian - Evaluates Jacobian matrix.
Input Parameters:
. snes - the SNES context
. x - input vector
. ptr - optional user-defined context, as set by SNESSetJacobian()
Output Parameters:
. A - Jacobian matrix
. B - optionally different preconditioning matrix
. flag - flag indicating matrix structure
Notes:
Due to grid point reordering with DMDAs, we must always work
with the local grid points, and then transform them to the new
global numbering with the "ltog" mapping (via DMDAGetGlobalIndices()).
We cannot work directly with the global numbers for the original
uniprocessor grid!
*/
PetscErrorCode FormJacobian(SNES snes,Vec X,Mat *J,Mat *B,MatStructure *flag,void *ptr)
{
AppCtx *user = (AppCtx*)ptr; /* user-defined application context */
Mat jac = *B; /* Jacobian matrix */
Vec localX = user->localX; /* local vector */
PetscErrorCode ierr;
PetscInt *ltog; /* local-to-global mapping */
PetscInt i,j,row,mx,my,col[5];
PetscInt nloc,xs,ys,xm,ym,gxs,gys,gxm,gym,grow;
PetscScalar two = 2.0,one = 1.0,lambda,v[5],hx,hy,hxdhy,hydhx,sc,*x;
mx = user->mx; my = user->my; lambda = user->param;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
sc = hx*hy; hxdhy = hx/hy; hydhx = hy/hx;
/*
Scatter ghost points to local vector,using the 2-step process
DMGlobalToLocalBegin(),DMGlobalToLocalEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = DMGlobalToLocalBegin(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
/*
Get pointer to vector data
*/
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,PETSC_NULL,&gxm,&gym,PETSC_NULL);CHKERRQ(ierr);
/*
Get the global node numbers for all local nodes, including ghost points
*/
ierr = DMDAGetGlobalIndices(user->da,&nloc,<og);CHKERRQ(ierr);
/*
Compute entries for the locally owned part of the Jacobian.
- Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors. The "grow"
parameter computed below specifies the global row number
corresponding to each local grid point.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Always specify global row and columns of matrix entries.
- Here, we set all entries for a particular row at once.
*/
for (j=ys; j<ys+ym; j++) {
row = (j - gys)*gxm + xs - gxs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
/* boundary points */
if (i == 0 || j == 0 || i == mx-1 || j == my-1) {
ierr = MatSetValues(jac,1,&grow,1,&grow,&one,INSERT_VALUES);CHKERRQ(ierr);
continue;
}
/* interior grid points */
v[0] = -hxdhy; col[0] = ltog[row - gxm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy) - sc*lambda*exp(x[row]); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + gxm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Set flag to indicate that the Jacobian matrix retains an identical
nonzero structure throughout all nonlinear iterations (although the
values of the entries change). Thus, we can save some work in setting
up the preconditioner (e.g., no need to redo symbolic factorization for
ILU/ICC preconditioners).
- If the nonzero structure of the matrix is different during
successive linear solves, then the flag DIFFERENT_NONZERO_PATTERN
must be used instead. If you are unsure whether the matrix
structure has changed or not, use the flag DIFFERENT_NONZERO_PATTERN.
- Caution: If you specify SAME_NONZERO_PATTERN, PETSc
believes your assertion and does not check the structure
of the matrix. If you erroneously claim that the structure
is the same when it actually is not, the new preconditioner
will not function correctly. Thus, use this optimization
feature with caution!
*/
*flag = SAME_NONZERO_PATTERN;
/*
Tell the matrix we will never add a new nonzero location to the
matrix. If we do it will generate an error.
*/
/* ierr = MatSetOption(jac,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr); */
return 0;
}
/*
ComputeJacobian - Evaluates Jacobian matrix.
Input Parameters:
. x - input vector
. user - user-defined application context
Output Parameters:
. jac - Jacobian matrix
. flag - flag indicating matrix structure
Notes:
Due to grid point reordering with DMDAs, we must always work
with the local grid points, and then transform them to the new
global numbering with the "ltog" mapping (via DMDAGetGlobalIndices()).
We cannot work directly with the global numbers for the original
uniprocessor grid!
*/
PetscErrorCode ComputeJacobian(AppCtx *user,Vec X,Mat jac)
{
PetscErrorCode ierr;
Vec localX = user->localX; /* local vector */
const PetscInt *ltog; /* local-to-global mapping */
PetscInt i,j,row,mx,my,col[5];
PetscInt nloc,xs,ys,xm,ym,gxs,gys,gxm,gym,grow;
PetscScalar two = 2.0,one = 1.0,lambda,v[5],hx,hy,hxdhy,hydhx,sc,*x;
mx = user->mx; my = user->my; lambda = user->param;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
sc = hx*hy; hxdhy = hx/hy; hydhx = hy/hx;
/*
Scatter ghost points to local vector, using the 2-step process
DMGlobalToLocalBegin(), DMGlobalToLocalEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = DMGlobalToLocalBegin(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
/*
Get pointer to vector data
*/
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->da,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
/*
Get the global node numbers for all local nodes, including ghost points
*/
ierr = DMDAGetGlobalIndices(user->da,&nloc,<og);CHKERRQ(ierr);
/*
Compute entries for the locally owned part of the Jacobian.
- Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors. The "grow"
parameter computed below specifies the global row number
corresponding to each local grid point.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Always specify global row and columns of matrix entries.
- Here, we set all entries for a particular row at once.
*/
for (j=ys; j<ys+ym; j++) {
row = (j - gys)*gxm + xs - gxs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
/* boundary points */
if (i == 0 || j == 0 || i == mx-1 || j == my-1) {
ierr = MatSetValues(jac,1,&grow,1,&grow,&one,INSERT_VALUES);CHKERRQ(ierr);
continue;
}
/* interior grid points */
v[0] = -hxdhy; col[0] = ltog[row - gxm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy) - sc*lambda*PetscExpScalar(x[row]); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + gxm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = DMDARestoreGlobalIndices(user->da,&nloc,<og);CHKERRQ(ierr);
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscErrorCode ierr;
PetscInt M = 10,N = 8,m = PETSC_DECIDE;
PetscInt s=2,w=2,n = PETSC_DECIDE,nloc,l,i,j,kk;
PetscInt Xs,Xm,Ys,Ym,iloc,*iglobal,*ltog;
PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL;
PetscBool testorder = PETSC_FALSE,flg;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by= DMDA_BOUNDARY_NONE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDAStencilType st = DMDA_STENCIL_BOX;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,400,&viewer);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(PETSC_NULL,"-NX",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NY",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-w",&w,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xperiodic",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) bx = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yperiodic",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) by = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xghosted",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) bx = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yghosted",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) by = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-box",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-testorder",&testorder,PETSC_NULL);CHKERRQ(ierr);
/*
Test putting two nodes in x and y on each processor, exact last processor
in x and y gets the rest.
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-distribute",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc(m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc(n*sizeof(PetscInt),&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,st,M,N,m,n,w,s,lx,ly,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!testorder) { /* turn off printing when testing ordering mappings */
ierr = PetscPrintf (PETSC_COMM_WORLD,"\nGlobal Vectors:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf (PETSC_COMM_WORLD,"\n\n");CHKERRQ(ierr);
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-local_print",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (testorder) {
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,PETSC_NULL,&Xm,&Ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(da,&nloc,<og);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = PetscMalloc(nloc*sizeof(PetscInt),&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscFPrintf(PETSC_COMM_SELF,stdout,"[%d] Problem with mapping: j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",
rank,j,i,l,ltog[iloc+l],iglobal[kk]);}
kk++;
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 3,N = 5,P=3,s=1,w=2,nloc,l,i,j,k,kk,m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE;
PetscErrorCode ierr;
PetscInt Xs,Xm,Ys,Ym,Zs,Zm,iloc,*iglobal;
const PetscInt *ltog;
PetscInt *lx = NULL,*ly = NULL,*lz = NULL;
PetscBool test_order = PETSC_FALSE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE,bz = DM_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
AO ao;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,300,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(NULL,"-NX",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-NY",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-NZ",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-p",&p,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-w",&w,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-star",&flg,NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-box",&flg,NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-xperiodic",&flg,NULL);CHKERRQ(ierr);
if (flg) bx = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-xghosted",&flg,NULL);CHKERRQ(ierr);
if (flg) bx = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-xnonghosted",&flg,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-yperiodic",&flg,NULL);CHKERRQ(ierr);
if (flg) by = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-yghosted",&flg,NULL);CHKERRQ(ierr);
if (flg) by = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-ynonghosted",&flg,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-zperiodic",&flg,NULL);CHKERRQ(ierr);
if (flg) bz = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-zghosted",&flg,NULL);CHKERRQ(ierr);
if (flg) bz = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-znonghosted",&flg,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-testorder",&test_order,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-distribute",&flg,NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) lx[i] = 4;
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) ly[i] = 2;
ly[n-1] = N - 2*(n-1);
if (p == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -p option with -distribute option");
ierr = PetscMalloc1(p,&lz);CHKERRQ(ierr);
for (i=0; i<p-1; i++) lz[i] = 2;
lz[p-1] = P - 2*(p-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,m,n,p,w,s,lx,ly,lz,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = PetscFree(lz);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!test_order) { /* turn off printing when testing ordering mappings */
if (M*N*P<40) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (test_order) {
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,&Zs,&Xm,&Ym,&Zm);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(da,&nloc,<og);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
/* ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = PetscMalloc1(nloc,&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y, then z) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscPrintf(MPI_COMM_WORLD,"[%D] Problem with mapping: z=%D, j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",
rank,k,j,i,l,ltog[iloc+l],iglobal[kk]);
}
kk++;
}
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
ierr = DMDARestoreGlobalIndices(da,&nloc,<og);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
