/*@
DMDAVecRestoreArrayDOFRead - Restores a multiple dimension array obtained with DMDAVecGetArrayDOFRead()
Not Collective
Input Parameter:
+ da - the distributed array
. vec - the vector, either a vector the same size as one obtained with
DMCreateGlobalVector() or DMCreateLocalVector()
- array - the array
Level: intermediate
.keywords: distributed array, get, corners, nodes, local indices, coordinates
.seealso: DMDAGetGhostCorners(), DMDAGetCorners(), VecGetArray(), VecRestoreArray(), DMDAVecGetArray(), DMDAVecGetArrayDOF(), DMDAVecRestoreArrayDOF()
@*/
PetscErrorCode DMDAVecRestoreArrayDOFRead(DM da,Vec vec,void *array)
{
PetscErrorCode ierr;
PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof;
PetscFunctionBegin;
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gxm,&gym,&gzm);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,&dim,0,0,0,0,0,0,&dof,0,0,0,0,0);CHKERRQ(ierr);
/* Handle case where user passes in global vector as opposed to local */
ierr = VecGetLocalSize(vec,&N);CHKERRQ(ierr);
if (N == xm*ym*zm*dof) {
gxm = xm;
gym = ym;
gzm = zm;
gxs = xs;
gys = ys;
gzs = zs;
}
if (dim == 1) {
ierr = VecRestoreArray2dRead(vec,gxm,dof,gxs,0,(PetscScalar***)array);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = VecRestoreArray3dRead(vec,gym,gxm,dof,gys,gxs,0,(PetscScalar****)array);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = VecRestoreArray4dRead(vec,gzm,gym,gxm,dof,gzs,gys,gxs,0,(PetscScalar*****)array);CHKERRQ(ierr);
} else SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"DMDA dimension not 1, 2, or 3, it is %D\n",dim);
PetscFunctionReturn(0);
}
static PetscErrorCode DMDAGetElements_1D(DM dm,PetscInt *nel,PetscInt *nen,const PetscInt *e[])
{
PetscErrorCode ierr;
DM_DA *da = (DM_DA*)dm->data;
PetscInt i,xs,xe,Xs,Xe;
PetscInt cnt=0;
PetscFunctionBegin;
if (!da->e) {
if (!da->s) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Cannot get elements for DMDA with zero stencil width");
ierr = DMDAGetCorners(dm,&xs,0,0,&xe,0,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(dm,&Xs,0,0,&Xe,0,0);CHKERRQ(ierr);
xe += xs; Xe += Xs; if (xs != Xs) xs -= 1;
da->ne = 1*(xe - xs - 1);
ierr = PetscMalloc1(1 + 2*da->ne,&da->e);CHKERRQ(ierr);
for (i=xs; i<xe-1; i++) {
da->e[cnt++] = (i-Xs);
da->e[cnt++] = (i-Xs+1);
}
}
*nel = da->ne;
*nen = 2;
*e = da->e;
PetscFunctionReturn(0);
}
/*
FormInitialGuess - Computes an initial approximation to the solution.
Input Parameters:
. user - user-defined application context
. X - vector
Output Parameters:
X - vector
*/
PetscErrorCode FormInitialGuess(AppCtx *user,Vec X)
{
PetscErrorCode ierr;
PetscInt i, j, k, mx = user->mx, my = user->my;
PetscInt xs, ys, xm, ym, gxm, gym, gxs, gys, xe, ye;
PetscReal hx = 1.0/(mx+1), hy = 1.0/(my+1), temp, val;
PetscFunctionBegin;
/* Get local mesh boundaries */
ierr = DMDAGetCorners(user->dm,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->dm,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
/* Compute initial guess over locally owned part of mesh */
xe = xs+xm;
ye = ys+ym;
for (j=ys; j<ye; j++) { /* for (j=0; j<my; j++) */
temp = PetscMin(j+1,my-j)*hy;
for (i=xs; i<xe; i++) { /* for (i=0; i<mx; i++) */
k = (j-gys)*gxm + i-gxs;
val = PetscMin((PetscMin(i+1,mx-i))*hx,temp);
ierr = VecSetValuesLocal(X,1,&k,&val,ADD_VALUES);CHKERRQ(ierr);
}
}
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormInitialGuess - Forms initial approximation.
Input Parameters:
user - user-defined application context
X - vector
Output Parameter:
X - vector
*/
PetscErrorCode FormInitialGuess(AppCtx *user,Vec X)
{
PetscInt i,j,row,mx,my,xs,ys,xm,ym,gxm,gym,gxs,gys;
PetscErrorCode ierr;
PetscReal one = 1.0,lambda,temp1,temp,hx,hy,hxdhy,hydhx,sc;
PetscScalar *x;
Vec localX = user->localX;
mx = user->mx; my = user->my; lambda = user->param;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
sc = hx*hy*lambda; hxdhy = hx/hy; hydhx = hy/hx;
temp1 = lambda/(lambda + one);
/*
Get a pointer to vector data.
- For default PETSc vectors,VecGetArray() returns a pointer to
the data array. Otherwise, the routine is implementation dependent.
- You MUST call VecRestoreArray() when you no longer need access to
the array.
*/
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/*
Get local grid boundaries (for 2-dimensional DMDA):
xs, ys - starting grid indices (no ghost points)
xm, ym - widths of local grid (no ghost points)
gxs, gys - starting grid indices (including ghost points)
gxm, gym - widths of local grid (including ghost points)
*/
ierr = DMDAGetCorners(user->da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,PETSC_NULL,&gxm,&gym,PETSC_NULL);CHKERRQ(ierr);
/*
Compute initial guess over the locally owned part of the grid
*/
for (j=ys; j<ys+ym; j++) {
temp = (PetscReal)(PetscMin(j,my-j-1))*hy;
for (i=xs; i<xs+xm; i++) {
row = i - gxs + (j - gys)*gxm;
if (i == 0 || j == 0 || i == mx-1 || j == my-1) {
x[row] = 0.0;
continue;
}
x[row] = temp1*PetscSqrtReal(PetscMin((PetscReal)(PetscMin(i,mx-i-1))*hx,temp));
}
}
/*
Restore vector
*/
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
/*
Insert values into global vector
*/
ierr = DMLocalToGlobalBegin(user->da,localX,INSERT_VALUES,X);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(user->da,localX,INSERT_VALUES,X);CHKERRQ(ierr);
return 0;
}
/*
ComputeFunction - Evaluates nonlinear function, F(x).
Input Parameters:
. X - input vector
. user - user-defined application context
Output Parameter:
. F - function vector
*/
PetscErrorCode ComputeFunction(AppCtx *user,Vec X,Vec F)
{
PetscErrorCode ierr;
PetscInt i,j,row,mx,my,xs,ys,xm,ym,gxs,gys,gxm,gym;
PetscReal two = 2.0,one = 1.0,lambda,hx,hy,hxdhy,hydhx,sc;
PetscScalar u,uxx,uyy,*x,*f;
Vec localX = user->localX;
mx = user->mx; my = user->my; lambda = user->param;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
sc = hx*hy*lambda; hxdhy = hx/hy; hydhx = hy/hx;
/*
Scatter ghost points to local vector, using the 2-step process
DMGlobalToLocalBegin(), DMGlobalToLocalEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = DMGlobalToLocalBegin(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
/*
Get pointers to vector data
*/
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
ierr = VecGetArray(F,&f);CHKERRQ(ierr);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->da,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
/*
Compute function over the locally owned part of the grid
*/
for (j=ys; j<ys+ym; j++) {
row = (j - gys)*gxm + xs - gxs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
if (i == 0 || j == 0 || i == mx-1 || j == my-1) {
f[row] = x[row];
continue;
}
u = x[row];
uxx = (two*u - x[row-1] - x[row+1])*hydhx;
uyy = (two*u - x[row-gxm] - x[row+gxm])*hxdhy;
f[row] = uxx + uyy - sc*PetscExpScalar(u);
}
}
/*
Restore vectors
*/
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = VecRestoreArray(F,&f);CHKERRQ(ierr);
ierr = PetscLogFlops(11.0*ym*xm);CHKERRQ(ierr);
return 0;
}
PetscErrorCode VTKIO_VTI_Write(FD3D_Parameters *p, DM da, const char fieldname[], const char file_prefix[],const char local_file_prefix[])
{
MPI_Comm comm;
PetscMPIInt size,rank;
char vtk_filename[PETSC_MAX_PATH_LEN];
FILE *vtk_fp = NULL;
PetscInt M,N,P,si,sj,sk,nx,ny,nz;
PetscInt i,dofs;
PetscErrorCode ierr;
PetscFunctionBeginUser;
/* only master generates this file */
PetscObjectGetComm((PetscObject)da,&comm);
MPI_Comm_size(comm,&size);
MPI_Comm_rank(comm,&rank);
if (rank != 0) PetscFunctionReturn(0);
/* create file name */
ierr = PetscSNPrintf(vtk_filename,sizeof(vtk_filename),"%s.pvti",file_prefix);CHKERRQ(ierr);
vtk_fp = fopen(vtk_filename,"w");
if (!vtk_fp) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"Cannot open file = %s \n",vtk_filename);
/* (VTK) generate pvts header */
PetscFPrintf(PETSC_COMM_SELF,vtk_fp,"<?xml version=\"1.0\"?>\n");
#if defined(PETSC_WORDS_BIGENDIAN)
PetscFPrintf(PETSC_COMM_SELF,vtk_fp,"<VTKFile type=\"PImageData\" version=\"0.1\" byte_order=\"BigEndian\">\n");
#else
PetscFPrintf(PETSC_COMM_SELF,vtk_fp,"<VTKFile type=\"PImageData\" version=\"0.1\" byte_order=\"LittleEndian\">\n");
#endif
/* define size of the nodal mesh based on the cell DM */
ierr = DMDAGetInfo(da,0,&M,&N,&P,0,0,0,&dofs,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&si,&sj,&sk,&nx,&ny,&nz);CHKERRQ(ierr);
PetscFPrintf(PETSC_COMM_SELF,vtk_fp," <PImageData GhostLevel=\"4\" WholeExtent=\"%d %d %d %d %d %d\" Origin=\"0 0 0\" Spacing=\"%f %f %f\">\n",0,M-1,0,N-1,0,P-1,p->delta_xyz, p->delta_xyz,p->delta_xyz); /* note overlap = 1 for Q1 */
/* DUMP THE CELL REFERENCES */
PetscFPrintf(PETSC_COMM_SELF,vtk_fp," <PCellData>\n");
PetscFPrintf(PETSC_COMM_SELF,vtk_fp," </PCellData>\n");
PetscFPrintf(PETSC_COMM_SELF,vtk_fp," <PPointData>\n");
PetscFPrintf(PETSC_COMM_SELF,vtk_fp," <PDataArray type=\"Float64\" Name=\"%s\" NumberOfComponents=\"1\"/>\n",fieldname);
PetscFPrintf(PETSC_COMM_SELF,vtk_fp," </PPointData>\n");
/* write out the parallel information */
ierr = VTKIO_VTI_PieceExtend(vtk_fp,2,da,local_file_prefix);CHKERRQ(ierr);
/* close the file */
PetscFPrintf(PETSC_COMM_SELF,vtk_fp," </PImageData>\n");
PetscFPrintf(PETSC_COMM_SELF,vtk_fp,"</VTKFile>\n");
if (vtk_fp) {
fclose(vtk_fp);
vtk_fp = NULL;
}
PetscFunctionReturn(0);
}
PETSC_EXTERN void PETSC_STDCALL dmdagetghostcorners_(DM *da,PetscInt *x,PetscInt *y,PetscInt *z,PetscInt *m,PetscInt *n,PetscInt *p, int *ierr )
{
CHKFORTRANNULLINTEGER(y);
CHKFORTRANNULLINTEGER(z);
CHKFORTRANNULLINTEGER(n);
CHKFORTRANNULLINTEGER(p);
*ierr = DMDAGetGhostCorners(*da,x,y,z,m,n,p);
}
static PetscErrorCode DMDAGetElements_3D(DM dm,PetscInt *nel,PetscInt *nen,const PetscInt *e[])
{
PetscErrorCode ierr;
DM_DA *da = (DM_DA*)dm->data;
PetscInt i,xs,xe,Xs,Xe;
PetscInt j,ys,ye,Ys,Ye;
PetscInt k,zs,ze,Zs,Ze;
PetscInt cnt=0, cell[8], ns=6, nn=4;
PetscInt c, split[] = {0,1,3,7,
0,1,7,4,
1,2,3,7,
1,2,7,6,
1,4,5,7,
1,5,6,7};
PetscFunctionBegin;
if (!da->e) {
if (!da->s) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Cannot get elements for DMDA with zero stencil width");
if (da->elementtype == DMDA_ELEMENT_P1) {ns=6; nn=4;}
if (da->elementtype == DMDA_ELEMENT_Q1) {ns=1; nn=8;}
ierr = DMDAGetCorners(dm,&xs,&ys,&zs,&xe,&ye,&ze);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(dm,&Xs,&Ys,&Zs,&Xe,&Ye,&Ze);CHKERRQ(ierr);
xe += xs; Xe += Xs; if (xs != Xs) xs -= 1;
ye += ys; Ye += Ys; if (ys != Ys) ys -= 1;
ze += zs; Ze += Zs; if (zs != Zs) zs -= 1;
da->ne = ns*(xe - xs - 1)*(ye - ys - 1)*(ze - zs - 1);
ierr = PetscMalloc1(1 + nn*da->ne,&da->e);CHKERRQ(ierr);
for (k=zs; k<ze-1; k++) {
for (j=ys; j<ye-1; j++) {
for (i=xs; i<xe-1; i++) {
cell[0] = (i-Xs ) + (j-Ys )*(Xe-Xs) + (k-Zs )*(Xe-Xs)*(Ye-Ys);
cell[1] = (i-Xs+1) + (j-Ys )*(Xe-Xs) + (k-Zs )*(Xe-Xs)*(Ye-Ys);
cell[2] = (i-Xs+1) + (j-Ys+1)*(Xe-Xs) + (k-Zs )*(Xe-Xs)*(Ye-Ys);
cell[3] = (i-Xs ) + (j-Ys+1)*(Xe-Xs) + (k-Zs )*(Xe-Xs)*(Ye-Ys);
cell[4] = (i-Xs ) + (j-Ys )*(Xe-Xs) + (k-Zs+1)*(Xe-Xs)*(Ye-Ys);
cell[5] = (i-Xs+1) + (j-Ys )*(Xe-Xs) + (k-Zs+1)*(Xe-Xs)*(Ye-Ys);
cell[6] = (i-Xs+1) + (j-Ys+1)*(Xe-Xs) + (k-Zs+1)*(Xe-Xs)*(Ye-Ys);
cell[7] = (i-Xs ) + (j-Ys+1)*(Xe-Xs) + (k-Zs+1)*(Xe-Xs)*(Ye-Ys);
if (da->elementtype == DMDA_ELEMENT_P1) {
for (c=0; c<ns*nn; c++) da->e[cnt++] = cell[split[c]];
}
if (da->elementtype == DMDA_ELEMENT_Q1) {
for (c=0; c<ns*nn; c++) da->e[cnt++] = cell[c];
}
}
}
}
}
*nel = da->ne;
*nen = nn;
*e = da->e;
PetscFunctionReturn(0);
}
PetscErrorCode FormJacobian_Grid(AppCtx *user,GridCtx *grid,Vec X, Mat *J,Mat *B)
{
Mat jac = *J;
PetscErrorCode ierr;
PetscInt i, j, row, mx, my, xs, ys, xm, ym, Xs, Ys, Xm, Ym, col[5], nloc, *ltog, grow;
PetscScalar two = 2.0, one = 1.0, lambda, v[5], hx, hy, hxdhy, hydhx, sc, *x, value;
Vec localX = grid->localX;
mx = grid->mx; my = grid->my; lambda = user->param;
hx = one/(double)(mx-1); hy = one/(double)(my-1);
sc = hx*hy; hxdhy = hx/hy; hydhx = hy/hx;
/* Get ghost points */
ierr = DMGlobalToLocalBegin(grid->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(grid->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMDAGetCorners(grid->da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(grid->da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(grid->da,&nloc,<og);CHKERRQ(ierr);
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/* Evaluate Jacobian of function */
for (j=ys; j<ys+ym; j++) {
row = (j - Ys)*Xm + xs - Xs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
if (i > 0 && i < mx-1 && j > 0 && j < my-1) {
v[0] = -hxdhy; col[0] = ltog[row - Xm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy) - sc*lambda*exp(x[row]); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + Xm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else if ((i > 0 && i < mx-1) || (j > 0 && j < my-1)){
value = .5*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
} else {
value = .25*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
int FormJacobian_Grid(GridCtx *grid,Mat *J)
{
Mat jac = *J;
PetscErrorCode ierr;
PetscInt i,j,row,mx,my,xs,ys,xm,ym,Xs,Ys,Xm,Ym,col[5];
PetscInt grow;
const PetscInt *ltog;
PetscScalar two = 2.0,one = 1.0,v[5],hx,hy,hxdhy,hydhx,value;
ISLocalToGlobalMapping ltogm;
mx = grid->mx; my = grid->my;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
hxdhy = hx/hy; hydhx = hy/hx;
/* Get ghost points */
ierr = DMDAGetCorners(grid->da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(grid->da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(grid->da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
/* Evaluate Jacobian of function */
for (j=ys; j<ys+ym; j++) {
row = (j - Ys)*Xm + xs - Xs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
if (i > 0 && i < mx-1 && j > 0 && j < my-1) {
v[0] = -hxdhy; col[0] = ltog[row - Xm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + Xm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else if ((i > 0 && i < mx-1) || (j > 0 && j < my-1)) {
value = .5*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
} else {
value = .25*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
PetscErrorCode FormFunction(SNES snes,Vec X,Vec F,void *ptr)
{
AppCtx *user = (AppCtx *) ptr;
PetscInt i, j, row, mx, my, xs, ys, xm, ym, Xs, Ys, Xm, Ym;
PetscErrorCode ierr;
double two = 2.0, one = 1.0, lambda,hx, hy, hxdhy, hydhx,sc;
PetscScalar u, uxx, uyy, *x,*f;
Vec localX = user->fine.localX, localF = user->fine.localF;
mx = user->fine.mx; my = user->fine.my; lambda = user->param;
hx = one/(double)(mx-1); hy = one/(double)(my-1);
sc = hx*hy*lambda; hxdhy = hx/hy; hydhx = hy/hx;
/* Get ghost points */
ierr = DMGlobalToLocalBegin(user->fine.da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->fine.da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMDAGetCorners(user->fine.da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->fine.da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
ierr = VecGetArray(localF,&f);CHKERRQ(ierr);
/* Evaluate function */
for (j=ys; j<ys+ym; j++) {
row = (j - Ys)*Xm + xs - Xs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
if (i > 0 && i < mx-1 && j > 0 && j < my-1) {
u = x[row];
uxx = (two*u - x[row-1] - x[row+1])*hydhx;
uyy = (two*u - x[row-Xm] - x[row+Xm])*hxdhy;
f[row] = uxx + uyy - sc*exp(u);
} else if ((i > 0 && i < mx-1) || (j > 0 && j < my-1)){
f[row] = .5*two*(hydhx + hxdhy)*x[row];
} else {
f[row] = .25*two*(hydhx + hxdhy)*x[row];
}
}
}
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = VecRestoreArray(localF,&f);CHKERRQ(ierr);
/* Insert values into global vector */
ierr = DMLocalToGlobalBegin(user->fine.da,localF,INSERT_VALUES,F);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(user->fine.da,localF,INSERT_VALUES,F);CHKERRQ(ierr);
ierr = PetscLogFlops(11.0*ym*xm);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode DMDAGetElements_2D(DM dm,PetscInt *nel,PetscInt *nen,const PetscInt *e[])
{
PetscErrorCode ierr;
DM_DA *da = (DM_DA*)dm->data;
PetscInt i,xs,xe,Xs,Xe;
PetscInt j,ys,ye,Ys,Ye;
PetscInt cnt=0, cell[4], ns=2, nn=3;
PetscInt c, split[] = {0,1,3,
2,3,1};
PetscFunctionBegin;
if (!da->e) {
if (da->elementtype == DMDA_ELEMENT_P1) {ns=2; nn=3;}
if (da->elementtype == DMDA_ELEMENT_Q1) {ns=1; nn=4;}
ierr = DMDAGetCorners(dm,&xs,&ys,0,&xe,&ye,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(dm,&Xs,&Ys,0,&Xe,&Ye,0);CHKERRQ(ierr);
xe += xs; Xe += Xs; if (xs != Xs) xs -= 1;
ye += ys; Ye += Ys; if (ys != Ys) ys -= 1;
da->ne = ns*(xe - xs - 1)*(ye - ys - 1);
ierr = PetscMalloc((1 + nn*da->ne)*sizeof(PetscInt),&da->e);CHKERRQ(ierr);
for (j=ys; j<ye-1; j++) {
for (i=xs; i<xe-1; i++) {
cell[0] = (i-Xs ) + (j-Ys )*(Xe-Xs);
cell[1] = (i-Xs+1) + (j-Ys )*(Xe-Xs);
cell[2] = (i-Xs+1) + (j-Ys+1)*(Xe-Xs);
cell[3] = (i-Xs ) + (j-Ys+1)*(Xe-Xs);
if (da->elementtype == DMDA_ELEMENT_P1) {
for (c=0; c<ns*nn; c++)
da->e[cnt++] = cell[split[c]];
}
if (da->elementtype == DMDA_ELEMENT_Q1) {
for (c=0; c<ns*nn; c++)
da->e[cnt++] = cell[c];
}
}
}
}
*nel = da->ne;
*nen = nn;
*e = da->e;
PetscFunctionReturn(0);
}
PetscErrorCode FormInitialGuess1(AppCtx *user,Vec X)
{
PetscInt i, j, row, mx, my, xs, ys, xm, ym, Xm, Ym, Xs, Ys;
PetscErrorCode ierr;
double one = 1.0, lambda, temp1, temp, hx, hy;
/* double hxdhy, hydhx,sc; */
PetscScalar *x;
Vec localX = user->fine.localX;
mx = user->fine.mx; my = user->fine.my; lambda = user->param;
hx = one/(double)(mx-1); hy = one/(double)(my-1);
/* sc = hx*hy*lambda; hxdhy = hx/hy; hydhx = hy/hx; */
temp1 = lambda/(lambda + one);
/* Get ghost points */
ierr = DMDAGetCorners(user->fine.da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->fine.da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/* Compute initial guess */
for (j=ys; j<ys+ym; j++) {
temp = (double)(PetscMin(j,my-j-1))*hy;
for (i=xs; i<xs+xm; i++) {
row = i - Xs + (j - Ys)*Xm;
if (i == 0 || j == 0 || i == mx-1 || j == my-1 ) {
x[row] = 0.0;
continue;
}
x[row] = temp1*PetscSqrtReal( PetscMin( (double)(PetscMin(i,mx-i-1))*hx,temp) );
}
}
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
/* Insert values into global vector */
ierr = DMLocalToGlobalBegin(user->fine.da,localX,INSERT_VALUES,X);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(user->fine.da,localX,INSERT_VALUES,X);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode DMDAGetElements_1D(DM dm,PetscInt *nel,PetscInt *nen,const PetscInt *e[])
{
PetscErrorCode ierr;
DM_DA *da = (DM_DA*)dm->data;
PetscInt i,xs,xe,Xs,Xe;
PetscInt cnt=0;
PetscFunctionBegin;
if (!da->e) {
ierr = DMDAGetCorners(dm,&xs,0,0,&xe,0,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(dm,&Xs,0,0,&Xe,0,0);CHKERRQ(ierr);
xe += xs; Xe += Xs; if (xs != Xs) xs -= 1;
da->ne = 1*(xe - xs - 1);
ierr = PetscMalloc((1 + 2*da->ne)*sizeof(PetscInt),&da->e);CHKERRQ(ierr);
for (i=xs; i<xe-1; i++) {
da->e[cnt++] = (i-Xs );
da->e[cnt++] = (i-Xs+1);
}
}
*nel = da->ne;
*nen = 2;
*e = da->e;
PetscFunctionReturn(0);
}
PetscErrorCode ComputeB(AppCtx* user)
{
PetscErrorCode ierr;
PetscInt i,j,k;
PetscInt nx,ny,xs,xm,gxs,gxm,ys,ym,gys,gym;
PetscReal two=2.0, pi=4.0*atan(1.0);
PetscReal hx,hy,ehxhy;
PetscReal temp,*b;
PetscReal ecc=user->ecc;
nx=user->nx;
ny=user->ny;
hx=two*pi/(nx+1.0);
hy=two*user->b/(ny+1.0);
ehxhy = ecc*hx*hy;
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->dm,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->dm,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
/* Compute the linear term in the objective function */
ierr = VecGetArray(user->B,&b);CHKERRQ(ierr);
for (i=xs; i<xs+xm; i++){
temp=PetscSinScalar((i+1)*hx);
for (j=ys; j<ys+ym; j++){
k=xm*(j-ys)+(i-xs);
b[k]= - ehxhy*temp;
}
}
ierr = VecRestoreArray(user->B,&b);CHKERRQ(ierr);
ierr = PetscLogFlops(5*xm*ym+3*xm);CHKERRQ(ierr);
return 0;
}
/*@C
DMDAVecGetArray - Returns a multiple dimension array that shares data with
the underlying vector and is indexed using the global dimensions.
Logically collective on Vec
Input Parameter:
+ da - the distributed array
- vec - the vector, either a vector the same size as one obtained with DMCreateGlobalVector() or DMCreateLocalVector()
Output Parameter:
. array - the array
Notes:
Call DMDAVecRestoreArray() once you have finished accessing the vector entries.
In C, the indexing is "backwards" from what expects: array[k][j][i] NOT array[i][j][k]!
If vec is a local vector (obtained with DMCreateLocalVector() etc) then the ghost point locations are accessible. If it is
a global vector then the ghost points are not accessible. Of course with the local vector you will have had to do the
appropriate DMGlobalToLocalBegin() and DMGlobalToLocalEnd() to have correct values in the ghost locations.
Fortran Notes: From Fortran use DMDAVecGetArrayF90() and pass for the array type PetscScalar,pointer :: array(:,...,:) of the appropriate
dimension. For a DMDA created with a dof of 1 use the dimension of the DMDA, for a DMDA created with a dof greater than 1 use one more than the
dimension of the DMDA. The order of the indices is array(xs:xs+xm-1,ys:ys+ym-1,zs:zs+zm-1) (when dof is 1) otherwise
array(0:dof-1,xs:xs+xm-1,ys:ys+ym-1,zs:zs+zm-1) where the values are obtained from
DMDAGetCorners() for a global array or DMDAGetGhostCorners() for a local array. Include petsc/finclude/petscdmda.h90 to access this routine.
Due to bugs in the compiler DMDAVecGetArrayF90() does not work with gfortran versions before 4.5
Level: intermediate
.keywords: distributed array, get, corners, nodes, local indices, coordinates
.seealso: DMDAGetGhostCorners(), DMDAGetCorners(), VecGetArray(), VecRestoreArray(), DMDAVecRestoreArray(), DMDAVecRestoreArrayDOF()
DMDAVecGetArrayDOF()
@*/
PetscErrorCode DMDAVecGetArray(DM da,Vec vec,void *array)
{
PetscErrorCode ierr;
PetscInt xs,ys,zs,xm,ym,zm,gxs,gys,gzs,gxm,gym,gzm,N,dim,dof;
PetscFunctionBegin;
PetscValidHeaderSpecific(da, DM_CLASSID, 1);
PetscValidHeaderSpecific(vec, VEC_CLASSID, 2);
PetscValidPointer(array, 3);
if (da->defaultSection) {
ierr = VecGetArray(vec,(PetscScalar**)array);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gxm,&gym,&gzm);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,&dim,0,0,0,0,0,0,&dof,0,0,0,0,0);CHKERRQ(ierr);
/* Handle case where user passes in global vector as opposed to local */
ierr = VecGetLocalSize(vec,&N);CHKERRQ(ierr);
if (N == xm*ym*zm*dof) {
gxm = xm;
gym = ym;
gzm = zm;
gxs = xs;
gys = ys;
gzs = zs;
} else if (N != gxm*gym*gzm*dof) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Vector local size %D is not compatible with DMDA local sizes %D %D\n",N,xm*ym*zm*dof,gxm*gym*gzm*dof);
if (dim == 1) {
ierr = VecGetArray1d(vec,gxm*dof,gxs*dof,(PetscScalar**)array);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = VecGetArray2d(vec,gym,gxm*dof,gys,gxs*dof,(PetscScalar***)array);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = VecGetArray3d(vec,gzm,gym,gxm*dof,gzs,gys,gxs*dof,(PetscScalar****)array);CHKERRQ(ierr);
} else SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_CORRUPT,"DMDA dimension not 1, 2, or 3, it is %D\n",dim);
PetscFunctionReturn(0);
}
PetscErrorCode FormFunctionGradient(Tao tao, Vec X, PetscReal *fcn,Vec G,void *ptr)
{
AppCtx* user=(AppCtx*)ptr;
PetscErrorCode ierr;
PetscInt i,j,k,kk;
PetscInt col[5],row,nx,ny,xs,xm,gxs,gxm,ys,ym,gys,gym;
PetscReal one=1.0, two=2.0, six=6.0,pi=4.0*atan(1.0);
PetscReal hx,hy,hxhy,hxhx,hyhy;
PetscReal xi,v[5];
PetscReal ecc=user->ecc, trule1,trule2,trule3,trule4,trule5,trule6;
PetscReal vmiddle, vup, vdown, vleft, vright;
PetscReal tt,f1,f2;
PetscReal *x,*g,zero=0.0;
Vec localX;
nx=user->nx;
ny=user->ny;
hx=two*pi/(nx+1.0);
hy=two*user->b/(ny+1.0);
hxhy=hx*hy;
hxhx=one/(hx*hx);
hyhy=one/(hy*hy);
ierr = DMGetLocalVector(user->dm,&localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(user->dm,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->dm,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = VecSet(G, zero);CHKERRQ(ierr);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->dm,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->dm,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
ierr = VecGetArray(G,&g);CHKERRQ(ierr);
for (i=xs; i< xs+xm; i++){
xi=(i+1)*hx;
trule1=hxhy*( p(xi,ecc) + p(xi+hx,ecc) + p(xi,ecc) ) / six; /* L(i,j) */
trule2=hxhy*( p(xi,ecc) + p(xi-hx,ecc) + p(xi,ecc) ) / six; /* U(i,j) */
trule3=hxhy*( p(xi,ecc) + p(xi+hx,ecc) + p(xi+hx,ecc) ) / six; /* U(i+1,j) */
trule4=hxhy*( p(xi,ecc) + p(xi-hx,ecc) + p(xi-hx,ecc) ) / six; /* L(i-1,j) */
trule5=trule1; /* L(i,j-1) */
trule6=trule2; /* U(i,j+1) */
vdown=-(trule5+trule2)*hyhy;
vleft=-hxhx*(trule2+trule4);
vright= -hxhx*(trule1+trule3);
vup=-hyhy*(trule1+trule6);
vmiddle=(hxhx)*(trule1+trule2+trule3+trule4)+hyhy*(trule1+trule2+trule5+trule6);
for (j=ys; j<ys+ym; j++){
row=(j-gys)*gxm + (i-gxs);
v[0]=0; v[1]=0; v[2]=0; v[3]=0; v[4]=0;
k=0;
if (j>gys){
v[k]=vdown; col[k]=row - gxm; k++;
}
if (i>gxs){
v[k]= vleft; col[k]=row - 1; k++;
}
v[k]= vmiddle; col[k]=row; k++;
if (i+1 < gxs+gxm){
v[k]= vright; col[k]=row+1; k++;
}
if (j+1 <gys+gym){
v[k]= vup; col[k] = row+gxm; k++;
}
tt=0;
for (kk=0;kk<k;kk++){
tt+=v[kk]*x[col[kk]];
}
row=(j-ys)*xm + (i-xs);
g[row]=tt;
}
}
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = VecRestoreArray(G,&g);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(user->dm,&localX);CHKERRQ(ierr);
ierr = VecDot(X,G,&f1);CHKERRQ(ierr);
ierr = VecDot(user->B,X,&f2);CHKERRQ(ierr);
ierr = VecAXPY(G, one, user->B);CHKERRQ(ierr);
*fcn = f1/2.0 + f2;
ierr = PetscLogFlops((91 + 10*ym) * xm);CHKERRQ(ierr);
return 0;
}
/*
ComputeJacobian - Evaluates Jacobian matrix.
Input Parameters:
. x - input vector
. user - user-defined application context
Output Parameters:
. jac - Jacobian matrix
. flag - flag indicating matrix structure
Notes:
Due to grid point reordering with DMDAs, we must always work
with the local grid points, and then transform them to the new
global numbering with the "ltog" mapping
We cannot work directly with the global numbers for the original
uniprocessor grid!
*/
PetscErrorCode ComputeJacobian(AppCtx *user,Vec X,Mat jac)
{
PetscErrorCode ierr;
Vec localX = user->localX; /* local vector */
const PetscInt *ltog; /* local-to-global mapping */
PetscInt i,j,row,mx,my,col[5];
PetscInt xs,ys,xm,ym,gxs,gys,gxm,gym,grow;
PetscScalar two = 2.0,one = 1.0,lambda,v[5],hx,hy,hxdhy,hydhx,sc,*x;
ISLocalToGlobalMapping ltogm;
mx = user->mx; my = user->my; lambda = user->param;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
sc = hx*hy; hxdhy = hx/hy; hydhx = hy/hx;
/*
Scatter ghost points to local vector, using the 2-step process
DMGlobalToLocalBegin(), DMGlobalToLocalEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = DMGlobalToLocalBegin(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
/*
Get pointer to vector data
*/
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->da,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
/*
Get the global node numbers for all local nodes, including ghost points
*/
ierr = DMGetLocalToGlobalMapping(user->da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
/*
Compute entries for the locally owned part of the Jacobian.
- Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors. The "grow"
parameter computed below specifies the global row number
corresponding to each local grid point.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Always specify global row and columns of matrix entries.
- Here, we set all entries for a particular row at once.
*/
for (j=ys; j<ys+ym; j++) {
row = (j - gys)*gxm + xs - gxs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
/* boundary points */
if (i == 0 || j == 0 || i == mx-1 || j == my-1) {
ierr = MatSetValues(jac,1,&grow,1,&grow,&one,INSERT_VALUES);CHKERRQ(ierr);
continue;
}
/* interior grid points */
v[0] = -hxdhy; col[0] = ltog[row - gxm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy) - sc*lambda*PetscExpScalar(x[row]); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + gxm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode MatSetupDM_HYPREStruct(Mat mat,DM da)
{
PetscErrorCode ierr;
Mat_HYPREStruct *ex = (Mat_HYPREStruct*) mat->data;
PetscInt dim,dof,sw[3],nx,ny,nz,ilower[3],iupper[3],ssize,i;
DMBoundaryType px,py,pz;
DMDAStencilType st;
ISLocalToGlobalMapping ltog;
PetscFunctionBegin;
ex->da = da;
ierr = PetscObjectReference((PetscObject)da);CHKERRQ(ierr);
ierr = DMDAGetInfo(ex->da,&dim,0,0,0,0,0,0,&dof,&sw[0],&px,&py,&pz,&st);CHKERRQ(ierr);
ierr = DMDAGetCorners(ex->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);CHKERRQ(ierr);
iupper[0] += ilower[0] - 1;
iupper[1] += ilower[1] - 1;
iupper[2] += ilower[2] - 1;
/* the hypre_Box is used to zero out the matrix entries in MatZeroValues() */
ex->hbox.imin[0] = ilower[0];
ex->hbox.imin[1] = ilower[1];
ex->hbox.imin[2] = ilower[2];
ex->hbox.imax[0] = iupper[0];
ex->hbox.imax[1] = iupper[1];
ex->hbox.imax[2] = iupper[2];
/* create the hypre grid object and set its information */
if (dof > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Currently only support for scalar problems");
if (px || py || pz) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add periodic support by calling HYPRE_StructGridSetPeriodic()");
PetscStackCallStandard(HYPRE_StructGridCreate,(ex->hcomm,dim,&ex->hgrid));
PetscStackCallStandard(HYPRE_StructGridSetExtents,(ex->hgrid,(HYPRE_Int *)ilower,(HYPRE_Int *)iupper));
PetscStackCallStandard(HYPRE_StructGridAssemble,(ex->hgrid));
sw[1] = sw[0];
sw[2] = sw[1];
PetscStackCallStandard(HYPRE_StructGridSetNumGhost,(ex->hgrid,(HYPRE_Int *)sw));
/* create the hypre stencil object and set its information */
if (sw[0] > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for wider stencils");
if (st == DMDA_STENCIL_BOX) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for box stencils");
if (dim == 1) {
PetscInt offsets[3][1] = {{-1},{0},{1}};
ssize = 3;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
} else if (dim == 2) {
PetscInt offsets[5][2] = {{0,-1},{-1,0},{0,0},{1,0},{0,1}};
ssize = 5;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
} else if (dim == 3) {
PetscInt offsets[7][3] = {{0,0,-1},{0,-1,0},{-1,0,0},{0,0,0},{1,0,0},{0,1,0},{0,0,1}};
ssize = 7;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
}
/* create the HYPRE vector for rhs and solution */
PetscStackCallStandard(HYPRE_StructVectorCreate,(ex->hcomm,ex->hgrid,&ex->hb));
PetscStackCallStandard(HYPRE_StructVectorCreate,(ex->hcomm,ex->hgrid,&ex->hx));
PetscStackCallStandard(HYPRE_StructVectorInitialize,(ex->hb));
PetscStackCallStandard(HYPRE_StructVectorInitialize,(ex->hx));
PetscStackCallStandard(HYPRE_StructVectorAssemble,(ex->hb));
PetscStackCallStandard(HYPRE_StructVectorAssemble,(ex->hx));
/* create the hypre matrix object and set its information */
PetscStackCallStandard(HYPRE_StructMatrixCreate,(ex->hcomm,ex->hgrid,ex->hstencil,&ex->hmat));
PetscStackCallStandard(HYPRE_StructGridDestroy,(ex->hgrid));
PetscStackCallStandard(HYPRE_StructStencilDestroy,(ex->hstencil));
if (ex->needsinitialization) {
PetscStackCallStandard(HYPRE_StructMatrixInitialize,(ex->hmat));
ex->needsinitialization = PETSC_FALSE;
}
/* set the global and local sizes of the matrix */
ierr = DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);CHKERRQ(ierr);
ierr = MatSetSizes(mat,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->rmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->cmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
if (dim == 3) {
mat->ops->setvalueslocal = MatSetValuesLocal_HYPREStruct_3d;
mat->ops->zerorowslocal = MatZeroRowsLocal_HYPREStruct_3d;
mat->ops->zeroentries = MatZeroEntries_HYPREStruct_3d;
ierr = MatZeroEntries_HYPREStruct_3d(mat);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only support for 3d DMDA currently");
/* get values that will be used repeatedly in MatSetValuesLocal() and MatZeroRowsLocal() repeatedly */
ierr = MatGetOwnershipRange(mat,&ex->rstart,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(ex->da,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog, (const PetscInt **) &ex->gindices);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(ex->da,0,0,0,&ex->gnx,&ex->gnxgny,0);CHKERRQ(ierr);
ex->gnxgny *= ex->gnx;
ierr = DMDAGetCorners(ex->da,&ex->xs,&ex->ys,&ex->zs,&ex->nx,&ex->ny,0);CHKERRQ(ierr);
ex->nxny = ex->nx*ex->ny;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscErrorCode ierr;
PetscInt M = 10,N = 8,m = PETSC_DECIDE;
PetscInt s =2,w=2,n = PETSC_DECIDE,nloc,l,i,j,kk;
PetscInt Xs,Xm,Ys,Ym,iloc,*iglobal;
const PetscInt *ltog;
PetscInt *lx = NULL,*ly = NULL;
PetscBool testorder = PETSC_FALSE,flg;
DMBoundaryType bx = DM_BOUNDARY_NONE,by= DM_BOUNDARY_NONE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDAStencilType st = DMDA_STENCIL_BOX;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,400,&viewer);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(NULL,NULL,"-NX",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-NY",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-w",&w,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xghosted",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yghosted",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-star",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-box",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-testorder",&testorder,NULL);CHKERRQ(ierr);
/*
Test putting two nodes in x and y on each processor, exact last processor
in x and y gets the rest.
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-distribute",&flg,NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,st,M,N,m,n,w,s,lx,ly,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!testorder) { /* turn off printing when testing ordering mappings */
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vectors:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n");CHKERRQ(ierr);
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (testorder) {
ISLocalToGlobalMapping ltogm;
ierr = DMGetLocalToGlobalMapping(da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltogm,&nloc);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,NULL,&Xm,&Ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = PetscMalloc1(nloc,&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscFPrintf(PETSC_COMM_SELF,stdout,"[%d] Problem with mapping: j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",rank,j,i,l,ltog[iloc+l],iglobal[kk]);CHKERRQ(ierr);
}
kk++;
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*
Forms the interpolation (and restriction) operator from
coarse grid to fine.
*/
PetscErrorCode FormInterpolation(AppCtx *user)
{
PetscErrorCode ierr;
PetscInt i,j,i_start,m_fine,j_start,m,n,*idx;
PetscInt m_ghost,n_ghost,*idx_c,m_ghost_c,n_ghost_c,m_coarse;
PetscInt row,i_start_ghost,j_start_ghost,cols[4], m_c;
PetscInt nc,ratio = user->ratio,m_c_local,m_fine_local;
PetscInt i_c,j_c,i_start_c,j_start_c,n_c,i_start_ghost_c,j_start_ghost_c,col;
PetscScalar v[4],x,y, one = 1.0;
Mat mat;
Vec Rscale;
ierr = DMDAGetCorners(user->fine.da,&i_start,&j_start,0,&m,&n,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->fine.da,&i_start_ghost,&j_start_ghost,0,&m_ghost,&n_ghost,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(user->fine.da,PETSC_NULL,&idx);CHKERRQ(ierr);
ierr = DMDAGetCorners(user->coarse.da,&i_start_c,&j_start_c,0,&m_c,&n_c,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->coarse.da,&i_start_ghost_c,&j_start_ghost_c,0,&m_ghost_c,&n_ghost_c,0);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(user->coarse.da,PETSC_NULL,&idx_c);CHKERRQ(ierr);
/* create interpolation matrix */
ierr = VecGetLocalSize(user->fine.x,&m_fine_local);CHKERRQ(ierr);
ierr = VecGetLocalSize(user->coarse.x,&m_c_local);CHKERRQ(ierr);
ierr = VecGetSize(user->fine.x,&m_fine);CHKERRQ(ierr);
ierr = VecGetSize(user->coarse.x,&m_coarse);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,m_fine_local,m_c_local,m_fine,m_coarse,
5,0,3,0,&mat);CHKERRQ(ierr);
/* loop over local fine grid nodes setting interpolation for those*/
for ( j=j_start; j<j_start+n; j++ ) {
for ( i=i_start; i<i_start+m; i++ ) {
/* convert to local "natural" numbering and then to PETSc global numbering */
row = idx[m_ghost*(j-j_start_ghost) + (i-i_start_ghost)];
i_c = (i/ratio); /* coarse grid node to left of fine grid node */
j_c = (j/ratio); /* coarse grid node below fine grid node */
/*
Only include those interpolation points that are truly
nonzero. Note this is very important for final grid lines
in x and y directions; since they have no right/top neighbors
*/
x = ((double)(i - i_c*ratio))/((double)ratio);
y = ((double)(j - j_c*ratio))/((double)ratio);
nc = 0;
/* one left and below; or we are right on it */
if (j_c < j_start_ghost_c || j_c > j_start_ghost_c+n_ghost_c) {
SETERRQ3(PETSC_COMM_SELF,1,"Sorry j %D %D %D",j_c,j_start_ghost_c,j_start_ghost_c+n_ghost_c);
}
if (i_c < i_start_ghost_c || i_c > i_start_ghost_c+m_ghost_c) {
SETERRQ3(PETSC_COMM_SELF,1,"Sorry i %D %D %D",i_c,i_start_ghost_c,i_start_ghost_c+m_ghost_c);
}
col = m_ghost_c*(j_c-j_start_ghost_c) + (i_c-i_start_ghost_c);
cols[nc] = idx_c[col];
v[nc++] = x*y - x - y + 1.0;
/* one right and below */
if (i_c*ratio != i) {
cols[nc] = idx_c[col+1];
v[nc++] = -x*y + x;
}
/* one left and above */
if (j_c*ratio != j) {
cols[nc] = idx_c[col+m_ghost_c];
v[nc++] = -x*y + y;
}
/* one right and above */
if (j_c*ratio != j && i_c*ratio != i) {
cols[nc] = idx_c[col+m_ghost_c+1];
v[nc++] = x*y;
}
ierr = MatSetValues(mat,1,&row,nc,cols,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecDuplicate(user->coarse.x,&Rscale);CHKERRQ(ierr);
ierr = VecSet(user->fine.x,one);CHKERRQ(ierr);
ierr = MatMultTranspose(mat,user->fine.x,Rscale);CHKERRQ(ierr);
ierr = VecReciprocal(Rscale);CHKERRQ(ierr);
user->Rscale = Rscale;
user->R = mat;
return 0;
}
/*
FormFunctionGradient - Evaluates the function and corresponding gradient.
Input Parameters:
tao - the Tao context
X - the input vector
ptr - optional user-defined context, as set by TaoSetObjectiveAndGradientRoutine()
Output Parameters:
f - the newly evaluated function
G - the newly evaluated gradient
*/
PetscErrorCode FormFunctionGradient(Tao tao,Vec X,PetscReal *f,Vec G,void *ptr){
AppCtx *user = (AppCtx *)ptr;
PetscErrorCode ierr;
PetscInt i,j,k,ind;
PetscInt xe,ye,xsm,ysm,xep,yep;
PetscInt xs, ys, xm, ym, gxm, gym, gxs, gys;
PetscInt mx = user->mx, my = user->my;
PetscReal three = 3.0, zero = 0.0, *x, floc, cdiv3 = user->param/three;
PetscReal p5 = 0.5, area, val, flin, fquad;
PetscReal v,vb,vl,vr,vt,dvdx,dvdy;
PetscReal hx = 1.0/(user->mx + 1);
PetscReal hy = 1.0/(user->my + 1);
Vec localX = user->localX;
PetscFunctionBegin;
/* Initialize */
flin = fquad = zero;
ierr = VecSet(G, zero);CHKERRQ(ierr);
/*
Scatter ghost points to local vector,using the 2-step process
DMGlobalToLocalBegin(),DMGlobalToLocalEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = DMGlobalToLocalBegin(user->dm,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->dm,X,INSERT_VALUES,localX);CHKERRQ(ierr);
/* Get pointer to vector data */
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/* Get local mesh boundaries */
ierr = DMDAGetCorners(user->dm,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->dm,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
/* Set local loop dimensions */
xe = xs+xm;
ye = ys+ym;
if (xs == 0) xsm = xs-1;
else xsm = xs;
if (ys == 0) ysm = ys-1;
else ysm = ys;
if (xe == mx) xep = xe+1;
else xep = xe;
if (ye == my) yep = ye+1;
else yep = ye;
/* Compute local gradient contributions over the lower triangular elements */
for (j=ysm; j<ye; j++) { /* for (j=-1; j<my; j++) */
for (i=xsm; i<xe; i++) { /* for (i=-1; i<mx; i++) */
k = (j-gys)*gxm + i-gxs;
v = zero;
vr = zero;
vt = zero;
if (i >= 0 && j >= 0) v = x[k];
if (i < mx-1 && j > -1) vr = x[k+1];
if (i > -1 && j < my-1) vt = x[k+gxm];
dvdx = (vr-v)/hx;
dvdy = (vt-v)/hy;
if (i != -1 && j != -1) {
ind = k; val = - dvdx/hx - dvdy/hy - cdiv3;
ierr = VecSetValuesLocal(G,1,&k,&val,ADD_VALUES);CHKERRQ(ierr);
}
if (i != mx-1 && j != -1) {
ind = k+1; val = dvdx/hx - cdiv3;
ierr = VecSetValuesLocal(G,1,&ind,&val,ADD_VALUES);CHKERRQ(ierr);
}
if (i != -1 && j != my-1) {
ind = k+gxm; val = dvdy/hy - cdiv3;
ierr = VecSetValuesLocal(G,1,&ind,&val,ADD_VALUES);CHKERRQ(ierr);
}
fquad += dvdx*dvdx + dvdy*dvdy;
flin -= cdiv3 * (v + vr + vt);
}
}
/* Compute local gradient contributions over the upper triangular elements */
for (j=ys; j<yep; j++) { /* for (j=0; j<=my; j++) */
for (i=xs; i<xep; i++) { /* for (i=0; i<=mx; i++) */
k = (j-gys)*gxm + i-gxs;
vb = zero;
vl = zero;
v = zero;
if (i < mx && j > 0) vb = x[k-gxm];
if (i > 0 && j < my) vl = x[k-1];
if (i < mx && j < my) v = x[k];
dvdx = (v-vl)/hx;
dvdy = (v-vb)/hy;
if (i != mx && j != 0) {
ind = k-gxm; val = - dvdy/hy - cdiv3;
ierr = VecSetValuesLocal(G,1,&ind,&val,ADD_VALUES);CHKERRQ(ierr);
}
if (i != 0 && j != my) {
ind = k-1; val = - dvdx/hx - cdiv3;
ierr = VecSetValuesLocal(G,1,&ind,&val,ADD_VALUES);CHKERRQ(ierr);
}
if (i != mx && j != my) {
ind = k; val = dvdx/hx + dvdy/hy - cdiv3;
ierr = VecSetValuesLocal(G,1,&ind,&val,ADD_VALUES);CHKERRQ(ierr);
}
fquad += dvdx*dvdx + dvdy*dvdy;
flin -= cdiv3 * (vb + vl + v);
}
}
/* Restore vector */
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
/* Assemble gradient vector */
ierr = VecAssemblyBegin(G);CHKERRQ(ierr);
ierr = VecAssemblyEnd(G);CHKERRQ(ierr);
/* Scale the gradient */
area = p5*hx*hy;
floc = area * (p5 * fquad + flin);
ierr = VecScale(G, area);CHKERRQ(ierr);
/* Sum function contributions from all processes */
ierr = (PetscErrorCode)MPI_Allreduce((void*)&floc,(void*)f,1,MPIU_REAL,MPIU_SUM,MPI_COMM_WORLD);CHKERRQ(ierr);
ierr=PetscLogFlops((ye-ysm)*(xe-xsm)*20+(xep-xs)*(yep-ys)*16);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 3,N = 5,P=3,s=1,w=2,nloc,l,i,j,k,kk,m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE;
PetscErrorCode ierr;
PetscInt Xs,Xm,Ys,Ym,Zs,Zm,iloc,*ltog,*iglobal;
PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL,*lz = PETSC_NULL;
PetscBool test_order = PETSC_FALSE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
AO ao;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,300,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-NX",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NY",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NZ",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-w",&w,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-box",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xnonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-ynonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-znonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-testorder",&test_order,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-distribute",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc(m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc(n*sizeof(PetscInt),&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
if (p == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -p option with -distribute option");
ierr = PetscMalloc(p*sizeof(PetscInt),&lz);CHKERRQ(ierr);
for (i=0; i<p-1; i++) { lz[i] = 2;}
lz[p-1] = P - 2*(p-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,m,n,p,w,s,lx,ly,lz,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = PetscFree(lz);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!test_order) { /* turn off printing when testing ordering mappings */
if (M*N*P<40) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-local_print",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (test_order) {
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,&Zs,&Xm,&Ym,&Zm);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(da,&nloc,<og);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
/* ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = PetscMalloc(nloc*sizeof(PetscInt),&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y, then z) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscPrintf(MPI_COMM_WORLD,"[%D] Problem with mapping: z=%D, j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",
rank,k,j,i,l,ltog[iloc+l],iglobal[kk]);
}
kk++;
}
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
//------------------------------------------------------------------------------
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
DM da;
Vec ug, ul;
PetscInt i, ibeg, nloc, nx=200;
const PetscInt sw = 3, ndof = 1; // stencil width
PetscMPIInt rank, size;
double cfl = 0.4;
ierr = PetscInitialize(&argc, &argv, (char*)0, help); CHKERRQ(ierr);
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
MPI_Comm_size(PETSC_COMM_WORLD, &size);
ierr = DMDACreate1d(PETSC_COMM_WORLD, DM_BOUNDARY_PERIODIC, nx, ndof, sw, NULL, &da); CHKERRQ(ierr);
ierr = DMSetFromOptions(da); CHKERRQ(ierr);
ierr = DMSetUp(da); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da, &ug); CHKERRQ(ierr);
ierr = DMDAGetCorners(da, &ibeg, 0, 0, &nloc, 0, 0); CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&nx,0,0,0,0,0,0,0,0,0,0,0); CHKERRQ(ierr);
PetscReal dx = (xmax - xmin) / (PetscReal)(nx);
PetscPrintf(PETSC_COMM_WORLD,"nx = %d, dx = %e\n", nx, dx);
for(i=ibeg; i<ibeg+nloc; ++i)
{
PetscReal x = xmin + i*dx;
PetscReal v = initcond(x);
ierr = VecSetValues(ug,1,&i,&v,INSERT_VALUES); CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(ug); CHKERRQ(ierr);
ierr = VecAssemblyEnd(ug); CHKERRQ(ierr);
savesol(nx, dx, ug);
// Get local view
ierr = DMGetLocalVector(da, &ul); CHKERRQ(ierr);
PetscInt il, nl;
ierr = DMDAGetGhostCorners(da,&il,0,0,&nl,0,0); CHKERRQ(ierr);
double res[nloc], uold[nloc];
double dt = cfl * dx;
double lam= dt/dx;
double tfinal = 2.0, t = 0.0;
while(t < tfinal)
{
if(t+dt > tfinal)
{
dt = tfinal - t;
lam = dt/dx;
}
for(int rk=0; rk<3; ++rk)
{
ierr = DMGlobalToLocalBegin(da, ug, INSERT_VALUES, ul); CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da, ug, INSERT_VALUES, ul); CHKERRQ(ierr);
PetscScalar *u;
ierr = DMDAVecGetArrayRead(da, ul, &u); CHKERRQ(ierr);
PetscScalar *unew;
ierr = DMDAVecGetArray(da, ug, &unew); CHKERRQ(ierr);
if(rk==0)
for(i=ibeg; i<ibeg+nloc; ++i) uold[i-ibeg] = u[i];
for(i=0; i<nloc; ++i)
res[i] = 0.0;
// Loop over faces and compute flux
for(i=0; i<nloc+1; ++i)
{
// face between j-1, j
int j = il+sw+i;
int jm1 = j-1;
int jm2 = j-2;
int jm3 = j-3;
int jp1 = j+1;
double uleft = weno5(u[jm3],u[jm2],u[jm1],u[j],u[jp1]);
double flux = uleft;
if(i==0)
{
res[i] -= flux;
}
else if(i==nloc)
{
res[i-1] += flux;
}
else
{
res[i] -= flux;
res[i-1] += flux;
}
}
// Update solution
for(i=ibeg; i<ibeg+nloc; ++i)
unew[i] = ark[rk]*uold[i-ibeg] + (1-ark[rk])*(u[i] - lam * res[i-ibeg]);
ierr = DMDAVecRestoreArrayRead(da, ul, &u); CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(da, ug, &unew); CHKERRQ(ierr);
}
t += dt;
PetscPrintf(PETSC_COMM_WORLD,"t = %f\n", t);
}
savesol(nx, dx, ug);
// Destroy everything before finishing
ierr = DMDestroy(&da); CHKERRQ(ierr);
ierr = VecDestroy(&ug); CHKERRQ(ierr);
ierr = PetscFinalize(); CHKERRQ(ierr);
}
PetscErrorCode FACreate(FA *infa)
{
FA fa;
PetscMPIInt rank;
PetscInt tonglobal,globalrstart,x,nx,y,ny,*tonatural,i,j,*to,*from,offt[3];
PetscInt *fromnatural,fromnglobal,nscat,nlocal,cntl1,cntl2,cntl3,*indices;
PetscErrorCode ierr;
/* Each DMDA manages the local vector for the portion of region 1, 2, and 3 for that processor
Each DMDA can belong on any subset (overlapping between DMDA's or not) of processors
For processes that a particular DMDA does not exist on, the corresponding comm should be set to zero
*/
DM da1 = 0,da2 = 0,da3 = 0;
/*
v1, v2, v3 represent the local vector for a single DMDA
*/
Vec vl1 = 0,vl2 = 0,vl3 = 0, vg1 = 0, vg2 = 0,vg3 = 0;
/*
globalvec and friends represent the global vectors that are used for the PETSc solvers
localvec represents the concatenation of the (up to) 3 local vectors; vl1, vl2, vl3
tovec and friends represent intermediate vectors that are ONLY used for setting up the
final communication patterns. Once this setup routine is complete they are destroyed.
The tovec is like the globalvec EXCEPT it has redundant locations for the ghost points
between regions 2+3 and 1.
*/
AO toao,globalao;
IS tois,globalis,is;
Vec tovec,globalvec,localvec;
VecScatter vscat;
PetscScalar *globalarray,*localarray,*toarray;
ierr = PetscNew(struct _p_FA,&fa);CHKERRQ(ierr);
/*
fa->sw is the stencil width
fa->p1 is the width of region 1, fa->p2 the width of region 2 (must be the same)
fa->r1 height of region 1
fa->r2 height of region 2
fa->r2 is also the height of region 3-4
(fa->p1 - fa->p2)/2 is the width of both region 3 and region 4
*/
fa->p1 = 24;
fa->p2 = 15;
fa->r1 = 6;
fa->r2 = 6;
fa->sw = 1;
fa->r1g = fa->r1 + fa->sw;
fa->r2g = fa->r2 + fa->sw;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
fa->comm[0] = PETSC_COMM_WORLD;
fa->comm[1] = PETSC_COMM_WORLD;
fa->comm[2] = PETSC_COMM_WORLD;
/* Test case with different communicators */
/* Normally one would use MPI_Comm routines to build MPI communicators on which you wish to partition the DMDAs*/
/*
if (!rank) {
fa->comm[0] = PETSC_COMM_SELF;
fa->comm[1] = 0;
fa->comm[2] = 0;
} else if (rank == 1) {
fa->comm[0] = 0;
fa->comm[1] = PETSC_COMM_SELF;
fa->comm[2] = 0;
} else {
fa->comm[0] = 0;
fa->comm[1] = 0;
fa->comm[2] = PETSC_COMM_SELF;
} */
if (fa->p2 > fa->p1 - 3) SETERRQ(PETSC_COMM_SELF,1,"Width of region fa->p2 must be at least 3 less then width of region 1");
if (!((fa->p2 - fa->p1) % 2)) SETERRQ(PETSC_COMM_SELF,1,"width of region 3 must NOT be divisible by 2");
if (fa->comm[1]) {
ierr = DMDACreate2d(fa->comm[1],DMDA_BOUNDARY_PERIODIC,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p2,fa->r2g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da2);CHKERRQ(ierr);
ierr = DMGetLocalVector(da2,&vl2);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da2,&vg2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = DMDACreate2d(fa->comm[2],DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p1-fa->p2,fa->r2g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da3);CHKERRQ(ierr);
ierr = DMGetLocalVector(da3,&vl3);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da3,&vg3);CHKERRQ(ierr);
}
if (fa->comm[0]) {
ierr = DMDACreate2d(fa->comm[0],DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p1,fa->r1g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da1);CHKERRQ(ierr);
ierr = DMGetLocalVector(da1,&vl1);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da1,&vg1);CHKERRQ(ierr);
}
/* count the number of unknowns owned on each processor and determine the starting point of each processors ownership
for global vector with redundancy */
tonglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Number of unknowns owned %d\n",rank,tonglobal);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
/* Get tonatural number for each node */
ierr = PetscMalloc((tonglobal+1)*sizeof(PetscInt),&tonatural);CHKERRQ(ierr);
tonglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
tonatural[tonglobal++] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) tonatural[tonglobal++] = x + i + fa->p1*(y + j);
else tonatural[tonglobal++] = fa->p2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
tonatural[tonglobal++] = fa->p1*fa->r2g + x + i + fa->p1*(y + j);
}
}
}
/* ierr = PetscIntView(tonglobal,tonatural,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = AOCreateBasic(PETSC_COMM_WORLD,tonglobal,tonatural,0,&toao);CHKERRQ(ierr);
ierr = PetscFree(tonatural);CHKERRQ(ierr);
/* count the number of unknowns owned on each processor and determine the starting point of each processors ownership
for global vector without redundancy */
fromnglobal = 0;
fa->offg[1] = 0;
offt[1] = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
offt[2] = nx*ny;
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fromnglobal += nx*ny;
fa->offg[2] = fromnglobal;
} else {
offt[2] = 0;
fa->offg[2] = 0;
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
offt[0] = offt[2] + nx*ny;
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fromnglobal += nx*ny;
fa->offg[0] = fromnglobal;
} else {
offt[0] = offt[2];
fa->offg[0] = fromnglobal;
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y == 0) ny--; /* includes the ghost points on the lower side */
fromnglobal += nx*ny;
}
ierr = MPI_Scan(&fromnglobal,&globalrstart,1,MPIU_INT,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
globalrstart -= fromnglobal;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Number of unknowns owned %d\n",rank,fromnglobal);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
/* Get fromnatural number for each node */
ierr = PetscMalloc((fromnglobal+1)*sizeof(PetscInt),&fromnatural);CHKERRQ(ierr);
fromnglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fa->xg[1] = x; fa->yg[1] = y; fa->mg[1] = nx; fa->ng[1] = ny;
ierr = DMDAGetGhostCorners(da2,&fa->xl[1],&fa->yl[1],0,&fa->ml[1],&fa->nl[1],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
fromnatural[fromnglobal++] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fa->xg[2] = x; fa->yg[2] = y; fa->mg[2] = nx; fa->ng[2] = ny;
ierr = DMDAGetGhostCorners(da3,&fa->xl[2],&fa->yl[2],0,&fa->ml[2],&fa->nl[2],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) fromnatural[fromnglobal++] = x + i + fa->p1*(y + j);
else fromnatural[fromnglobal++] = fa->p2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y == 0) ny--; /* includes the ghost points on the lower side */
else y--;
fa->xg[0] = x; fa->yg[0] = y; fa->mg[0] = nx; fa->ng[0] = ny;
ierr = DMDAGetGhostCorners(da1,&fa->xl[0],&fa->yl[0],0,&fa->ml[0],&fa->nl[0],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
fromnatural[fromnglobal++] = fa->p1*fa->r2 + x + i + fa->p1*(y + j);
}
}
}
/*ierr = PetscIntView(fromnglobal,fromnatural,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
ierr = AOCreateBasic(PETSC_COMM_WORLD,fromnglobal,fromnatural,0,&globalao);CHKERRQ(ierr);
ierr = PetscFree(fromnatural);CHKERRQ(ierr);
/* ---------------------------------------------------*/
/* Create the scatter that updates 1 from 2 and 3 and 3 and 2 from 1 */
/* currently handles stencil width of 1 ONLY */
ierr = PetscMalloc(tonglobal*sizeof(PetscInt),&to);CHKERRQ(ierr);
ierr = PetscMalloc(tonglobal*sizeof(PetscInt),&from);CHKERRQ(ierr);
nscat = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
to[nscat] = from[nscat] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);nscat++;
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) {
to[nscat] = from[nscat] = x + i + fa->p1*(y + j);nscat++;
} else {
to[nscat] = from[nscat] = fa->p2 + x + i + fa->p1*(y + j);nscat++;
}
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
to[nscat] = fa->p1*fa->r2g + x + i + fa->p1*(y + j);
from[nscat++] = fa->p1*(fa->r2 - 1) + x + i + fa->p1*(y + j);
}
}
}
ierr = AOApplicationToPetsc(toao,nscat,to);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(globalao,nscat,from);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,nscat,to,PETSC_COPY_VALUES,&tois);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,nscat,from,PETSC_COPY_VALUES,&globalis);CHKERRQ(ierr);
ierr = PetscFree(to);CHKERRQ(ierr);
ierr = PetscFree(from);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,tonglobal,PETSC_DETERMINE,&tovec);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,fromnglobal,PETSC_DETERMINE,&globalvec);CHKERRQ(ierr);
ierr = VecScatterCreate(globalvec,globalis,tovec,tois,&vscat);CHKERRQ(ierr);
ierr = ISDestroy(&tois);CHKERRQ(ierr);
ierr = ISDestroy(&globalis);CHKERRQ(ierr);
ierr = AODestroy(&globalao);CHKERRQ(ierr);
ierr = AODestroy(&toao);CHKERRQ(ierr);
/* fill up global vector without redundant values with PETSc global numbering */
ierr = VecGetArray(globalvec,&globalarray);CHKERRQ(ierr);
for (i=0; i<fromnglobal; i++) {
globalarray[i] = globalrstart + i;
}
ierr = VecRestoreArray(globalvec,&globalarray);CHKERRQ(ierr);
/* scatter PETSc global indices to redundant valueed array */
ierr = VecScatterBegin(vscat,globalvec,tovec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,globalvec,tovec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Create local vector that is the concatenation of the local vectors */
nlocal = 0;
cntl1 = cntl2 = cntl3 = 0;
if (fa->comm[1]) {
ierr = VecGetSize(vl2,&cntl2);CHKERRQ(ierr);
nlocal += cntl2;
}
if (fa->comm[2]) {
ierr = VecGetSize(vl3,&cntl3);CHKERRQ(ierr);
nlocal += cntl3;
}
if (fa->comm[0]) {
ierr = VecGetSize(vl1,&cntl1);CHKERRQ(ierr);
nlocal += cntl1;
}
fa->offl[0] = cntl2 + cntl3;
fa->offl[1] = 0;
fa->offl[2] = cntl2;
ierr = VecCreateSeq(PETSC_COMM_SELF,nlocal,&localvec);CHKERRQ(ierr);
/* cheat so that vl1, vl2, vl3 shared array memory with localvec */
ierr = VecGetArray(localvec,&localarray);CHKERRQ(ierr);
ierr = VecGetArray(tovec,&toarray);CHKERRQ(ierr);
if (fa->comm[1]) {
ierr = VecPlaceArray(vl2,localarray+fa->offl[1]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg2,toarray+offt[1]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da2,vg2,INSERT_VALUES,vl2);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da2,vg2,INSERT_VALUES,vl2);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da2,&vg2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = VecPlaceArray(vl3,localarray+fa->offl[2]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg3,toarray+offt[2]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da3,vg3,INSERT_VALUES,vl3);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da3,vg3,INSERT_VALUES,vl3);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da3,&vg3);CHKERRQ(ierr);
}
if (fa->comm[0]) {
ierr = VecPlaceArray(vl1,localarray+fa->offl[0]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg1,toarray+offt[0]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da1,vg1,INSERT_VALUES,vl1);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da1,vg1,INSERT_VALUES,vl1);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da1,&vg1);CHKERRQ(ierr);
}
ierr = VecRestoreArray(localvec,&localarray);CHKERRQ(ierr);
ierr = VecRestoreArray(tovec,&toarray);CHKERRQ(ierr);
/* no longer need the redundant vector and VecScatter to it */
ierr = VecScatterDestroy(&vscat);CHKERRQ(ierr);
ierr = VecDestroy(&tovec);CHKERRQ(ierr);
/* Create final scatter that goes directly from globalvec to localvec */
/* this is the one to be used in the application code */
ierr = PetscMalloc(nlocal*sizeof(PetscInt),&indices);CHKERRQ(ierr);
ierr = VecGetArray(localvec,&localarray);CHKERRQ(ierr);
for (i=0; i<nlocal; i++) {
indices[i] = (PetscInt) (localarray[i]);
}
ierr = VecRestoreArray(localvec,&localarray);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_WORLD,2,nlocal,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = PetscFree(indices);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,2*nlocal,&fa->l);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,2*fromnglobal,PETSC_DETERMINE,&fa->g);CHKERRQ(ierr);
ierr = VecScatterCreate(fa->g,is,fa->l,NULL,&fa->vscat);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = VecDestroy(&globalvec);CHKERRQ(ierr);
ierr = VecDestroy(&localvec);CHKERRQ(ierr);
if (fa->comm[0]) {
ierr = DMRestoreLocalVector(da1,&vl1);CHKERRQ(ierr);
ierr = DMDestroy(&da1);CHKERRQ(ierr);
}
if (fa->comm[1]) {
ierr = DMRestoreLocalVector(da2,&vl2);CHKERRQ(ierr);
ierr = DMDestroy(&da2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = DMRestoreLocalVector(da3,&vl3);CHKERRQ(ierr);
ierr = DMDestroy(&da3);CHKERRQ(ierr);
}
*infa = fa;
PetscFunctionReturn(0);
}
/*
DMDAGetFaceInterpolation - Gets the interpolation for a face based coarse space
*/
PetscErrorCode DMDAGetFaceInterpolation(DM da,PC_Exotic *exotic,Mat Aglobal,MatReuse reuse,Mat *P)
{
PetscErrorCode ierr;
PetscInt dim,i,j,k,m,n,p,dof,Nint,Nface,Nwire,Nsurf,*Iint,*Isurf,cint = 0,csurf = 0,istart,jstart,kstart,*II,N,c = 0;
PetscInt mwidth,nwidth,pwidth,cnt,mp,np,pp,Ntotal,gl[6],*globals,Ng,*IIint,*IIsurf,Nt;
Mat Xint, Xsurf,Xint_tmp;
IS isint,issurf,is,row,col;
ISLocalToGlobalMapping ltg;
MPI_Comm comm;
Mat A,Aii,Ais,Asi,*Aholder,iAii;
MatFactorInfo info;
PetscScalar *xsurf,*xint;
#if defined(PETSC_USE_DEBUG_foo)
PetscScalar tmp;
#endif
PetscTable ht;
PetscFunctionBegin;
ierr = DMDAGetInfo(da,&dim,0,0,0,&mp,&np,&pp,&dof,0,0,0,0,0);
CHKERRQ(ierr);
if (dof != 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only for single field problems");
if (dim != 3) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only coded for 3d problems");
ierr = DMDAGetCorners(da,0,0,0,&m,&n,&p);
CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&istart,&jstart,&kstart,&mwidth,&nwidth,&pwidth);
CHKERRQ(ierr);
istart = istart ? -1 : 0;
jstart = jstart ? -1 : 0;
kstart = kstart ? -1 : 0;
/*
the columns of P are the interpolation of each coarse grid point (one for each vertex and edge)
to all the local degrees of freedom (this includes the vertices, edges and faces).
Xint are the subset of the interpolation into the interior
Xface are the interpolation onto faces but not into the interior
Xsurf are the interpolation onto the vertices and edges (the surfbasket)
Xint
Symbolically one could write P = (Xface) after interchanging the rows to match the natural ordering on the domain
Xsurf
*/
N = (m - istart)*(n - jstart)*(p - kstart);
Nint = (m-2-istart)*(n-2-jstart)*(p-2-kstart);
Nface = 2*((m-2-istart)*(n-2-jstart) + (m-2-istart)*(p-2-kstart) + (n-2-jstart)*(p-2-kstart));
Nwire = 4*((m-2-istart) + (n-2-jstart) + (p-2-kstart)) + 8;
Nsurf = Nface + Nwire;
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nint,6,NULL,&Xint);
CHKERRQ(ierr);
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nsurf,6,NULL,&Xsurf);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/*
Require that all 12 edges and 6 faces have at least one grid point. Otherwise some of the columns of
Xsurf will be all zero (thus making the coarse matrix singular).
*/
if (m-istart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in X direction must be at least 3");
if (n-jstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Y direction must be at least 3");
if (p-kstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Z direction must be at least 3");
cnt = 0;
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 0*Nsurf] = 1;
}
for (k=1; k<p-1-kstart; k++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 1*Nsurf] = 1;
for (j=1; j<n-1-jstart; j++) {
xsurf[cnt++ + 2*Nsurf] = 1;
/* these are the interior nodes */
xsurf[cnt++ + 3*Nsurf] = 1;
}
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 4*Nsurf] = 1;
}
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 5*Nsurf] = 1;
}
#if defined(PETSC_USE_DEBUG_foo)
for (i=0; i<Nsurf; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xsurf[i+j*Nsurf];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xsurf interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
#endif
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/* ierr = MatView(Xsurf,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
/*
I are the indices for all the needed vertices (in global numbering)
Iint are the indices for the interior values, I surf for the surface values
(This is just for the part of the global matrix obtained with MatGetSubMatrix(), it
is NOT the local DMDA ordering.)
IIint and IIsurf are the same as the Iint, Isurf except they are in the global numbering
*/
#define Endpoint(a,start,b) (a == 0 || a == (b-1-start))
ierr = PetscMalloc3(N,&II,Nint,&Iint,Nsurf,&Isurf);
CHKERRQ(ierr);
ierr = PetscMalloc2(Nint,&IIint,Nsurf,&IIsurf);
CHKERRQ(ierr);
for (k=0; k<p-kstart; k++) {
for (j=0; j<n-jstart; j++) {
for (i=0; i<m-istart; i++) {
II[c++] = i + j*mwidth + k*mwidth*nwidth;
if (!Endpoint(i,istart,m) && !Endpoint(j,jstart,n) && !Endpoint(k,kstart,p)) {
IIint[cint] = i + j*mwidth + k*mwidth*nwidth;
Iint[cint++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
} else {
IIsurf[csurf] = i + j*mwidth + k*mwidth*nwidth;
Isurf[csurf++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
}
}
}
}
if (c != N) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"c != N");
if (cint != Nint) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"cint != Nint");
if (csurf != Nsurf) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"csurf != Nsurf");
ierr = DMGetLocalToGlobalMapping(da,<g);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,N,II,II);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nint,IIint,IIint);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nsurf,IIsurf,IIsurf);
CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)da,&comm);
CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,N,II,PETSC_COPY_VALUES,&is);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nint,Iint,PETSC_COPY_VALUES,&isint);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nsurf,Isurf,PETSC_COPY_VALUES,&issurf);
CHKERRQ(ierr);
ierr = PetscFree3(II,Iint,Isurf);
CHKERRQ(ierr);
ierr = ISSort(is);
CHKERRQ(ierr);
ierr = MatGetSubMatrices(Aglobal,1,&is,&is,MAT_INITIAL_MATRIX,&Aholder);
CHKERRQ(ierr);
A = *Aholder;
ierr = PetscFree(Aholder);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,isint,MAT_INITIAL_MATRIX,&Aii);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,issurf,MAT_INITIAL_MATRIX,&Ais);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,issurf,isint,MAT_INITIAL_MATRIX,&Asi);
CHKERRQ(ierr);
/*
Solve for the interpolation onto the interior Xint
*/
ierr = MatMatMult(Ais,Xsurf,MAT_INITIAL_MATRIX,PETSC_DETERMINE,&Xint_tmp);
CHKERRQ(ierr);
ierr = MatScale(Xint_tmp,-1.0);
CHKERRQ(ierr);
if (exotic->directSolve) {
ierr = MatGetFactor(Aii,MATSOLVERPETSC,MAT_FACTOR_LU,&iAii);
CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&info);
CHKERRQ(ierr);
ierr = MatGetOrdering(Aii,MATORDERINGND,&row,&col);
CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(iAii,Aii,row,col,&info);
CHKERRQ(ierr);
ierr = ISDestroy(&row);
CHKERRQ(ierr);
ierr = ISDestroy(&col);
CHKERRQ(ierr);
ierr = MatLUFactorNumeric(iAii,Aii,&info);
CHKERRQ(ierr);
ierr = MatMatSolve(iAii,Xint_tmp,Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&iAii);
CHKERRQ(ierr);
} else {
Vec b,x;
PetscScalar *xint_tmp;
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&x);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&b);
CHKERRQ(ierr);
ierr = KSPSetOperators(exotic->ksp,Aii,Aii);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = VecPlaceArray(x,xint+i*Nint);
CHKERRQ(ierr);
ierr = VecPlaceArray(b,xint_tmp+i*Nint);
CHKERRQ(ierr);
ierr = KSPSolve(exotic->ksp,b,x);
CHKERRQ(ierr);
ierr = VecResetArray(x);
CHKERRQ(ierr);
ierr = VecResetArray(b);
CHKERRQ(ierr);
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&b);
CHKERRQ(ierr);
}
ierr = MatDestroy(&Xint_tmp);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
for (i=0; i<Nint; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xint[i+j*Nint];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xint interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
/* ierr =MatView(Xint,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
#endif
/* total faces */
Ntotal = mp*np*(pp+1) + mp*pp*(np+1) + np*pp*(mp+1);
/*
For each vertex, edge, face on process (in the same orderings as used above) determine its local number including ghost points
*/
cnt = 0;
{
gl[cnt++] = mwidth+1;
}
{
{
gl[cnt++] = mwidth*nwidth+1;
}
{
gl[cnt++] = mwidth*nwidth + mwidth; /* these are the interior nodes */ gl[cnt++] = mwidth*nwidth + mwidth+m-istart-1;
}
{
gl[cnt++] = mwidth*nwidth+mwidth*(n-jstart-1)+1;
}
}
{
gl[cnt++] = mwidth*nwidth*(p-kstart-1) + mwidth+1;
}
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* convert that to global numbering and get them on all processes */
ierr = ISLocalToGlobalMappingApply(ltg,6,gl,gl);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
ierr = PetscMalloc1(6*mp*np*pp,&globals);
CHKERRQ(ierr);
ierr = MPI_Allgather(gl,6,MPIU_INT,globals,6,MPIU_INT,PetscObjectComm((PetscObject)da));
CHKERRQ(ierr);
/* Number the coarse grid points from 0 to Ntotal */
ierr = MatGetSize(Aglobal,&Nt,NULL);
CHKERRQ(ierr);
ierr = PetscTableCreate(Ntotal/3,Nt+1,&ht);
CHKERRQ(ierr);
for (i=0; i<6*mp*np*pp; i++) {
ierr = PetscTableAddCount(ht,globals[i]+1);
CHKERRQ(ierr);
}
ierr = PetscTableGetCount(ht,&cnt);
CHKERRQ(ierr);
if (cnt != Ntotal) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Hash table size %D not equal to total number coarse grid points %D",cnt,Ntotal);
ierr = PetscFree(globals);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = PetscTableFind(ht,gl[i]+1,&gl[i]);
CHKERRQ(ierr);
gl[i]--;
}
ierr = PetscTableDestroy(&ht);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* construct global interpolation matrix */
ierr = MatGetLocalSize(Aglobal,&Ng,NULL);
CHKERRQ(ierr);
if (reuse == MAT_INITIAL_MATRIX) {
ierr = MatCreateAIJ(PetscObjectComm((PetscObject)da),Ng,PETSC_DECIDE,PETSC_DECIDE,Ntotal,Nint+Nsurf,NULL,Nint,NULL,P);
CHKERRQ(ierr);
} else {
ierr = MatZeroEntries(*P);
CHKERRQ(ierr);
}
ierr = MatSetOption(*P,MAT_ROW_ORIENTED,PETSC_FALSE);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nint,IIint,6,gl,xint,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nsurf,IIsurf,6,gl,xsurf,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = PetscFree2(IIint,IIsurf);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
{
Vec x,y;
PetscScalar *yy;
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),Ng,PETSC_DETERMINE,&y);
CHKERRQ(ierr);
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),PETSC_DETERMINE,Ntotal,&x);
CHKERRQ(ierr);
ierr = VecSet(x,1.0);
CHKERRQ(ierr);
ierr = MatMult(*P,x,y);
CHKERRQ(ierr);
ierr = VecGetArray(y,&yy);
CHKERRQ(ierr);
for (i=0; i<Ng; i++) {
if (PetscAbsScalar(yy[i]-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong p interpolation at i %D value %g",i,(double)PetscAbsScalar(yy[i]));
}
ierr = VecRestoreArray(y,&yy);
CHKERRQ(ierr);
ierr = VecDestroy(x);
CHKERRQ(ierr);
ierr = VecDestroy(y);
CHKERRQ(ierr);
}
#endif
ierr = MatDestroy(&Aii);
CHKERRQ(ierr);
ierr = MatDestroy(&Ais);
CHKERRQ(ierr);
ierr = MatDestroy(&Asi);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = ISDestroy(&is);
CHKERRQ(ierr);
ierr = ISDestroy(&isint);
CHKERRQ(ierr);
ierr = ISDestroy(&issurf);
CHKERRQ(ierr);
ierr = MatDestroy(&Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&Xsurf);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FormHessian(Tao tao, Vec X, Mat A, Mat Hpre, void*ctx)
{
AppCtx *user= (AppCtx*) ctx;
PetscErrorCode ierr;
PetscInt i,j,k;
PetscInt col[5],row;
PetscInt xs,xm,gxs,gxm,ys,ym,gys,gym;
PetscReal v[5];
PetscReal hx=1.0/(user->mx+1), hy=1.0/(user->my+1), hxhx=1.0/(hx*hx), hyhy=1.0/(hy*hy), area=0.5*hx*hy;
/* Compute the quadratic term in the objective function */
/*
Get local grid boundaries
*/
PetscFunctionBegin;
ierr = DMDAGetCorners(user->dm,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->dm,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++){
for (i=xs; i< xs+xm; i++){
row=(j-gys)*gxm + (i-gxs);
k=0;
if (j>gys){
v[k]=-2*hyhy; col[k]=row - gxm; k++;
}
if (i>gxs){
v[k]= -2*hxhx; col[k]=row - 1; k++;
}
v[k]= 4.0*(hxhx+hyhy); col[k]=row; k++;
if (i+1 < gxs+gxm){
v[k]= -2.0*hxhx; col[k]=row+1; k++;
}
if (j+1 <gys+gym){
v[k]= -2*hyhy; col[k] = row+gxm; k++;
}
ierr = MatSetValuesLocal(A,1,&row,k,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Tell the matrix we will never add a new nonzero location to the
matrix. If we do it will generate an error.
*/
ierr = MatScale(A,area);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscLogFlops(9*xm*ym+49*xm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormHessian computes the quadratic term in the quadratic objective function
Notice that the objective function in this problem is quadratic (therefore a constant
hessian). If using a nonquadratic solver, then you might want to reconsider this function
*/
PetscErrorCode FormHessian(Tao tao,Vec X,Mat hes, Mat Hpre, void *ptr)
{
AppCtx* user=(AppCtx*)ptr;
PetscErrorCode ierr;
PetscInt i,j,k;
PetscInt col[5],row,nx,ny,xs,xm,gxs,gxm,ys,ym,gys,gym;
PetscReal one=1.0, two=2.0, six=6.0,pi=4.0*atan(1.0);
PetscReal hx,hy,hxhy,hxhx,hyhy;
PetscReal xi,v[5];
PetscReal ecc=user->ecc, trule1,trule2,trule3,trule4,trule5,trule6;
PetscReal vmiddle, vup, vdown, vleft, vright;
PetscBool assembled;
nx=user->nx;
ny=user->ny;
hx=two*pi/(nx+1.0);
hy=two*user->b/(ny+1.0);
hxhy=hx*hy;
hxhx=one/(hx*hx);
hyhy=one/(hy*hy);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->dm,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->dm,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
ierr = MatAssembled(hes,&assembled);CHKERRQ(ierr);
if (assembled){ierr = MatZeroEntries(hes);CHKERRQ(ierr);}
for (i=xs; i< xs+xm; i++){
xi=(i+1)*hx;
trule1=hxhy*( p(xi,ecc) + p(xi+hx,ecc) + p(xi,ecc) ) / six; /* L(i,j) */
trule2=hxhy*( p(xi,ecc) + p(xi-hx,ecc) + p(xi,ecc) ) / six; /* U(i,j) */
trule3=hxhy*( p(xi,ecc) + p(xi+hx,ecc) + p(xi+hx,ecc) ) / six; /* U(i+1,j) */
trule4=hxhy*( p(xi,ecc) + p(xi-hx,ecc) + p(xi-hx,ecc) ) / six; /* L(i-1,j) */
trule5=trule1; /* L(i,j-1) */
trule6=trule2; /* U(i,j+1) */
vdown=-(trule5+trule2)*hyhy;
vleft=-hxhx*(trule2+trule4);
vright= -hxhx*(trule1+trule3);
vup=-hyhy*(trule1+trule6);
vmiddle=(hxhx)*(trule1+trule2+trule3+trule4)+hyhy*(trule1+trule2+trule5+trule6);
v[0]=0; v[1]=0; v[2]=0; v[3]=0; v[4]=0;
for (j=ys; j<ys+ym; j++){
row=(j-gys)*gxm + (i-gxs);
k=0;
if (j>gys){
v[k]=vdown; col[k]=row - gxm; k++;
}
if (i>gxs){
v[k]= vleft; col[k]=row - 1; k++;
}
v[k]= vmiddle; col[k]=row; k++;
if (i+1 < gxs+gxm){
v[k]= vright; col[k]=row+1; k++;
}
if (j+1 <gys+gym){
v[k]= vup; col[k] = row+gxm; k++;
}
ierr = MatSetValuesLocal(hes,1,&row,k,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = MatAssemblyBegin(hes,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(hes,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Tell the matrix we will never add a new nonzero location to the
matrix. If we do it will generate an error.
*/
ierr = MatSetOption(hes,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatSetOption(hes,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscLogFlops(9*xm*ym+49*xm);CHKERRQ(ierr);
ierr = MatNorm(hes,NORM_1,&hx);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode MatSetupDM_HYPRESStruct(Mat mat,DM da)
{
PetscErrorCode ierr;
Mat_HYPRESStruct *ex = (Mat_HYPRESStruct*) mat->data;
PetscInt dim,dof,sw[3],nx,ny,nz;
PetscInt ilower[3],iupper[3],ssize,i;
DMBoundaryType px,py,pz;
DMDAStencilType st;
PetscInt nparts= 1; /* assuming only one part */
PetscInt part = 0;
ISLocalToGlobalMapping ltog;
PetscFunctionBegin;
ex->da = da;
ierr = PetscObjectReference((PetscObject)da);CHKERRQ(ierr);
ierr = DMDAGetInfo(ex->da,&dim,0,0,0,0,0,0,&dof,&sw[0],&px,&py,&pz,&st);CHKERRQ(ierr);
ierr = DMDAGetCorners(ex->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);CHKERRQ(ierr);
iupper[0] += ilower[0] - 1;
iupper[1] += ilower[1] - 1;
iupper[2] += ilower[2] - 1;
/* the hypre_Box is used to zero out the matrix entries in MatZeroValues() */
ex->hbox.imin[0] = ilower[0];
ex->hbox.imin[1] = ilower[1];
ex->hbox.imin[2] = ilower[2];
ex->hbox.imax[0] = iupper[0];
ex->hbox.imax[1] = iupper[1];
ex->hbox.imax[2] = iupper[2];
ex->dofs_order = 0;
/* assuming that the same number of dofs on each gridpoint. Also assume all cell-centred based */
ex->nvars= dof;
/* create the hypre grid object and set its information */
if (px || py || pz) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add periodic support by calling HYPRE_SStructGridSetPeriodic()");
PetscStackCallStandard(HYPRE_SStructGridCreate,(ex->hcomm,dim,nparts,&ex->ss_grid));
PetscStackCallStandard(HYPRE_SStructGridSetExtents,(ex->ss_grid,part,ex->hbox.imin,ex->hbox.imax));
{
HYPRE_SStructVariable *vartypes;
ierr = PetscMalloc1(ex->nvars,&vartypes);CHKERRQ(ierr);
for (i= 0; i< ex->nvars; i++) vartypes[i]= HYPRE_SSTRUCT_VARIABLE_CELL;
PetscStackCallStandard(HYPRE_SStructGridSetVariables,(ex->ss_grid, part, ex->nvars,vartypes));
ierr = PetscFree(vartypes);CHKERRQ(ierr);
}
PetscStackCallStandard(HYPRE_SStructGridAssemble,(ex->ss_grid));
sw[1] = sw[0];
sw[2] = sw[1];
/* PetscStackCallStandard(HYPRE_SStructGridSetNumGhost,(ex->ss_grid,sw)); */
/* create the hypre stencil object and set its information */
if (sw[0] > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for wider stencils");
if (st == DMDA_STENCIL_BOX) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for box stencils");
if (dim == 1) {
PetscInt offsets[3][1] = {{-1},{0},{1}};
PetscInt j, cnt;
ssize = 3*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i = 0; i < (ex->nvars); i++) {
for (j = 0; j < 3; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
} else if (dim == 2) {
PetscInt offsets[5][2] = {{0,-1},{-1,0},{0,0},{1,0},{0,1}};
PetscInt j, cnt;
ssize = 5*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i= 0; i< (ex->nvars); i++) {
for (j= 0; j< 5; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
} else if (dim == 3) {
PetscInt offsets[7][3] = {{0,0,-1},{0,-1,0},{-1,0,0},{0,0,0},{1,0,0},{0,1,0},{0,0,1}};
PetscInt j, cnt;
ssize = 7*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i= 0; i< (ex->nvars); i++) {
for (j= 0; j< 7; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
}
/* create the HYPRE graph */
PetscStackCallStandard(HYPRE_SStructGraphCreate,(ex->hcomm, ex->ss_grid, &(ex->ss_graph)));
/* set the stencil graph. Note that each variable has the same graph. This means that each
variable couples to all the other variable and with the same stencil pattern. */
for (i= 0; i< (ex->nvars); i++) {
PetscStackCallStandard(HYPRE_SStructGraphSetStencil,(ex->ss_graph,part,i,ex->ss_stencil));
}
PetscStackCallStandard(HYPRE_SStructGraphAssemble,(ex->ss_graph));
/* create the HYPRE sstruct vectors for rhs and solution */
PetscStackCallStandard(HYPRE_SStructVectorCreate,(ex->hcomm,ex->ss_grid,&ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorCreate,(ex->hcomm,ex->ss_grid,&ex->ss_x));
PetscStackCallStandard(HYPRE_SStructVectorInitialize,(ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorInitialize,(ex->ss_x));
PetscStackCallStandard(HYPRE_SStructVectorAssemble,(ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorAssemble,(ex->ss_x));
/* create the hypre matrix object and set its information */
PetscStackCallStandard(HYPRE_SStructMatrixCreate,(ex->hcomm,ex->ss_graph,&ex->ss_mat));
PetscStackCallStandard(HYPRE_SStructGridDestroy,(ex->ss_grid));
PetscStackCallStandard(HYPRE_SStructStencilDestroy,(ex->ss_stencil));
if (ex->needsinitialization) {
PetscStackCallStandard(HYPRE_SStructMatrixInitialize,(ex->ss_mat));
ex->needsinitialization = PETSC_FALSE;
}
/* set the global and local sizes of the matrix */
ierr = DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);CHKERRQ(ierr);
ierr = MatSetSizes(mat,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->rmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->cmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
if (dim == 3) {
mat->ops->setvalueslocal = MatSetValuesLocal_HYPRESStruct_3d;
mat->ops->zerorowslocal = MatZeroRowsLocal_HYPRESStruct_3d;
mat->ops->zeroentries = MatZeroEntries_HYPRESStruct_3d;
ierr = MatZeroEntries_HYPRESStruct_3d(mat);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only support for 3d DMDA currently");
/* get values that will be used repeatedly in MatSetValuesLocal() and MatZeroRowsLocal() repeatedly */
ierr = MatGetOwnershipRange(mat,&ex->rstart,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(ex->da,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog, (const PetscInt **) &ex->gindices);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(ex->da,0,0,0,&ex->gnx,&ex->gnxgny,&ex->gnxgnygnz);CHKERRQ(ierr);
ex->gnxgny *= ex->gnx;
ex->gnxgnygnz *= ex->gnxgny;
ierr = DMDAGetCorners(ex->da,&ex->xs,&ex->ys,&ex->zs,&ex->nx,&ex->ny,&ex->nz);CHKERRQ(ierr);
ex->nxny = ex->nx*ex->ny;
ex->nxnynz = ex->nz*ex->nxny;
PetscFunctionReturn(0);
}
/*
FormJacobian - Evaluates Jacobian matrix.
Input Parameters:
. snes - the SNES context
. x - input vector
. ptr - optional user-defined context, as set by SNESSetJacobian()
Output Parameters:
. A - Jacobian matrix
. B - optionally different preconditioning matrix
. flag - flag indicating matrix structure
Notes:
Due to grid point reordering with DMDAs, we must always work
with the local grid points, and then transform them to the new
global numbering with the "ltog" mapping (via DMDAGetGlobalIndices()).
We cannot work directly with the global numbers for the original
uniprocessor grid!
*/
PetscErrorCode FormJacobian(SNES snes,Vec X,Mat *J,Mat *B,MatStructure *flag,void *ptr)
{
AppCtx *user = (AppCtx*)ptr; /* user-defined application context */
Mat jac = *B; /* Jacobian matrix */
Vec localX = user->localX; /* local vector */
PetscErrorCode ierr;
PetscInt *ltog; /* local-to-global mapping */
PetscInt i,j,row,mx,my,col[5];
PetscInt nloc,xs,ys,xm,ym,gxs,gys,gxm,gym,grow;
PetscScalar two = 2.0,one = 1.0,lambda,v[5],hx,hy,hxdhy,hydhx,sc,*x;
mx = user->mx; my = user->my; lambda = user->param;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
sc = hx*hy; hxdhy = hx/hy; hydhx = hy/hx;
/*
Scatter ghost points to local vector,using the 2-step process
DMGlobalToLocalBegin(),DMGlobalToLocalEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = DMGlobalToLocalBegin(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
/*
Get pointer to vector data
*/
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,PETSC_NULL,&gxm,&gym,PETSC_NULL);CHKERRQ(ierr);
/*
Get the global node numbers for all local nodes, including ghost points
*/
ierr = DMDAGetGlobalIndices(user->da,&nloc,<og);CHKERRQ(ierr);
/*
Compute entries for the locally owned part of the Jacobian.
- Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors. The "grow"
parameter computed below specifies the global row number
corresponding to each local grid point.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Always specify global row and columns of matrix entries.
- Here, we set all entries for a particular row at once.
*/
for (j=ys; j<ys+ym; j++) {
row = (j - gys)*gxm + xs - gxs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
/* boundary points */
if (i == 0 || j == 0 || i == mx-1 || j == my-1) {
ierr = MatSetValues(jac,1,&grow,1,&grow,&one,INSERT_VALUES);CHKERRQ(ierr);
continue;
}
/* interior grid points */
v[0] = -hxdhy; col[0] = ltog[row - gxm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy) - sc*lambda*exp(x[row]); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + gxm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Set flag to indicate that the Jacobian matrix retains an identical
nonzero structure throughout all nonlinear iterations (although the
values of the entries change). Thus, we can save some work in setting
up the preconditioner (e.g., no need to redo symbolic factorization for
ILU/ICC preconditioners).
- If the nonzero structure of the matrix is different during
successive linear solves, then the flag DIFFERENT_NONZERO_PATTERN
must be used instead. If you are unsure whether the matrix
structure has changed or not, use the flag DIFFERENT_NONZERO_PATTERN.
- Caution: If you specify SAME_NONZERO_PATTERN, PETSc
believes your assertion and does not check the structure
of the matrix. If you erroneously claim that the structure
is the same when it actually is not, the new preconditioner
will not function correctly. Thus, use this optimization
feature with caution!
*/
*flag = SAME_NONZERO_PATTERN;
/*
Tell the matrix we will never add a new nonzero location to the
matrix. If we do it will generate an error.
*/
/* ierr = MatSetOption(jac,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr); */
return 0;
}