int main(int argc,char **argv)
{
DM da; /* distributed array */
Vec x,b,u; /* approx solution, RHS, exact solution */
Mat A; /* linear system matrix */
KSP ksp; /* linear solver context */
PetscRandom rctx; /* random number generator context */
PetscReal norm; /* norm of solution error */
PetscInt i,j,its;
PetscErrorCode ierr;
PetscBool flg = PETSC_FALSE;
PetscLogStage stage;
DMDALocalInfo info;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/*
Create distributed array to handle parallel distribution.
The problem size will default to 8 by 7, but this can be
changed using -da_grid_x M -da_grid_y N
*/
ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_NONE, DMDA_BOUNDARY_NONE,DMDA_STENCIL_STAR,-8,-7,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compute the matrix and right-hand-side vector that define
the linear system, Ax = b.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create parallel matrix preallocated according to the DMDA, format AIJ by default.
To use symmetric storage, run with -dm_mat_type sbaij -mat_ignore_lower_triangular
*/
ierr = DMCreateMatrix(da,MATAIJ,&A);CHKERRQ(ierr);
/*
Set matrix elements for the 2-D, five-point stencil in parallel.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Rows and columns are specified by the stencil
- Entries are normalized for a domain [0,1]x[0,1]
*/
ierr = PetscLogStageRegister("Assembly", &stage);CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
for (j=info.ys; j<info.ys+info.ym; j++) {
for (i=info.xs; i<info.xs+info.xm; i++) {
PetscReal hx = 1./info.mx,hy = 1./info.my;
MatStencil row = {0},col[5] = {{0}};
PetscScalar v[5];
PetscInt ncols = 0;
row.j = j; row.i = i;
col[ncols].j = j; col[ncols].i = i; v[ncols++] = 2*(hx/hy + hy/hx);
/* boundaries */
if (i>0) {col[ncols].j = j; col[ncols].i = i-1; v[ncols++] = -hy/hx;}
if (i<info.mx-1) {col[ncols].j = j; col[ncols].i = i+1; v[ncols++] = -hy/hx;}
if (j>0) {col[ncols].j = j-1; col[ncols].i = i; v[ncols++] = -hx/hy;}
if (j<info.my-1) {col[ncols].j = j+1; col[ncols].i = i; v[ncols++] = -hx/hy;}
ierr = MatSetValuesStencil(A,1,&row,ncols,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd()
Computations can be done while messages are in transition
by placing code between these two statements.
*/
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
/*
Create parallel vectors compatible with the DMDA.
*/
ierr = DMCreateGlobalVector(da,&u);CHKERRQ(ierr);
ierr = VecDuplicate(u,&b);CHKERRQ(ierr);
ierr = VecDuplicate(u,&x);CHKERRQ(ierr);
/*
Set exact solution; then compute right-hand-side vector.
By default we use an exact solution of a vector with all
elements of 1.0; Alternatively, using the runtime option
-random_sol forms a solution vector with random components.
*/
ierr = PetscOptionsGetBool(PETSC_NULL,"-random_exact_sol",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr);
ierr = VecSetRandom(u,rctx);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);
} else {
ierr = VecSet(u,1.);CHKERRQ(ierr);
}
ierr = MatMult(A,u,b);CHKERRQ(ierr);
/*
View the exact solution vector if desired
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-view_exact_sol",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {ierr = VecView(u,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the linear solver and set various options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
/*
Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
These options will override those specified above as long as
KSPSetFromOptions() is called _after_ any other customization
routines.
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Check solution and clean up
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Check the error
*/
ierr = VecAXPY(x,-1.,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
/*
Print convergence information. PetscPrintf() produces a single
print statement from all processes that share a communicator.
An alternative is PetscFPrintf(), which prints to a file.
*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %G iterations %D\n",norm,its);CHKERRQ(ierr);
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/*
Always call PetscFinalize() before exiting a program. This routine
- finalizes the PETSc libraries as well as MPI
- provides summary and diagnostic information if certain runtime
options are chosen (e.g., -log_summary).
*/
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode SNESComputeJacobian_DMDA(SNES snes,Vec X,Mat *A,Mat *B,MatStructure *mstr,void *ctx)
{
PetscErrorCode ierr;
DM dm;
DMSNES_DA *dmdasnes = (DMSNES_DA*)ctx;
DMDALocalInfo info;
Vec Xloc;
void *x;
PetscFunctionBegin;
if (!dmdasnes->residuallocal) SETERRQ(PetscObjectComm((PetscObject)snes),PETSC_ERR_PLIB,"Corrupt context");
ierr = SNESGetDM(snes,&dm);CHKERRQ(ierr);
if (dmdasnes->jacobianlocal) {
ierr = DMGetLocalVector(dm,&Xloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr);
ierr = DMDAVecGetArray(dm,Xloc,&x);CHKERRQ(ierr);
CHKMEMQ;
ierr = (*dmdasnes->jacobianlocal)(&info,x,*A,*B,mstr,dmdasnes->jacobianlocalctx);CHKERRQ(ierr);
CHKMEMQ;
ierr = DMDAVecRestoreArray(dm,Xloc,&x);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm,&Xloc);CHKERRQ(ierr);
} else {
MatFDColoring fdcoloring;
ierr = PetscObjectQuery((PetscObject)dm,"DMDASNES_FDCOLORING",(PetscObject*)&fdcoloring);CHKERRQ(ierr);
if (!fdcoloring) {
ISColoring coloring;
ierr = DMCreateColoring(dm,dm->coloringtype,&coloring);CHKERRQ(ierr);
ierr = MatFDColoringCreate(*B,coloring,&fdcoloring);CHKERRQ(ierr);
ierr = ISColoringDestroy(&coloring);CHKERRQ(ierr);
switch (dm->coloringtype) {
case IS_COLORING_GLOBAL:
ierr = MatFDColoringSetFunction(fdcoloring,(PetscErrorCode (*)(void))SNESComputeFunction_DMDA,dmdasnes);CHKERRQ(ierr);
break;
default: SETERRQ1(PetscObjectComm((PetscObject)snes),PETSC_ERR_SUP,"No support for coloring type '%s'",ISColoringTypes[dm->coloringtype]);
}
ierr = PetscObjectSetOptionsPrefix((PetscObject)fdcoloring,((PetscObject)dm)->prefix);CHKERRQ(ierr);
ierr = MatFDColoringSetFromOptions(fdcoloring);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject)dm,"DMDASNES_FDCOLORING",(PetscObject)fdcoloring);CHKERRQ(ierr);
ierr = PetscObjectDereference((PetscObject)fdcoloring);CHKERRQ(ierr);
/* The following breaks an ugly reference counting loop that deserves a paragraph. MatFDColoringApply() will call
* VecDuplicate() with the state Vec and store inside the MatFDColoring. This Vec will duplicate the Vec, but the
* MatFDColoring is composed with the DM. We dereference the DM here so that the reference count will eventually
* drop to 0. Note the code in DMDestroy() that exits early for a negative reference count. That code path will be
* taken when the PetscObjectList for the Vec inside MatFDColoring is destroyed.
*/
ierr = PetscObjectDereference((PetscObject)dm);CHKERRQ(ierr);
}
*mstr = SAME_NONZERO_PATTERN;
ierr = MatFDColoringApply(*B,fdcoloring,X,mstr,snes);CHKERRQ(ierr);
}
/* This will be redundant if the user called both, but it's too common to forget. */
if (*A != *B) {
ierr = MatAssemblyBegin(*A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode DMView_DA_2d(DM da,PetscViewer viewer)
{
PetscErrorCode ierr;
PetscMPIInt rank;
PetscBool iascii,isdraw,isbinary;
DM_DA *dd = (DM_DA*)da->data;
#if defined(PETSC_HAVE_MATLAB_ENGINE)
PetscBool ismatlab;
#endif
PetscFunctionBegin;
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)da),&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERBINARY,&isbinary);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MATLAB_ENGINE)
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERMATLAB,&ismatlab);CHKERRQ(ierr);
#endif
if (iascii) {
PetscViewerFormat format;
ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
if (format != PETSC_VIEWER_ASCII_VTK && format != PETSC_VIEWER_ASCII_VTK_CELL) {
DMDALocalInfo info;
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %D N %D m %D n %D w %D s %D\n",rank,dd->M,dd->N,dd->m,dd->n,dd->w,dd->s);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %D %D, Y range of indices: %D %D\n",info.xs,info.xs+info.xm,info.ys,info.ys+info.ym);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);CHKERRQ(ierr);
} else {
ierr = DMView_DA_VTK(da,viewer);CHKERRQ(ierr);
}
} else if (isdraw) {
PetscDraw draw;
double ymin = -1*dd->s-1,ymax = dd->N+dd->s;
double xmin = -1*dd->s-1,xmax = dd->M+dd->s;
double x,y;
PetscInt base;
const PetscInt *idx;
char node[10];
PetscBool isnull;
ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr);
ierr = PetscDrawIsNull(draw,&isnull);CHKERRQ(ierr); if (isnull) PetscFunctionReturn(0);
if (!da->coordinates) {
ierr = PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);CHKERRQ(ierr);
}
ierr = PetscDrawSynchronizedClear(draw);CHKERRQ(ierr);
/* first processor draw all node lines */
if (!rank) {
ymin = 0.0; ymax = dd->N - 1;
for (xmin=0; xmin<dd->M; xmin++) {
ierr = PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);CHKERRQ(ierr);
}
xmin = 0.0; xmax = dd->M - 1;
for (ymin=0; ymin<dd->N; ymin++) {
ierr = PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);CHKERRQ(ierr);
}
}
ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr);
ierr = PetscDrawPause(draw);CHKERRQ(ierr);
/* draw my box */
ymin = dd->ys; ymax = dd->ye - 1; xmin = dd->xs/dd->w;
xmax =(dd->xe-1)/dd->w;
ierr = PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);CHKERRQ(ierr);
ierr = PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);CHKERRQ(ierr);
ierr = PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);CHKERRQ(ierr);
ierr = PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);CHKERRQ(ierr);
/* put in numbers */
base = (dd->base)/dd->w;
for (y=ymin; y<=ymax; y++) {
for (x=xmin; x<=xmax; x++) {
sprintf(node,"%d",(int)base++);
ierr = PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);CHKERRQ(ierr);
}
}
ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr);
ierr = PetscDrawPause(draw);CHKERRQ(ierr);
/* overlay ghost numbers, useful for error checking */
/* put in numbers */
base = 0;
ierr = ISLocalToGlobalMappingGetIndices(da->ltogmapb,&idx);CHKERRQ(ierr);
ymin = dd->Ys; ymax = dd->Ye; xmin = dd->Xs; xmax = dd->Xe;
for (y=ymin; y<ymax; y++) {
for (x=xmin; x<xmax; x++) {
if ((base % dd->w) == 0) {
sprintf(node,"%d",(int)(idx[base]));
ierr = PetscDrawString(draw,x/dd->w,y,PETSC_DRAW_BLUE,node);CHKERRQ(ierr);
}
base++;
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(da->ltogmapb,&idx);
ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr);
ierr = PetscDrawPause(draw);CHKERRQ(ierr);
} else if (isbinary) {
ierr = DMView_DA_Binary(da,viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MATLAB_ENGINE)
} else if (ismatlab) {
ierr = DMView_DA_Matlab(da,viewer);CHKERRQ(ierr);
#endif
}
PetscFunctionReturn(0);
}
PetscErrorCode FormMatrix(DM da,Mat jac)
{
PetscErrorCode ierr;
PetscInt i,j,nrows = 0;
MatStencil col[5],row,*rows;
PetscScalar v[5],hx,hy,hxdhy,hydhx;
DMDALocalInfo info;
PetscFunctionBeginUser;
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
hx = 1.0/(PetscReal)(info.mx-1);
hy = 1.0/(PetscReal)(info.my-1);
hxdhy = hx/hy;
hydhx = hy/hx;
ierr = PetscMalloc(info.ym*info.xm*sizeof(MatStencil),&rows);CHKERRQ(ierr);
/*
Compute entries for the locally owned part of the Jacobian.
- Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Here, we set all entries for a particular row at once.
- We can set matrix entries either using either
MatSetValuesLocal() or MatSetValues(), as discussed above.
*/
for (j=info.ys; j<info.ys+info.ym; j++) {
for (i=info.xs; i<info.xs+info.xm; i++) {
row.j = j; row.i = i;
/* boundary points */
if (i == 0 || j == 0 || i == info.mx-1 || j == info.my-1) {
v[0] = 2.0*(hydhx + hxdhy);
ierr = MatSetValuesStencil(jac,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
rows[nrows].i = i;
rows[nrows++].j = j;
} else {
/* interior grid points */
v[0] = -hxdhy; col[0].j = j - 1; col[0].i = i;
v[1] = -hydhx; col[1].j = j; col[1].i = i-1;
v[2] = 2.0*(hydhx + hxdhy); col[2].j = row.j; col[2].i = row.i;
v[3] = -hydhx; col[3].j = j; col[3].i = i+1;
v[4] = -hxdhy; col[4].j = j + 1; col[4].i = i;
ierr = MatSetValuesStencil(jac,1,&row,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
*/
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatZeroRowsColumnsStencil(jac,nrows,rows,2.0*(hydhx + hxdhy),NULL,NULL);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
/*
Tell the matrix we will never add a new nonzero location to the
matrix. If we do, it will generate an error.
*/
ierr = MatSetOption(jac,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMDAVTKWriteAll_VTS(DM da,PetscViewer viewer)
{
#if defined(PETSC_USE_REAL_SINGLE)
const char precision[] = "Float32";
#elif defined(PETSC_USE_REAL_DOUBLE)
const char precision[] = "Float64";
#else
const char precision[] = "UnknownPrecision";
#endif
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*)viewer->data;
PetscViewerVTKObjectLink link;
FILE *fp;
PetscMPIInt rank,size,tag;
DMDALocalInfo info;
PetscInt dim,mx,my,mz,bs,boffset,maxnnodes,i,j,k,f,r;
PetscInt rloc[6],(*grloc)[6] = NULL;
PetscScalar *array,*array2;
PetscReal gmin[3],gmax[3];
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr);
#if defined(PETSC_USE_COMPLEX)
SETERRQ(comm,PETSC_ERR_SUP,"Complex values not supported");
#endif
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,&dim, &mx,&my,&mz, 0,0,0, &bs,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMDAGetBoundingBox(da,gmin,gmax);CHKERRQ(ierr);
ierr = PetscFOpen(comm,vtk->filename,"wb",&fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"<?xml version=\"1.0\"?>\n");CHKERRQ(ierr);
#if defined(PETSC_WORDS_BIGENDIAN)
ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"StructuredGrid\" version=\"0.1\" byte_order=\"BigEndian\">\n");CHKERRQ(ierr);
#else
ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"StructuredGrid\" version=\"0.1\" byte_order=\"LittleEndian\">\n");CHKERRQ(ierr);
#endif
ierr = PetscFPrintf(comm,fp," <StructuredGrid WholeExtent=\"%D %D %D %D %D %D\">\n",0,mx-1,0,my-1,0,mz-1);CHKERRQ(ierr);
if (!rank) {ierr = PetscMalloc(size*6*sizeof(PetscInt),&grloc);CHKERRQ(ierr);}
rloc[0] = info.xs;
rloc[1] = info.xm;
rloc[2] = info.ys;
rloc[3] = info.ym;
rloc[4] = info.zs;
rloc[5] = info.zm;
ierr = MPI_Gather(rloc,6,MPIU_INT,&grloc[0][0],6,MPIU_INT,0,comm);CHKERRQ(ierr);
/* Write XML header */
maxnnodes = 0; /* Used for the temporary array size on rank 0 */
boffset = 0; /* Offset into binary file */
for (r=0; r<size; r++) {
PetscInt xs=-1,xm=-1,ys=-1,ym=-1,zs=-1,zm=-1,nnodes = 0;
if (!rank) {
xs = grloc[r][0];
xm = grloc[r][1];
ys = grloc[r][2];
ym = grloc[r][3];
zs = grloc[r][4];
zm = grloc[r][5];
nnodes = xm*ym*zm;
}
maxnnodes = PetscMax(maxnnodes,nnodes);
#if 0
switch (dim) {
case 1:
ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,0,0,0,0);CHKERRQ(ierr);
break;
case 2:
ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm,ys+ym-1,xs,xs+xm-1,0,0);CHKERRQ(ierr);
break;
case 3:
ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,ys,ys+ym-1,zs,zs+zm-1);CHKERRQ(ierr);
break;
default: SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"No support for dimension %D",dim);
}
#endif
ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,ys,ys+ym-1,zs,zs+zm-1);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <Points>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"Position\" NumberOfComponents=\"3\" format=\"appended\" offset=\"%D\" />\n",precision,boffset);CHKERRQ(ierr);
boffset += 3*nnodes*sizeof(PetscScalar) + sizeof(int);
ierr = PetscFPrintf(comm,fp," </Points>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <PointData Scalars=\"ScalarPointData\">\n");CHKERRQ(ierr);
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
const char *vecname = "";
if (((PetscObject)X)->name || link != vtk->link) { /* If the object is already named, use it. If it is past the first link, name it to disambiguate. */
ierr = PetscObjectGetName((PetscObject)X,&vecname);CHKERRQ(ierr);
}
for (i=0; i<bs; i++) {
char buf[256];
const char *fieldname;
ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr);
if (!fieldname) {
ierr = PetscSNPrintf(buf,sizeof(buf),"Unnamed%D",i);CHKERRQ(ierr);
fieldname = buf;
}
ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"%s%s\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",precision,vecname,fieldname,boffset);CHKERRQ(ierr);
boffset += nnodes*sizeof(PetscScalar) + sizeof(int);
}
}
ierr = PetscFPrintf(comm,fp," </PointData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," </Piece>\n");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm,fp," </StructuredGrid>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <AppendedData encoding=\"raw\">\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"_");CHKERRQ(ierr);
/* Now write the arrays. */
tag = ((PetscObject)viewer)->tag;
ierr = PetscMalloc2(maxnnodes*PetscMax(3,bs),PetscScalar,&array,maxnnodes*3,PetscScalar,&array2);CHKERRQ(ierr);
for (r=0; r<size; r++) {
MPI_Status status;
PetscInt xs=-1,xm=-1,ys=-1,ym=-1,zs=-1,zm=-1,nnodes = 0;
if (!rank) {
xs = grloc[r][0];
xm = grloc[r][1];
ys = grloc[r][2];
ym = grloc[r][3];
zs = grloc[r][4];
zm = grloc[r][5];
nnodes = xm*ym*zm;
} else if (r == rank) {
nnodes = info.xm*info.ym*info.zm;
}
{ /* Write the coordinates */
Vec Coords;
ierr = DMGetCoordinates(da,&Coords);CHKERRQ(ierr);
if (Coords) {
const PetscScalar *coords;
ierr = VecGetArrayRead(Coords,&coords);CHKERRQ(ierr);
if (!rank) {
if (r) {
PetscMPIInt nn;
ierr = MPI_Recv(array,nnodes*dim,MPIU_SCALAR,r,tag,comm,&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_SCALAR,&nn);CHKERRQ(ierr);
if (nn != nnodes*dim) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Array size mismatch");
} else {
ierr = PetscMemcpy(array,coords,nnodes*dim*sizeof(PetscScalar));CHKERRQ(ierr);
}
/* Transpose coordinates to VTK (C-style) ordering */
for (k=0; k<zm; k++) {
for (j=0; j<ym; j++) {
for (i=0; i<xm; i++) {
PetscInt Iloc = i+xm*(j+ym*k);
array2[Iloc*3+0] = array[Iloc*dim + 0];
array2[Iloc*3+1] = dim > 1 ? array[Iloc*dim + 1] : 0;
array2[Iloc*3+2] = dim > 2 ? array[Iloc*dim + 2] : 0;
}
}
}
} else if (r == rank) {
ierr = MPI_Send((void*)coords,nnodes*dim,MPIU_SCALAR,0,tag,comm);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(Coords,&coords);CHKERRQ(ierr);
} else { /* Fabricate some coordinates using grid index */
for (k=0; k<zm; k++) {
for (j=0; j<ym; j++) {
for (i=0; i<xm; i++) {
PetscInt Iloc = i+xm*(j+ym*k);
array2[Iloc*3+0] = xs+i;
array2[Iloc*3+1] = ys+j;
array2[Iloc*3+2] = zs+k;
}
}
}
}
ierr = PetscViewerVTKFWrite(viewer,fp,array2,nnodes*3,PETSC_SCALAR);CHKERRQ(ierr);
}
/* Write each of the objects queued up for this file */
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
const PetscScalar *x;
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
if (!rank) {
if (r) {
PetscMPIInt nn;
ierr = MPI_Recv(array,nnodes*bs,MPIU_SCALAR,r,tag,comm,&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_SCALAR,&nn);CHKERRQ(ierr);
if (nn != nnodes*bs) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Array size mismatch receiving from rank %D",r);
} else {
ierr = PetscMemcpy(array,x,nnodes*bs*sizeof(PetscScalar));CHKERRQ(ierr);
}
for (f=0; f<bs; f++) {
/* Extract and transpose the f'th field */
for (k=0; k<zm; k++) {
for (j=0; j<ym; j++) {
for (i=0; i<xm; i++) {
PetscInt Iloc = i+xm*(j+ym*k);
array2[Iloc] = array[Iloc*bs + f];
}
}
}
ierr = PetscViewerVTKFWrite(viewer,fp,array2,nnodes,PETSC_SCALAR);CHKERRQ(ierr);
}
} else if (r == rank) {
ierr = MPI_Send((void*)x,nnodes*bs,MPIU_SCALAR,0,tag,comm);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
}
}
ierr = PetscFree2(array,array2);CHKERRQ(ierr);
ierr = PetscFree(grloc);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"\n </AppendedData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"</VTKFile>\n");CHKERRQ(ierr);
ierr = PetscFClose(comm,fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da,*subda;
PetscInt i,dim=3;
PetscMPIInt size,rank;
Vec v;
Vec slvec,sgvec;
IS *ois,*iis;
VecScatter oscata;
VecScatter *iscat,*oscat,*gscat;
DMDALocalInfo info;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dim",&dim,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,-4,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMCreateDomainDecomposition(da,NULL,NULL,&iis,&ois,&subda);CHKERRQ(ierr);
ierr = DMCreateDomainDecompositionScatters(da,1,subda,&iscat,&oscat,&gscat);CHKERRQ(ierr);
{
DMDALocalInfo subinfo;
MatStencil lower,upper;
IS patchis,subpatchis;
Vec smallvec;
Vec largevec;
VecScatter patchscat;
ierr = DMDAGetLocalInfo(subda[0],&subinfo);CHKERRQ(ierr);
lower.i = info.xs;
lower.j = info.ys;
lower.k = info.zs;
upper.i = info.xs+info.xm;
upper.j = info.ys+info.ym;
upper.k = info.zs+info.zm;
/* test the patch IS as a thing to scatter to/from */
ierr = DMDACreatePatchIS(da,&lower,&upper,&patchis);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&smallvec);CHKERRQ(ierr);
ierr = VecSetSizes(smallvec,info.dof*(upper.i - lower.i)*(upper.j - lower.j)*(upper.k - lower.k),PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(smallvec);CHKERRQ(ierr);
ierr = VecScatterCreate(smallvec,NULL,largevec,patchis,&patchscat);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],smallvec);CHKERRQ(ierr);
ierr = VecSet(largevec,0);CHKERRQ(ierr);
ierr = VecScatterBegin(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = ISView(patchis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecScatterView(patchscat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecView(smallvec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecView(largevec,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&smallvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = ISDestroy(&patchis);CHKERRQ(ierr);
ierr = VecScatterDestroy(&patchscat);CHKERRQ(ierr);
}
/* view the various parts */
{
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMView(subda[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&v);CHKERRQ(ierr);
/* test filling outer between the big DM and the small ones with the IS scatter*/
ierr = VecScatterCreate(v,ois[0],sgvec,NULL,&oscata);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],sgvec);CHKERRQ(ierr);
ierr = VecScatterBegin(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* test the local-to-local scatter */
/* fill up the local subdomain and then add them together */
ierr = FillLocalSubdomain(da,v);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test ghost scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test overlap scattering backwards */
ierr = DMLocalToGlobalBegin(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test interior scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test matrix allocation */
for (i = 0; i < size; i++) {
if (i == rank) {
Mat m;
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMSetMatType(subda[0],MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(subda[0],&m);CHKERRQ(ierr);
ierr = MatView(m,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&m);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMRestoreLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&v);CHKERRQ(ierr);
}
ierr = DMDestroy(&subda[0]);CHKERRQ(ierr);
ierr = ISDestroy(&ois[0]);CHKERRQ(ierr);
ierr = ISDestroy(&iis[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&iscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&gscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscata);CHKERRQ(ierr);
ierr = PetscFree(iscat);CHKERRQ(ierr);
ierr = PetscFree(oscat);CHKERRQ(ierr);
ierr = PetscFree(gscat);CHKERRQ(ierr);
ierr = PetscFree(oscata);CHKERRQ(ierr);
ierr = PetscFree(subda);CHKERRQ(ierr);
ierr = PetscFree(ois);CHKERRQ(ierr);
ierr = PetscFree(iis);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode NonlinearGS(SNES snes, Vec X, Vec B, void *ctx)
{
DMDALocalInfo info;
Field **x,**b;
PetscErrorCode ierr;
Vec localX, localB;
DM da;
PetscInt xints,xinte,yints,yinte,i,j,k,l;
PetscInt max_its,tot_its;
PetscInt sweeps;
PetscReal rtol,atol,stol;
PetscReal hx,hy,dhx,dhy,hxdhy,hydhx;
PetscReal grashof,prandtl,lid;
PetscScalar u,uxx,uyy,vx,vy,avx,avy,vxp,vxm,vyp,vym;
PetscScalar fu, fv, fomega, ftemp;
PetscScalar dfudu;
PetscScalar dfvdv;
PetscScalar dfodu, dfodv, dfodo;
PetscScalar dftdu, dftdv, dftdt;
PetscScalar yu=0, yv=0, yo=0, yt=0;
PetscScalar bjiu, bjiv, bjiomega, bjitemp;
PetscBool ptconverged;
PetscReal pfnorm,pfnorm0,pynorm,pxnorm;
AppCtx *user = (AppCtx*)ctx;
PetscFunctionBeginUser;
grashof = user->grashof;
prandtl = user->prandtl;
lid = user->lidvelocity;
tot_its = 0;
ierr = SNESNGSGetTolerances(snes,&rtol,&atol,&stol,&max_its);CHKERRQ(ierr);
ierr = SNESNGSGetSweeps(snes,&sweeps);CHKERRQ(ierr);
ierr = SNESGetDM(snes,(DM*)&da);CHKERRQ(ierr);
ierr = DMGetLocalVector(da,&localX);CHKERRQ(ierr);
if (B) {
ierr = DMGetLocalVector(da,&localB);CHKERRQ(ierr);
}
/*
Scatter ghost points to local vector, using the 2-step process
DMGlobalToLocalBegin(), DMGlobalToLocalEnd().
*/
ierr = DMGlobalToLocalBegin(da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
if (B) {
ierr = DMGlobalToLocalBegin(da,B,INSERT_VALUES,localB);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,B,INSERT_VALUES,localB);CHKERRQ(ierr);
}
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMDAVecGetArray(da,localX,&x);CHKERRQ(ierr);
if (B) {
ierr = DMDAVecGetArrayRead(da,localB,&b);CHKERRQ(ierr);
}
/* looks like a combination of the formfunction / formjacobian routines */
dhx = (PetscReal)(info.mx-1);dhy = (PetscReal)(info.my-1);
hx = 1.0/dhx; hy = 1.0/dhy;
hxdhy = hx*dhy; hydhx = hy*dhx;
xints = info.xs; xinte = info.xs+info.xm; yints = info.ys; yinte = info.ys+info.ym;
/* Set the boundary conditions on the momentum equations */
/* Test whether we are on the bottom edge of the global array */
if (yints == 0) {
j = 0;
/* bottom edge */
for (i=info.xs; i<info.xs+info.xm; i++) {
if (B) {
bjiu = b[j][i].u;
bjiv = b[j][i].v;
} else {
bjiu = 0.0;
bjiv = 0.0;
}
x[j][i].u = 0.0 + bjiu;
x[j][i].v = 0.0 + bjiv;
}
}
/* Test whether we are on the top edge of the global array */
if (yinte == info.my) {
j = info.my - 1;
/* top edge */
for (i=info.xs; i<info.xs+info.xm; i++) {
if (B) {
bjiu = b[j][i].u;
bjiv = b[j][i].v;
} else {
bjiu = 0.0;
bjiv = 0.0;
}
x[j][i].u = lid + bjiu;
x[j][i].v = bjiv;
}
}
/* Test whether we are on the left edge of the global array */
if (xints == 0) {
i = 0;
/* left edge */
for (j=info.ys; j<info.ys+info.ym; j++) {
if (B) {
bjiu = b[j][i].u;
bjiv = b[j][i].v;
} else {
bjiu = 0.0;
bjiv = 0.0;
}
x[j][i].u = 0.0 + bjiu;
x[j][i].v = 0.0 + bjiv;
}
}
/* Test whether we are on the right edge of the global array */
if (xinte == info.mx) {
i = info.mx - 1;
/* right edge */
for (j=info.ys; j<info.ys+info.ym; j++) {
if (B) {
bjiu = b[j][i].u;
bjiv = b[j][i].v;
} else {
bjiu = 0.0;
bjiv = 0.0;
}
x[j][i].u = 0.0 + bjiu;
x[j][i].v = 0.0 + bjiv;
}
}
for (k=0; k < sweeps; k++) {
for (j=info.ys; j<info.ys + info.ym; j++) {
for (i=info.xs; i<info.xs + info.xm; i++) {
ptconverged = PETSC_FALSE;
pfnorm0 = 0.0;
fu = 0.0;
fv = 0.0;
fomega = 0.0;
ftemp = 0.0;
for (l = 0; l < max_its && !ptconverged; l++) {
if (B) {
bjiu = b[j][i].u;
bjiv = b[j][i].v;
bjiomega = b[j][i].omega;
bjitemp = b[j][i].temp;
} else {
bjiu = 0.0;
bjiv = 0.0;
bjiomega = 0.0;
bjitemp = 0.0;
}
if (i != 0 && i != info.mx - 1 && j != 0 && j != info.my-1) {
/* U velocity */
u = x[j][i].u;
uxx = (2.0*u - x[j][i-1].u - x[j][i+1].u)*hydhx;
uyy = (2.0*u - x[j-1][i].u - x[j+1][i].u)*hxdhy;
fu = uxx + uyy - .5*(x[j+1][i].omega-x[j-1][i].omega)*hx - bjiu;
dfudu = 2.0*(hydhx + hxdhy);
/* V velocity */
u = x[j][i].v;
uxx = (2.0*u - x[j][i-1].v - x[j][i+1].v)*hydhx;
uyy = (2.0*u - x[j-1][i].v - x[j+1][i].v)*hxdhy;
fv = uxx + uyy + .5*(x[j][i+1].omega-x[j][i-1].omega)*hy - bjiv;
dfvdv = 2.0*(hydhx + hxdhy);
/*
convective coefficients for upwinding
*/
vx = x[j][i].u; avx = PetscAbsScalar(vx);
vxp = .5*(vx+avx); vxm = .5*(vx-avx);
vy = x[j][i].v; avy = PetscAbsScalar(vy);
vyp = .5*(vy+avy); vym = .5*(vy-avy);
/* Omega */
u = x[j][i].omega;
uxx = (2.0*u - x[j][i-1].omega - x[j][i+1].omega)*hydhx;
uyy = (2.0*u - x[j-1][i].omega - x[j+1][i].omega)*hxdhy;
fomega = uxx + uyy + (vxp*(u - x[j][i-1].omega) + vxm*(x[j][i+1].omega - u))*hy +
(vyp*(u - x[j-1][i].omega) + vym*(x[j+1][i].omega - u))*hx -
.5*grashof*(x[j][i+1].temp - x[j][i-1].temp)*hy - bjiomega;
/* convective coefficient derivatives */
dfodo = 2.0*(hydhx + hxdhy) + ((vxp - vxm)*hy + (vyp - vym)*hx);
if (PetscRealPart(vx) > 0.0) dfodu = (u - x[j][i-1].omega)*hy;
else dfodu = (x[j][i+1].omega - u)*hy;
if (PetscRealPart(vy) > 0.0) dfodv = (u - x[j-1][i].omega)*hx;
else dfodv = (x[j+1][i].omega - u)*hx;
/* Temperature */
u = x[j][i].temp;
uxx = (2.0*u - x[j][i-1].temp - x[j][i+1].temp)*hydhx;
uyy = (2.0*u - x[j-1][i].temp - x[j+1][i].temp)*hxdhy;
ftemp = uxx + uyy + prandtl*((vxp*(u - x[j][i-1].temp) + vxm*(x[j][i+1].temp - u))*hy + (vyp*(u - x[j-1][i].temp) + vym*(x[j+1][i].temp - u))*hx) - bjitemp;
dftdt = 2.0*(hydhx + hxdhy) + prandtl*((vxp - vxm)*hy + (vyp - vym)*hx);
if (PetscRealPart(vx) > 0.0) dftdu = prandtl*(u - x[j][i-1].temp)*hy;
else dftdu = prandtl*(x[j][i+1].temp - u)*hy;
if (PetscRealPart(vy) > 0.0) dftdv = prandtl*(u - x[j-1][i].temp)*hx;
else dftdv = prandtl*(x[j+1][i].temp - u)*hx;
/* invert the system:
[ dfu / du 0 0 0 ][yu] = [fu]
[ 0 dfv / dv 0 0 ][yv] [fv]
[ dfo / du dfo / dv dfo / do 0 ][yo] [fo]
[ dft / du dft / dv 0 dft / dt ][yt] [ft]
by simple back-substitution
*/
yu = fu / dfudu;
yv = fv / dfvdv;
yo = (fomega - (dfodu*yu + dfodv*yv)) / dfodo;
yt = (ftemp - (dftdu*yu + dftdv*yv)) / dftdt;
x[j][i].u = x[j][i].u - yu;
x[j][i].v = x[j][i].v - yv;
x[j][i].temp = x[j][i].temp - yt;
x[j][i].omega = x[j][i].omega - yo;
}
if (i == 0) {
fomega = x[j][i].omega - (x[j][i+1].v - x[j][i].v)*dhx - bjiomega;
ftemp = x[j][i].temp - bjitemp;
yo = fomega;
yt = ftemp;
x[j][i].omega = x[j][i].omega - fomega;
x[j][i].temp = x[j][i].temp - ftemp;
}
if (i == info.mx - 1) {
fomega = x[j][i].omega - (x[j][i].v - x[j][i-1].v)*dhx - bjiomega;
ftemp = x[j][i].temp - (PetscReal)(grashof>0) - bjitemp;
yo = fomega;
yt = ftemp;
x[j][i].omega = x[j][i].omega - fomega;
x[j][i].temp = x[j][i].temp - ftemp;
}
if (j == 0) {
fomega = x[j][i].omega + (x[j+1][i].u - x[j][i].u)*dhy - bjiomega;
ftemp = x[j][i].temp-x[j+1][i].temp - bjitemp;
yo = fomega;
yt = ftemp;
x[j][i].omega = x[j][i].omega - fomega;
x[j][i].temp = x[j][i].temp - ftemp;
}
if (j == info.my - 1) {
fomega = x[j][i].omega + (x[j][i].u - x[j-1][i].u)*dhy - bjiomega;
ftemp = x[j][i].temp-x[j-1][i].temp - bjitemp;
yo = fomega;
yt = ftemp;
x[j][i].omega = x[j][i].omega - fomega;
x[j][i].temp = x[j][i].temp - ftemp;
}
tot_its++;
pfnorm = PetscRealPart(fu*fu + fv*fv + fomega*fomega + ftemp*ftemp);
pfnorm = PetscSqrtReal(pfnorm);
pynorm = PetscRealPart(yu*yu + yv*yv + yo*yo + yt*yt);
pynorm = PetscSqrtReal(pynorm);
pxnorm = PetscRealPart(x[j][i].u*x[j][i].u + x[j][i].v*x[j][i].v + x[j][i].omega*x[j][i].omega + x[j][i].temp*x[j][i].temp);
pxnorm = PetscSqrtReal(pxnorm);
if (l == 0) pfnorm0 = pfnorm;
if (rtol*pfnorm0 >pfnorm ||
atol > pfnorm ||
pxnorm*stol > pynorm) ptconverged = PETSC_TRUE;
}
}
}
}
ierr = DMDAVecRestoreArray(da,localX,&x);CHKERRQ(ierr);
if (B) {
ierr = DMDAVecRestoreArrayRead(da,localB,&b);CHKERRQ(ierr);
}
ierr = DMLocalToGlobalBegin(da,localX,INSERT_VALUES,X);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,localX,INSERT_VALUES,X);CHKERRQ(ierr);
ierr = PetscLogFlops(tot_its*(84.0 + 41.0 + 26.0));CHKERRQ(ierr);
ierr = DMRestoreLocalVector(da,&localX);CHKERRQ(ierr);
if (B) {
ierr = DMRestoreLocalVector(da,&localB);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode WASHIFunction(TS ts,PetscReal t,Vec X,Vec Xdot,Vec F,void* ctx)
{
PetscErrorCode ierr;
Wash wash=(Wash)ctx;
DM networkdm;
Vec localX,localXdot,localF;
const PetscInt *cone;
PetscInt vfrom,vto,offsetfrom,offsetto,type,varoffset,voffset;
PetscInt v,vStart,vEnd,e,eStart,eEnd,pipeoffset;
PetscBool ghost;
PetscScalar *farr,*vf,*juncx,*juncf;
Pipe pipe;
PipeField *pipex,*pipexdot,*pipef;
DMDALocalInfo info;
Junction junction;
MPI_Comm comm;
PetscMPIInt rank,size;
const PetscScalar *xarr,*xdotarr;
DMNetworkComponentGenericDataType *nwarr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)ts,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = VecSet(F,0.0);CHKERRQ(ierr);
localX = wash->localX;
localXdot = wash->localXdot;
ierr = TSGetDM(ts,&networkdm);CHKERRQ(ierr);
ierr = DMGetLocalVector(networkdm,&localF);CHKERRQ(ierr);
ierr = VecSet(localF,0.0);CHKERRQ(ierr);
/* update ghost values of locaX and locaXdot */
ierr = DMGlobalToLocalBegin(networkdm,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(networkdm,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(networkdm,Xdot,INSERT_VALUES,localXdot);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(networkdm,Xdot,INSERT_VALUES,localXdot);CHKERRQ(ierr);
ierr = VecGetArrayRead(localX,&xarr);CHKERRQ(ierr);
ierr = VecGetArrayRead(localXdot,&xdotarr);CHKERRQ(ierr);
ierr = VecGetArray(localF,&farr);CHKERRQ(ierr);
/* Initialize localF = localX at non-ghost vertices */
ierr = DMNetworkGetVertexRange(networkdm,&vStart,&vEnd);CHKERRQ(ierr);
for (v=vStart; v<vEnd; v++) {
ierr = DMNetworkIsGhostVertex(networkdm,v,&ghost);CHKERRQ(ierr);
if (!ghost) {
ierr = DMNetworkGetVariableOffset(networkdm,v,&varoffset);CHKERRQ(ierr);
juncx = (PetscScalar*)(xarr+varoffset);
juncf = (PetscScalar*)(farr+varoffset);
juncf[0] = juncx[0];
juncf[1] = juncx[1];
}
}
/* Get component(application) data array */
ierr = DMNetworkGetComponentDataArray(networkdm,&nwarr);CHKERRQ(ierr);
/* Edge */
ierr = DMNetworkGetEdgeRange(networkdm,&eStart,&eEnd);CHKERRQ(ierr);
for (e=eStart; e<eEnd; e++) {
ierr = DMNetworkGetComponentTypeOffset(networkdm,e,0,&type,&pipeoffset);CHKERRQ(ierr);
ierr = DMNetworkGetVariableOffset(networkdm,e,&varoffset);CHKERRQ(ierr);
pipe = (Pipe)(nwarr + pipeoffset);
pipex = (PipeField*)(xarr + varoffset);
pipexdot = (PipeField*)(xdotarr + varoffset);
pipef = (PipeField*)(farr + varoffset);
/* Get boundary values H0 and QL from connected vertices */
ierr = DMNetworkGetConnectedNodes(networkdm,e,&cone);CHKERRQ(ierr);
vfrom = cone[0]; /* local ordering */
vto = cone[1];
ierr = DMNetworkGetVariableOffset(networkdm,vfrom,&offsetfrom);CHKERRQ(ierr);
ierr = DMNetworkGetVariableOffset(networkdm,vto,&offsetto);CHKERRQ(ierr);
if (pipe->boundary.Q0 == PIPE_CHARACTERISTIC) {
pipe->boundary.H0 = (xarr+offsetfrom)[1]; /* h_from */
} else {
pipe->boundary.Q0 = (xarr+offsetfrom)[0]; /* q_from */
}
if (pipe->boundary.HL == PIPE_CHARACTERISTIC) {
pipe->boundary.QL = (xarr+offsetto)[0]; /* q_to */
} else {
pipe->boundary.HL = (xarr+offsetto)[1]; /* h_to */
}
/* Evaluate PipeIFunctionLocal() */
ierr = DMDAGetLocalInfo(pipe->da,&info);CHKERRQ(ierr);
ierr = PipeIFunctionLocal(&info, t, pipex, pipexdot, pipef, pipe);CHKERRQ(ierr);
/* Set F at vfrom */
vf = (PetscScalar*)(farr+offsetfrom);
if (pipe->boundary.Q0 == PIPE_CHARACTERISTIC) {
vf[0] -= pipex[0].q; /* q_vfrom - q[0] */
} else {
vf[1] -= pipex[0].h; /* h_vfrom - h[0] */
}
/* Set F at vto */
vf = (PetscScalar*)(farr+offsetto);
if (pipe->boundary.HL == PIPE_CHARACTERISTIC) {
vf[1] -= pipex[pipe->nnodes-1].h; /* h_vto - h[last] */
} else {
vf[0] -= pipex[pipe->nnodes-1].q; /* q_vto - q[last] */
}
}
/* Set F at boundary vertices */
for (v=vStart; v<vEnd; v++) {
ierr = DMNetworkGetComponentTypeOffset(networkdm,v,0,&type,&voffset);CHKERRQ(ierr);
ierr = DMNetworkGetVariableOffset(networkdm,v,&varoffset);CHKERRQ(ierr);
junction = (Junction)(nwarr + voffset);
juncf = (PetscScalar *)(farr + varoffset);
if (junction->isEnd == -1) {
juncf[1] -= wash->H0;
} else if (junction->isEnd == 1) {
juncf[0] -= wash->QL;
}
}
ierr = VecRestoreArrayRead(localX,&xarr);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(localXdot,&xdotarr);CHKERRQ(ierr);
ierr = VecRestoreArray(localF,&farr);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(networkdm,localF,ADD_VALUES,F);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(networkdm,localF,ADD_VALUES,F);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(networkdm,&localF);CHKERRQ(ierr);
/* ierr = VecView(F,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
ObsCtx user;
SNES snes;
DM da;
Vec u, /* solution */
Xu; /* upper bound */
DMDALocalInfo info;
PetscReal error1,errorinf;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,11,11,/* default to 10x10 grid */
PETSC_DECIDE,PETSC_DECIDE, /* number of processors in each dimension */1,/* dof = 1 */1,/* s = 1; stencil extends out one cell */
NULL,NULL,/* do not specify processor decomposition */&da);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&u);CHKERRQ(ierr);
ierr = VecDuplicate(u,&(user.uexact));CHKERRQ(ierr);
ierr = VecDuplicate(u,&(user.psi));CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(da,-2.0,2.0,-2.0,2.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMSetApplicationContext(da,&user);CHKERRQ(ierr);
ierr = FormPsiAndExactSoln(da);CHKERRQ(ierr);
ierr = VecSet(u,0.0);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
ierr = SNESSetDM(snes,da);CHKERRQ(ierr);
ierr = SNESSetApplicationContext(snes,&user);CHKERRQ(ierr);
ierr = SNESSetType(snes,SNESVINEWTONRSLS);CHKERRQ(ierr);
/* set upper and lower bound constraints for VI */
ierr = VecDuplicate(u,&Xu);CHKERRQ(ierr);
ierr = VecSet(Xu,PETSC_INFINITY);CHKERRQ(ierr);
ierr = SNESVISetVariableBounds(snes,user.psi,Xu);CHKERRQ(ierr);
ierr = VecDestroy(&Xu);CHKERRQ(ierr);
ierr = DMDASNESSetFunctionLocal(da,INSERT_VALUES,(PetscErrorCode (*)(DMDALocalInfo*,void*,void*,void*))FormFunctionLocal,&user);CHKERRQ(ierr);
ierr = DMDASNESSetJacobianLocal(da,(PetscErrorCode (*)(DMDALocalInfo*,void*,Mat,Mat,void*))FormJacobianLocal,&user);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
/* report on setup */
ierr = DMDAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"setup done: grid Mx,My = %D,%D with spacing dx,dy = %.4f,%.4f\n",
info.mx,info.my,(double)(4.0/(PetscReal)(info.mx-1)),(double)(4.0/(PetscReal)(info.my-1)));CHKERRQ(ierr);
/* solve nonlinear system */
ierr = SNESSolve(snes,NULL,u);CHKERRQ(ierr);
/* compare to exact */
ierr = VecAXPY(u,-1.0,user.uexact);CHKERRQ(ierr); /* u <- u - uexact */
ierr = VecNorm(u,NORM_1,&error1);CHKERRQ(ierr);
error1 /= (PetscReal)info.mx * (PetscReal)info.my;
ierr = VecNorm(u,NORM_INFINITY,&errorinf);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"errors: av |u-uexact| = %.3e |u-uexact|_inf = %.3e\n",(double)error1,(double)errorinf);CHKERRQ(ierr);
/* Free work space. */
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&(user.psi));CHKERRQ(ierr);
ierr = VecDestroy(&(user.uexact));CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
static PetscErrorCode FormJacobian_All(SNES snes,Vec X,Mat J,Mat B,void *ctx)
{
User user = (User)ctx;
DM dau,dak;
DMDALocalInfo infou,infok;
PetscScalar *u,*k;
PetscErrorCode ierr;
Vec Uloc,Kloc;
PetscFunctionBeginUser;
ierr = DMCompositeGetEntries(user->pack,&dau,&dak);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(dau,&infou);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(dak,&infok);CHKERRQ(ierr);
ierr = DMCompositeGetLocalVectors(user->pack,&Uloc,&Kloc);CHKERRQ(ierr);
switch (user->ptype) {
case 0:
ierr = DMGlobalToLocalBegin(dau,X,INSERT_VALUES,Uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd (dau,X,INSERT_VALUES,Uloc);CHKERRQ(ierr);
ierr = DMDAVecGetArray(dau,Uloc,&u);CHKERRQ(ierr);
ierr = DMDAVecGetArray(dak,user->Kloc,&k);CHKERRQ(ierr);
ierr = FormJacobianLocal_U(user,&infou,u,k,B);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(dau,Uloc,&u);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(dak,user->Kloc,&k);CHKERRQ(ierr);
break;
case 1:
ierr = DMGlobalToLocalBegin(dak,X,INSERT_VALUES,Kloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd (dak,X,INSERT_VALUES,Kloc);CHKERRQ(ierr);
ierr = DMDAVecGetArray(dau,user->Uloc,&u);CHKERRQ(ierr);
ierr = DMDAVecGetArray(dak,Kloc,&k);CHKERRQ(ierr);
ierr = FormJacobianLocal_K(user,&infok,u,k,B);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(dau,user->Uloc,&u);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(dak,Kloc,&k);CHKERRQ(ierr);
break;
case 2: {
Mat Buu,Buk,Bku,Bkk;
IS *is;
ierr = DMCompositeScatter(user->pack,X,Uloc,Kloc);CHKERRQ(ierr);
ierr = DMDAVecGetArray(dau,Uloc,&u);CHKERRQ(ierr);
ierr = DMDAVecGetArray(dak,Kloc,&k);CHKERRQ(ierr);
ierr = DMCompositeGetLocalISs(user->pack,&is);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(B,is[0],is[0],&Buu);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(B,is[0],is[1],&Buk);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(B,is[1],is[0],&Bku);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(B,is[1],is[1],&Bkk);CHKERRQ(ierr);
ierr = FormJacobianLocal_U(user,&infou,u,k,Buu);CHKERRQ(ierr);
ierr = FormJacobianLocal_UK(user,&infou,&infok,u,k,Buk);CHKERRQ(ierr);
ierr = FormJacobianLocal_KU(user,&infou,&infok,u,k,Bku);CHKERRQ(ierr);
ierr = FormJacobianLocal_K(user,&infok,u,k,Bkk);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(B,is[0],is[0],&Buu);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(B,is[0],is[1],&Buk);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(B,is[1],is[0],&Bku);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(B,is[1],is[1],&Bkk);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(dau,Uloc,&u);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(dak,Kloc,&k);CHKERRQ(ierr);
ierr = ISDestroy(&is[0]);CHKERRQ(ierr);
ierr = ISDestroy(&is[1]);CHKERRQ(ierr);
ierr = PetscFree(is);CHKERRQ(ierr);
} break;
}
ierr = DMCompositeRestoreLocalVectors(user->pack,&Uloc,&Kloc);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (J != B) {
ierr = MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*
Fills the local vector problem on the subdomain from the global problem.
Right now this assumes one subdomain per processor.
*/
PetscErrorCode DMCreateDomainDecompositionScatters_DA(DM dm,PetscInt nsubdms,DM *subdms,VecScatter **iscat,VecScatter **oscat, VecScatter **lscat)
{
PetscErrorCode ierr;
DMDALocalInfo info,subinfo;
DM subdm;
MatStencil upper,lower;
IS idis,isis,odis,osis,gdis;
Vec svec,dvec,slvec;
PetscInt xm,ym,zm,xs,ys,zs;
PetscInt i;
PetscFunctionBegin;
/* allocate the arrays of scatters */
if (iscat) {ierr = PetscMalloc1(nsubdms,iscat);CHKERRQ(ierr);}
if (oscat) {ierr = PetscMalloc1(nsubdms,oscat);CHKERRQ(ierr);}
if (lscat) {ierr = PetscMalloc1(nsubdms,lscat);CHKERRQ(ierr);}
ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr);
for (i = 0; i < nsubdms; i++) {
subdm = subdms[i];
ierr = DMDAGetLocalInfo(subdm,&subinfo);CHKERRQ(ierr);
ierr = DMDAGetNonOverlappingRegion(subdm,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
/* create the global and subdomain index sets for the inner domain */
lower.i = xs;
lower.j = ys;
lower.k = zs;
upper.i = xs+xm;
upper.j = ys+ym;
upper.k = zs+zm;
ierr = DMDACreatePatchIS(dm,&lower,&upper,&idis);CHKERRQ(ierr);
ierr = DMDACreatePatchIS(subdm,&lower,&upper,&isis);CHKERRQ(ierr);
/* create the global and subdomain index sets for the outer subdomain */
lower.i = subinfo.xs;
lower.j = subinfo.ys;
lower.k = subinfo.zs;
upper.i = subinfo.xs+subinfo.xm;
upper.j = subinfo.ys+subinfo.ym;
upper.k = subinfo.zs+subinfo.zm;
ierr = DMDACreatePatchIS(dm,&lower,&upper,&odis);CHKERRQ(ierr);
ierr = DMDACreatePatchIS(subdm,&lower,&upper,&osis);CHKERRQ(ierr);
/* global and subdomain ISes for the local indices of the subdomain */
/* todo - make this not loop over at nonperiodic boundaries, which will be more involved */
lower.i = subinfo.gxs;
lower.j = subinfo.gys;
lower.k = subinfo.gzs;
upper.i = subinfo.gxs+subinfo.gxm;
upper.j = subinfo.gys+subinfo.gym;
upper.k = subinfo.gzs+subinfo.gzm;
ierr = DMDACreatePatchIS(dm,&lower,&upper,&gdis);CHKERRQ(ierr);
/* form the scatter */
ierr = DMGetGlobalVector(dm,&dvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(subdm,&svec);CHKERRQ(ierr);
ierr = DMGetLocalVector(subdm,&slvec);CHKERRQ(ierr);
if (iscat) {ierr = VecScatterCreate(dvec,idis,svec,isis,&(*iscat)[i]);CHKERRQ(ierr);}
if (oscat) {ierr = VecScatterCreate(dvec,odis,svec,osis,&(*oscat)[i]);CHKERRQ(ierr);}
if (lscat) {ierr = VecScatterCreate(dvec,gdis,slvec,NULL,&(*lscat)[i]);CHKERRQ(ierr);}
ierr = DMRestoreGlobalVector(dm,&dvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(subdm,&svec);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(subdm,&slvec);CHKERRQ(ierr);
ierr = ISDestroy(&idis);CHKERRQ(ierr);
ierr = ISDestroy(&isis);CHKERRQ(ierr);
ierr = ISDestroy(&odis);CHKERRQ(ierr);
ierr = ISDestroy(&osis);CHKERRQ(ierr);
ierr = ISDestroy(&gdis);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode DMDASubDomainDA_Private(DM dm, PetscInt *nlocal, DM **sdm)
{
DM *da;
PetscInt dim,size,i,j,k,idx;
PetscErrorCode ierr;
DMDALocalInfo info;
PetscInt xsize,ysize,zsize;
PetscInt xo,yo,zo;
PetscInt xs,ys,zs;
PetscInt xm=1,ym=1,zm=1;
PetscInt xol,yol,zol;
PetscInt m=1,n=1,p=1;
PetscInt M,N,P;
PetscInt pm,mtmp;
PetscFunctionBegin;
ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr);
ierr = DMDAGetOverlap(dm,&xol,&yol,&zol);CHKERRQ(ierr);
ierr = DMDAGetNumLocalSubDomains(dm,&size);CHKERRQ(ierr);
ierr = PetscMalloc1(size,&da);CHKERRQ(ierr);
if (nlocal) *nlocal = size;
dim = info.dim;
M = info.xm;
N = info.ym;
P = info.zm;
if (dim == 1) {
m = size;
} else if (dim == 2) {
m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)N)));
while (m > 0) {
n = size/m;
if (m*n*p == size) break;
m--;
}
} else if (dim == 3) {
n = (PetscInt)(0.5 + PetscPowReal(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.))); if (!n) n = 1;
while (n > 0) {
pm = size/n;
if (n*pm == size) break;
n--;
}
if (!n) n = 1;
m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
if (!m) m = 1;
while (m > 0) {
p = size/(m*n);
if (m*n*p == size) break;
m--;
}
if (M > P && m < p) {mtmp = m; m = p; p = mtmp;}
}
zs = info.zs;
idx = 0;
for (k = 0; k < p; k++) {
ys = info.ys;
for (j = 0; j < n; j++) {
xs = info.xs;
for (i = 0; i < m; i++) {
if (dim == 1) {
xm = M/m + ((M % m) > i);
} else if (dim == 2) {
xm = M/m + ((M % m) > i);
ym = N/n + ((N % n) > j);
} else if (dim == 3) {
xm = M/m + ((M % m) > i);
ym = N/n + ((N % n) > j);
zm = P/p + ((P % p) > k);
}
xsize = xm;
ysize = ym;
zsize = zm;
xo = xs;
yo = ys;
zo = zs;
ierr = DMDACreate(PETSC_COMM_SELF,&(da[idx]));CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da[idx],"sub_");CHKERRQ(ierr);
ierr = DMSetDimension(da[idx], info.dim);CHKERRQ(ierr);
ierr = DMDASetDof(da[idx], info.dof);CHKERRQ(ierr);
ierr = DMDASetStencilType(da[idx],info.st);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da[idx],info.sw);CHKERRQ(ierr);
if (info.bx == DM_BOUNDARY_PERIODIC || (xs != 0)) {
xsize += xol;
xo -= xol;
}
if (info.by == DM_BOUNDARY_PERIODIC || (ys != 0)) {
ysize += yol;
yo -= yol;
}
if (info.bz == DM_BOUNDARY_PERIODIC || (zs != 0)) {
zsize += zol;
zo -= zol;
}
if (info.bx == DM_BOUNDARY_PERIODIC || (xs+xm != info.mx)) xsize += xol;
if (info.by == DM_BOUNDARY_PERIODIC || (ys+ym != info.my)) ysize += yol;
if (info.bz == DM_BOUNDARY_PERIODIC || (zs+zm != info.mz)) zsize += zol;
if (info.bx != DM_BOUNDARY_PERIODIC) {
if (xo < 0) {
xsize += xo;
xo = 0;
}
if (xo+xsize > info.mx-1) {
xsize -= xo+xsize - info.mx;
}
}
if (info.by != DM_BOUNDARY_PERIODIC) {
if (yo < 0) {
ysize += yo;
yo = 0;
}
if (yo+ysize > info.my-1) {
ysize -= yo+ysize - info.my;
}
}
if (info.bz != DM_BOUNDARY_PERIODIC) {
if (zo < 0) {
zsize += zo;
zo = 0;
}
if (zo+zsize > info.mz-1) {
zsize -= zo+zsize - info.mz;
}
}
ierr = DMDASetSizes(da[idx], xsize, ysize, zsize);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da[idx], 1, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da[idx], DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
/* set up as a block instead */
ierr = DMSetUp(da[idx]);CHKERRQ(ierr);
/* nonoverlapping region */
ierr = DMDASetNonOverlappingRegion(da[idx],xs,ys,zs,xm,ym,zm);CHKERRQ(ierr);
/* this alters the behavior of DMDAGetInfo, DMDAGetLocalInfo, DMDAGetCorners, and DMDAGetGhostedCorners and should be used with care */
ierr = DMDASetOffset(da[idx],xo,yo,zo,info.mx,info.my,info.mz);CHKERRQ(ierr);
xs += xm;
idx++;
}
ys += ym;
}
zs += zm;
}
*sdm = da;
PetscFunctionReturn(0);
}
/*
L_2Error - Integrate the L_2 error of our solution over each face
*/
PetscErrorCode L_2Error(DM da, Vec fVec, PetscReal *error, AppCtx *user)
{
DMDALocalInfo info;
Vec fLocalVec;
Field **f;
Field u, uExact, uLocal[4];
PetscScalar hx, hy, hxhy, x, y, phi[3];
PetscInt i, j, q;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DMDAGetLocalInfo(da, &info);CHKERRQ(ierr);
ierr = DMGetLocalVector(da, &fLocalVec);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,fVec, INSERT_VALUES, fLocalVec);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,fVec, INSERT_VALUES, fLocalVec);CHKERRQ(ierr);
ierr = DMDAVecGetArray(da, fLocalVec, &f);CHKERRQ(ierr);
*error = 0.0;
hx = 1.0/(PetscReal)(info.mx-1);
hy = 1.0/(PetscReal)(info.my-1);
hxhy = hx*hy;
for (j = info.ys; j < info.ys+info.ym-1; j++) {
for (i = info.xs; i < info.xs+info.xm-1; i++) {
uLocal[0] = f[j][i];
uLocal[1] = f[j][i+1];
uLocal[2] = f[j+1][i+1];
uLocal[3] = f[j+1][i];
/* Lower element */
for (q = 0; q < 4; q++) {
phi[0] = 1.0 - quadPoints[q*2] - quadPoints[q*2+1];
phi[1] = quadPoints[q*2];
phi[2] = quadPoints[q*2+1];
u.u = uLocal[0].u*phi[0]+ uLocal[1].u*phi[1] + uLocal[3].u*phi[2];
u.v = uLocal[0].v*phi[0]+ uLocal[1].v*phi[1] + uLocal[3].v*phi[2];
u.p = uLocal[0].p*phi[0]+ uLocal[1].p*phi[1] + uLocal[3].p*phi[2];
x = (quadPoints[q*2] + (PetscReal)i)*hx;
y = (quadPoints[q*2+1] + (PetscReal)j)*hy;
ierr = ExactSolution(PetscAbsScalar(x), PetscAbsScalar(y), &uExact);CHKERRQ(ierr);
*error += PetscAbsScalar(hxhy*quadWeights[q]*((u.u - uExact.u)*(u.u - uExact.u) + (u.v - uExact.v)*(u.v - uExact.v) + (u.p - uExact.p)*(u.p - uExact.p)));
}
/* Upper element */
/*
The affine map from the lower to the upper is
/ x_U \ = / -1 0 \ / x_L \ + / hx \
\ y_U / \ 0 -1 / \ y_L / \ hy /
*/
for (q = 0; q < 4; q++) {
phi[0] = 1.0 - quadPoints[q*2] - quadPoints[q*2+1];
phi[1] = quadPoints[q*2];
phi[2] = quadPoints[q*2+1];
u.u = uLocal[2].u*phi[0]+ uLocal[3].u*phi[1] + uLocal[1].u*phi[2];
u.v = uLocal[2].v*phi[0]+ uLocal[3].v*phi[1] + uLocal[1].v*phi[2];
u.p = uLocal[0].p*phi[0]+ uLocal[1].p*phi[1] + uLocal[3].p*phi[2];
x = (1.0 - quadPoints[q*2] + (PetscReal)i)*hx;
y = (1.0 - quadPoints[q*2+1] + (PetscReal)j)*hy;
ierr = ExactSolution(PetscAbsScalar(x), PetscAbsScalar(y), &uExact);CHKERRQ(ierr);
*error += PetscAbsScalar(hxhy*quadWeights[q]*((u.u - uExact.u)*(u.u - uExact.u) + (u.v - uExact.v)*(u.v - uExact.v) + (u.p - uExact.p)*(u.p - uExact.p)));
}
}
}
ierr = DMDAVecRestoreArray(da, fLocalVec, &f);CHKERRQ(ierr);
/* ierr = DMLocalToGlobalBegin(da,xLocalVec,ADD_VALUES,xVec);CHKERRQ(ierr); */
/* ierr = DMLocalToGlobalEnd(da,xLocalVec,ADD_VALUES,xVec);CHKERRQ(ierr); */
ierr = DMRestoreLocalVector(da, &fLocalVec);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
