int main(int argc,char **argv)
{
DM dm;
Vec X,Y;
PetscErrorCode ierr;
PetscInt dof = 10;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-128,-128,-128,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,1,NULL,NULL,NULL,&dm);CHKERRQ(ierr);
ierr = DMSetFromOptions(dm);CHKERRQ(ierr);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = PetscMemoryTrace("DMDACreate3d ");CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dm,&X);CHKERRQ(ierr);
ierr = PetscMemoryTrace("DMCreateGlobalVector");
ierr = DMCreateGlobalVector(dm,&Y);CHKERRQ(ierr);
ierr = PetscMemoryTrace("DMCreateGlobalVector");CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&Y);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
inline void
DMCreate<3
>(MPI_Comm const& comm,
DMBoundaryTypeVector<3> const& boundaryTypes,
DMDAStencilType const& stencilType,
VectorDi<3> const& globalSize,
VectorDi<3> const& processorSize,
int const& dof,
int const& stencilWidth,
VectorDConstPetscIntPointer<3> const& localSizes,
DM* da) {
DMDACreate3d(comm,
boundaryTypes(0),
boundaryTypes(1),
boundaryTypes(2),
stencilType,
globalSize(0),
globalSize(1),
globalSize(2),
processorSize(0),
processorSize(1),
processorSize(2),
dof,
stencilWidth,
localSizes(0).get(),
localSizes(1).get(),
localSizes(2).get(),
da);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
KSP ksp;
PetscReal norm;
DM da;
Vec x,b,r;
Mat A;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);
CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_STAR,-7,-7,-7,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0,0,&da);
CHKERRQ(ierr);
ierr = DMSetInitialGuess(da,ComputeInitialGuess);
CHKERRQ(ierr);
ierr = KSPSetComputeRHS(ksp,ComputeRHS,PETSC_NULL);
CHKERRQ(ierr);
ierr = KSPSetComputeOperators(ksp,ComputeMatrix,PETSC_NULL);
CHKERRQ(ierr);
ierr = KSPSetDM(ksp,da);
CHKERRQ(ierr);
ierr = DMDestroy(&da);
CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);
CHKERRQ(ierr);
ierr = KSPSolve(ksp,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
ierr = KSPGetSolution(ksp,&x);
CHKERRQ(ierr);
ierr = KSPGetRhs(ksp,&b);
CHKERRQ(ierr);
ierr = VecDuplicate(b,&r);
CHKERRQ(ierr);
ierr = KSPGetOperators(ksp,&A,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
ierr = MatMult(A,x,r);
CHKERRQ(ierr);
ierr = VecAXPY(r,-1.0,b);
CHKERRQ(ierr);
ierr = VecNorm(r,NORM_2,&norm);
CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Residual norm %G\n",norm);
CHKERRQ(ierr);
ierr = VecDestroy(&r);
CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);
CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv) {
PetscErrorCode ierr;
SNES snes;
KSP ksp;
Vec u, uexact;
FishCtx user;
DMDALocalInfo info;
double errnorm;
PetscInitialize(&argc,&argv,NULL,help);
user.printevals = PETSC_FALSE;
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, "f3_", "options for fish3", ""); CHKERRQ(ierr);
ierr = PetscOptionsBool("-printevals","residual and Jacobian routines report grid",
"fish3.c",user.printevals,&user.printevals,NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd(); CHKERRQ(ierr);
ierr = DMDACreate3d(COMM,
DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,
DMDA_STENCIL_STAR,
3,3,3,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
1,1,
NULL,NULL,NULL,&(user.da)); CHKERRQ(ierr);
ierr = DMSetFromOptions(user.da); CHKERRQ(ierr);
ierr = DMSetUp(user.da); CHKERRQ(ierr);
ierr = DMSetApplicationContext(user.da,&user);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(user.da,&info); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(user.da,&u);CHKERRQ(ierr);
ierr = VecDuplicate(u,&uexact);CHKERRQ(ierr);
ierr = VecDuplicate(u,&(user.b));CHKERRQ(ierr);
ierr = formExactRHS(&info,uexact,user.b,&user); CHKERRQ(ierr);
ierr = SNESCreate(COMM,&snes); CHKERRQ(ierr);
ierr = SNESSetDM(snes,user.da); CHKERRQ(ierr);
ierr = DMDASNESSetFunctionLocal(user.da,INSERT_VALUES,
(DMDASNESFunction)FormFunctionLocal,&user); CHKERRQ(ierr);
ierr = DMDASNESSetJacobianLocal(user.da,
(DMDASNESJacobian)FormJacobianLocal,&user); CHKERRQ(ierr);
ierr = SNESGetKSP(snes,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPCG); CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes); CHKERRQ(ierr);
ierr = VecSet(u,0.0); CHKERRQ(ierr);
ierr = SNESSolve(snes,NULL,u); CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,uexact); CHKERRQ(ierr); // u <- u + (-1.0) uexact
ierr = VecNorm(u,NORM_INFINITY,&errnorm); CHKERRQ(ierr);
ierr = PetscPrintf(COMM,"on %d x %d x %d grid: error |u-uexact|_inf = %g\n",
info.mx,info.my,info.mz,errnorm); CHKERRQ(ierr);
VecDestroy(&u); VecDestroy(&uexact); VecDestroy(&(user.b));
SNESDestroy(&snes); DMDestroy(&(user.da));
PetscFinalize();
return 0;
}
/*@
DMDAGetReducedDMDA - Gets the DMDA with the same layout but with fewer or more fields
Collective on DMDA
Input Parameters:
+ da - the distributed array
- nfields - number of fields in new DMDA
Output Parameter:
. nda - the new DMDA
Level: intermediate
.keywords: distributed array, get, corners, nodes, local indices, coordinates
.seealso: DMDAGetGhostCorners(), DMSetCoordinates(), DMDASetUniformCoordinates(), DMGetCoordinates(), DMDAGetGhostedCoordinates()
@*/
PetscErrorCode DMDAGetReducedDMDA(DM da,PetscInt nfields,DM *nda)
{
PetscErrorCode ierr;
DM_DA *dd = (DM_DA*)da->data;
PetscInt s,m,n,p,M,N,P,dim,Mo,No,Po;
const PetscInt *lx,*ly,*lz;
DMBoundaryType bx,by,bz;
DMDAStencilType stencil_type;
PetscInt ox,oy,oz;
PetscInt cl,rl;
PetscFunctionBegin;
dim = da->dim;
M = dd->M;
N = dd->N;
P = dd->P;
m = dd->m;
n = dd->n;
p = dd->p;
s = dd->s;
bx = dd->bx;
by = dd->by;
bz = dd->bz;
stencil_type = dd->stencil_type;
ierr = DMDAGetOwnershipRanges(da,&lx,&ly,&lz);CHKERRQ(ierr);
if (dim == 1) {
ierr = DMDACreate1d(PetscObjectComm((PetscObject)da),bx,M,nfields,s,dd->lx,nda);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = DMDACreate2d(PetscObjectComm((PetscObject)da),bx,by,stencil_type,M,N,m,n,nfields,s,lx,ly,nda);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PetscObjectComm((PetscObject)da),bx,by,bz,stencil_type,M,N,P,m,n,p,nfields,s,lx,ly,lz,nda);CHKERRQ(ierr);
}
ierr = DMSetUp(*nda);CHKERRQ(ierr);
if (da->coordinates) {
ierr = PetscObjectReference((PetscObject)da->coordinates);CHKERRQ(ierr);
(*nda)->coordinates = da->coordinates;
}
/* allow for getting a reduced DA corresponding to a domain decomposition */
ierr = DMDAGetOffset(da,&ox,&oy,&oz,&Mo,&No,&Po);CHKERRQ(ierr);
ierr = DMDASetOffset(*nda,ox,oy,oz,Mo,No,Po);CHKERRQ(ierr);
/* allow for getting a reduced DA corresponding to a coarsened DA */
ierr = DMGetCoarsenLevel(da,&cl);CHKERRQ(ierr);
ierr = DMGetRefineLevel(da,&rl);CHKERRQ(ierr);
(*nda)->levelup = rl;
(*nda)->leveldown = cl;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE,M = 3,N = 5,P=3,s=1;
PetscInt *lx = NULL,*ly = NULL,*lz = NULL;
PetscErrorCode ierr;
PetscBool flg = PETSC_FALSE;
DM da;
Vec local,global,vslice;
PetscScalar value;
DMDABoundaryType wrap = DMDA_XYPERIODIC;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
VecScatter scatter;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-P",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-p",&p,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-star",&flg,NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,wrap,stencil_type,M,N,P,m,n,p,1,s,
lx,ly,lz,&da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = GenerateSliceScatter(da,&scatter,&vslice);CHKERRQ(ierr);
/* Put the value rank+1 into all locations of vslice and transfer back to global vector */
value = 1.0 + rank;
ierr = VecSet(vslice,value);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,vslice,global,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,vslice,global,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode TestQ2Q1DA( void )
{
DM Q2_da,Q1_da,cda;
PetscInt mx,my,mz;
Vec coords,gcoords,gcoords2;
PetscErrorCode ierr;
mx=7;
my=11;
mz=13;
ierr=DMDACreate3d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,mx,my,mz,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,2,0,0,0,&Q2_da);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(Q2_da,-1.0,1.0,-2.0,2.0,-3.0,3.0);CHKERRQ(ierr);
ierr = DMDAGetCoordinates(Q2_da,&coords);CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,mx,my,mz,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,0,0,0,&Q1_da);CHKERRQ(ierr);
ierr = DMDASetCoordinates(Q1_da,coords);CHKERRQ(ierr);
/* Get ghost coordinates one way */
ierr = DMDAGetGhostedCoordinates(Q1_da,&gcoords);CHKERRQ(ierr);
/* And another */
ierr = DMDAGetCoordinates(Q1_da,&coords);CHKERRQ(ierr);
ierr = DMDAGetCoordinateDA(Q1_da,&cda);CHKERRQ(ierr);
ierr = DMGetLocalVector(cda,&gcoords2);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(cda,coords,INSERT_VALUES,gcoords2);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(cda,coords,INSERT_VALUES,gcoords2);CHKERRQ(ierr);
ierr = CompareGhostedCoords(gcoords,gcoords2);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(cda,&gcoords2);CHKERRQ(ierr);
ierr = VecScale(coords,10.0);CHKERRQ(ierr);
ierr = VecScale(gcoords,10.0);CHKERRQ(ierr);
ierr = DMDAGetGhostedCoordinates(Q1_da,&gcoords2);CHKERRQ(ierr);
ierr = CompareGhostedCoords(gcoords,gcoords2);CHKERRQ(ierr);
ierr = DMDestroy(&Q2_da);CHKERRQ(ierr);
ierr = DMDestroy(&Q1_da);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscInt M = 5,N = 4,P = 3, m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE,dim = 1;
PetscErrorCode ierr;
DM dac,daf;
DMDABoundaryType bx=DMDA_BOUNDARY_NONE,by=DMDA_BOUNDARY_NONE,bz=DMDA_BOUNDARY_NONE;
DMDAStencilType stype = DMDA_STENCIL_BOX;
Mat A;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-P",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dim",&dim,PETSC_NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
if (dim == 1) {
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,1,1,PETSC_NULL,&dac);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,stype,M,N,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,&dac);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stype,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,PETSC_NULL,&dac);CHKERRQ(ierr);
}
ierr = DMRefine(dac,PETSC_COMM_WORLD,&daf);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(dac,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
if (dim == 1) {
ierr = SetCoordinates1d(daf);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = SetCoordinates2d(daf);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = SetCoordinates3d(daf);CHKERRQ(ierr);
}
ierr = DMCreateInterpolation(dac,daf,&A,0);CHKERRQ(ierr);
ierr = MatViewFromOptions(A,"-mat_view");CHKERRQ(ierr);
/* Free memory */
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscInt M = 3,N = 4,P = 2,s = 1,w = 2,i, m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE;
PetscErrorCode ierr;
DM da;
Mat mat;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
PetscBool flg = PETSC_FALSE;
MatStencil idx[2],idy[2];
PetscScalar *values;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-P",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-p",&p,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-w",&w,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-star",&flg,NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,stencil_type,M,N,P,m,n,p,w,s,
0,0,0,&da);CHKERRQ(ierr);
ierr = DMSetMatType(da,MATMPIBAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,&mat);CHKERRQ(ierr);
idx[0].i = 1; idx[0].j = 1; idx[0].k = 0;
idx[1].i = 2; idx[1].j = 1; idx[1].k = 0;
idy[0].i = 1; idy[0].j = 2; idy[0].k = 0;
idy[1].i = 2; idy[1].j = 2; idy[1].k = 0;
ierr = PetscMalloc1(2*2*w*w,&values);CHKERRQ(ierr);
for (i=0; i<2*2*w*w; i++) values[i] = i;
ierr = MatSetValuesBlockedStencil(mat,2,idx,2,idy,values,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Free memory */
ierr = PetscFree(values);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt M = -2, N = -3, P = 4,stencil_width = 1, dof = 1,m,n,p,xstart,ystart,zstart,i,j,k,c;
DM da;
Vec global,local;
PetscScalar ****vglobal;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
PetscFunctionBeginUser;
ierr = PetscOptionsGetInt(0,"-stencil_width",&stencil_width,0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(0,"-dof",&dof,0);CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_MIRROR,DM_BOUNDARY_MIRROR,DM_BOUNDARY_MIRROR,DMDA_STENCIL_STAR,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,NULL,NULL,NULL,&da);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xstart,&ystart,&zstart,&m,&n,&p);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMDAVecGetArrayDOF(da,global,&vglobal);CHKERRQ(ierr);
for (k=zstart; k<zstart+p; k++) {
for (j=ystart; j<ystart+n; j++) {
for (i=xstart; i<xstart+m; i++) {
for (c=0; c<dof; c++) {
vglobal[k][j][i][c] = 1000*k + 100*j + 10*(i+1) + c;
}
}
}
}
ierr = DMDAVecRestoreArrayDOF(da,global,&vglobal);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = PetscSequentialPhaseBegin(PETSC_COMM_WORLD,1);CHKERRQ(ierr);
ierr = VecView(local,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscSequentialPhaseEnd(PETSC_COMM_WORLD,1);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode ComputeMapping(DomainData dd,ISLocalToGlobalMapping *isg2lmap)
{
PetscErrorCode ierr;
DM da;
AO ao;
DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE, bz = DM_BOUNDARY_NONE;
DMDAStencilType stype = DMDA_STENCIL_BOX;
ISLocalToGlobalMapping temp_isg2lmap;
PetscInt i,j,k,ig,jg,kg,lindex,gindex,localsize;
PetscInt *global_indices;
PetscFunctionBeginUser;
/* Not an efficient mapping: this function computes a very simple lexicographic mapping
just to illustrate the creation of a MATIS object */
localsize = dd.xm_l*dd.ym_l*dd.zm_l;
ierr = PetscMalloc1(localsize,&global_indices);CHKERRQ(ierr);
for (k=0; k<dd.zm_l; k++) {
kg=dd.startz+k;
for (j=0; j<dd.ym_l; j++) {
jg=dd.starty+j;
for (i=0; i<dd.xm_l; i++) {
ig =dd.startx+i;
lindex =k*dd.xm_l*dd.ym_l+j*dd.xm_l+i;
gindex =kg*dd.xm*dd.ym+jg*dd.xm+ig;
global_indices[lindex]=gindex;
}
}
}
if (dd.dim==3) {
ierr = DMDACreate3d(dd.gcomm,bx,by,bz,stype,dd.xm,dd.ym,dd.zm,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dd.dim==2) {
ierr = DMDACreate2d(dd.gcomm,bx,by,stype,dd.xm,dd.ym,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(dd.gcomm,bx,dd.xm,1,1,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDASetAOType(da,AOMEMORYSCALABLE);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,dd.xm_l*dd.ym_l*dd.zm_l,global_indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(dd.gcomm,1,localsize,global_indices,PETSC_OWN_POINTER,&temp_isg2lmap);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
*isg2lmap = temp_isg2lmap;
PetscFunctionReturn(0);
}
PetscErrorCode TestDMDAVec(PetscBool usempiio)
{
DM dm;
Vec x_ref,x_test;
PetscBool skipheader = PETSC_TRUE;
PetscErrorCode ierr;
PetscFunctionBeginUser;
if (!usempiio) { ierr = PetscPrintf(PETSC_COMM_WORLD,"%s\n",__FUNCT__);CHKERRQ(ierr); }
else { ierr = PetscPrintf(PETSC_COMM_WORLD,"%s [using mpi-io]\n",__FUNCT__);CHKERRQ(ierr); }
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,
DMDA_STENCIL_BOX,DMDA_I,DMDA_J,DMDA_K,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
3,2,NULL,NULL,NULL,&dm);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dm,&x_ref);CHKERRQ(ierr);
ierr = DMDAVecGenerateEntries(dm,x_ref);CHKERRQ(ierr);
if (!usempiio) {
ierr = MyVecDump("dmda.pbvec",skipheader,PETSC_FALSE,x_ref);CHKERRQ(ierr);
} else {
ierr = MyVecDump("dmda-mpiio.pbvec",skipheader,PETSC_TRUE,x_ref);CHKERRQ(ierr);
}
ierr = DMCreateGlobalVector(dm,&x_test);CHKERRQ(ierr);
if (!usempiio) {
ierr = MyVecLoad("dmda.pbvec",skipheader,usempiio,x_test);CHKERRQ(ierr);
} else {
ierr = MyVecLoad("dmda-mpiio.pbvec",skipheader,usempiio,x_test);CHKERRQ(ierr);
}
ierr = VecCompare(x_ref,x_test);CHKERRQ(ierr);
if (!usempiio) {
ierr = HeaderlessBinaryReadCheck(dm,"dmda.pbvec");CHKERRQ(ierr);
} else {
ierr = HeaderlessBinaryReadCheck(dm,"dmda-mpiio.pbvec");CHKERRQ(ierr);
}
ierr = VecDestroy(&x_ref);CHKERRQ(ierr);
ierr = VecDestroy(&x_test);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
example is used for testing the subroutine MatLoad_MPI_DA\n\n";
#include <petscdm.h>
#include <petscdmda.h>
int main(int argc,char **argv)
{
PetscInt X = 10,Y = 8,Z=8;
PetscErrorCode ierr;
DM da;
PetscViewer viewer;
Mat A;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"temp.dat",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(NULL,NULL,"-X",&X,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Y",&Y,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Z",&Z,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,X,Y,Z,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMSetMatType(da,MATMPIAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,&A);CHKERRQ(ierr);
ierr = MatShift(A,X);CHKERRQ(ierr);
ierr = MatView(A,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"temp.dat",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,&A);CHKERRQ(ierr);
ierr = MatLoad(A,viewer);CHKERRQ(ierr);
/* Free memory */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscInt N = 3,M=2,P=4,dof=1,rstart,rend,i;
PetscInt stencil_width = 2;
PetscErrorCode ierr;
PetscMPIInt rank;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_STAR;
DM da;
Vec global,local;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-P",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-stencil_width",&stencil_width,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-stencil_type",(PetscInt*)&stencil_type,NULL);CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,0,&da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(global,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {ierr = VecSetValue(global,i,(PetscReal)(i + 100*rank),INSERT_VALUES);CHKERRQ(ierr);}
ierr = VecAssemblyBegin(global);CHKERRQ(ierr);
ierr = VecAssemblyEnd(global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = VecSet(local,-1);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
if (!rank) {ierr = VecView(local,0);CHKERRQ(ierr);}
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscInt M = 6,N = 5,P = 4, m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE,i,j,k,is,js,ks,in,jen,kn;
PetscErrorCode ierr;
DM da;
Vec local,global;
PetscScalar ****l;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_BOX,M,N,P,m,n,p,2,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&is,&js,&ks,&in,&jen,&kn);CHKERRQ(ierr);
ierr = DMDAVecGetArrayDOF(da,local,&l);CHKERRQ(ierr);
for (i=is; i<is+in; i++) {
for (j=js; j<js+jen; j++) {
for (k=ks; k<ks+kn; k++) {
l[k][j][i][0] = 2*(i + j*M + k*M*N);
l[k][j][i][1] = 2*(i + j*M + k*M*N) + 1;
}
}
}
ierr = DMDAVecRestoreArrayDOF(da,local,&l);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,ADD_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,ADD_VALUES,global);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Free memory */
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char *argv[])
{
PetscErrorCode ierr;
DM da;
PetscInt dim = 2,m,n,p,i;
const PetscInt *lx,*ly,*lz;
PetscMPIInt rank,size;
ierr = PetscInitialize(&argc,&argv,0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,0,"-dim",&dim,0);CHKERRQ(ierr);
switch (dim) {
case 2:
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR, 3,5,PETSC_DECIDE,PETSC_DECIDE,2,1,NULL,NULL,&da);CHKERRQ(ierr);
break;
case 3:
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR, 3,5,7,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,2,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
break;
default: SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"No support for %D dimensions",dim);
}
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da, 0, 0,0,0, &m,&n,&p, 0,0, 0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetOwnershipRanges(da,&lx,&ly,&lz);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
for (i=0; i<size; i++) {
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
if (i == rank) {
ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_SELF,"[%d] lx ly%s\n",rank,dim>2 ? " lz" : "");CHKERRQ(ierr);
ierr = PetscIntView(m,lx,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscIntView(n,ly,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
if (dim > 2) {ierr = PetscIntView(n,lz,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);}
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscErrorCode ierr;
PetscFunctionBegin;
switch(dim) {
case 2:
ierr = DMDACreate2d(comm, DMDA_BOUNDARY_NONE, DMDA_BOUNDARY_NONE, DMDA_STENCIL_STAR, -4, -4, PETSC_DECIDE, PETSC_DECIDE, 2, 1, PETSC_NULL, PETSC_NULL, dm);CHKERRQ(ierr);
break;
case 3:
ierr = DMDACreate3d(comm, DMDA_BOUNDARY_NONE, DMDA_BOUNDARY_NONE, DMDA_BOUNDARY_NONE, DMDA_STENCIL_STAR, -4, -4, -4, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE, 2, 1, PETSC_NULL, PETSC_NULL, PETSC_NULL, dm);CHKERRQ(ierr);
break;
default:
SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Could not create mesh for dimension %d", dim);
}
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
ierr = DMDASetFieldName(*dm, 0, "velocity");CHKERRQ(ierr);
ierr = DMDASetFieldName(*dm, 1, "pressure");CHKERRQ(ierr);
user->dm = *dm;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
KSP ksp;
DM da;
PetscReal norm;
PetscErrorCode ierr;
PetscInt i,j,k,mx,my,mz,xm,ym,zm,xs,ys,zs;
PetscScalar Hx,Hy,Hz;
PetscScalar ***array;
Vec x,b,r;
Mat J;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-12,-12,-12,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0,0,&da);CHKERRQ(ierr);
ierr = DMDASetInterpolationType(da, DMDA_Q0);CHKERRQ(ierr);
ierr = KSPSetDM(ksp,da);CHKERRQ(ierr);
ierr = KSPSetComputeRHS(ksp,ComputeRHS,NULL);CHKERRQ(ierr);
ierr = KSPSetComputeOperators(ksp,ComputeMatrix,NULL);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSolve(ksp,NULL,NULL);CHKERRQ(ierr);
ierr = KSPGetSolution(ksp,&x);CHKERRQ(ierr);
ierr = KSPGetRhs(ksp,&b);CHKERRQ(ierr);
ierr = KSPGetOperators(ksp,NULL,&J);CHKERRQ(ierr);
ierr = VecDuplicate(b,&r);CHKERRQ(ierr);
ierr = MatMult(J,x,r);CHKERRQ(ierr);
ierr = VecAXPY(r,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(r,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Residual norm %g\n",(double)norm);CHKERRQ(ierr);
ierr = DMDAGetInfo(da, 0, &mx, &my, &mz, 0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(mx);
Hy = 1.0 / (PetscReal)(my);
Hz = 1.0 / (PetscReal)(mz);
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
ierr = DMDAVecGetArray(da, x, &array);CHKERRQ(ierr);
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
array[k][j][i] -=
PetscCosScalar(2*PETSC_PI*(((PetscReal)i+0.5)*Hx))*
PetscCosScalar(2*PETSC_PI*(((PetscReal)j+0.5)*Hy))*
PetscCosScalar(2*PETSC_PI*(((PetscReal)k+0.5)*Hz));
}
}
}
ierr = DMDAVecRestoreArray(da, x, &array);CHKERRQ(ierr);
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_INFINITY,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error norm %g\n",(double)norm);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_1,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error norm %g\n",(double)(norm/((PetscReal)(mx)*(PetscReal)(my)*(PetscReal)(mz))));CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error norm %g\n",(double)(norm/((PetscReal)(mx)*(PetscReal)(my)*(PetscReal)(mz))));CHKERRQ(ierr);
ierr = VecDestroy(&r);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da,*subda;
PetscInt i,dim=3;
PetscMPIInt size,rank;
Vec v;
Vec slvec,sgvec;
IS *ois,*iis;
VecScatter oscata;
VecScatter *iscat,*oscat,*gscat;
DMDALocalInfo info;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-dim",&dim,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,-4,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMCreateDomainDecomposition(da,NULL,NULL,&iis,&ois,&subda);CHKERRQ(ierr);
ierr = DMCreateDomainDecompositionScatters(da,1,subda,&iscat,&oscat,&gscat);CHKERRQ(ierr);
{
DMDALocalInfo subinfo;
MatStencil lower,upper;
IS patchis,subpatchis;
Vec smallvec;
Vec largevec;
VecScatter patchscat;
ierr = DMDAGetLocalInfo(subda[0],&subinfo);CHKERRQ(ierr);
lower.i = info.xs;
lower.j = info.ys;
lower.k = info.zs;
upper.i = info.xs+info.xm;
upper.j = info.ys+info.ym;
upper.k = info.zs+info.zm;
/* test the patch IS as a thing to scatter to/from */
ierr = DMDACreatePatchIS(da,&lower,&upper,&patchis);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&smallvec);CHKERRQ(ierr);
ierr = VecSetSizes(smallvec,info.dof*(upper.i - lower.i)*(upper.j - lower.j)*(upper.k - lower.k),PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(smallvec);CHKERRQ(ierr);
ierr = VecScatterCreate(smallvec,NULL,largevec,patchis,&patchscat);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],smallvec);CHKERRQ(ierr);
ierr = VecSet(largevec,0);CHKERRQ(ierr);
ierr = VecScatterBegin(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = ISView(patchis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecScatterView(patchscat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecView(smallvec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecView(largevec,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&smallvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = ISDestroy(&patchis);CHKERRQ(ierr);
ierr = VecScatterDestroy(&patchscat);CHKERRQ(ierr);
}
/* view the various parts */
{
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMView(subda[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&v);CHKERRQ(ierr);
/* test filling outer between the big DM and the small ones with the IS scatter*/
ierr = VecScatterCreate(v,ois[0],sgvec,NULL,&oscata);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],sgvec);CHKERRQ(ierr);
ierr = VecScatterBegin(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* test the local-to-local scatter */
/* fill up the local subdomain and then add them together */
ierr = FillLocalSubdomain(da,v);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test ghost scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test overlap scattering backwards */
ierr = DMLocalToGlobalBegin(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test interior scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test matrix allocation */
for (i = 0; i < size; i++) {
if (i == rank) {
Mat m;
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMSetMatType(subda[0],MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(subda[0],&m);CHKERRQ(ierr);
ierr = MatView(m,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&m);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMRestoreLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&v);CHKERRQ(ierr);
}
ierr = DMDestroy(&subda[0]);CHKERRQ(ierr);
ierr = ISDestroy(&ois[0]);CHKERRQ(ierr);
ierr = ISDestroy(&iis[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&iscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&gscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscata);CHKERRQ(ierr);
ierr = PetscFree(iscat);CHKERRQ(ierr);
ierr = PetscFree(oscat);CHKERRQ(ierr);
ierr = PetscFree(gscat);CHKERRQ(ierr);
ierr = PetscFree(oscata);CHKERRQ(ierr);
ierr = PetscFree(subda);CHKERRQ(ierr);
ierr = PetscFree(ois);CHKERRQ(ierr);
ierr = PetscFree(iis);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
AppCtx user; /* user-defined work context */
PetscInt mx,my,its;
PetscErrorCode ierr;
MPI_Comm comm;
SNES snes;
DM da;
Vec x,X,b;
PetscBool youngflg,poissonflg,muflg,lambdaflg,view=PETSC_FALSE,viewline=PETSC_FALSE;
PetscReal poisson=0.2,young=4e4;
char filename[PETSC_MAX_PATH_LEN] = "ex16.vts";
char filename_def[PETSC_MAX_PATH_LEN] = "ex16_def.vts";
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = FormElements();CHKERRQ(ierr);
comm = PETSC_COMM_WORLD;
ierr = SNESCreate(comm,&snes);CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_BOX,11,2,2,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = SNESSetDM(snes,(DM)da);CHKERRQ(ierr);
ierr = SNESSetNGS(snes,NonlinearGS,&user);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,&my,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);CHKERRQ(ierr);
user.loading = 0.0;
user.arc = PETSC_PI/3.;
user.mu = 4.0;
user.lambda = 1.0;
user.rad = 100.0;
user.height = 3.;
user.width = 1.;
user.ploading = -5e3;
ierr = PetscOptionsGetReal(NULL,NULL,"-arc",&user.arc,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-mu",&user.mu,&muflg);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-lambda",&user.lambda,&lambdaflg);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-rad",&user.rad,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-height",&user.height,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-width",&user.width,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-loading",&user.loading,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-ploading",&user.ploading,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-poisson",&poisson,&poissonflg);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(NULL,NULL,"-young",&young,&youngflg);CHKERRQ(ierr);
if ((youngflg || poissonflg) || !(muflg || lambdaflg)) {
/* set the lame' parameters based upon the poisson ratio and young's modulus */
user.lambda = poisson*young / ((1. + poisson)*(1. - 2.*poisson));
user.mu = young/(2.*(1. + poisson));
}
ierr = PetscOptionsGetBool(NULL,NULL,"-view",&view,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-view_line",&viewline,NULL);CHKERRQ(ierr);
ierr = DMDASetFieldName(da,0,"x_disp");CHKERRQ(ierr);
ierr = DMDASetFieldName(da,1,"y_disp");CHKERRQ(ierr);
ierr = DMDASetFieldName(da,2,"z_disp");CHKERRQ(ierr);
ierr = DMSetApplicationContext(da,&user);CHKERRQ(ierr);
ierr = DMDASNESSetFunctionLocal(da,INSERT_VALUES,(PetscErrorCode (*)(DMDALocalInfo*,void*,void*,void*))FormFunctionLocal,&user);CHKERRQ(ierr);
ierr = DMDASNESSetJacobianLocal(da,(DMDASNESJacobian)FormJacobianLocal,&user);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = FormCoordinates(da,&user);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&b);CHKERRQ(ierr);
ierr = InitialGuess(da,&user,x);CHKERRQ(ierr);
ierr = FormRHS(da,&user,b);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"lambda: %f mu: %f\n",(double)user.lambda,(double)user.mu);CHKERRQ(ierr);
/* show a cross-section of the initial state */
if (viewline) {
ierr = DisplayLine(snes,x);CHKERRQ(ierr);
}
/* get the loaded configuration */
ierr = SNESSolve(snes,b,x);CHKERRQ(ierr);
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Number of SNES iterations = %D\n", its);CHKERRQ(ierr);
ierr = SNESGetSolution(snes,&X);CHKERRQ(ierr);
/* show a cross-section of the final state */
if (viewline) {
ierr = DisplayLine(snes,X);CHKERRQ(ierr);
}
if (view) {
PetscViewer viewer;
Vec coords;
ierr = PetscViewerVTKOpen(PETSC_COMM_WORLD,filename,FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
ierr = VecView(x,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = DMGetCoordinates(da,&coords);CHKERRQ(ierr);
ierr = VecAXPY(coords,1.0,x);CHKERRQ(ierr);
ierr = PetscViewerVTKOpen(PETSC_COMM_WORLD,filename_def,FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
ierr = VecView(x,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscMPIInt rank,size;
PetscInt N = 6,M=8,P=5,dof=1;
PetscInt stencil_width=1,pt=0,st=0;
PetscErrorCode ierr;
PetscBool flg2,flg3,isbinary,mpiio;
DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE,bz = DM_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_STAR;
DM da,da2;
Vec global1,global2;
PetscScalar mone = -1.0;
PetscReal norm;
PetscViewer viewer;
PetscRandom rdm;
#if defined(PETSC_HAVE_HDF5)
PetscBool ishdf5;
#endif
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-P",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-stencil_width",&stencil_width,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-periodic",&pt,NULL);CHKERRQ(ierr);
if (pt == 1) bx = DM_BOUNDARY_PERIODIC;
if (pt == 2) by = DM_BOUNDARY_PERIODIC;
if (pt == 4) {bx = DM_BOUNDARY_PERIODIC; by = DM_BOUNDARY_PERIODIC;}
ierr = PetscOptionsGetInt(NULL,NULL,"-stencil_type",&st,NULL);CHKERRQ(ierr);
stencil_type = (DMDAStencilType) st;
ierr = PetscOptionsHasName(NULL,NULL,"-oned",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL,"-twod",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL,"-threed",&flg3);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL,"-binary",&isbinary);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HDF5)
ierr = PetscOptionsHasName(NULL,NULL,"-hdf5",&ishdf5);CHKERRQ(ierr);
#endif
ierr = PetscOptionsHasName(NULL,NULL,"-mpiio",&mpiio);CHKERRQ(ierr);
if (flg2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,stencil_type,M,N,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,&da);CHKERRQ(ierr);
} else if (flg3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,0,&da);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,dof,stencil_width,NULL,&da);CHKERRQ(ierr);
}
ierr = DMCreateGlobalVector(da,&global1);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)global1,"Test_Vec");CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = VecSetRandom(global1,rdm);CHKERRQ(ierr);
if (isbinary) {
if (mpiio) {
ierr = PetscOptionsSetValue(NULL,"-viewer_binary_mpiio","");CHKERRQ(ierr);
}
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"temp",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HDF5)
} else if (ishdf5) {
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"temp",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
#endif
} else SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Invalid Viewer : Run with -binary or -hdf5 option\n");
ierr = VecView(global1,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Global vector written to temp file is \n");CHKERRQ(ierr);
ierr = VecView(global1,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (flg2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,stencil_type,M,N,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,&da2);CHKERRQ(ierr);
} else if (flg3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,0,&da2);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,dof,stencil_width,NULL,&da2);CHKERRQ(ierr);
}
if (isbinary) {
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"temp",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HDF5)
} else if (ishdf5) {
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"temp",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
#endif
} else SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Invalid Viewer : Run with -binary or -hdf5 option\n");
ierr = DMCreateGlobalVector(da2,&global2);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)global2,"Test_Vec");CHKERRQ(ierr);
ierr = VecLoad(global2,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Global vector read from temp file is \n");CHKERRQ(ierr);
ierr = VecView(global2,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecAXPY(global2,mone,global1);CHKERRQ(ierr);
ierr = VecNorm(global2,NORM_MAX,&norm);CHKERRQ(ierr);
if (norm != 0.0) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"ex23: Norm of difference %g should be zero\n",(double)norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Number of processors %d\n",size);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," M,N,P,dof %D %D %D %D\n",M,N,P,dof);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," stencil_width %D stencil_type %d periodic %d\n",stencil_width,(int)stencil_type,(int)pt);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," dimension %d\n",1 + (int) flg2 + (int) flg3);CHKERRQ(ierr);
}
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = DMDestroy(&da2);CHKERRQ(ierr);
ierr = VecDestroy(&global1);CHKERRQ(ierr);
ierr = VecDestroy(&global2);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscMPIInt size;
PetscInt N = 6,m=PETSC_DECIDE,n=PETSC_DECIDE,p=PETSC_DECIDE,M=8,dof=1,stencil_width=1,P=5,pt = 0,st = 0;
PetscErrorCode ierr;
PetscBool flg2,flg3;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_STAR;
DM da;
Vec global1,global2,global3,global4;
PetscScalar mone = -1.0;
PetscReal norm;
PetscViewer viewer;
PetscRandom rdm;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-P",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-stencil_width",&stencil_width,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-periodic",&pt,PETSC_NULL);CHKERRQ(ierr);
if (pt == 1) bx = DMDA_BOUNDARY_PERIODIC;
if (pt == 2) by = DMDA_BOUNDARY_PERIODIC;
if (pt == 3) {bx = DMDA_BOUNDARY_PERIODIC; by = DMDA_BOUNDARY_PERIODIC;}
if (pt == 4) bz = DMDA_BOUNDARY_PERIODIC;
ierr = PetscOptionsGetInt(PETSC_NULL,"-stencil_type",&st,PETSC_NULL);CHKERRQ(ierr);
stencil_type = (DMDAStencilType) st;
ierr = PetscOptionsHasName(PETSC_NULL,"-one",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-two",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-three",&flg3);CHKERRQ(ierr);
if (flg2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,stencil_type,M,N,m,n,dof,stencil_width,0,0,&da);CHKERRQ(ierr);
} else if (flg3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,m,n,p,dof,stencil_width,0,0,0,&da);CHKERRQ(ierr);
}
else {
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,dof,stencil_width,PETSC_NULL,&da);CHKERRQ(ierr);
}
ierr = DMCreateGlobalVector(da,&global1);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global2);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global3);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global4);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"temp",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
ierr = VecSetRandom(global1,rdm);CHKERRQ(ierr);
ierr = VecView(global1,viewer);CHKERRQ(ierr);
ierr = VecSetRandom(global3,rdm);CHKERRQ(ierr);
ierr = VecView(global3,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"temp",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = VecLoad(global2,viewer);CHKERRQ(ierr);
ierr = VecLoad(global4,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecAXPY(global2,mone,global1);CHKERRQ(ierr);
ierr = VecNorm(global2,NORM_MAX,&norm);CHKERRQ(ierr);
if (norm != 0.0) {
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"ex23: Norm of difference %G should be zero\n",norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Number of processors %d\n",size);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," M,N,P,dof %D %D %D %D\n",M,N,P,dof);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," stencil_width %D stencil_type %d periodic %d\n",stencil_width,(int)stencil_type,(int)bx);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," dimension %d\n",1 + (int) flg2 + (int) flg3);CHKERRQ(ierr);
}
ierr = VecAXPY(global4,mone,global3);CHKERRQ(ierr);
ierr = VecNorm(global4,NORM_MAX,&norm);CHKERRQ(ierr);
if (norm != 0.0) {
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"ex23: Norm of difference %G should be zero\n",norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Number of processors %d\n",size);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," M,N,P,dof %D %D %D %D\n",M,N,P,dof);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," stencil_width %D stencil_type %d periodic %d\n",stencil_width,(int)stencil_type,(int)bx);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," dimension %d\n",1 + (int) flg2 + (int) flg3);CHKERRQ(ierr);
}
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = VecDestroy(&global1);CHKERRQ(ierr);
ierr = VecDestroy(&global2);CHKERRQ(ierr);
ierr = VecDestroy(&global3);CHKERRQ(ierr);
ierr = VecDestroy(&global4);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscSolver::PetscSolver(FlowField & flowField, Parameters & parameters):
LinearSolver(flowField, parameters), _ctx(parameters, flowField){
// Set the type of boundary nodes of the system
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,
by = DMDA_BOUNDARY_NONE,
bz = DMDA_BOUNDARY_NONE;
if (parameters.walls.typeLeft==PERIODIC){
bx = DMDA_BOUNDARY_PERIODIC;
}
if (parameters.walls.typeBottom==PERIODIC){
by = DMDA_BOUNDARY_PERIODIC;
}
if (parameters.walls.typeFront==PERIODIC){
bz = DMDA_BOUNDARY_PERIODIC;
}
KSPCreate(PETSC_COMM_WORLD,&_ksp);
PetscErrorCode (*computeMatrix)(KSP, Mat, Mat, MatStructure*, void*) = NULL;
if (_parameters.geometry.dim == 2){
computeMatrix = computeMatrix2D;
DMDACreate2d(PETSC_COMM_WORLD, bx, by, DMDA_STENCIL_STAR,
parameters.geometry.sizeX+2, parameters.geometry.sizeY+2,
parameters.parallel.numProcessors[0],
parameters.parallel.numProcessors[1],
1, 2,
_parameters.parallel.sizes[0],
_parameters.parallel.sizes[1],
&_da);
} else if (_parameters.geometry.dim == 3){
computeMatrix = computeMatrix3D;
DMDACreate3d(PETSC_COMM_WORLD, bx, by, bz, DMDA_STENCIL_STAR,
parameters.geometry.sizeX + 2, parameters.geometry.sizeY + 2,
parameters.geometry.sizeZ + 2,
parameters.parallel.numProcessors[0],
parameters.parallel.numProcessors[1],
parameters.parallel.numProcessors[2],
1, 2, // Degrees of freedom and stencil length
_parameters.parallel.sizes[0],
_parameters.parallel.sizes[1],
_parameters.parallel.sizes[2],
&_da);
}
// Find out what are the corners of the subdomain
DMDAGetCorners(_da, &_firstX, &_firstY, &_firstZ, &_lengthX, &_lengthY, &_lengthZ);
// Current function to assign the limits
createLimits(parameters, _da, _limitsX, _limitsY, _limitsZ);
// Set the rank in the context. Necessary since PETSc declares the rank of the neighbors under
// periodic conditions as the rank of the process itself. So the rank must be known to properly
// set matrices and RHS vectors
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
// Set offsets to fix where the results of the pressure will be written in the flow field
if (_firstX == 0){
_offsetX = 1;
} else {
_offsetX = 2;
}
if (_firstY == 0){
_offsetY = 1;
} else {
_offsetY = 2;
}
if (_firstZ == 0){
_offsetZ = 1;
} else {
_offsetZ = 2;
}
// Set a pointer to the limits in the context, so that they can be used by the function
_ctx.setLimits(_limitsX, _limitsY, _limitsZ);
// Determine whether a process writes a boundary on the system.
// Right now, it only depends on the position of the array. The identity of the neighbors will
// become significant only in the iterators, where it will apply a boundary condition only if
// the rank is invalid, so that there is no neighbor
_ctx.setAsBoundary = 0;
if (parameters.parallel.indices[0] == 0){
_ctx.setAsBoundary += LEFT_WALL_BIT;
}
if (parameters.parallel.indices[0] == parameters.parallel.numProcessors[0]-1){
_ctx.setAsBoundary += RIGHT_WALL_BIT;
}
if (parameters.parallel.indices[1] == 0){
_ctx.setAsBoundary += BOTTOM_WALL_BIT;
}
if (parameters.parallel.indices[1] == parameters.parallel.numProcessors[1]-1){
_ctx.setAsBoundary += TOP_WALL_BIT;
}
if (parameters.parallel.indices[2] == 0){
_ctx.setAsBoundary += FRONT_WALL_BIT;
}
if (parameters.parallel.indices[2] == parameters.parallel.numProcessors[2]-1){
_ctx.setAsBoundary += BACK_WALL_BIT;
}
// Set displacements to deal with periodic boundaries if necessary.
// If the boundary is periodic, it will take information from positions beyond the ghost cells,
// since they are used only for parallel communication. Otherwise, PETSc deals with
// communication of the pressure.
int Nx = parameters.geometry.sizeX + 2;
int Ny = parameters.geometry.sizeY + 2;
int Nz = parameters.geometry.sizeZ + 2;
if (parameters.walls.typeLeft==PERIODIC){
_ctx.displacement[0] = -2;
_ctx.displacement[1] = Nx+1;
} else {
_ctx.displacement[0] = 1;
_ctx.displacement[1] = Nx-2;
}
if (parameters.walls.typeBottom==PERIODIC){
_ctx.displacement[2] = -2;
_ctx.displacement[3] = Ny+1;
} else {
_ctx.displacement[2] = 1;
_ctx.displacement[3] = Ny-2;
}
if (parameters.walls.typeFront==PERIODIC){
_ctx.displacement[4] = -2;
_ctx.displacement[5] = Nz+1;
} else {
_ctx.displacement[4] = 1;
_ctx.displacement[5] = Nz-2;
}
DMCreateGlobalVector(_da, &_x);
KSPSetDM(_ksp, _da);
KSPSetComputeOperators(_ksp, computeMatrix, &_ctx);
KSPSetType(_ksp,KSPFGMRES);
KSPSetFromOptions(_ksp);
KSPSetInitialGuessNonzero(_ksp,PETSC_TRUE);
}
PetscInt main(PetscInt argc,char **args)
{
typedef enum {RANDOM, CONSTANT, TANH, NUM_FUNCS} FuncType;
const char *funcNames[NUM_FUNCS] = {"random", "constant", "tanh"};
Mat A, AA;
PetscMPIInt size;
PetscInt N,i, stencil=1,dof=3;
PetscInt dim[3] = {10,10,10}, ndim = 3;
Vec coords,x,y,z,xx, yy, zz;
Vec xxsplit[DOF], yysplit[DOF], zzsplit[DOF];
PetscReal h[3];
PetscScalar s;
PetscRandom rdm;
PetscReal norm, enorm;
PetscInt func;
FuncType function = TANH;
DM da, da1, coordsda;
PetscBool view_x = PETSC_FALSE, view_y = PETSC_FALSE, view_z = PETSC_FALSE;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&args,(char *)0,help);CHKERRQ(ierr);
#if !defined(PETSC_USE_COMPLEX)
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This example requires complex numbers");
#endif
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This is a uniprocessor example only!");
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, PETSC_NULL, "USFFT Options", "ex27");CHKERRQ(ierr);
ierr = PetscOptionsEList("-function", "Function type", "ex27", funcNames, NUM_FUNCS, funcNames[function], &func, PETSC_NULL);CHKERRQ(ierr);
function = (FuncType) func;
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-view_x",&view_x,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-view_y",&view_y,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-view_z",&view_z,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetIntArray(PETSC_NULL,"-dim",dim,&ndim,PETSC_NULL);CHKERRQ(ierr);
// DMDA with the correct fiber dimension
ierr = DMDACreate3d(PETSC_COMM_SELF,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_STAR,
dim[0], dim[1], dim[2],
PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE,
dof, stencil,
PETSC_NULL, PETSC_NULL, PETSC_NULL,
&da);
CHKERRQ(ierr);
// DMDA with fiber dimension 1 for split fields
ierr = DMDACreate3d(PETSC_COMM_SELF,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_STAR,
dim[0], dim[1], dim[2],
PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE,
1, stencil,
PETSC_NULL, PETSC_NULL, PETSC_NULL,
&da1);
CHKERRQ(ierr);
// Coordinates
ierr = DMDAGetCoordinateDA(da, &coordsda);
ierr = DMGetGlobalVector(coordsda, &coords);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coords, "Grid coordinates");CHKERRQ(ierr);
for(i = 0, N = 1; i < 3; i++) {
h[i] = 1.0/dim[i];
PetscScalar *a;
ierr = VecGetArray(coords, &a);CHKERRQ(ierr);
PetscInt j,k,n = 0;
for(i = 0; i < 3; ++i) {
for(j = 0; j < dim[i]; ++j){
for(k = 0; k < 3; ++k) {
a[n] = j*h[i]; // coordinate along the j-th point in the i-th dimension
++n;
}
}
}
ierr = VecRestoreArray(coords, &a);CHKERRQ(ierr);
}
ierr = DMDASetCoordinates(da, coords);CHKERRQ(ierr);
ierr = VecDestroy(&coords);CHKERRQ(ierr);
// Work vectors
ierr = DMGetGlobalVector(da, &x);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Real space vector");CHKERRQ(ierr);
ierr = DMGetGlobalVector(da, &xx);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) xx, "Real space vector");CHKERRQ(ierr);
ierr = DMGetGlobalVector(da, &y);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, "USFFT frequency space vector");CHKERRQ(ierr);
ierr = DMGetGlobalVector(da, &yy);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) yy, "FFTW frequency space vector");CHKERRQ(ierr);
ierr = DMGetGlobalVector(da, &z);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) z, "USFFT reconstructed vector");CHKERRQ(ierr);
ierr = DMGetGlobalVector(da, &zz);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) zz, "FFTW reconstructed vector");CHKERRQ(ierr);
// Split vectors for FFTW
for(int ii = 0; ii < 3; ++ii) {
ierr = DMGetGlobalVector(da1, &xxsplit[ii]);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) xxsplit[ii], "Real space split vector");CHKERRQ(ierr);
ierr = DMGetGlobalVector(da1, &yysplit[ii]);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) yysplit[ii], "FFTW frequency space split vector");CHKERRQ(ierr);
ierr = DMGetGlobalVector(da1, &zzsplit[ii]);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) zzsplit[ii], "FFTW reconstructed split vector");CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_SELF, "%3-D: USFFT on vector of ");CHKERRQ(ierr);
for(i = 0, N = 1; i < 3; i++) {
ierr = PetscPrintf(PETSC_COMM_SELF, "dim[%d] = %d ",i,dim[i]);CHKERRQ(ierr);
N *= dim[i];
}
ierr = PetscPrintf(PETSC_COMM_SELF, "; total size %d \n",N);CHKERRQ(ierr);
if (function == RANDOM) {
ierr = PetscRandomCreate(PETSC_COMM_SELF, &rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = VecSetRandom(x, rdm);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
}
else if (function == CONSTANT) {
ierr = VecSet(x, 1.0);CHKERRQ(ierr);
}
else if (function == TANH) {
PetscScalar *a;
ierr = VecGetArray(x, &a);CHKERRQ(ierr);
PetscInt j,k = 0;
for(i = 0; i < 3; ++i) {
for(j = 0; j < dim[i]; ++j) {
a[k] = tanh((j - dim[i]/2.0)*(10.0/dim[i]));
++k;
}
}
ierr = VecRestoreArray(x, &a);CHKERRQ(ierr);
}
if(view_x) {
ierr = VecView(x, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecCopy(x,xx);CHKERRQ(ierr);
// Split xx
ierr = VecStrideGatherAll(xx,xxsplit, INSERT_VALUES);CHKERRQ(ierr); //YES! 'Gather' means 'split' (or maybe 'scatter'?)!
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "|x|_2 = %g\n",norm);CHKERRQ(ierr);
/* create USFFT object */
ierr = MatCreateSeqUSFFT(da,da,&A);CHKERRQ(ierr);
/* create FFTW object */
ierr = MatCreateSeqFFTW(PETSC_COMM_SELF,3,dim,&AA);CHKERRQ(ierr);
/* apply USFFT and FFTW FORWARD "preemptively", so the fftw_plans can be reused on different vectors */
ierr = MatMult(A,x,z);CHKERRQ(ierr);
for(int ii = 0; ii < 3; ++ii) {
ierr = MatMult(AA,xxsplit[ii],zzsplit[ii]);CHKERRQ(ierr);
}
// Now apply USFFT and FFTW forward several (3) times
for (i=0; i<3; ++i){
ierr = MatMult(A,x,y);CHKERRQ(ierr);
for(int ii = 0; ii < 3; ++ii) {
ierr = MatMult(AA,xxsplit[ii],yysplit[ii]);CHKERRQ(ierr);
}
ierr = MatMultTranspose(A,y,z);CHKERRQ(ierr);
for(int ii = 0; ii < 3; ++ii) {
ierr = MatMult(AA,yysplit[ii],zzsplit[ii]);CHKERRQ(ierr);
}
}
// Unsplit yy
ierr = VecStrideScatterAll(yysplit, yy, INSERT_VALUES);CHKERRQ(ierr); //YES! 'Scatter' means 'collect' (or maybe 'gather'?)!
// Unsplit zz
ierr = VecStrideScatterAll(zzsplit, zz, INSERT_VALUES);CHKERRQ(ierr); //YES! 'Scatter' means 'collect' (or maybe 'gather'?)!
if(view_y) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "y = \n");CHKERRQ(ierr);
ierr = VecView(y, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "yy = \n");CHKERRQ(ierr);
ierr = VecView(yy, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
if(view_z) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "z = \n");CHKERRQ(ierr);
ierr = VecView(z, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "zz = \n");CHKERRQ(ierr);
ierr = VecView(zz, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* compare x and z. USFFT computes an unnormalized DFT, thus z = N*x */
s = 1.0/(PetscReal)N;
ierr = VecScale(z,s);CHKERRQ(ierr);
ierr = VecAXPY(x,-1.0,z);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_1,&enorm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "|x-z| = %g\n",enorm);CHKERRQ(ierr);
/* compare xx and zz. FFTW computes an unnormalized DFT, thus zz = N*x */
s = 1.0/(PetscReal)N;
ierr = VecScale(zz,s);CHKERRQ(ierr);
ierr = VecAXPY(xx,-1.0,zz);CHKERRQ(ierr);
ierr = VecNorm(xx,NORM_1,&enorm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "|xx-zz| = %g\n",enorm);CHKERRQ(ierr);
/* compare y and yy: USFFT and FFTW results*/
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
ierr = VecAXPY(y,-1.0,yy);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_1,&enorm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "|y|_2 = %g\n",norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "|y-yy| = %g\n",enorm);CHKERRQ(ierr);
/* compare z and zz: USFFT and FFTW results*/
ierr = VecNorm(z,NORM_2,&norm);CHKERRQ(ierr);
ierr = VecAXPY(z,-1.0,zz);CHKERRQ(ierr);
ierr = VecNorm(z,NORM_1,&enorm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "|z|_2 = %g\n",norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "|z-zz| = %g\n",enorm);CHKERRQ(ierr);
/* free spaces */
ierr = DMRestoreGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&xx);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&y);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&yy);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&z);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&zz);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode test1_DAInjection3d(PetscInt mx, PetscInt my, PetscInt mz)
{
PetscErrorCode ierr;
DM dac,daf;
PetscViewer vv;
Vec ac,af;
PetscInt periodicity;
DMBoundaryType bx,by,bz;
PetscFunctionBeginUser;
bx = DM_BOUNDARY_NONE;
by = DM_BOUNDARY_NONE;
bz = DM_BOUNDARY_NONE;
periodicity = 0;
ierr = PetscOptionsGetInt(NULL,NULL,"-periodic", &periodicity, NULL);CHKERRQ(ierr);
if (periodicity==1) {
bx = DM_BOUNDARY_PERIODIC;
} else if (periodicity==2) {
by = DM_BOUNDARY_PERIODIC;
} else if (periodicity==3) {
bz = DM_BOUNDARY_PERIODIC;
}
ierr = DMDACreate3d(PETSC_COMM_WORLD, bx,by,bz, DMDA_STENCIL_BOX,mx+1, my+1,mz+1,PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE,1, /* 1 dof */
1, /* stencil = 1 */NULL,NULL,NULL,&daf);CHKERRQ(ierr);
ierr = DMSetFromOptions(daf);CHKERRQ(ierr);
ierr = DMSetUp(daf);CHKERRQ(ierr);
ierr = DMCoarsen(daf,MPI_COMM_NULL,&dac);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(dac, -1.0,1.0, -1.0,1.0, -1.0,1.0);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(daf, -1.0,1.0, -1.0,1.0, -1.0,1.0);CHKERRQ(ierr);
{
DM cdaf,cdac;
Vec coordsc,coordsf,coordsf2;
Mat inject;
VecScatter vscat;
Mat interp;
PetscReal norm;
ierr = DMGetCoordinateDM(dac,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(daf,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(dac,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinates(daf,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInjection(cdac,cdaf,&inject);CHKERRQ(ierr);
ierr = MatScatterGetVecScatter(inject,&vscat);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat ,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = MatDestroy(&inject);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&interp,NULL);CHKERRQ(ierr);
ierr = VecDuplicate(coordsf,&coordsf2);CHKERRQ(ierr);
ierr = MatInterpolate(interp,coordsc,coordsf2);CHKERRQ(ierr);
ierr = VecAXPY(coordsf2,-1.0,coordsf);CHKERRQ(ierr);
ierr = VecNorm(coordsf2,NORM_MAX,&norm);CHKERRQ(ierr);
/* The fine coordinates are only reproduced in certain cases */
if (!bx && !by && !bz && norm > PETSC_SQRT_MACHINE_EPSILON) {ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm %g\n",(double)norm);CHKERRQ(ierr);}
ierr = VecDestroy(&coordsf2);CHKERRQ(ierr);
ierr = MatDestroy(&interp);CHKERRQ(ierr);
}
if (0) {
ierr = DMCreateGlobalVector(dac,&ac);CHKERRQ(ierr);
ierr = VecZeroEntries(ac);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&af);CHKERRQ(ierr);
ierr = VecZeroEntries(af);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "dac_7.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(dac, vv);CHKERRQ(ierr);
ierr = VecView(ac, vv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "daf_7.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(daf, vv);CHKERRQ(ierr);
ierr = VecView(af, vv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = VecDestroy(&ac);CHKERRQ(ierr);
ierr = VecDestroy(&af);CHKERRQ(ierr);
}
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv) {
PetscErrorCode ierr;
SNES snes;
Vec u, uexact;
double err, uexnorm;
DMDALocalInfo info;
Ctx user;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = configureCtx(&user);
CHKERRQ(ierr);
//STARTDMDA
ierr = DMDACreate3d(PETSC_COMM_WORLD,
DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_PERIODIC,
DMDA_STENCIL_STAR, 3,3,3, PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
1, 1, NULL,NULL,NULL,
&user.da);
CHKERRQ(ierr);
//ENDDMDA
ierr = DMSetFromOptions(user.da);
CHKERRQ(ierr);
ierr = DMSetUp(user.da);
CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(user.da,-1.0,1.0,-1.0,1.0,-1.0,1.0);
CHKERRQ(ierr);
ierr = DMSetApplicationContext(user.da,&user);
CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(user.da,&info);
CHKERRQ(ierr);
if ((info.mx < 2) || (info.my < 2) || (info.mz < 3)) {
SETERRQ(PETSC_COMM_WORLD,1,"grid too coarse: require (mx,my,mz) > (2,2,3)");
}
ierr = DMCreateGlobalVector(user.da,&uexact);
CHKERRQ(ierr);
ierr = VecDuplicate(uexact,&user.f);
CHKERRQ(ierr);
ierr = VecDuplicate(uexact,&user.g);
CHKERRQ(ierr);
ierr = formUexFG(&info,&user,uexact);
CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);
CHKERRQ(ierr);
ierr = SNESSetDM(snes,user.da);
CHKERRQ(ierr);
ierr = DMDASNESSetFunctionLocal(user.da,INSERT_VALUES,
(DMDASNESFunction)FormFunctionLocal,&user);
CHKERRQ(ierr);
ierr = DMDASNESSetJacobianLocal(user.da,
(DMDASNESJacobian)FormJacobianLocal,&user);
CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);
CHKERRQ(ierr);
ierr = VecDuplicate(uexact,&u);
CHKERRQ(ierr);
ierr = VecCopy(user.g,u);
CHKERRQ(ierr); // g has zeros except at bdry
ierr = SNESSolve(snes,NULL,u);
CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,uexact);
CHKERRQ(ierr); // u <- u + (-1.0) uxact
ierr = VecNorm(u,NORM_2,&err);
CHKERRQ(ierr);
ierr = VecNorm(uexact,NORM_2,&uexnorm);
CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"done on %d x %d x %d grid with eps=%g: error |u-uexact|_2/|uexact|_2 = %g\n",
info.mx,info.my,info.mz,user.eps,err/uexnorm);
CHKERRQ(ierr);
VecDestroy(&u);
VecDestroy(&uexact);
VecDestroy(&user.f);
VecDestroy(&user.g);
SNESDestroy(&snes);
DMDestroy(&user.da);
ierr = PetscFinalize();
CHKERRQ(ierr);
return 0;
}
/*
Mm_ratio - ration of grid lines between fine and coarse grids.
*/
int main(int argc,char **argv)
{
PetscErrorCode ierr;
AppCtx user;
PetscInt Npx=PETSC_DECIDE,Npy=PETSC_DECIDE,Npz=PETSC_DECIDE;
PetscMPIInt size,rank;
PetscInt m,n,M,N,i,nrows;
PetscScalar one = 1.0;
PetscReal fill=2.0;
Mat A,A_tmp,P,C,C1,C2;
PetscScalar *array,none = -1.0,alpha;
Vec x,v1,v2,v3,v4;
PetscReal norm,norm_tmp,norm_tmp1,tol=100.*PETSC_MACHINE_EPSILON;
PetscRandom rdm;
PetscBool Test_MatMatMult=PETSC_TRUE,Test_MatPtAP=PETSC_TRUE,Test_3D=PETSC_TRUE,flg;
const PetscInt *ia,*ja;
ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr;
ierr = PetscOptionsGetReal(NULL,NULL,"-tol",&tol,NULL);CHKERRQ(ierr);
user.ratio = 2;
user.coarse.mx = 20; user.coarse.my = 20; user.coarse.mz = 20;
ierr = PetscOptionsGetInt(NULL,NULL,"-Mx",&user.coarse.mx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-My",&user.coarse.my,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Mz",&user.coarse.mz,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ratio",&user.ratio,NULL);CHKERRQ(ierr);
if (user.coarse.mz) Test_3D = PETSC_TRUE;
user.fine.mx = user.ratio*(user.coarse.mx-1)+1;
user.fine.my = user.ratio*(user.coarse.my-1)+1;
user.fine.mz = user.ratio*(user.coarse.mz-1)+1;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npx",&Npx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npy",&Npy,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npz",&Npz,NULL);CHKERRQ(ierr);
/* Set up distributed array for fine grid */
if (!Test_3D) {
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,Npx,Npy,1,1,NULL,NULL,&user.fine.da);CHKERRQ(ierr);
} else {
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,user.fine.mz,Npx,Npy,Npz,1,1,NULL,NULL,NULL,&user.fine.da);CHKERRQ(ierr);
}
ierr = DMSetFromOptions(user.fine.da);CHKERRQ(ierr);
ierr = DMSetUp(user.fine.da);CHKERRQ(ierr);
/* Test DMCreateMatrix() */
/*------------------------------------------------------------*/
ierr = DMSetMatType(user.fine.da,MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.fine.da,&A);CHKERRQ(ierr);
ierr = DMSetMatType(user.fine.da,MATBAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.fine.da,&C);CHKERRQ(ierr);
ierr = MatConvert(C,MATAIJ,MAT_INITIAL_MATRIX,&A_tmp);CHKERRQ(ierr); /* not work for mpisbaij matrix! */
ierr = MatEqual(A,A_tmp,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"A != C");
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A_tmp);CHKERRQ(ierr);
/*------------------------------------------------------------*/
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
/* if (!rank) printf("A %d, %d\n",M,N); */
/* set val=one to A */
if (size == 1) {
ierr = MatGetRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(A,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(A,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
} else {
Mat AA,AB;
ierr = MatMPIAIJGetSeqAIJ(A,&AA,&AB,NULL);CHKERRQ(ierr);
ierr = MatGetRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(AA,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(AA,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
ierr = MatGetRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(AB,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(AB,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
}
/* ierr = MatView(A, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Set up distributed array for coarse grid */
if (!Test_3D) {
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,Npx,Npy,1,1,NULL,NULL,&user.coarse.da);CHKERRQ(ierr);
} else {
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,user.coarse.mz,Npx,Npy,Npz,
1,1,NULL,NULL,NULL,&user.coarse.da);CHKERRQ(ierr);
}
ierr = DMSetFromOptions(user.coarse.da);CHKERRQ(ierr);
ierr = DMSetUp(user.coarse.da);CHKERRQ(ierr);
/* Create interpolation between the fine and coarse grids */
ierr = DMCreateInterpolation(user.coarse.da,user.fine.da,&P,NULL);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,&m,&n);CHKERRQ(ierr);
ierr = MatGetSize(P,&M,&N);CHKERRQ(ierr);
/* if (!rank) printf("P %d, %d\n",M,N); */
/* Create vectors v1 and v2 that are compatible with A */
ierr = VecCreate(PETSC_COMM_WORLD,&v1);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&m,NULL);CHKERRQ(ierr);
ierr = VecSetSizes(v1,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(v1);CHKERRQ(ierr);
ierr = VecDuplicate(v1,&v2);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
/* Test MatMatMult(): C = A*P */
/*----------------------------*/
if (Test_MatMatMult) {
ierr = MatDuplicate(A,MAT_COPY_VALUES,&A_tmp);CHKERRQ(ierr);
ierr = MatMatMult(A_tmp,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr);
/* Test MAT_REUSE_MATRIX - reuse symbolic C */
alpha=1.0;
for (i=0; i<2; i++) {
alpha -= 0.1;
ierr = MatScale(A_tmp,alpha);CHKERRQ(ierr);
ierr = MatMatMult(A_tmp,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr);
}
/* Free intermediate data structures created for reuse of C=Pt*A*P */
ierr = MatFreeIntermediateDataStructures(C);CHKERRQ(ierr);
/* Test MatDuplicate() */
/*----------------------------*/
ierr = MatDuplicate(C,MAT_COPY_VALUES,&C1);CHKERRQ(ierr);
ierr = MatDuplicate(C1,MAT_COPY_VALUES,&C2);CHKERRQ(ierr);
ierr = MatDestroy(&C1);CHKERRQ(ierr);
ierr = MatDestroy(&C2);CHKERRQ(ierr);
/* Create vector x that is compatible with P */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,NULL,&n);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
norm = 0.0;
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(P,x,v1);CHKERRQ(ierr);
ierr = MatMult(A_tmp,v1,v2);CHKERRQ(ierr); /* v2 = A*P*x */
ierr = MatMult(C,x,v1);CHKERRQ(ierr); /* v1 = C*x */
ierr = VecAXPY(v1,none,v2);CHKERRQ(ierr);
ierr = VecNorm(v1,NORM_1,&norm_tmp);CHKERRQ(ierr);
ierr = VecNorm(v2,NORM_1,&norm_tmp1);CHKERRQ(ierr);
norm_tmp /= norm_tmp1;
if (norm_tmp > norm) norm = norm_tmp;
}
if (norm >= tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMatMult(), |v1 - v2|/|v2|: %g\n",(double)norm);CHKERRQ(ierr);
}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A_tmp);CHKERRQ(ierr);
}
/* Test P^T * A * P - MatPtAP() */
/*------------------------------*/
if (Test_MatPtAP) {
ierr = MatPtAP(A,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr);
ierr = MatGetLocalSize(C,&m,&n);CHKERRQ(ierr);
/* Test MAT_REUSE_MATRIX - reuse symbolic C */
alpha=1.0;
for (i=0; i<1; i++) {
alpha -= 0.1;
ierr = MatScale(A,alpha);CHKERRQ(ierr);
ierr = MatPtAP(A,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr);
}
/* Free intermediate data structures created for reuse of C=Pt*A*P */
ierr = MatFreeIntermediateDataStructures(C);CHKERRQ(ierr);
/* Test MatDuplicate() */
/*----------------------------*/
ierr = MatDuplicate(C,MAT_COPY_VALUES,&C1);CHKERRQ(ierr);
ierr = MatDuplicate(C1,MAT_COPY_VALUES,&C2);CHKERRQ(ierr);
ierr = MatDestroy(&C1);CHKERRQ(ierr);
ierr = MatDestroy(&C2);CHKERRQ(ierr);
/* Create vector x that is compatible with P */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,&m,&n);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&v3);CHKERRQ(ierr);
ierr = VecSetSizes(v3,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(v3);CHKERRQ(ierr);
ierr = VecDuplicate(v3,&v4);CHKERRQ(ierr);
norm = 0.0;
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(P,x,v1);CHKERRQ(ierr);
ierr = MatMult(A,v1,v2);CHKERRQ(ierr); /* v2 = A*P*x */
ierr = MatMultTranspose(P,v2,v3);CHKERRQ(ierr); /* v3 = Pt*A*P*x */
ierr = MatMult(C,x,v4);CHKERRQ(ierr); /* v3 = C*x */
ierr = VecAXPY(v4,none,v3);CHKERRQ(ierr);
ierr = VecNorm(v4,NORM_1,&norm_tmp);CHKERRQ(ierr);
ierr = VecNorm(v3,NORM_1,&norm_tmp1);CHKERRQ(ierr);
norm_tmp /= norm_tmp1;
if (norm_tmp > norm) norm = norm_tmp;
}
if (norm >= tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatPtAP(), |v3 - v4|/|v3|: %g\n",(double)norm);CHKERRQ(ierr);
}
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = VecDestroy(&v3);CHKERRQ(ierr);
ierr = VecDestroy(&v4);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
}
/* Clean up */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = VecDestroy(&v1);CHKERRQ(ierr);
ierr = VecDestroy(&v2);CHKERRQ(ierr);
ierr = DMDestroy(&user.fine.da);CHKERRQ(ierr);
ierr = DMDestroy(&user.coarse.da);CHKERRQ(ierr);
ierr = MatDestroy(&P);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 3,N = 5,P=3,s=1,w=2,nloc,l,i,j,k,kk,m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE;
PetscErrorCode ierr;
PetscInt Xs,Xm,Ys,Ym,Zs,Zm,iloc,*ltog,*iglobal;
PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL,*lz = PETSC_NULL;
PetscBool test_order = PETSC_FALSE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
AO ao;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,300,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-NX",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NY",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NZ",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-w",&w,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-box",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xnonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-ynonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-znonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-testorder",&test_order,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-distribute",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc(m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc(n*sizeof(PetscInt),&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
if (p == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -p option with -distribute option");
ierr = PetscMalloc(p*sizeof(PetscInt),&lz);CHKERRQ(ierr);
for (i=0; i<p-1; i++) { lz[i] = 2;}
lz[p-1] = P - 2*(p-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,m,n,p,w,s,lx,ly,lz,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = PetscFree(lz);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!test_order) { /* turn off printing when testing ordering mappings */
if (M*N*P<40) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-local_print",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (test_order) {
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,&Zs,&Xm,&Ym,&Zm);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(da,&nloc,<og);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
/* ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = PetscMalloc(nloc*sizeof(PetscInt),&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y, then z) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscPrintf(MPI_COMM_WORLD,"[%D] Problem with mapping: z=%D, j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",
rank,k,j,i,l,ltog[iloc+l],iglobal[kk]);
}
kk++;
}
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M=8,dof=1,stencil_width=1,i,start,end,P=5,N = 6,m=PETSC_DECIDE,n=PETSC_DECIDE,p=PETSC_DECIDE,pt = 0,st = 0;
PetscErrorCode ierr;
PetscBool flg = PETSC_FALSE,flg2,flg3;
DMDABoundaryType periodic;
DMDAStencilType stencil_type;
DM da;
Vec local,global,local_copy;
PetscScalar value;
PetscReal norm,work;
PetscViewer viewer;
char filename[64];
FILE *file;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-stencil_width",&stencil_width,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-periodic",&pt,NULL);CHKERRQ(ierr);
periodic = (DMDABoundaryType) pt;
ierr = PetscOptionsGetInt(NULL,"-stencil_type",&st,NULL);CHKERRQ(ierr);
stencil_type = (DMDAStencilType) st;
ierr = PetscOptionsHasName(NULL,"-2d",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,"-3d",&flg3);CHKERRQ(ierr);
if (flg2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,periodic,periodic,stencil_type,M,N,m,n,dof,stencil_width,
NULL,NULL,&da);CHKERRQ(ierr);
} else if (flg3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,periodic,periodic,periodic,stencil_type,M,N,P,m,n,p,dof,stencil_width,
NULL,NULL,NULL,&da);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(PETSC_COMM_WORLD,periodic,M,dof,stencil_width,NULL,&da);CHKERRQ(ierr);
}
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = VecDuplicate(local,&local_copy);CHKERRQ(ierr);
/* zero out vectors so that ghostpoints are zero */
value = 0;
ierr = VecSet(local,value);CHKERRQ(ierr);
ierr = VecSet(local_copy,value);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(global,&start,&end);CHKERRQ(ierr);
for (i=start; i<end; i++) {
value = i + 1;
ierr = VecSetValues(global,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(global);CHKERRQ(ierr);
ierr = VecAssemblyEnd(global);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMDALocalToLocalBegin(da,local,INSERT_VALUES,local_copy);CHKERRQ(ierr);
ierr = DMDALocalToLocalEnd(da,local,INSERT_VALUES,local_copy);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-save",&flg,NULL);CHKERRQ(ierr);
if (flg) {
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
sprintf(filename,"local.%d",rank);
ierr = PetscViewerASCIIOpen(PETSC_COMM_SELF,filename,&viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIGetPointer(viewer,&file);CHKERRQ(ierr);
ierr = VecView(local,viewer);CHKERRQ(ierr);
fprintf(file,"Vector with correct ghost points\n");
ierr = VecView(local_copy,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = VecAXPY(local_copy,-1.0,local);CHKERRQ(ierr);
ierr = VecNorm(local_copy,NORM_MAX,&work);CHKERRQ(ierr);
ierr = MPI_Allreduce(&work,&norm,1,MPIU_REAL,MPIU_MAX,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of difference %G should be zero\n",norm);CHKERRQ(ierr);
ierr = VecDestroy(&local_copy);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscInt i,M = 3,N = 5,P=3,s=1,w=2,m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE;
PetscErrorCode ierr;
PetscInt *lx = NULL,*ly = NULL,*lz = NULL;
DM da;
PetscBool flg = PETSC_FALSE,test_order = PETSC_FALSE;
ISColoring coloring;
Mat mat;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
Vec lvec,dvec;
MatFDColoring fdcoloring;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-P",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-p",&p,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-w",&w,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-star",&flg,NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
ierr = PetscOptionsGetBool(NULL,"-test_order",&test_order,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-distribute",&flg,NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) lx[i] = 4;
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) ly[i] = 2;
ly[n-1] = N - 2*(n-1);
if (p == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -p option with -distribute option");
ierr = PetscMalloc1(p,&lz);CHKERRQ(ierr);
for (i=0; i<p-1; i++) lz[i] = 2;
lz[p-1] = P - 2*(p-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,stencil_type,M,N,P,m,n,p,w,s,lx,ly,lz,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = PetscFree(lz);CHKERRQ(ierr);
ierr = DMSetMatType(da,MATMPIAIJ);CHKERRQ(ierr);
ierr = DMCreateColoring(da,IS_COLORING_GLOBAL,&coloring);CHKERRQ(ierr);
ierr = DMSetMatType(da,MATMPIAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,&mat);CHKERRQ(ierr);
ierr = MatFDColoringCreate(mat,coloring,&fdcoloring);CHKERRQ(ierr);
ierr = MatFDColoringSetUp(mat,coloring,fdcoloring);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&dvec);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&lvec);CHKERRQ(ierr);
/* Free memory */
ierr = MatFDColoringDestroy(&fdcoloring);CHKERRQ(ierr);
ierr = VecDestroy(&dvec);CHKERRQ(ierr);
ierr = VecDestroy(&lvec);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = ISColoringDestroy(&coloring);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
