PetscErrorCode DMSetFromOptions_Mesh(DM dm)
{
PetscBool flg;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = PetscOptionsHead("DMMesh Options");CHKERRQ(ierr);
/* Handle DMMesh refinement */
/* Handle associated vectors */
/* Handle viewing */
ierr = PetscOptionsBool("-dm_mesh_view_vtk", "Output mesh in VTK format", "DMView", PETSC_FALSE, &flg, PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer viewer;
ierr = PetscViewerCreate(((PetscObject) dm)->comm, &viewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(viewer, "mesh.vtk");CHKERRQ(ierr);
ierr = DMView(dm, viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = PetscOptionsBool("-dm_mesh_view", "Exhaustive mesh description", "DMView", PETSC_FALSE, &flg, PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer viewer;
ierr = PetscViewerCreate(((PetscObject) dm)->comm, &viewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(viewer, PETSCVIEWERASCII);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_INFO_DETAIL);CHKERRQ(ierr);
ierr = DMView(dm, viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = PetscOptionsTail();CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
PetscErrorCode ierr;
Vec U,cv,eta;
DM da,da2;
PetscViewer viewer,view_vtk_cv,view_vtk_eta;
char filename[PETSC_MAX_PATH_LEN],cv_filename[PETSC_MAX_PATH_LEN],eta_filename[PETSC_MAX_PATH_LEN];
PetscBool flg,sflg = PETSC_FALSE;
PetscInt i,n=10000;
PetscInt seed;
PetscInitialize(&argc,&argv, (char*)0, help);
ierr = PetscOptionsSetValue("-viewer_binary_skip_info","true");CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-f",filename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) {
ierr = PetscOptionsGetInt(PETSC_NULL,"-random_seed",&seed,&sflg);CHKERRQ(ierr);
if (!sflg) {
ierr = PetscStrcpy(filename,"ex61.data");CHKERRQ(ierr);
} else {
sprintf(filename,"ex61.data.%d",seed);
}
}
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,filename,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
/* Get physics and time parameters */
ierr = DMCreate(PETSC_COMM_WORLD,&da);CHKERRQ(ierr);
ierr = DMLoad(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&U);CHKERRQ(ierr);
ierr = DMDAGetReducedDA(da,1,&da2);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da2,&cv);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da2,&eta);CHKERRQ(ierr);
for (i=0; i<n; i++) {
/* when this fails it simply means the file is finished */
ierr = VecLoad(U,viewer);CHKERRQ(ierr);
ierr = VecStrideGather(U,1,cv,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecStrideGather(U,4,eta,INSERT_VALUES);CHKERRQ(ierr);
sprintf(cv_filename,"%s_cv_%d.vtk",filename,i);
sprintf(eta_filename,"%s_eta_%d.vtk",filename,i);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,cv_filename,&view_vtk_cv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,eta_filename,&view_vtk_eta);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(view_vtk_cv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(view_vtk_eta, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(da2,view_vtk_cv);CHKERRQ(ierr);
ierr = DMView(da2,view_vtk_eta);CHKERRQ(ierr);
ierr = VecView(cv,view_vtk_cv);CHKERRQ(ierr);
ierr = VecView(eta,view_vtk_eta);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&view_vtk_cv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&view_vtk_eta);CHKERRQ(ierr);
}
ierr = VecDestroy(&U);CHKERRQ(ierr);
ierr = VecDestroy(&cv);CHKERRQ(ierr);
ierr = VecDestroy(&eta);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = DMDestroy(&da2);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
int main(int argc, char **argv)
{
DM dm;
char typeString[256] = {'\0'};
PetscViewer viewer = NULL;
PetscBool flg;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, NULL,help);if (ierr) return ierr;
ierr = DMCreate(PETSC_COMM_WORLD, &dm);CHKERRQ(ierr);
ierr = PetscStrncpy(typeString,DMFOREST,256);CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"DM Forest example options",NULL);CHKERRQ(ierr);
ierr = PetscOptionsString("-dm_type","The type of the dm",NULL,DMFOREST,typeString,sizeof(typeString),NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = DMSetType(dm,(DMType) typeString);CHKERRQ(ierr);
ierr = DMSetFromOptions(dm);CHKERRQ(ierr);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = PetscOptionsGetViewer(PETSC_COMM_WORLD,NULL,"-dm_view",&viewer,NULL,&flg);CHKERRQ(ierr);
if (flg) {
ierr = DMView(dm,viewer);CHKERRQ(ierr);
}
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode DMView_Composite(DM dm,PetscViewer v)
{
PetscErrorCode ierr;
PetscBool iascii;
DM_Composite *com = (DM_Composite *)dm->data;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)v,PETSCVIEWERASCII,&iascii);CHKERRQ(ierr);
if (iascii) {
struct DMCompositeLink *lnk = com->next;
PetscInt i;
ierr = PetscViewerASCIIPrintf(v,"DM (%s)\n",((PetscObject)dm)->prefix?((PetscObject)dm)->prefix:"no prefix");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(v," contains %D DMs\n",com->nDM);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v);CHKERRQ(ierr);
for (i=0; lnk; lnk=lnk->next,i++) {
ierr = PetscViewerASCIIPrintf(v,"Link %D: DM of type %s\n",i,((PetscObject)lnk->dm)->type_name);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v);CHKERRQ(ierr);
ierr = DMView(lnk->dm,v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v);CHKERRQ(ierr);
}
ierr = PetscViewerASCIIPopTab(v);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode DMView_Network(DM dm, PetscViewer viewer)
{
PetscErrorCode ierr;
DM_Network *network = (DM_Network*) dm->data;
PetscFunctionBegin;
ierr = DMView(network->plex,viewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMMGView - prints information on a DA based multi-level preconditioner
Collective on DMMG and PetscViewer
Input Parameter:
+ dmmg - the context
- viewer - the viewer
Level: advanced
.seealso DMMGCreate(), DMMGDestroy(), DMMGSetMatType()
@*/
PetscErrorCode PETSCSNES_DLLEXPORT DMMGView(DMMG *dmmg,PetscViewer viewer)
{
PetscErrorCode ierr;
PetscInt i,nlevels = dmmg[0]->nlevels;
PetscMPIInt flag;
MPI_Comm comm;
PetscTruth iascii,isbinary;
PetscFunctionBegin;
PetscValidPointer(dmmg,1);
PetscValidHeaderSpecific(viewer,PETSC_VIEWER_COOKIE,2);
ierr = PetscObjectGetComm((PetscObject)viewer,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_compare(comm,dmmg[0]->comm,&flag);CHKERRQ(ierr);
if (flag != MPI_CONGRUENT && flag != MPI_IDENT) {
SETERRQ(PETSC_ERR_ARG_NOTSAMECOMM,"Different communicators in the DMMG and the PetscViewer");
}
ierr = PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);CHKERRQ(ierr);
ierr = PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_BINARY,&isbinary);CHKERRQ(ierr);
if (isbinary) {
for (i=0; i<nlevels; i++) {
ierr = MatView(dmmg[i]->J,viewer);CHKERRQ(ierr);
}
for (i=1; i<nlevels; i++) {
ierr = MatView(dmmg[i]->R,viewer);CHKERRQ(ierr);
}
} else {
if (iascii) {
ierr = PetscViewerASCIIPrintf(viewer,"DMMG Object with %D levels\n",nlevels);CHKERRQ(ierr);
if (dmmg[0]->isctype == IS_COLORING_GLOBAL) {
ierr = PetscViewerASCIIPrintf(viewer,"Using global (nonghosted) Jacobian coloring computation\n");CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"Using ghosted Jacobian coloring computation\n");CHKERRQ(ierr);
}
}
for (i=0; i<nlevels; i++) {
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = DMView(dmmg[i]->dm,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
}
if (iascii) {
ierr = PetscViewerASCIIPrintf(viewer,"Using matrix type %s\n",dmmg[nlevels-1]->mtype);CHKERRQ(ierr);
}
if (DMMGGetKSP(dmmg)) {
ierr = KSPView(DMMGGetKSP(dmmg),viewer);CHKERRQ(ierr);
} else if (DMMGGetSNES(dmmg)) {
ierr = SNESView(DMMGGetSNES(dmmg),viewer);CHKERRQ(ierr);
} else if (iascii) {
ierr = PetscViewerASCIIPrintf(viewer,"DMMG does not have a SNES or KSP set\n");CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE,M = 3,N = 5,P=3,s=1;
PetscInt *lx = NULL,*ly = NULL,*lz = NULL;
PetscErrorCode ierr;
PetscBool flg = PETSC_FALSE;
DM da;
Vec local,global,vslice;
PetscScalar value;
DMDABoundaryType wrap = DMDA_XYPERIODIC;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
VecScatter scatter;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-P",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-p",&p,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-star",&flg,NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,wrap,stencil_type,M,N,P,m,n,p,1,s,
lx,ly,lz,&da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = GenerateSliceScatter(da,&scatter,&vslice);CHKERRQ(ierr);
/* Put the value rank+1 into all locations of vslice and transfer back to global vector */
value = 1.0 + rank;
ierr = VecSet(vslice,value);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,vslice,global,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,vslice,global,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 10,N = 8,m = PETSC_DECIDE,n = PETSC_DECIDE;
PetscErrorCode ierr;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,300,300,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_BOX,M,N,m,n,1,1,NULL,NULL,&da);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
value = -3.0;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank+1;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,ADD_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,ADD_VALUES,global);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc, char **argv)
{
DM dm;
AppCtx user; /* user-defined work context */
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, PETSC_NULL, help);CHKERRQ(ierr);
ierr = ProcessOptions(PETSC_COMM_WORLD, &user);CHKERRQ(ierr);
ierr = DMPatchCreateGrid(PETSC_COMM_WORLD, user.dim, user.patchSize, user.commSize, user.gridSize, &dm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) dm, "Patch Mesh");CHKERRQ(ierr);
ierr = DMSetFromOptions(dm);CHKERRQ(ierr);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = DMView(dm, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMPatchSolve(dm);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode DMPatchView_Ascii(DM dm, PetscViewer viewer)
{
DM_Patch *mesh = (DM_Patch*) dm->data;
PetscViewerFormat format;
const char *name;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
/* if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) */
ierr = PetscObjectGetName((PetscObject) dm, &name);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "Patch DM %s\n", name);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "Coarse DM\n");CHKERRQ(ierr);
ierr = DMView(mesh->dmCoarse, viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = -10,N = -8;
PetscErrorCode ierr;
PetscBool flg = PETSC_FALSE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE;
DMDAStencilType stype = DMDA_STENCIL_BOX;
#if defined(PETSC_HAVE_MATLAB_ENGINE)
PetscViewer mviewer;
PetscMPIInt size;
#endif
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,300,300,&viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MATLAB_ENGINE)
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size == 1) {
ierr = PetscViewerMatlabOpen(PETSC_COMM_WORLD,"tmp.mat",FILE_MODE_WRITE,&mviewer);CHKERRQ(ierr);
}
#endif
ierr = PetscOptionsGetBool(NULL,NULL,"-star_stencil",&flg,NULL);CHKERRQ(ierr);
if (flg) stype = DMDA_STENCIL_STAR;
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,stype,M,N,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
value = -3.0;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank+1;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,ADD_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,ADD_VALUES,global);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL, "-view_global", &flg,NULL);CHKERRQ(ierr);
if (flg) { /* view global vector in natural ordering */
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = VecView(global,viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MATLAB_ENGINE)
if (size == 1) {
ierr = DMView(da,mviewer);CHKERRQ(ierr);
ierr = VecView(global,mviewer);CHKERRQ(ierr);
}
#endif
/* Free memory */
#if defined(PETSC_HAVE_MATLAB_ENGINE)
if (size == 1) {
ierr = PetscViewerDestroy(&mviewer);CHKERRQ(ierr);
}
#endif
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode test1_DAInjection3d(PetscInt mx, PetscInt my, PetscInt mz)
{
PetscErrorCode ierr;
DM dac,daf;
PetscViewer vv;
Vec ac,af;
PetscInt periodicity;
DMBoundaryType bx,by,bz;
PetscFunctionBeginUser;
bx = DM_BOUNDARY_NONE;
by = DM_BOUNDARY_NONE;
bz = DM_BOUNDARY_NONE;
periodicity = 0;
ierr = PetscOptionsGetInt(NULL,NULL,"-periodic", &periodicity, NULL);CHKERRQ(ierr);
if (periodicity==1) {
bx = DM_BOUNDARY_PERIODIC;
} else if (periodicity==2) {
by = DM_BOUNDARY_PERIODIC;
} else if (periodicity==3) {
bz = DM_BOUNDARY_PERIODIC;
}
ierr = DMDACreate3d(PETSC_COMM_WORLD, bx,by,bz, DMDA_STENCIL_BOX,mx+1, my+1,mz+1,PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE,1, /* 1 dof */
1, /* stencil = 1 */NULL,NULL,NULL,&daf);CHKERRQ(ierr);
ierr = DMSetFromOptions(daf);CHKERRQ(ierr);
ierr = DMSetUp(daf);CHKERRQ(ierr);
ierr = DMCoarsen(daf,MPI_COMM_NULL,&dac);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(dac, -1.0,1.0, -1.0,1.0, -1.0,1.0);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(daf, -1.0,1.0, -1.0,1.0, -1.0,1.0);CHKERRQ(ierr);
{
DM cdaf,cdac;
Vec coordsc,coordsf,coordsf2;
Mat inject;
VecScatter vscat;
Mat interp;
PetscReal norm;
ierr = DMGetCoordinateDM(dac,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(daf,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(dac,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinates(daf,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInjection(cdac,cdaf,&inject);CHKERRQ(ierr);
ierr = MatScatterGetVecScatter(inject,&vscat);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat ,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = MatDestroy(&inject);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&interp,NULL);CHKERRQ(ierr);
ierr = VecDuplicate(coordsf,&coordsf2);CHKERRQ(ierr);
ierr = MatInterpolate(interp,coordsc,coordsf2);CHKERRQ(ierr);
ierr = VecAXPY(coordsf2,-1.0,coordsf);CHKERRQ(ierr);
ierr = VecNorm(coordsf2,NORM_MAX,&norm);CHKERRQ(ierr);
/* The fine coordinates are only reproduced in certain cases */
if (!bx && !by && !bz && norm > PETSC_SQRT_MACHINE_EPSILON) {ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm %g\n",(double)norm);CHKERRQ(ierr);}
ierr = VecDestroy(&coordsf2);CHKERRQ(ierr);
ierr = MatDestroy(&interp);CHKERRQ(ierr);
}
if (0) {
ierr = DMCreateGlobalVector(dac,&ac);CHKERRQ(ierr);
ierr = VecZeroEntries(ac);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&af);CHKERRQ(ierr);
ierr = VecZeroEntries(af);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "dac_7.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(dac, vv);CHKERRQ(ierr);
ierr = VecView(ac, vv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "daf_7.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(daf, vv);CHKERRQ(ierr);
ierr = VecView(af, vv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = VecDestroy(&ac);CHKERRQ(ierr);
ierr = VecDestroy(&af);CHKERRQ(ierr);
}
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da,*subda;
PetscInt i,dim=3;
PetscMPIInt size,rank;
Vec v;
Vec slvec,sgvec;
IS *ois,*iis;
VecScatter oscata;
VecScatter *iscat,*oscat,*gscat;
DMDALocalInfo info;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-dim",&dim,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,-4,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMCreateDomainDecomposition(da,NULL,NULL,&iis,&ois,&subda);CHKERRQ(ierr);
ierr = DMCreateDomainDecompositionScatters(da,1,subda,&iscat,&oscat,&gscat);CHKERRQ(ierr);
{
DMDALocalInfo subinfo;
MatStencil lower,upper;
IS patchis,subpatchis;
Vec smallvec;
Vec largevec;
VecScatter patchscat;
ierr = DMDAGetLocalInfo(subda[0],&subinfo);CHKERRQ(ierr);
lower.i = info.xs;
lower.j = info.ys;
lower.k = info.zs;
upper.i = info.xs+info.xm;
upper.j = info.ys+info.ym;
upper.k = info.zs+info.zm;
/* test the patch IS as a thing to scatter to/from */
ierr = DMDACreatePatchIS(da,&lower,&upper,&patchis);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&smallvec);CHKERRQ(ierr);
ierr = VecSetSizes(smallvec,info.dof*(upper.i - lower.i)*(upper.j - lower.j)*(upper.k - lower.k),PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(smallvec);CHKERRQ(ierr);
ierr = VecScatterCreate(smallvec,NULL,largevec,patchis,&patchscat);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],smallvec);CHKERRQ(ierr);
ierr = VecSet(largevec,0);CHKERRQ(ierr);
ierr = VecScatterBegin(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = ISView(patchis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecScatterView(patchscat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecView(smallvec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecView(largevec,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&smallvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = ISDestroy(&patchis);CHKERRQ(ierr);
ierr = VecScatterDestroy(&patchscat);CHKERRQ(ierr);
}
/* view the various parts */
{
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMView(subda[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&v);CHKERRQ(ierr);
/* test filling outer between the big DM and the small ones with the IS scatter*/
ierr = VecScatterCreate(v,ois[0],sgvec,NULL,&oscata);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],sgvec);CHKERRQ(ierr);
ierr = VecScatterBegin(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* test the local-to-local scatter */
/* fill up the local subdomain and then add them together */
ierr = FillLocalSubdomain(da,v);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test ghost scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test overlap scattering backwards */
ierr = DMLocalToGlobalBegin(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test interior scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test matrix allocation */
for (i = 0; i < size; i++) {
if (i == rank) {
Mat m;
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMSetMatType(subda[0],MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(subda[0],&m);CHKERRQ(ierr);
ierr = MatView(m,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&m);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMRestoreLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&v);CHKERRQ(ierr);
}
ierr = DMDestroy(&subda[0]);CHKERRQ(ierr);
ierr = ISDestroy(&ois[0]);CHKERRQ(ierr);
ierr = ISDestroy(&iis[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&iscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&gscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscata);CHKERRQ(ierr);
ierr = PetscFree(iscat);CHKERRQ(ierr);
ierr = PetscFree(oscat);CHKERRQ(ierr);
ierr = PetscFree(gscat);CHKERRQ(ierr);
ierr = PetscFree(oscata);CHKERRQ(ierr);
ierr = PetscFree(subda);CHKERRQ(ierr);
ierr = PetscFree(ois);CHKERRQ(ierr);
ierr = PetscFree(iis);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscErrorCode ierr;
PetscInt M = 10,N = 8,m = PETSC_DECIDE;
PetscInt s =2,w=2,n = PETSC_DECIDE,nloc,l,i,j,kk;
PetscInt Xs,Xm,Ys,Ym,iloc,*iglobal;
const PetscInt *ltog;
PetscInt *lx = NULL,*ly = NULL;
PetscBool testorder = PETSC_FALSE,flg;
DMBoundaryType bx = DM_BOUNDARY_NONE,by= DM_BOUNDARY_NONE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDAStencilType st = DMDA_STENCIL_BOX;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,400,&viewer);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(NULL,NULL,"-NX",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-NY",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-w",&w,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xghosted",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yghosted",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-star",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-box",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-testorder",&testorder,NULL);CHKERRQ(ierr);
/*
Test putting two nodes in x and y on each processor, exact last processor
in x and y gets the rest.
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-distribute",&flg,NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,st,M,N,m,n,w,s,lx,ly,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!testorder) { /* turn off printing when testing ordering mappings */
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vectors:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n");CHKERRQ(ierr);
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (testorder) {
ISLocalToGlobalMapping ltogm;
ierr = DMGetLocalToGlobalMapping(da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltogm,&nloc);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,NULL,&Xm,&Ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = PetscMalloc1(nloc,&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscFPrintf(PETSC_COMM_SELF,stdout,"[%d] Problem with mapping: j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",rank,j,i,l,ltog[iloc+l],iglobal[kk]);CHKERRQ(ierr);
}
kk++;
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscBool interpolate = user->interpolate;
PetscReal refinementLimit = user->refinementLimit;
PetscBool cellSimplex = user->cellSimplex;
const char *filename = user->filename;
PetscInt triSizes_n2[2] = {4, 4};
PetscInt triPoints_n2[8] = {3, 5, 6, 7, 0, 1, 2, 4};
PetscInt triSizes_n8[8] = {1, 1, 1, 1, 1, 1, 1, 1};
PetscInt triPoints_n8[8] = {0, 1, 2, 3, 4, 5, 6, 7};
PetscInt quadSizes[2] = {2, 2};
PetscInt quadPoints[4] = {2, 3, 0, 1};
const PetscInt cells[3] = {2, 2, 2};
size_t len;
PetscMPIInt rank, numProcs;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogEventBegin(user->createMeshEvent,0,0,0,0);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
if (len) {ierr = DMPlexCreateFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);}
else if (cellSimplex) {ierr = DMPlexCreateBoxMesh(comm, dim, dim == 2 ? 2 : 1, interpolate, dm);CHKERRQ(ierr);}
else {ierr = DMPlexCreateHexBoxMesh(comm, dim, cells, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, dm);CHKERRQ(ierr);}
{
DM refinedMesh = NULL;
DM distributedMesh = NULL;
if (user->testPartition) {
const PetscInt *sizes = NULL;
const PetscInt *points = NULL;
PetscPartitioner part;
if (!rank) {
if (dim == 2 && cellSimplex && numProcs == 2) {
sizes = triSizes_n2; points = triPoints_n2;
} else if (dim == 2 && cellSimplex && numProcs == 8) {
sizes = triSizes_n8; points = triPoints_n8;
} else if (dim == 2 && !cellSimplex && numProcs == 2) {
sizes = quadSizes; points = quadPoints;
}
}
ierr = DMPlexGetPartitioner(*dm, &part);CHKERRQ(ierr);
ierr = PetscPartitionerSetType(part, PETSCPARTITIONERSHELL);CHKERRQ(ierr);
ierr = PetscPartitionerShellSetPartition(part, numProcs, sizes, points);CHKERRQ(ierr);
}
/* Distribute mesh over processes */
ierr = DMPlexDistribute(*dm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
if (distributedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = distributedMesh;
}
/* Refine mesh using a volume constraint */
ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
ierr = DMPlexSetRefinementLimit(*dm, refinementLimit);CHKERRQ(ierr);
ierr = DMRefine(*dm, comm, &refinedMesh);CHKERRQ(ierr);
if (refinedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = refinedMesh;
}
}
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
if (user->overlap) {
DM overlapMesh = NULL;
/* Add the level-1 overlap to refined mesh */
ierr = DMPlexDistributeOverlap(*dm, 1, NULL, &overlapMesh);CHKERRQ(ierr);
if (overlapMesh) {
ierr = DMView(overlapMesh, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = overlapMesh;
}
}
ierr = PetscObjectSetName((PetscObject) *dm, "Simplicial Mesh");CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
ierr = PetscLogEventEnd(user->createMeshEvent,0,0,0,0);CHKERRQ(ierr);
user->dm = *dm;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 13,s=1,dof=1;
DMDABoundaryType bx = DMDA_BOUNDARY_PERIODIC;
PetscErrorCode ierr;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
PetscDraw draw;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Create viewers */
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",280,480,600,200,&viewer);CHKERRQ(ierr);
ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr);
ierr = PetscDrawSetDoubleBuffer(draw);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetEnum(NULL,"-wrap",DMDABoundaryTypes,(PetscEnum*)&bx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-s",&s,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,dof,s,NULL,&da);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank+1;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = VecView(global,viewer);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ISLocalToGlobalMapping is;
ierr = PetscViewerASCIISynchronizedAllow(PETSC_VIEWER_STDOUT_WORLD,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal to global mapping: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(da,&is);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(is,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode DMView_AKKT(DM dm, PetscViewer v) {
DM_AKKT* kkt = (DM_AKKT*)(dm->data);
PetscErrorCode ierr;
PetscBool isascii;
PetscInt i, tab, vtab;
const char* name, *prefix;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)v, PETSCVIEWERASCII, &isascii); CHKERRQ(ierr);
if(!isascii) SETERRQ(((PetscObject)dm)->comm, PETSC_ERR_SUP, "No support for non-ASCII viewers");
ierr = PetscObjectGetTabLevel((PetscObject)dm, &tab); CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)dm, &name); CHKERRQ(ierr);
ierr = PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix); CHKERRQ(ierr);
ierr = PetscViewerASCIIUseTabs(v,PETSC_TRUE); CHKERRQ(ierr);
ierr = PetscViewerASCIIGetTab(v,&vtab); CHKERRQ(ierr);
ierr = PetscViewerASCIISetTab(v,tab); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(v, "DM Algebraic KKT, name: %s, prefix: %s\n", ((PetscObject)dm)->name, ((PetscObject)dm)->prefix); CHKERRQ(ierr);
if(kkt->dm) {
ierr = PetscViewerASCIIPrintf(v, "DM:\n"); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = DMView(kkt->dm,v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
if(kkt->Aff) {
ierr = PetscViewerASCIIPrintf(v, "Aff:\n"); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = MatView(kkt->Aff,v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
if(kkt->dname) {
ierr = PetscViewerASCIIPrintf(v, "Decomposition, name %s:\n"); CHKERRQ(ierr);
}
for(i = 0; i < 2; ++i) {
const char* label;
if(i == 0) {
label = "Primal";
}
else {
label = "Dual";
}
if(kkt->names[i]) {
ierr = PetscViewerASCIIPrintf(v, "%s, name %s:\n", label, kkt->names[i]); CHKERRQ(ierr);
}
if(kkt->isf[i]){
ierr = PetscViewerASCIIPrintf(v, "%s, IS:\n",label); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = ISView(kkt->isf[i],v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
if(kkt->dmf[i]){
ierr = PetscViewerASCIIPrintf(v, "%s, DM:\n", label); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = DMView(kkt->dmf[i],v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
}
if(kkt->Pfc) {
ierr = PetscViewerASCIIPrintf(v, "Prolongation:\n"); CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(v); CHKERRQ(ierr);
ierr = MatView(kkt->Pfc,v); CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(v); CHKERRQ(ierr);
}
ierr = PetscViewerASCIISetTab(v,vtab); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscBool interpolate = user->interpolate;
PetscReal refinementLimit = user->refinementLimit;
PetscBool cellSimplex = user->cellSimplex;
PetscBool cellWedge = user->cellWedge;
PetscBool simplex2tensor = user->simplex2tensor;
const char *filename = user->filename;
const char *bdfilename = user->bdfilename;
const char *extfilename = user->extfilename;
PetscBool testp4est_seq = user->testp4est[0];
PetscBool testp4est_par = user->testp4est[1];
PetscInt triSizes_n2[2] = {4, 4};
PetscInt triPoints_n2[8] = {3, 5, 6, 7, 0, 1, 2, 4};
PetscInt triSizes_n8[8] = {1, 1, 1, 1, 1, 1, 1, 1};
PetscInt triPoints_n8[8] = {0, 1, 2, 3, 4, 5, 6, 7};
PetscInt quadSizes[2] = {2, 2};
PetscInt quadPoints[4] = {2, 3, 0, 1};
PetscInt gmshSizes_n3[3] = {14, 14, 14};
PetscInt gmshPoints_n3[42] = {1, 2, 4, 5, 9, 10, 11, 15, 16, 20, 21, 27, 28, 29,
3, 8, 12, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
0, 6, 7, 13, 14, 17, 18, 19, 22, 23, 24, 25, 26, 41};
PetscInt fluentSizes_n3[3] = {50, 50, 50};
PetscInt fluentPoints_n3[150] = { 5, 6, 7, 8, 12, 14, 16, 34, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 48, 50, 51, 80, 81, 89,
91, 93, 94, 95, 96, 97, 98, 99, 100, 101, 104, 121, 122, 124, 125, 126, 127, 128, 129, 131, 133, 143, 144, 145, 147,
1, 3, 4, 9, 10, 17, 18, 19, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 47, 61, 71, 72, 73, 74,
75, 76, 77, 78, 79, 86, 87, 88, 90, 92, 113, 115, 116, 117, 118, 119, 120, 123, 138, 140, 141, 142, 146, 148, 149,
0, 2, 11, 13, 15, 20, 21, 22, 23, 49, 52, 53, 54, 55, 56, 57, 58, 59, 60, 62, 63, 64, 65, 66, 67,
68, 69, 70, 82, 83, 84, 85, 102, 103, 105, 106, 107, 108, 109, 110, 111, 112, 114, 130, 132, 134, 135, 136, 137, 139};
size_t len, bdlen, extlen;
PetscMPIInt rank, size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogEventBegin(user->createMeshEvent,0,0,0,0);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
ierr = PetscStrlen(bdfilename, &bdlen);CHKERRQ(ierr);
ierr = PetscStrlen(extfilename, &extlen);CHKERRQ(ierr);
ierr = PetscLogStagePush(user->stages[STAGE_LOAD]);CHKERRQ(ierr);
if (len) {
ierr = DMPlexCreateFromFile(comm, filename, interpolate, dm);CHKERRQ(ierr);
} else if (bdlen) {
DM boundary;
ierr = DMPlexCreateFromFile(comm, bdfilename, interpolate, &boundary);CHKERRQ(ierr);
ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
ierr = DMDestroy(&boundary);CHKERRQ(ierr);
} else if (extlen) {
DM edm;
ierr = DMPlexCreateFromFile(comm, extfilename, interpolate, &edm);CHKERRQ(ierr);
ierr = DMPlexExtrude(edm, user->extrude_layers, user->extrude_thickness, PETSC_TRUE, interpolate, dm);CHKERRQ(ierr);
ierr = DMDestroy(&edm);CHKERRQ(ierr);
} else {
switch (user->domainShape) {
case BOX:
if (cellWedge) {
if (dim != 3) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Dimension must be 3 for a wedge mesh, not %D", dim);
ierr = DMPlexCreateWedgeBoxMesh(comm, user->domainBoxSizes, user->domainBoxL, user->domainBoxU, user->periodicity, PETSC_FALSE, interpolate, dm);CHKERRQ(ierr);
} else {
ierr = DMPlexCreateBoxMesh(comm, dim, cellSimplex, user->domainBoxSizes, user->domainBoxL, user->domainBoxU, user->periodicity, interpolate, dm);CHKERRQ(ierr);
}
break;
case CYLINDER:
if (cellSimplex) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Cannot mesh a cylinder with simplices");
if (dim != 3) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Dimension must be 3 for a cylinder mesh, not %D", dim);
if (cellWedge) {
ierr = DMPlexCreateWedgeCylinderMesh(comm, 6, interpolate, dm);CHKERRQ(ierr);
} else {
ierr = DMPlexCreateHexCylinderMesh(comm, 3, user->periodicity[2], dm);CHKERRQ(ierr);
}
break;
default: SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Unknown domain shape %D", user->domainShape);
}
}
ierr = DMLocalizeCoordinates(*dm);CHKERRQ(ierr); /* needed for periodic */
ierr = DMViewFromOptions(*dm,NULL,"-init_dm_view");CHKERRQ(ierr);
ierr = DMGetDimension(*dm,&dim);CHKERRQ(ierr);
if (testp4est_seq) {
#if defined(PETSC_HAVE_P4EST)
DM dmConv = NULL;
ierr = DMPlexSetRefinementUniform(*dm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMPlexRefineSimplexToTensor(*dm, &dmConv);CHKERRQ(ierr);
if (dmConv) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
user->cellSimplex = PETSC_FALSE;
ierr = DMConvert(*dm,dim == 2 ? DMP4EST : DMP8EST,&dmConv);CHKERRQ(ierr);
if (dmConv) {
ierr = DMSetFromOptions(dmConv);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-conv_seq_1_dm_view");CHKERRQ(ierr);
ierr = DMConvert(*dm,DMPLEX,&dmConv);CHKERRQ(ierr);
if (dmConv) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
ierr = DMViewFromOptions(*dm, NULL, "-conv_seq_2_dm_view");CHKERRQ(ierr);
#else
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Recompile with --download-p4est");
#endif
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (!testp4est_seq) {
DM refinedMesh = NULL;
DM distributedMesh = NULL;
if (user->testPartition) {
const PetscInt *sizes = NULL;
const PetscInt *points = NULL;
PetscPartitioner part;
if (!rank) {
if (dim == 2 && cellSimplex && size == 2) {
sizes = triSizes_n2; points = triPoints_n2;
} else if (dim == 2 && cellSimplex && size == 8) {
sizes = triSizes_n8; points = triPoints_n8;
} else if (dim == 2 && !cellSimplex && size == 2) {
sizes = quadSizes; points = quadPoints;
} else if (dim == 2 && size == 3) {
PetscInt Nc;
ierr = DMPlexGetHeightStratum(*dm, 0, NULL, &Nc);CHKERRQ(ierr);
if (Nc == 42) { /* Gmsh 3 & 4 */
sizes = gmshSizes_n3; points = gmshPoints_n3;
} else if (Nc == 150) { /* Fluent 1 */
sizes = fluentSizes_n3; points = fluentPoints_n3;
} else if (Nc == 42) { /* Med 1 */
} else if (Nc == 161) { /* Med 3 */
}
}
}
ierr = DMPlexGetPartitioner(*dm, &part);CHKERRQ(ierr);
ierr = PetscPartitionerSetType(part, PETSCPARTITIONERSHELL);CHKERRQ(ierr);
ierr = PetscPartitionerShellSetPartition(part, size, sizes, points);CHKERRQ(ierr);
} else {
PetscPartitioner part;
ierr = DMPlexGetPartitioner(*dm,&part);CHKERRQ(ierr);
ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
}
/* Distribute mesh over processes */
ierr = PetscLogStagePush(user->stages[STAGE_DISTRIBUTE]);CHKERRQ(ierr);
ierr = DMPlexDistribute(*dm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
if (distributedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = distributedMesh;
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-distributed_dm_view");CHKERRQ(ierr);
/* Refine mesh using a volume constraint */
ierr = PetscLogStagePush(user->stages[STAGE_REFINE]);CHKERRQ(ierr);
ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
ierr = DMPlexSetRefinementLimit(*dm, refinementLimit);CHKERRQ(ierr);
ierr = DMRefine(*dm, comm, &refinedMesh);CHKERRQ(ierr);
if (refinedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = refinedMesh;
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
}
ierr = PetscLogStagePush(user->stages[STAGE_REFINE]);CHKERRQ(ierr);
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (testp4est_par) {
#if defined(PETSC_HAVE_P4EST)
DM dmConv = NULL;
ierr = DMPlexSetRefinementUniform(*dm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMPlexRefineSimplexToTensor(*dm, &dmConv);CHKERRQ(ierr);
if (dmConv) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
user->cellSimplex = PETSC_FALSE;
ierr = DMConvert(*dm,dim == 2 ? DMP4EST : DMP8EST,&dmConv);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject) dmConv, "conv_par_1_");CHKERRQ(ierr);
if (dmConv) {
ierr = DMSetFromOptions(dmConv);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
ierr = DMConvert(*dm,DMPLEX,&dmConv);CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject) dmConv, "conv_par_2_");CHKERRQ(ierr);
if (dmConv) {
ierr = DMSetFromOptions(dmConv);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = dmConv;
}
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
#else
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Recompile with --download-p4est");
#endif
}
if (user->overlap) {
DM overlapMesh = NULL;
/* Add the level-1 overlap to refined mesh */
ierr = PetscLogStagePush(user->stages[STAGE_OVERLAP]);CHKERRQ(ierr);
ierr = DMPlexDistributeOverlap(*dm, 1, NULL, &overlapMesh);CHKERRQ(ierr);
if (overlapMesh) {
ierr = DMView(overlapMesh, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = overlapMesh;
}
ierr = PetscLogStagePop();CHKERRQ(ierr);
}
if (simplex2tensor) {
DM rdm = NULL;
ierr = DMPlexSetRefinementUniform(*dm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMPlexRefineSimplexToTensor(*dm, &rdm);CHKERRQ(ierr);
if (rdm) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = rdm;
}
user->cellSimplex = PETSC_FALSE;
}
ierr = PetscObjectSetName((PetscObject) *dm, "Simplicial Mesh");CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
ierr = PetscLogEventEnd(user->createMeshEvent,0,0,0,0);CHKERRQ(ierr);
user->dm = *dm;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da;
/* Initialize the Petsc context */
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 1);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* test moving data in and out */
ierr = PetscFinalize();
return 0;
}
PetscErrorCode da_test_RefineCoords3D(PetscInt mx,PetscInt my,PetscInt mz)
{
PetscErrorCode ierr;
DM dac,daf;
PetscViewer vv;
Vec ac,af;
PetscInt map_id,Mx,My,Mz;
Mat II,INTERP;
Vec scale;
PetscBool output = PETSC_FALSE;
PetscFunctionBeginUser;
ierr = DMDACreate3d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,
mx+1, my+1,mz+1,
PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE,
1, /* 1 dof */
1, /* stencil = 1 */
PETSC_NULL,PETSC_NULL,PETSC_NULL,
&dac );CHKERRQ(ierr);
ierr = DMSetFromOptions(dac);CHKERRQ(ierr);
ierr = DMRefine(dac,MPI_COMM_NULL,&daf);CHKERRQ(ierr);
ierr = DMDAGetInfo(daf,0,&Mx,&My,&Mz,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Mx--; My--; Mz--;
ierr = DMDASetUniformCoordinates(dac, -1.0,1.0, -1.0,1.0, -1.0,1.0 );CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(daf, -1.0,1.0, -1.0,1.0, -1.0,1.0 );CHKERRQ(ierr);
/* apply trilinear mappings */
/*ierr = DAApplyTrilinearMapping(dac);CHKERRQ(ierr);*/
/* apply conformal mappings */
map_id = 0;
ierr = PetscOptionsGetInt( PETSC_NULL,"-cmap", &map_id,PETSC_NULL );CHKERRQ(ierr);
if (map_id >= 1) {
ierr = DAApplyConformalMapping(dac,map_id);CHKERRQ(ierr);
}
{
DM cdaf,cdac;
Vec coordsc,coordsf;
ierr = DMGetCoordinateDM(dac,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(daf,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(dac,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinates(daf,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&II,&scale);CHKERRQ(ierr);
ierr = MatInterpolate(II,coordsc,coordsf);CHKERRQ(ierr);
ierr = MatDestroy(&II);CHKERRQ(ierr);
ierr = VecDestroy(&scale);CHKERRQ(ierr);
}
ierr = DMCreateInterpolation(dac,daf,&INTERP,PETSC_NULL);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac,&ac);CHKERRQ(ierr);
ierr = VecZeroEntries(ac);CHKERRQ(ierr);
ierr = DADefineXLinearField3D(dac,ac);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&af);CHKERRQ(ierr);
ierr = VecZeroEntries(af);CHKERRQ(ierr);
ierr = MatMult(INTERP,ac, af);CHKERRQ(ierr);
{
Vec afexact;
PetscReal nrm;
PetscInt N;
ierr = DMCreateGlobalVector(daf,&afexact);CHKERRQ(ierr);
ierr = VecZeroEntries(afexact);CHKERRQ(ierr);
ierr = DADefineXLinearField3D(daf,afexact);CHKERRQ(ierr);
ierr = VecAXPY(afexact,-1.0,af);CHKERRQ(ierr); /* af <= af - afinterp */
ierr = VecNorm(afexact,NORM_2,&nrm);CHKERRQ(ierr);
ierr = VecGetSize(afexact,&N);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"[%D x %D x %D]=>[%D x %D x %D], interp err = %1.4e\n",mx,my,mz,Mx,My,Mz,nrm/sqrt((PetscReal)N) );
ierr = VecDestroy(&afexact);CHKERRQ(ierr);
}
PetscOptionsGetBool(PETSC_NULL,"-output",&output,PETSC_NULL);CHKERRQ(ierr);
if (output) {
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "dac_3D.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(dac, vv);CHKERRQ(ierr);
ierr = VecView(ac, vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "daf_3D.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(daf, vv);CHKERRQ(ierr);
ierr = VecView(af, vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
}
ierr = MatDestroy(&INTERP);CHKERRQ(ierr);
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
ierr = VecDestroy(&ac);CHKERRQ(ierr);
ierr = VecDestroy(&af);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode da_test_RefineCoords1D(PetscInt mx)
{
PetscErrorCode ierr;
DM dac,daf;
PetscViewer vv;
Vec ac,af;
PetscInt Mx;
Mat II,INTERP;
Vec scale;
PetscBool output = PETSC_FALSE;
PetscFunctionBeginUser;
ierr = DMDACreate1d( PETSC_COMM_WORLD, DMDA_BOUNDARY_NONE,
mx+1,
1, /* 1 dof */
1, /* stencil = 1 */
PETSC_NULL,
&dac );CHKERRQ(ierr);
ierr = DMSetFromOptions(dac);CHKERRQ(ierr);
ierr = DMRefine(dac,MPI_COMM_NULL,&daf);CHKERRQ(ierr);
ierr = DMDAGetInfo(daf,0,&Mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Mx--;
ierr = DMDASetUniformCoordinates(dac, -1.0,1.0, PETSC_DECIDE,PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE );CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(daf, -1.0,1.0, PETSC_DECIDE,PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE );CHKERRQ(ierr);
{
DM cdaf,cdac;
Vec coordsc,coordsf;
ierr = DMGetCoordinateDM(dac,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(daf,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(dac,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinates(daf,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&II,&scale);CHKERRQ(ierr);
ierr = MatInterpolate(II,coordsc,coordsf);CHKERRQ(ierr);
ierr = MatDestroy(&II);CHKERRQ(ierr);
ierr = VecDestroy(&scale);CHKERRQ(ierr);
}
ierr = DMCreateInterpolation(dac,daf,&INTERP,PETSC_NULL);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac,&ac);CHKERRQ(ierr);
ierr = VecSet(ac,66.99);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&af);CHKERRQ(ierr);
ierr = MatMult(INTERP,ac, af);CHKERRQ(ierr);
{
Vec afexact;
PetscReal nrm;
PetscInt N;
ierr = DMCreateGlobalVector(daf,&afexact);CHKERRQ(ierr);
ierr = VecSet(afexact,66.99);CHKERRQ(ierr);
ierr = VecAXPY(afexact,-1.0,af);CHKERRQ(ierr); /* af <= af - afinterp */
ierr = VecNorm(afexact,NORM_2,&nrm);CHKERRQ(ierr);
ierr = VecGetSize(afexact,&N);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"%D=>%D, interp err = %1.4e\n",mx,Mx,nrm/sqrt((PetscReal)N) );
ierr = VecDestroy(&afexact);CHKERRQ(ierr);
}
PetscOptionsGetBool(PETSC_NULL,"-output",&output,PETSC_NULL);CHKERRQ(ierr);
if (output) {
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "dac_1D.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(dac, vv);CHKERRQ(ierr);
ierr = VecView(ac, vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "daf_1D.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(daf, vv);CHKERRQ(ierr);
ierr = VecView(af, vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
}
ierr = MatDestroy(&INTERP);CHKERRQ(ierr);
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
ierr = VecDestroy(&ac);CHKERRQ(ierr);
ierr = VecDestroy(&af);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscInt M = 10,N = 8,dof=1,s=1,bx=0,by=0,i,n,j,k,m,wrap,xs,ys;
PetscErrorCode ierr;
DM da,dac;
PetscViewer viewer;
Vec local,global,coors;
PetscScalar ***xy,***aglobal;
PetscDraw draw;
char fname[16];
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
/* Create viewers */
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",PETSC_DECIDE,PETSC_DECIDE,600,200,&viewer);CHKERRQ(ierr);
ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr);
ierr = PetscDrawSetDoubleBuffer(draw);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-periodic_x",&wrap,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-periodic_y",&wrap,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,(DMBoundaryType)bx,(DMBoundaryType)by,DMDA_STENCIL_BOX,M,N,PETSC_DECIDE,PETSC_DECIDE,dof,s,NULL,NULL,&da);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(da,0.0,1.0,0.0,1.0,0.0,0.0);CHKERRQ(ierr);
for (i=0; i<dof; i++) {
sprintf(fname,"Field %d",(int)i);
ierr = DMDASetFieldName(da,i,fname);CHKERRQ(ierr);
}
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
ierr = DMGetCoordinates(da,&coors);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da,&dac);CHKERRQ(ierr);
/* Set values into global vectors */
ierr = DMDAVecGetArrayDOFRead(dac,coors,&xy);CHKERRQ(ierr);
ierr = DMDAVecGetArrayDOF(da,global,&aglobal);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xs,&ys,0,&m,&n,0);CHKERRQ(ierr);
for (k=0; k<dof; k++) {
for (j=ys; j<ys+n; j++) {
for (i=xs; i<xs+m; i++) {
aglobal[j][i][k] = PetscSinScalar(2.0*PETSC_PI*(k+1)*xy[j][i][0]);
}
}
}
ierr = DMDAVecRestoreArrayDOF(da,global,&aglobal);CHKERRQ(ierr);
ierr = DMDAVecRestoreArrayDOFRead(dac,coors,&xy);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = VecSet(global,0.0);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecView(global,viewer);CHKERRQ(ierr);
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode DMPatchSolve(DM dm)
{
MPI_Comm comm;
MPI_Comm commz;
DM dmc;
PetscSF sfz, sfzr;
Vec XC;
MatStencil patchSize, commSize, gridRank, lower, upper;
PetscInt M, N, P, i, j, k, l, m, n, p = 0;
PetscMPIInt rank, size;
PetscInt debug = 0;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMPatchGetCoarse(dm, &dmc);CHKERRQ(ierr);
ierr = DMPatchGetPatchSize(dm, &patchSize);CHKERRQ(ierr);
ierr = DMPatchGetCommSize(dm, &commSize);CHKERRQ(ierr);
ierr = DMPatchGetCommSize(dm, &commSize);CHKERRQ(ierr);
ierr = DMGetGlobalVector(dmc, &XC);CHKERRQ(ierr);
ierr = DMDAGetInfo(dmc, 0, &M, &N, &P, &l, &m, &n, 0,0,0,0,0,0);CHKERRQ(ierr);
M = PetscMax(M, 1); l = PetscMax(l, 1);
N = PetscMax(N, 1); m = PetscMax(m, 1);
P = PetscMax(P, 1); n = PetscMax(n, 1);
gridRank.i = rank % l;
gridRank.j = rank/l % m;
gridRank.k = rank/(l*m) % n;
if (commSize.i*commSize.j*commSize.k == size || commSize.i*commSize.j*commSize.k == 0) {
commSize.i = l; commSize.j = m; commSize.k = n;
commz = comm;
} else if (commSize.i*commSize.j*commSize.k == 1) {
commz = PETSC_COMM_SELF;
} else {
const PetscMPIInt newComm = ((gridRank.k/commSize.k)*(m/commSize.j) + gridRank.j/commSize.j)*(l/commSize.i) + (gridRank.i/commSize.i);
const PetscMPIInt newRank = ((gridRank.k%commSize.k)*commSize.j + gridRank.j%commSize.j)*commSize.i + (gridRank.i%commSize.i);
ierr = MPI_Comm_split(comm, newComm, newRank, &commz);CHKERRQ(ierr);
if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "Rank %d color %d key %d commz %d\n", rank, newComm, newRank, *((PetscMPIInt*) &commz));CHKERRQ(ierr);}
}
/*
Assumptions:
- patchSize divides gridSize
- commSize divides gridSize
- commSize divides l,m,n
Ignore multiple patches per rank for now
Multiple ranks per patch:
- l,m,n divides patchSize
- commSize divides patchSize
*/
for (k = 0; k < P; k += PetscMax(patchSize.k, 1)) {
for (j = 0; j < N; j += PetscMax(patchSize.j, 1)) {
for (i = 0; i < M; i += PetscMax(patchSize.i, 1), ++p) {
MPI_Comm commp = MPI_COMM_NULL;
DM dmz = NULL;
#if 0
DM dmf = NULL;
Mat interpz = NULL;
#endif
Vec XZ = NULL;
PetscScalar *xcarray = NULL;
PetscScalar *xzarray = NULL;
if ((gridRank.k/commSize.k == p/(l/commSize.i * m/commSize.j) % n/commSize.k) &&
(gridRank.j/commSize.j == p/(l/commSize.i) % m/commSize.j) &&
(gridRank.i/commSize.i == p % l/commSize.i)) {
if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "Rank %d is accepting Patch %d\n", rank, p);CHKERRQ(ierr);}
commp = commz;
}
/* Zoom to coarse patch */
lower.i = i; lower.j = j; lower.k = k;
upper.i = i + patchSize.i; upper.j = j + patchSize.j; upper.k = k + patchSize.k;
ierr = DMPatchZoom(dmc, XC, lower, upper, commp, &dmz, &sfz, &sfzr);CHKERRQ(ierr);
lower.c = 0; /* initialize member, otherwise compiler issues warnings */
upper.c = 0; /* initialize member, otherwise compiler issues warnings */
/* Debug */
ierr = PetscPrintf(comm, "Patch %d: (%d, %d, %d)--(%d, %d, %d)\n", p, lower.i, lower.j, lower.k, upper.i, upper.j, upper.k);CHKERRQ(ierr);
if (dmz) {ierr = DMView(dmz, PETSC_VIEWER_STDOUT_(commz));CHKERRQ(ierr);}
ierr = PetscSFView(sfz, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);
ierr = PetscSFView(sfzr, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);
/* Scatter Xcoarse -> Xzoom */
if (dmz) {ierr = DMGetGlobalVector(dmz, &XZ);CHKERRQ(ierr);}
if (XZ) {ierr = VecGetArray(XZ, &xzarray);CHKERRQ(ierr);}
ierr = VecGetArray(XC, &xcarray);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfz, MPIU_SCALAR, xcarray, xzarray);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfz, MPIU_SCALAR, xcarray, xzarray);CHKERRQ(ierr);
ierr = VecRestoreArray(XC, &xcarray);CHKERRQ(ierr);
if (XZ) {ierr = VecRestoreArray(XZ, &xzarray);CHKERRQ(ierr);}
#if 0
/* Interpolate Xzoom -> Xfine, note that this may be on subcomms */
ierr = DMRefine(dmz, MPI_COMM_NULL, &dmf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(dmz, dmf, &interpz, NULL);CHKERRQ(ierr);
ierr = DMInterpolate(dmz, interpz, dmf);CHKERRQ(ierr);
/* Smooth Xfine using two-step smoother, normal smoother plus Kaczmarz---moves back and forth from dmzoom to dmfine */
/* Compute residual Rfine */
/* Restrict Rfine to Rzoom_restricted */
#endif
/* Scatter Rzoom_restricted -> Rcoarse_restricted */
if (XZ) {ierr = VecGetArray(XZ, &xzarray);CHKERRQ(ierr);}
ierr = VecGetArray(XC, &xcarray);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sfzr, MPIU_SCALAR, xzarray, xcarray, MPIU_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfzr, MPIU_SCALAR, xzarray, xcarray, MPIU_SUM);CHKERRQ(ierr);
ierr = VecRestoreArray(XC, &xcarray);CHKERRQ(ierr);
if (XZ) {ierr = VecRestoreArray(XZ, &xzarray);CHKERRQ(ierr);}
if (dmz) {ierr = DMRestoreGlobalVector(dmz, &XZ);CHKERRQ(ierr);}
/* Compute global residual Rcoarse */
/* TauCoarse = Rcoarse - Rcoarse_restricted */
ierr = PetscSFDestroy(&sfz);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfzr);CHKERRQ(ierr);
ierr = DMDestroy(&dmz);CHKERRQ(ierr);
}
}
}
ierr = DMRestoreGlobalVector(dmc, &XC);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 3,N = 5,P=3,s=1,w=2,nloc,l,i,j,k,kk,m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE;
PetscErrorCode ierr;
PetscInt Xs,Xm,Ys,Ym,Zs,Zm,iloc,*ltog,*iglobal;
PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL,*lz = PETSC_NULL;
PetscBool test_order = PETSC_FALSE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
AO ao;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,300,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-NX",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NY",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NZ",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-w",&w,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-box",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xnonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-ynonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-znonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-testorder",&test_order,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-distribute",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc(m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc(n*sizeof(PetscInt),&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
if (p == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -p option with -distribute option");
ierr = PetscMalloc(p*sizeof(PetscInt),&lz);CHKERRQ(ierr);
for (i=0; i<p-1; i++) { lz[i] = 2;}
lz[p-1] = P - 2*(p-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,m,n,p,w,s,lx,ly,lz,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = PetscFree(lz);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!test_order) { /* turn off printing when testing ordering mappings */
if (M*N*P<40) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-local_print",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (test_order) {
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,&Zs,&Xm,&Ym,&Zm);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(da,&nloc,<og);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
/* ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = PetscMalloc(nloc*sizeof(PetscInt),&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y, then z) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscPrintf(MPI_COMM_WORLD,"[%D] Problem with mapping: z=%D, j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",
rank,k,j,i,l,ltog[iloc+l],iglobal[kk]);
}
kk++;
}
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}