PetscErrorCode DMCreateInterpolation_Composite(DM coarse,DM fine,Mat *A,Vec *v)
{
PetscErrorCode ierr;
PetscInt m,n,M,N,nDM,i;
struct DMCompositeLink *nextc;
struct DMCompositeLink *nextf;
Vec gcoarse,gfine,*vecs;
DM_Composite *comcoarse = (DM_Composite*)coarse->data;
DM_Composite *comfine = (DM_Composite*)fine->data;
Mat *mats;
PetscFunctionBegin;
PetscValidHeaderSpecific(coarse,DM_CLASSID,1);
PetscValidHeaderSpecific(fine,DM_CLASSID,2);
ierr = DMSetUp(coarse);CHKERRQ(ierr);
ierr = DMSetUp(fine);CHKERRQ(ierr);
/* use global vectors only for determining matrix layout */
ierr = DMGetGlobalVector(coarse,&gcoarse);CHKERRQ(ierr);
ierr = DMGetGlobalVector(fine,&gfine);CHKERRQ(ierr);
ierr = VecGetLocalSize(gcoarse,&n);CHKERRQ(ierr);
ierr = VecGetLocalSize(gfine,&m);CHKERRQ(ierr);
ierr = VecGetSize(gcoarse,&N);CHKERRQ(ierr);
ierr = VecGetSize(gfine,&M);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(coarse,&gcoarse);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(fine,&gfine);CHKERRQ(ierr);
nDM = comfine->nDM;
if (nDM != comcoarse->nDM) SETERRQ2(((PetscObject)fine)->comm,PETSC_ERR_ARG_INCOMP,"Fine DMComposite has %D entries, but coarse has %D",nDM,comcoarse->nDM);
ierr = PetscMalloc(nDM*nDM*sizeof(Mat),&mats);CHKERRQ(ierr);
ierr = PetscMemzero(mats,nDM*nDM*sizeof(Mat));CHKERRQ(ierr);
if (v) {
ierr = PetscMalloc(nDM*sizeof(Vec),&vecs);CHKERRQ(ierr);
ierr = PetscMemzero(vecs,nDM*sizeof(Vec));CHKERRQ(ierr);
}
/* loop over packed objects, handling one at at time */
for (nextc=comcoarse->next,nextf=comfine->next,i=0; nextc; nextc=nextc->next,nextf=nextf->next,i++) {
if (!v) {
ierr = DMCreateInterpolation(nextc->dm,nextf->dm,&mats[i*nDM+i],PETSC_NULL);CHKERRQ(ierr);
} else {
ierr = DMCreateInterpolation(nextc->dm,nextf->dm,&mats[i*nDM+i],&vecs[i]);CHKERRQ(ierr);
}
}
ierr = MatCreateNest(((PetscObject)fine)->comm,nDM,PETSC_NULL,nDM,PETSC_NULL,mats,A);CHKERRQ(ierr);
if (v) {
ierr = VecCreateNest(((PetscObject)fine)->comm,nDM,PETSC_NULL,vecs,v);CHKERRQ(ierr);
}
for (i=0; i<nDM*nDM; i++) {ierr = MatDestroy(&mats[i]);CHKERRQ(ierr);}
ierr = PetscFree(mats);CHKERRQ(ierr);
if (v) {
for (i=0; i<nDM; i++) {ierr = VecDestroy(&vecs[i]);CHKERRQ(ierr);}
ierr = PetscFree(vecs);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc,char *argv[])
{
Mat M;
Vec x,y;
PetscErrorCode ierr;
DM da,daf;
ierr = PetscInitialize(&argc,&argv,0,help);if (ierr) return ierr;
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,4,5,PETSC_DECIDE,PETSC_DECIDE,41,1,0,0,&da);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMRefine(da,PETSC_COMM_WORLD,&daf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(da,daf,&M,NULL);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&y);CHKERRQ(ierr);
ierr = MatMult(M,x,y);CHKERRQ(ierr);
ierr = MatMultTranspose(M,y,x);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = MatDestroy(&M);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
static PetscErrorCode CreateInterpolation(DM dm1,DM dm2,Mat *mat,Vec *vec)
{
DM da1,da2;
PetscErrorCode ierr;
ierr = DMShellGetContext(dm1,(void**)&da1);CHKERRQ(ierr);
ierr = DMShellGetContext(dm2,(void**)&da2);CHKERRQ(ierr);
ierr = DMCreateInterpolation(da1,da2,mat,vec);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode CreateRestriction(DM dm1,DM dm2,Mat *mat)
{
DM da1,da2;
PetscErrorCode ierr;
Mat tmat;
ierr = DMShellGetContext(dm1,(void**)&da1);CHKERRQ(ierr);
ierr = DMShellGetContext(dm2,(void**)&da2);CHKERRQ(ierr);
ierr = DMCreateInterpolation(da1,da2,&tmat,NULL);CHKERRQ(ierr);
ierr = MatTranspose(tmat,MAT_INITIAL_MATRIX,mat);CHKERRQ(ierr);
ierr = MatDestroy(&tmat);CHKERRQ(ierr);
return 0;
}
int main(int argc,char **argv)
{
PetscInt M = 5,N = 4,P = 3, m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE,dim = 1;
PetscErrorCode ierr;
DM dac,daf;
DMDABoundaryType bx=DMDA_BOUNDARY_NONE,by=DMDA_BOUNDARY_NONE,bz=DMDA_BOUNDARY_NONE;
DMDAStencilType stype = DMDA_STENCIL_BOX;
Mat A;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-P",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dim",&dim,PETSC_NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
if (dim == 1) {
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,1,1,PETSC_NULL,&dac);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,stype,M,N,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,&dac);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stype,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,PETSC_NULL,&dac);CHKERRQ(ierr);
}
ierr = DMRefine(dac,PETSC_COMM_WORLD,&daf);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(dac,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
if (dim == 1) {
ierr = SetCoordinates1d(daf);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = SetCoordinates2d(daf);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = SetCoordinates3d(daf);CHKERRQ(ierr);
}
ierr = DMCreateInterpolation(dac,daf,&A,0);CHKERRQ(ierr);
ierr = MatViewFromOptions(A,"-mat_view");CHKERRQ(ierr);
/* Free memory */
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscInt M = 14,dof = 1,s = 1,ratio = 2,dim = 2;
PetscErrorCode ierr;
DM da_c,da_f;
Vec v_c,v_f;
Mat I;
PetscScalar one = 1.0;
MPI_Comm comm_f, comm_c;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dim",&dim,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-sw",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-ratio",&ratio,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr);
comm_f = PETSC_COMM_WORLD;
ierr = DMDASplitComm2d(comm_f,M,M,s,&comm_c);CHKERRQ(ierr);
/* Set up the array */
if (dim == 2) {
ierr = DMDACreate2d(comm_c,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,M,M,PETSC_DECIDE,PETSC_DECIDE,dof,s,PETSC_NULL,PETSC_NULL,&da_c);CHKERRQ(ierr);
M = ratio*(M-1) + 1;
ierr = DMDACreate2d(comm_f,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,M,M,PETSC_DECIDE,PETSC_DECIDE,dof,s,PETSC_NULL,PETSC_NULL,&da_f);CHKERRQ(ierr);
} else if (dim == 3) {
;
}
ierr = DMCreateGlobalVector(da_c,&v_c);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da_f,&v_f);CHKERRQ(ierr);
ierr = VecSet(v_c,one);CHKERRQ(ierr);
ierr = DMCreateInterpolation(da_c,da_f,&I,PETSC_NULL);CHKERRQ(ierr);
ierr = MatInterpolate(I,v_c,v_f);CHKERRQ(ierr);
ierr = VecView(v_f,PETSC_VIEWER_STDOUT_(comm_f));CHKERRQ(ierr);
ierr = MatRestrict(I,v_f,v_c);CHKERRQ(ierr);
ierr = VecView(v_c,PETSC_VIEWER_STDOUT_(comm_c));CHKERRQ(ierr);
ierr = MatDestroy(&I);CHKERRQ(ierr);
ierr = VecDestroy(&v_c);CHKERRQ(ierr);
ierr = DMDestroy(&da_c);CHKERRQ(ierr);
ierr = VecDestroy(&v_f);CHKERRQ(ierr);
ierr = DMDestroy(&da_f);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode test1_DAInjection3d(PetscInt mx, PetscInt my, PetscInt mz)
{
PetscErrorCode ierr;
DM dac,daf;
PetscViewer vv;
Vec ac,af;
PetscInt periodicity;
DMBoundaryType bx,by,bz;
PetscFunctionBeginUser;
bx = DM_BOUNDARY_NONE;
by = DM_BOUNDARY_NONE;
bz = DM_BOUNDARY_NONE;
periodicity = 0;
ierr = PetscOptionsGetInt(NULL,NULL,"-periodic", &periodicity, NULL);CHKERRQ(ierr);
if (periodicity==1) {
bx = DM_BOUNDARY_PERIODIC;
} else if (periodicity==2) {
by = DM_BOUNDARY_PERIODIC;
} else if (periodicity==3) {
bz = DM_BOUNDARY_PERIODIC;
}
ierr = DMDACreate3d(PETSC_COMM_WORLD, bx,by,bz, DMDA_STENCIL_BOX,mx+1, my+1,mz+1,PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE,1, /* 1 dof */
1, /* stencil = 1 */NULL,NULL,NULL,&daf);CHKERRQ(ierr);
ierr = DMSetFromOptions(daf);CHKERRQ(ierr);
ierr = DMSetUp(daf);CHKERRQ(ierr);
ierr = DMCoarsen(daf,MPI_COMM_NULL,&dac);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(dac, -1.0,1.0, -1.0,1.0, -1.0,1.0);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(daf, -1.0,1.0, -1.0,1.0, -1.0,1.0);CHKERRQ(ierr);
{
DM cdaf,cdac;
Vec coordsc,coordsf,coordsf2;
Mat inject;
VecScatter vscat;
Mat interp;
PetscReal norm;
ierr = DMGetCoordinateDM(dac,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(daf,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(dac,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinates(daf,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInjection(cdac,cdaf,&inject);CHKERRQ(ierr);
ierr = MatScatterGetVecScatter(inject,&vscat);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat ,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = MatDestroy(&inject);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&interp,NULL);CHKERRQ(ierr);
ierr = VecDuplicate(coordsf,&coordsf2);CHKERRQ(ierr);
ierr = MatInterpolate(interp,coordsc,coordsf2);CHKERRQ(ierr);
ierr = VecAXPY(coordsf2,-1.0,coordsf);CHKERRQ(ierr);
ierr = VecNorm(coordsf2,NORM_MAX,&norm);CHKERRQ(ierr);
/* The fine coordinates are only reproduced in certain cases */
if (!bx && !by && !bz && norm > PETSC_SQRT_MACHINE_EPSILON) {ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm %g\n",(double)norm);CHKERRQ(ierr);}
ierr = VecDestroy(&coordsf2);CHKERRQ(ierr);
ierr = MatDestroy(&interp);CHKERRQ(ierr);
}
if (0) {
ierr = DMCreateGlobalVector(dac,&ac);CHKERRQ(ierr);
ierr = VecZeroEntries(ac);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&af);CHKERRQ(ierr);
ierr = VecZeroEntries(af);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "dac_7.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(dac, vv);CHKERRQ(ierr);
ierr = VecView(ac, vv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "daf_7.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(daf, vv);CHKERRQ(ierr);
ierr = VecView(af, vv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = VecDestroy(&ac);CHKERRQ(ierr);
ierr = VecDestroy(&af);CHKERRQ(ierr);
}
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMRefine_DA(DM da,MPI_Comm comm,DM *daref)
{
PetscErrorCode ierr;
PetscInt M,N,P,i;
DM da2;
DM_DA *dd = (DM_DA*)da->data,*dd2;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidPointer(daref,3);
if (dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
M = dd->refine_x*dd->M;
} else {
M = 1 + dd->refine_x*(dd->M - 1);
}
if (dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
if (dd->dim > 1) {
N = dd->refine_y*dd->N;
} else {
N = 1;
}
} else {
N = 1 + dd->refine_y*(dd->N - 1);
}
if (dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
if (dd->dim > 2) {
P = dd->refine_z*dd->P;
} else {
P = 1;
}
} else {
P = 1 + dd->refine_z*(dd->P - 1);
}
ierr = DMDACreate(((PetscObject)da)->comm,&da2);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da2,((PetscObject)da)->prefix);CHKERRQ(ierr);
ierr = DMDASetDim(da2,dd->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da2,M,N,P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da2,dd->m,dd->n,dd->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da2,dd->bx,dd->by,dd->bz);CHKERRQ(ierr);
ierr = DMDASetDof(da2,dd->w);CHKERRQ(ierr);
ierr = DMDASetStencilType(da2,dd->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da2,dd->s);CHKERRQ(ierr);
if (dd->dim == 3) {
PetscInt *lx,*ly,*lz;
ierr = PetscMalloc3(dd->m,PetscInt,&lx,dd->n,PetscInt,&ly,dd->p,PetscInt,&lz);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_z,dd->p,dd->lz,lz);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,lz);CHKERRQ(ierr);
ierr = PetscFree3(lx,ly,lz);CHKERRQ(ierr);
} else if (dd->dim == 2) {
PetscInt *lx,*ly;
ierr = PetscMalloc2(dd->m,PetscInt,&lx,dd->n,PetscInt,&ly);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree2(lx,ly);CHKERRQ(ierr);
} else if (dd->dim == 1) {
PetscInt *lx;
ierr = PetscMalloc(dd->m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
}
dd2 = (DM_DA*)da2->data;
/* allow overloaded (user replaced) operations to be inherited by refinement clones */
da2->ops->creatematrix = da->ops->creatematrix;
/* da2->ops->createinterpolation = da->ops->createinterpolation; this causes problem with SNESVI */
da2->ops->getcoloring = da->ops->getcoloring;
dd2->interptype = dd->interptype;
/* copy fill information if given */
if (dd->dfill) {
ierr = PetscMalloc((dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt),&dd2->dfill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->dfill,dd->dfill,(dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
if (dd->ofill) {
ierr = PetscMalloc((dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt),&dd2->ofill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->ofill,dd->ofill,(dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
/* copy the refine information */
dd2->coarsen_x = dd2->refine_x = dd->refine_x;
dd2->coarsen_y = dd2->refine_y = dd->refine_y;
dd2->coarsen_z = dd2->refine_z = dd->refine_z;
/* copy vector type information */
ierr = PetscFree(da2->vectype);CHKERRQ(ierr);
ierr = PetscStrallocpy(da->vectype,(char**)&da2->vectype);CHKERRQ(ierr);
dd2->lf = dd->lf;
dd2->lj = dd->lj;
da2->leveldown = da->leveldown;
da2->levelup = da->levelup + 1;
ierr = DMSetFromOptions(da2);CHKERRQ(ierr);
ierr = DMSetUp(da2);CHKERRQ(ierr);
ierr = DMViewFromOptions(da2,"-dm_view");CHKERRQ(ierr);
/* interpolate coordinates if they are set on the coarse grid */
if (da->coordinates) {
DM cdaf,cdac;
Vec coordsc,coordsf;
Mat II;
ierr = DMGetCoordinateDM(da,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinates(da,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da2,&cdaf);CHKERRQ(ierr);
/* force creation of the coordinate vector */
ierr = DMDASetUniformCoordinates(da2,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMGetCoordinates(da2,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&II,PETSC_NULL);CHKERRQ(ierr);
ierr = MatInterpolate(II,coordsc,coordsf);CHKERRQ(ierr);
ierr = MatDestroy(&II);CHKERRQ(ierr);
}
for (i=0; i<da->bs; i++) {
const char *fieldname;
ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr);
ierr = DMDASetFieldName(da2,i,fieldname);CHKERRQ(ierr);
}
*daref = da2;
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
#if !defined(PETSC_USE_COMPLEX)
PetscErrorCode ierr;
Vec x,yp1,yp2,yp3,yp4,ym1,ym2,ym3,ym4;
PetscReal *values;
PetscViewer viewer_in,viewer_outp1,viewer_outp2,viewer_outp3,viewer_outp4;
PetscViewer viewer_outm1,viewer_outm2,viewer_outm3,viewer_outm4;
DM daf,dac1,dac2,dac3,dac4,daf1,daf2,daf3,daf4;
Vec scaling_p1,scaling_p2,scaling_p3,scaling_p4;
Mat interp_p1,interp_p2,interp_p3,interp_p4,interp_m1,interp_m2,interp_m3,interp_m4;
#endif
PetscInitialize(&argc,&argv, (char*)0, help);
#if defined(PETSC_USE_COMPLEX)
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Not for complex numbers");
#else
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_PERIODIC,DM_BOUNDARY_PERIODIC,DMDA_STENCIL_BOX,1024,1024,PETSC_DECIDE,PETSC_DECIDE, 1, 1,NULL,NULL,&daf);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&x);CHKERRQ(ierr);
ierr = VecGetArray(x,&values);CHKERRQ(ierr);
ierr = DMCoarsen(daf,PETSC_COMM_WORLD,&dac1);CHKERRQ(ierr);
ierr = DMCoarsen(dac1,PETSC_COMM_WORLD,&dac2);CHKERRQ(ierr);
ierr = DMCoarsen(dac2,PETSC_COMM_WORLD,&dac3);CHKERRQ(ierr);
ierr = DMCoarsen(dac3,PETSC_COMM_WORLD,&dac4);CHKERRQ(ierr);
ierr = DMRefine(daf,PETSC_COMM_WORLD,&daf1);CHKERRQ(ierr);
ierr = DMRefine(daf1,PETSC_COMM_WORLD,&daf2);CHKERRQ(ierr);
ierr = DMRefine(daf2,PETSC_COMM_WORLD,&daf3);CHKERRQ(ierr);
ierr = DMRefine(daf3,PETSC_COMM_WORLD,&daf4);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac1,&yp1);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac2,&yp2);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac3,&yp3);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac4,&yp4);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf1,&ym1);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf2,&ym2);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf3,&ym3);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf4,&ym4);CHKERRQ(ierr);
ierr = DMCreateInterpolation(dac1,daf,&interp_p1,&scaling_p1);CHKERRQ(ierr);
ierr = DMCreateInterpolation(dac2,dac1,&interp_p2,&scaling_p2);CHKERRQ(ierr);
ierr = DMCreateInterpolation(dac3,dac2,&interp_p3,&scaling_p3);CHKERRQ(ierr);
ierr = DMCreateInterpolation(dac4,dac3,&interp_p4,&scaling_p4);CHKERRQ(ierr);
ierr = DMCreateInterpolation(daf,daf1,&interp_m1,NULL);CHKERRQ(ierr);
ierr = DMCreateInterpolation(daf1,daf2,&interp_m2,NULL);CHKERRQ(ierr);
ierr = DMCreateInterpolation(daf2,daf3,&interp_m3,NULL);CHKERRQ(ierr);
ierr = DMCreateInterpolation(daf3,daf4,&interp_m4,NULL);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phi",FILE_MODE_READ,&viewer_in);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer_in,values,1048576,PETSC_DOUBLE);CHKERRQ(ierr);
ierr = MatRestrict(interp_p1,x,yp1);
ierr = VecPointwiseMult(yp1,yp1,scaling_p1);CHKERRQ(ierr);
ierr = MatRestrict(interp_p2,yp1,yp2);
ierr = VecPointwiseMult(yp2,yp2,scaling_p2);CHKERRQ(ierr);
ierr = MatRestrict(interp_p3,yp2,yp3);
ierr = VecPointwiseMult(yp3,yp3,scaling_p3);CHKERRQ(ierr);
ierr = MatRestrict(interp_p4,yp3,yp4);
ierr = VecPointwiseMult(yp4,yp4,scaling_p4);CHKERRQ(ierr);
ierr = MatRestrict(interp_m1,x,ym1);
ierr = MatRestrict(interp_m2,ym1,ym2);
ierr = MatRestrict(interp_m3,ym2,ym3);
ierr = MatRestrict(interp_m4,ym3,ym4);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phi1",FILE_MODE_WRITE,&viewer_outp1);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phi2",FILE_MODE_WRITE,&viewer_outp2);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phi3",FILE_MODE_WRITE,&viewer_outp3);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phi4",FILE_MODE_WRITE,&viewer_outp4);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phim1",FILE_MODE_WRITE,&viewer_outm1);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phim2",FILE_MODE_WRITE,&viewer_outm2);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phim3",FILE_MODE_WRITE,&viewer_outm3);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"phim4",FILE_MODE_WRITE,&viewer_outm4);CHKERRQ(ierr);
ierr = VecView(yp1,viewer_outp1);CHKERRQ(ierr);
ierr = VecView(x,viewer_outp1);CHKERRQ(ierr);
ierr = VecView(yp2,viewer_outp2);CHKERRQ(ierr);
ierr = VecView(yp3,viewer_outp3);CHKERRQ(ierr);
ierr = VecView(yp4,viewer_outp4);CHKERRQ(ierr);
ierr = VecView(ym1,viewer_outm1);CHKERRQ(ierr);
ierr = VecView(ym2,viewer_outm2);CHKERRQ(ierr);
ierr = VecView(ym3,viewer_outm3);CHKERRQ(ierr);
ierr = VecView(ym4,viewer_outm4);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_in);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_outp1);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_outp2);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_outp3);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_outp4);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_outm1);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_outm2);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_outm3);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer_outm4);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&yp1);CHKERRQ(ierr);
ierr = VecDestroy(&yp2);CHKERRQ(ierr);
ierr = VecDestroy(&yp3);CHKERRQ(ierr);
ierr = VecDestroy(&yp4);CHKERRQ(ierr);
ierr = VecDestroy(&ym1);CHKERRQ(ierr);
ierr = VecDestroy(&ym2);CHKERRQ(ierr);
ierr = VecDestroy(&ym3);CHKERRQ(ierr);
ierr = VecDestroy(&ym4);CHKERRQ(ierr);
#endif
PetscFinalize();
return 0;
}
PetscErrorCode DMCoarsen_AKKT(DM dm, MPI_Comm comm, DM *cdm) {
PetscErrorCode ierr;
DM_AKKT* kkt = (DM_AKKT*)(dm->data);
Mat Acc; /* coarse-level KKT matrix */
Mat P0f0c, P1f1c; /* Primal and dual block prolongator */
DM dmc[2] = {PETSC_NULL, PETSC_NULL}; /* Coarse subDMs defining the block prolongators and the coarsened decomposition. */
PetscInt M0,N0,M1,N1; /* Sizes of P0f0c and P1f1c. */
PetscInt start0,end0,start1,end1; /* Ownership ranges for P0f0c and P1f1c. */
static Mat mats[4] = {PETSC_NULL, PETSC_NULL, PETSC_NULL, PETSC_NULL}; /* Used to construct MatNest out of pieces. */
IS isc[2]; /* Used to construct MatNest out of pieces and to define the coarsened decomposition. */
PetscFunctionBegin;
if(!cdm) PetscFunctionReturn(0);
if(kkt->cdm) {
ierr = PetscObjectReference((PetscObject)(kkt->cdm)); CHKERRQ(ierr);
*cdm = kkt->cdm;
PetscFunctionReturn(0);
}
/* Coarsen the 00 block with the attached DM and obtain the primal prolongator. */
if(kkt->dmf[0]) {
ierr = DMCoarsen(kkt->dmf[0], comm, dmc+0); CHKERRQ(ierr);
ierr = DMCreateInterpolation(dmc[0], kkt->dmf[0], &P0f0c, PETSC_NULL); CHKERRQ(ierr);
}
else SETERRQ(((PetscObject)dm)->comm, PETSC_ERR_ARG_WRONGSTATE, "Could not coarsen the primal block: primal subDM not set.");
/* Should P0f0c be transposed to act as a prolongator (i.e., to map from coarse to fine). */
ierr = MatGetSize(P0f0c, &M0, &N0); CHKERRQ(ierr);
if(M0 == N0) SETERRQ1(((PetscObject)dm)->comm, PETSC_ERR_ARG_WRONGSTATE,"Primal prolongator is square with size %D: cannot distinguish coarse from fine",M0);
if(M0 < N0) kkt->transposeP = PETSC_TRUE;
else kkt->transposeP = PETSC_FALSE;
/* See if the 11 block can be coarsened with an attached DM. If so, we are done. Otherwise, use GAMG to coarsen 11. */
if(kkt->dmf[1]) {
ierr = DMCoarsen(kkt->dmf[1], comm, &dmc[1]); CHKERRQ(ierr);
ierr = DMCreateInterpolation(dmc[1], kkt->dmf[1], &P1f1c, PETSC_NULL); CHKERRQ(ierr);
}
else {
ierr = DMCoarsen_AKKT_GAMG11(dm, P0f0c, &P1f1c); CHKERRQ(ierr);
}
/* Determine whether P1f1c should be transposed in order to act as a prolongator (i.e., to map from coarse to fine). */
ierr = MatGetSize(P1f1c, &M1, &N1); CHKERRQ(ierr);
if(M1 == N1) SETERRQ1(((PetscObject)dm)->comm, PETSC_ERR_ARG_WRONGSTATE, "Dual prlongator is square with size %D: cannot distinguish coarse from fine", M1);
if((M1 < N1 && !kkt->transposeP) || (M1 >= N1 && kkt->transposeP)) {
Mat P1f1ct;
ierr = MatTranspose(P1f1c, MAT_INITIAL_MATRIX, &P1f1ct); CHKERRQ(ierr);
ierr = MatDestroy(&P1f1c); CHKERRQ(ierr);
P1f1c = P1f1ct;
}
/* MatNest P0f0c, P1f1c together into Pfc. */
mats[0] = P0f0c; mats[3] = P1f1c;
ierr = MatGetOwnershipRange(P0f0c, &start0, &end0); CHKERRQ(ierr);
ierr = MatGetOwnershipRange(P1f1c, &start1, &end1); CHKERRQ(ierr);
ierr = ISCreateStride(((PetscObject)dm)->comm, end0-start0,start0,1,isc+0); CHKERRQ(ierr);
ierr = ISCreateStride(((PetscObject)dm)->comm, end1-start1,start1,1,isc+1); CHKERRQ(ierr);
if(kkt->transposeP) {
ierr = MatCreateNest(((PetscObject)dm)->comm,2,isc,2,kkt->isf,mats,&(kkt->Pfc)); CHKERRQ(ierr);
}
else {
ierr = MatCreateNest(((PetscObject)dm)->comm,2,kkt->isf,2,isc,mats,&(kkt->Pfc)); CHKERRQ(ierr);
}
ierr = MatDestroy(&P0f0c); CHKERRQ(ierr);
ierr = MatDestroy(&P1f1c); CHKERRQ(ierr);
/* Coarsening the underlying matrix and primal-dual decomposition. */
/*
We do not coarsen the underlying DM because
(a) Its coarsening may be incompatible with the specialized KKT-aware coarsening of the blocks defined here.
(b) Even if the coarsening the decomposition is compatible with the decomposition of the coarsening, we can
pick the former without loss of generality.
(c) Even if (b) is true, the embeddings (IS) of the coarsened subDMs are potentially different now from what
they would be in the coarsened DM; thus, embeddings would have to be supplied manually anyhow.
(d) In the typical situation we should only use the primal subDM for coarsening -- the whole point of
DMAKKT is that the dual block coarsening should be derived from the primal block coarsening for compatibility.
If we are given both subDMs, DMAKKT essentially becomes a version of DMComposite, in which case the composition
of the coarsened decomposition is by definition the coarsening of the whole system DM.
*/
/* Create the coarser DM. */
ierr = DMCreate(((PetscObject)dm)->comm, &(kkt->cdm)); CHKERRQ(ierr);
ierr = DMSetType(kkt->cdm, DMAKKT); CHKERRQ(ierr);
/* Coarsen the underlying matrix. */
ierr = MatPtAP(kkt->Aff, kkt->Pfc, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &Acc); CHKERRQ(ierr);
ierr = DMAKKTSetMatrix(dm, Acc); CHKERRQ(ierr);
/* Set the coarsened decomposition. */
ierr = DMAKKTSetFieldDecomposition(kkt->cdm, 2, (const char**)kkt->names, isc, dmc); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPatchSolve(DM dm)
{
MPI_Comm comm;
MPI_Comm commz;
DM dmc;
PetscSF sfz, sfzr;
Vec XC;
MatStencil patchSize, commSize, gridRank, lower, upper;
PetscInt M, N, P, i, j, k, l, m, n, p = 0;
PetscMPIInt rank, size;
PetscInt debug = 0;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMPatchGetCoarse(dm, &dmc);CHKERRQ(ierr);
ierr = DMPatchGetPatchSize(dm, &patchSize);CHKERRQ(ierr);
ierr = DMPatchGetCommSize(dm, &commSize);CHKERRQ(ierr);
ierr = DMPatchGetCommSize(dm, &commSize);CHKERRQ(ierr);
ierr = DMGetGlobalVector(dmc, &XC);CHKERRQ(ierr);
ierr = DMDAGetInfo(dmc, 0, &M, &N, &P, &l, &m, &n, 0,0,0,0,0,0);CHKERRQ(ierr);
M = PetscMax(M, 1); l = PetscMax(l, 1);
N = PetscMax(N, 1); m = PetscMax(m, 1);
P = PetscMax(P, 1); n = PetscMax(n, 1);
gridRank.i = rank % l;
gridRank.j = rank/l % m;
gridRank.k = rank/(l*m) % n;
if (commSize.i*commSize.j*commSize.k == size || commSize.i*commSize.j*commSize.k == 0) {
commSize.i = l; commSize.j = m; commSize.k = n;
commz = comm;
} else if (commSize.i*commSize.j*commSize.k == 1) {
commz = PETSC_COMM_SELF;
} else {
const PetscMPIInt newComm = ((gridRank.k/commSize.k)*(m/commSize.j) + gridRank.j/commSize.j)*(l/commSize.i) + (gridRank.i/commSize.i);
const PetscMPIInt newRank = ((gridRank.k%commSize.k)*commSize.j + gridRank.j%commSize.j)*commSize.i + (gridRank.i%commSize.i);
ierr = MPI_Comm_split(comm, newComm, newRank, &commz);CHKERRQ(ierr);
if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "Rank %d color %d key %d commz %d\n", rank, newComm, newRank, *((PetscMPIInt*) &commz));CHKERRQ(ierr);}
}
/*
Assumptions:
- patchSize divides gridSize
- commSize divides gridSize
- commSize divides l,m,n
Ignore multiple patches per rank for now
Multiple ranks per patch:
- l,m,n divides patchSize
- commSize divides patchSize
*/
for (k = 0; k < P; k += PetscMax(patchSize.k, 1)) {
for (j = 0; j < N; j += PetscMax(patchSize.j, 1)) {
for (i = 0; i < M; i += PetscMax(patchSize.i, 1), ++p) {
MPI_Comm commp = MPI_COMM_NULL;
DM dmz = NULL;
#if 0
DM dmf = NULL;
Mat interpz = NULL;
#endif
Vec XZ = NULL;
PetscScalar *xcarray = NULL;
PetscScalar *xzarray = NULL;
if ((gridRank.k/commSize.k == p/(l/commSize.i * m/commSize.j) % n/commSize.k) &&
(gridRank.j/commSize.j == p/(l/commSize.i) % m/commSize.j) &&
(gridRank.i/commSize.i == p % l/commSize.i)) {
if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "Rank %d is accepting Patch %d\n", rank, p);CHKERRQ(ierr);}
commp = commz;
}
/* Zoom to coarse patch */
lower.i = i; lower.j = j; lower.k = k;
upper.i = i + patchSize.i; upper.j = j + patchSize.j; upper.k = k + patchSize.k;
ierr = DMPatchZoom(dmc, XC, lower, upper, commp, &dmz, &sfz, &sfzr);CHKERRQ(ierr);
lower.c = 0; /* initialize member, otherwise compiler issues warnings */
upper.c = 0; /* initialize member, otherwise compiler issues warnings */
/* Debug */
ierr = PetscPrintf(comm, "Patch %d: (%d, %d, %d)--(%d, %d, %d)\n", p, lower.i, lower.j, lower.k, upper.i, upper.j, upper.k);CHKERRQ(ierr);
if (dmz) {ierr = DMView(dmz, PETSC_VIEWER_STDOUT_(commz));CHKERRQ(ierr);}
ierr = PetscSFView(sfz, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);
ierr = PetscSFView(sfzr, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);
/* Scatter Xcoarse -> Xzoom */
if (dmz) {ierr = DMGetGlobalVector(dmz, &XZ);CHKERRQ(ierr);}
if (XZ) {ierr = VecGetArray(XZ, &xzarray);CHKERRQ(ierr);}
ierr = VecGetArray(XC, &xcarray);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfz, MPIU_SCALAR, xcarray, xzarray);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfz, MPIU_SCALAR, xcarray, xzarray);CHKERRQ(ierr);
ierr = VecRestoreArray(XC, &xcarray);CHKERRQ(ierr);
if (XZ) {ierr = VecRestoreArray(XZ, &xzarray);CHKERRQ(ierr);}
#if 0
/* Interpolate Xzoom -> Xfine, note that this may be on subcomms */
ierr = DMRefine(dmz, MPI_COMM_NULL, &dmf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(dmz, dmf, &interpz, NULL);CHKERRQ(ierr);
ierr = DMInterpolate(dmz, interpz, dmf);CHKERRQ(ierr);
/* Smooth Xfine using two-step smoother, normal smoother plus Kaczmarz---moves back and forth from dmzoom to dmfine */
/* Compute residual Rfine */
/* Restrict Rfine to Rzoom_restricted */
#endif
/* Scatter Rzoom_restricted -> Rcoarse_restricted */
if (XZ) {ierr = VecGetArray(XZ, &xzarray);CHKERRQ(ierr);}
ierr = VecGetArray(XC, &xcarray);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sfzr, MPIU_SCALAR, xzarray, xcarray, MPIU_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfzr, MPIU_SCALAR, xzarray, xcarray, MPIU_SUM);CHKERRQ(ierr);
ierr = VecRestoreArray(XC, &xcarray);CHKERRQ(ierr);
if (XZ) {ierr = VecRestoreArray(XZ, &xzarray);CHKERRQ(ierr);}
if (dmz) {ierr = DMRestoreGlobalVector(dmz, &XZ);CHKERRQ(ierr);}
/* Compute global residual Rcoarse */
/* TauCoarse = Rcoarse - Rcoarse_restricted */
ierr = PetscSFDestroy(&sfz);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfzr);CHKERRQ(ierr);
ierr = DMDestroy(&dmz);CHKERRQ(ierr);
}
}
}
ierr = DMRestoreGlobalVector(dmc, &XC);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Mm_ratio - ration of grid lines between fine and coarse grids.
*/
int main(int argc,char **argv)
{
PetscErrorCode ierr;
AppCtx user;
PetscMPIInt size,rank;
PetscInt m,n,M,N,i,nrows;
PetscScalar one = 1.0;
PetscReal fill=2.0;
Mat A,P,R,C,PtAP;
PetscScalar *array;
PetscRandom rdm;
PetscBool Test_3D=PETSC_FALSE,flg;
const PetscInt *ia,*ja;
ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* Get size of fine grids and coarse grids */
user.ratio = 2;
user.coarse.mx = 4; user.coarse.my = 4; user.coarse.mz = 4;
ierr = PetscOptionsGetInt(NULL,NULL,"-Mx",&user.coarse.mx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-My",&user.coarse.my,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Mz",&user.coarse.mz,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ratio",&user.ratio,NULL);CHKERRQ(ierr);
if (user.coarse.mz) Test_3D = PETSC_TRUE;
user.fine.mx = user.ratio*(user.coarse.mx-1)+1;
user.fine.my = user.ratio*(user.coarse.my-1)+1;
user.fine.mz = user.ratio*(user.coarse.mz-1)+1;
if (!rank) {
if (!Test_3D) {
ierr = PetscPrintf(PETSC_COMM_SELF,"coarse grids: %D %D; fine grids: %D %D\n",user.coarse.mx,user.coarse.my,user.fine.mx,user.fine.my);CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_SELF,"coarse grids: %D %D %D; fine grids: %D %D %D\n",user.coarse.mx,user.coarse.my,user.coarse.mz,user.fine.mx,user.fine.my,user.fine.mz);CHKERRQ(ierr);
}
}
/* Set up distributed array for fine grid */
if (!Test_3D) {
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&user.fine.da);CHKERRQ(ierr);
} else {
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,user.fine.mz,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
1,1,NULL,NULL,NULL,&user.fine.da);CHKERRQ(ierr);
}
ierr = DMSetFromOptions(user.fine.da);CHKERRQ(ierr);
ierr = DMSetUp(user.fine.da);CHKERRQ(ierr);
/* Create and set A at fine grids */
ierr = DMSetMatType(user.fine.da,MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.fine.da,&A);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
/* set val=one to A (replace with random values!) */
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
if (size == 1) {
ierr = MatGetRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(A,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(A,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
} else {
Mat AA,AB;
ierr = MatMPIAIJGetSeqAIJ(A,&AA,&AB,NULL);CHKERRQ(ierr);
ierr = MatGetRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(AA,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(AA,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
ierr = MatGetRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(AB,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(AB,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
}
/* Set up distributed array for coarse grid */
if (!Test_3D) {
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&user.coarse.da);CHKERRQ(ierr);
} else {
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,user.coarse.mz,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,NULL,&user.coarse.da);CHKERRQ(ierr);
}
ierr = DMSetFromOptions(user.coarse.da);CHKERRQ(ierr);
ierr = DMSetUp(user.coarse.da);CHKERRQ(ierr);
/* Create interpolation between the fine and coarse grids */
ierr = DMCreateInterpolation(user.coarse.da,user.fine.da,&P,NULL);CHKERRQ(ierr);
/* Get R = P^T */
ierr = MatTranspose(P,MAT_INITIAL_MATRIX,&R);CHKERRQ(ierr);
/* C = R*A*P */
ierr = MatMatMatMult(R,A,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr);
ierr = MatMatMatMult(R,A,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr);
/* Test C == PtAP */
ierr = MatPtAP(A,P,MAT_INITIAL_MATRIX,fill,&PtAP);CHKERRQ(ierr);
ierr = MatPtAP(A,P,MAT_REUSE_MATRIX,fill,&PtAP);CHKERRQ(ierr);
ierr = MatEqual(C,PtAP,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"Matrices are not equal");
ierr = MatDestroy(&PtAP);CHKERRQ(ierr);
/* Clean up */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = DMDestroy(&user.fine.da);CHKERRQ(ierr);
ierr = DMDestroy(&user.coarse.da);CHKERRQ(ierr);
ierr = MatDestroy(&P);CHKERRQ(ierr);
ierr = MatDestroy(&R);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*
Mm_ratio - ration of grid lines between fine and coarse grids.
*/
int main(int argc,char **argv)
{
PetscErrorCode ierr;
AppCtx user;
PetscInt Npx=PETSC_DECIDE,Npy=PETSC_DECIDE,Npz=PETSC_DECIDE;
PetscMPIInt size,rank;
PetscInt m,n,M,N,i,nrows;
PetscScalar one = 1.0;
PetscReal fill=2.0;
Mat A,A_tmp,P,C,C1,C2;
PetscScalar *array,none = -1.0,alpha;
Vec x,v1,v2,v3,v4;
PetscReal norm,norm_tmp,norm_tmp1,tol=100.*PETSC_MACHINE_EPSILON;
PetscRandom rdm;
PetscBool Test_MatMatMult=PETSC_TRUE,Test_MatPtAP=PETSC_TRUE,Test_3D=PETSC_TRUE,flg;
const PetscInt *ia,*ja;
ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr;
ierr = PetscOptionsGetReal(NULL,NULL,"-tol",&tol,NULL);CHKERRQ(ierr);
user.ratio = 2;
user.coarse.mx = 20; user.coarse.my = 20; user.coarse.mz = 20;
ierr = PetscOptionsGetInt(NULL,NULL,"-Mx",&user.coarse.mx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-My",&user.coarse.my,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Mz",&user.coarse.mz,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ratio",&user.ratio,NULL);CHKERRQ(ierr);
if (user.coarse.mz) Test_3D = PETSC_TRUE;
user.fine.mx = user.ratio*(user.coarse.mx-1)+1;
user.fine.my = user.ratio*(user.coarse.my-1)+1;
user.fine.mz = user.ratio*(user.coarse.mz-1)+1;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npx",&Npx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npy",&Npy,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npz",&Npz,NULL);CHKERRQ(ierr);
/* Set up distributed array for fine grid */
if (!Test_3D) {
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,Npx,Npy,1,1,NULL,NULL,&user.fine.da);CHKERRQ(ierr);
} else {
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,user.fine.mz,Npx,Npy,Npz,1,1,NULL,NULL,NULL,&user.fine.da);CHKERRQ(ierr);
}
ierr = DMSetFromOptions(user.fine.da);CHKERRQ(ierr);
ierr = DMSetUp(user.fine.da);CHKERRQ(ierr);
/* Test DMCreateMatrix() */
/*------------------------------------------------------------*/
ierr = DMSetMatType(user.fine.da,MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.fine.da,&A);CHKERRQ(ierr);
ierr = DMSetMatType(user.fine.da,MATBAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.fine.da,&C);CHKERRQ(ierr);
ierr = MatConvert(C,MATAIJ,MAT_INITIAL_MATRIX,&A_tmp);CHKERRQ(ierr); /* not work for mpisbaij matrix! */
ierr = MatEqual(A,A_tmp,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"A != C");
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A_tmp);CHKERRQ(ierr);
/*------------------------------------------------------------*/
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
/* if (!rank) printf("A %d, %d\n",M,N); */
/* set val=one to A */
if (size == 1) {
ierr = MatGetRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(A,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(A,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
} else {
Mat AA,AB;
ierr = MatMPIAIJGetSeqAIJ(A,&AA,&AB,NULL);CHKERRQ(ierr);
ierr = MatGetRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(AA,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(AA,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
ierr = MatGetRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(AB,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(AB,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
}
/* ierr = MatView(A, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Set up distributed array for coarse grid */
if (!Test_3D) {
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,Npx,Npy,1,1,NULL,NULL,&user.coarse.da);CHKERRQ(ierr);
} else {
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,user.coarse.mz,Npx,Npy,Npz,
1,1,NULL,NULL,NULL,&user.coarse.da);CHKERRQ(ierr);
}
ierr = DMSetFromOptions(user.coarse.da);CHKERRQ(ierr);
ierr = DMSetUp(user.coarse.da);CHKERRQ(ierr);
/* Create interpolation between the fine and coarse grids */
ierr = DMCreateInterpolation(user.coarse.da,user.fine.da,&P,NULL);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,&m,&n);CHKERRQ(ierr);
ierr = MatGetSize(P,&M,&N);CHKERRQ(ierr);
/* if (!rank) printf("P %d, %d\n",M,N); */
/* Create vectors v1 and v2 that are compatible with A */
ierr = VecCreate(PETSC_COMM_WORLD,&v1);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&m,NULL);CHKERRQ(ierr);
ierr = VecSetSizes(v1,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(v1);CHKERRQ(ierr);
ierr = VecDuplicate(v1,&v2);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
/* Test MatMatMult(): C = A*P */
/*----------------------------*/
if (Test_MatMatMult) {
ierr = MatDuplicate(A,MAT_COPY_VALUES,&A_tmp);CHKERRQ(ierr);
ierr = MatMatMult(A_tmp,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr);
/* Test MAT_REUSE_MATRIX - reuse symbolic C */
alpha=1.0;
for (i=0; i<2; i++) {
alpha -= 0.1;
ierr = MatScale(A_tmp,alpha);CHKERRQ(ierr);
ierr = MatMatMult(A_tmp,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr);
}
/* Free intermediate data structures created for reuse of C=Pt*A*P */
ierr = MatFreeIntermediateDataStructures(C);CHKERRQ(ierr);
/* Test MatDuplicate() */
/*----------------------------*/
ierr = MatDuplicate(C,MAT_COPY_VALUES,&C1);CHKERRQ(ierr);
ierr = MatDuplicate(C1,MAT_COPY_VALUES,&C2);CHKERRQ(ierr);
ierr = MatDestroy(&C1);CHKERRQ(ierr);
ierr = MatDestroy(&C2);CHKERRQ(ierr);
/* Create vector x that is compatible with P */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,NULL,&n);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
norm = 0.0;
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(P,x,v1);CHKERRQ(ierr);
ierr = MatMult(A_tmp,v1,v2);CHKERRQ(ierr); /* v2 = A*P*x */
ierr = MatMult(C,x,v1);CHKERRQ(ierr); /* v1 = C*x */
ierr = VecAXPY(v1,none,v2);CHKERRQ(ierr);
ierr = VecNorm(v1,NORM_1,&norm_tmp);CHKERRQ(ierr);
ierr = VecNorm(v2,NORM_1,&norm_tmp1);CHKERRQ(ierr);
norm_tmp /= norm_tmp1;
if (norm_tmp > norm) norm = norm_tmp;
}
if (norm >= tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMatMult(), |v1 - v2|/|v2|: %g\n",(double)norm);CHKERRQ(ierr);
}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A_tmp);CHKERRQ(ierr);
}
/* Test P^T * A * P - MatPtAP() */
/*------------------------------*/
if (Test_MatPtAP) {
ierr = MatPtAP(A,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr);
ierr = MatGetLocalSize(C,&m,&n);CHKERRQ(ierr);
/* Test MAT_REUSE_MATRIX - reuse symbolic C */
alpha=1.0;
for (i=0; i<1; i++) {
alpha -= 0.1;
ierr = MatScale(A,alpha);CHKERRQ(ierr);
ierr = MatPtAP(A,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr);
}
/* Free intermediate data structures created for reuse of C=Pt*A*P */
ierr = MatFreeIntermediateDataStructures(C);CHKERRQ(ierr);
/* Test MatDuplicate() */
/*----------------------------*/
ierr = MatDuplicate(C,MAT_COPY_VALUES,&C1);CHKERRQ(ierr);
ierr = MatDuplicate(C1,MAT_COPY_VALUES,&C2);CHKERRQ(ierr);
ierr = MatDestroy(&C1);CHKERRQ(ierr);
ierr = MatDestroy(&C2);CHKERRQ(ierr);
/* Create vector x that is compatible with P */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,&m,&n);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&v3);CHKERRQ(ierr);
ierr = VecSetSizes(v3,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(v3);CHKERRQ(ierr);
ierr = VecDuplicate(v3,&v4);CHKERRQ(ierr);
norm = 0.0;
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(P,x,v1);CHKERRQ(ierr);
ierr = MatMult(A,v1,v2);CHKERRQ(ierr); /* v2 = A*P*x */
ierr = MatMultTranspose(P,v2,v3);CHKERRQ(ierr); /* v3 = Pt*A*P*x */
ierr = MatMult(C,x,v4);CHKERRQ(ierr); /* v3 = C*x */
ierr = VecAXPY(v4,none,v3);CHKERRQ(ierr);
ierr = VecNorm(v4,NORM_1,&norm_tmp);CHKERRQ(ierr);
ierr = VecNorm(v3,NORM_1,&norm_tmp1);CHKERRQ(ierr);
norm_tmp /= norm_tmp1;
if (norm_tmp > norm) norm = norm_tmp;
}
if (norm >= tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatPtAP(), |v3 - v4|/|v3|: %g\n",(double)norm);CHKERRQ(ierr);
}
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = VecDestroy(&v3);CHKERRQ(ierr);
ierr = VecDestroy(&v4);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
}
/* Clean up */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = VecDestroy(&v1);CHKERRQ(ierr);
ierr = VecDestroy(&v2);CHKERRQ(ierr);
ierr = DMDestroy(&user.fine.da);CHKERRQ(ierr);
ierr = DMDestroy(&user.coarse.da);CHKERRQ(ierr);
ierr = MatDestroy(&P);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode da_test_RefineCoords3D(PetscInt mx,PetscInt my,PetscInt mz)
{
PetscErrorCode ierr;
DM dac,daf;
PetscViewer vv;
Vec ac,af;
PetscInt map_id,Mx,My,Mz;
Mat II,INTERP;
Vec scale;
PetscBool output = PETSC_FALSE;
PetscFunctionBeginUser;
ierr = DMDACreate3d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,
mx+1, my+1,mz+1,
PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE,
1, /* 1 dof */
1, /* stencil = 1 */
PETSC_NULL,PETSC_NULL,PETSC_NULL,
&dac );CHKERRQ(ierr);
ierr = DMSetFromOptions(dac);CHKERRQ(ierr);
ierr = DMRefine(dac,MPI_COMM_NULL,&daf);CHKERRQ(ierr);
ierr = DMDAGetInfo(daf,0,&Mx,&My,&Mz,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Mx--; My--; Mz--;
ierr = DMDASetUniformCoordinates(dac, -1.0,1.0, -1.0,1.0, -1.0,1.0 );CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(daf, -1.0,1.0, -1.0,1.0, -1.0,1.0 );CHKERRQ(ierr);
/* apply trilinear mappings */
/*ierr = DAApplyTrilinearMapping(dac);CHKERRQ(ierr);*/
/* apply conformal mappings */
map_id = 0;
ierr = PetscOptionsGetInt( PETSC_NULL,"-cmap", &map_id,PETSC_NULL );CHKERRQ(ierr);
if (map_id >= 1) {
ierr = DAApplyConformalMapping(dac,map_id);CHKERRQ(ierr);
}
{
DM cdaf,cdac;
Vec coordsc,coordsf;
ierr = DMGetCoordinateDM(dac,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(daf,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(dac,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinates(daf,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&II,&scale);CHKERRQ(ierr);
ierr = MatInterpolate(II,coordsc,coordsf);CHKERRQ(ierr);
ierr = MatDestroy(&II);CHKERRQ(ierr);
ierr = VecDestroy(&scale);CHKERRQ(ierr);
}
ierr = DMCreateInterpolation(dac,daf,&INTERP,PETSC_NULL);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac,&ac);CHKERRQ(ierr);
ierr = VecZeroEntries(ac);CHKERRQ(ierr);
ierr = DADefineXLinearField3D(dac,ac);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&af);CHKERRQ(ierr);
ierr = VecZeroEntries(af);CHKERRQ(ierr);
ierr = MatMult(INTERP,ac, af);CHKERRQ(ierr);
{
Vec afexact;
PetscReal nrm;
PetscInt N;
ierr = DMCreateGlobalVector(daf,&afexact);CHKERRQ(ierr);
ierr = VecZeroEntries(afexact);CHKERRQ(ierr);
ierr = DADefineXLinearField3D(daf,afexact);CHKERRQ(ierr);
ierr = VecAXPY(afexact,-1.0,af);CHKERRQ(ierr); /* af <= af - afinterp */
ierr = VecNorm(afexact,NORM_2,&nrm);CHKERRQ(ierr);
ierr = VecGetSize(afexact,&N);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"[%D x %D x %D]=>[%D x %D x %D], interp err = %1.4e\n",mx,my,mz,Mx,My,Mz,nrm/sqrt((PetscReal)N) );
ierr = VecDestroy(&afexact);CHKERRQ(ierr);
}
PetscOptionsGetBool(PETSC_NULL,"-output",&output,PETSC_NULL);CHKERRQ(ierr);
if (output) {
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "dac_3D.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(dac, vv);CHKERRQ(ierr);
ierr = VecView(ac, vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "daf_3D.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(daf, vv);CHKERRQ(ierr);
ierr = VecView(af, vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
}
ierr = MatDestroy(&INTERP);CHKERRQ(ierr);
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
ierr = VecDestroy(&ac);CHKERRQ(ierr);
ierr = VecDestroy(&af);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode da_test_RefineCoords1D(PetscInt mx)
{
PetscErrorCode ierr;
DM dac,daf;
PetscViewer vv;
Vec ac,af;
PetscInt Mx;
Mat II,INTERP;
Vec scale;
PetscBool output = PETSC_FALSE;
PetscFunctionBeginUser;
ierr = DMDACreate1d( PETSC_COMM_WORLD, DMDA_BOUNDARY_NONE,
mx+1,
1, /* 1 dof */
1, /* stencil = 1 */
PETSC_NULL,
&dac );CHKERRQ(ierr);
ierr = DMSetFromOptions(dac);CHKERRQ(ierr);
ierr = DMRefine(dac,MPI_COMM_NULL,&daf);CHKERRQ(ierr);
ierr = DMDAGetInfo(daf,0,&Mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Mx--;
ierr = DMDASetUniformCoordinates(dac, -1.0,1.0, PETSC_DECIDE,PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE );CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(daf, -1.0,1.0, PETSC_DECIDE,PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE );CHKERRQ(ierr);
{
DM cdaf,cdac;
Vec coordsc,coordsf;
ierr = DMGetCoordinateDM(dac,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(daf,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(dac,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinates(daf,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&II,&scale);CHKERRQ(ierr);
ierr = MatInterpolate(II,coordsc,coordsf);CHKERRQ(ierr);
ierr = MatDestroy(&II);CHKERRQ(ierr);
ierr = VecDestroy(&scale);CHKERRQ(ierr);
}
ierr = DMCreateInterpolation(dac,daf,&INTERP,PETSC_NULL);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac,&ac);CHKERRQ(ierr);
ierr = VecSet(ac,66.99);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&af);CHKERRQ(ierr);
ierr = MatMult(INTERP,ac, af);CHKERRQ(ierr);
{
Vec afexact;
PetscReal nrm;
PetscInt N;
ierr = DMCreateGlobalVector(daf,&afexact);CHKERRQ(ierr);
ierr = VecSet(afexact,66.99);CHKERRQ(ierr);
ierr = VecAXPY(afexact,-1.0,af);CHKERRQ(ierr); /* af <= af - afinterp */
ierr = VecNorm(afexact,NORM_2,&nrm);CHKERRQ(ierr);
ierr = VecGetSize(afexact,&N);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"%D=>%D, interp err = %1.4e\n",mx,Mx,nrm/sqrt((PetscReal)N) );
ierr = VecDestroy(&afexact);CHKERRQ(ierr);
}
PetscOptionsGetBool(PETSC_NULL,"-output",&output,PETSC_NULL);CHKERRQ(ierr);
if (output) {
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "dac_1D.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(dac, vv);CHKERRQ(ierr);
ierr = VecView(ac, vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "daf_1D.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(daf, vv);CHKERRQ(ierr);
ierr = VecView(af, vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
}
ierr = MatDestroy(&INTERP);CHKERRQ(ierr);
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
ierr = VecDestroy(&ac);CHKERRQ(ierr);
ierr = VecDestroy(&af);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode SNESSetUp_FAS(SNES snes)
{
SNES_FAS *fas = (SNES_FAS*) snes->data;
PetscErrorCode ierr;
PetscInt dm_levels;
Vec vec_sol, vec_func, vec_sol_update, vec_rhs; /* preserve these if they're set through the reset */
SNES next;
PetscBool isFine;
SNESLineSearch linesearch;
SNESLineSearch slinesearch;
void *lsprectx,*lspostctx;
PetscErrorCode (*precheck)(SNESLineSearch,Vec,Vec,PetscBool*,void*);
PetscErrorCode (*postcheck)(SNESLineSearch,Vec,Vec,Vec,PetscBool*,PetscBool*,void*);
PetscFunctionBegin;
ierr = SNESFASCycleIsFine(snes, &isFine);CHKERRQ(ierr);
if (fas->usedmfornumberoflevels && isFine) {
ierr = DMGetRefineLevel(snes->dm,&dm_levels);CHKERRQ(ierr);
dm_levels++;
if (dm_levels > fas->levels) {
/* we don't want the solution and func vectors to be destroyed in the SNESReset when it's called in SNESFASSetLevels_FAS*/
vec_sol = snes->vec_sol;
vec_func = snes->vec_func;
vec_sol_update = snes->vec_sol_update;
vec_rhs = snes->vec_rhs;
snes->vec_sol = NULL;
snes->vec_func = NULL;
snes->vec_sol_update = NULL;
snes->vec_rhs = NULL;
/* reset the number of levels */
ierr = SNESFASSetLevels(snes,dm_levels,NULL);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
snes->vec_sol = vec_sol;
snes->vec_func = vec_func;
snes->vec_rhs = vec_rhs;
snes->vec_sol_update = vec_sol_update;
}
}
ierr = SNESFASCycleGetCorrection(snes, &next);CHKERRQ(ierr);
if (!isFine) snes->gridsequence = 0; /* no grid sequencing inside the multigrid hierarchy! */
ierr = SNESSetWorkVecs(snes, 2);CHKERRQ(ierr); /* work vectors used for intergrid transfers */
/* set up the smoothers if they haven't already been set up */
if (!fas->smoothd) {
ierr = SNESFASCycleCreateSmoother_Private(snes, &fas->smoothd);CHKERRQ(ierr);
}
if (snes->dm) {
/* set the smoother DMs properly */
if (fas->smoothu) ierr = SNESSetDM(fas->smoothu, snes->dm);CHKERRQ(ierr);
ierr = SNESSetDM(fas->smoothd, snes->dm);CHKERRQ(ierr);
/* construct EVERYTHING from the DM -- including the progressive set of smoothers */
if (next) {
/* for now -- assume the DM and the evaluation functions have been set externally */
if (!next->dm) {
ierr = DMCoarsen(snes->dm, PetscObjectComm((PetscObject)next), &next->dm);CHKERRQ(ierr);
ierr = SNESSetDM(next, next->dm);CHKERRQ(ierr);
}
/* set the interpolation and restriction from the DM */
if (!fas->interpolate) {
ierr = DMCreateInterpolation(next->dm, snes->dm, &fas->interpolate, &fas->rscale);CHKERRQ(ierr);
if (!fas->restrct) {
ierr = PetscObjectReference((PetscObject)fas->interpolate);CHKERRQ(ierr);
fas->restrct = fas->interpolate;
}
}
/* set the injection from the DM */
if (!fas->inject) {
ierr = DMCreateInjection(next->dm, snes->dm, &fas->inject);CHKERRQ(ierr);
}
}
}
/*pass the smoother, function, and jacobian up to the next level if it's not user set already */
if (fas->galerkin) {
if (next) {
ierr = SNESSetFunction(next, NULL, SNESFASGalerkinDefaultFunction, next);CHKERRQ(ierr);
}
if (fas->smoothd && fas->level != fas->levels - 1) {
ierr = SNESSetFunction(fas->smoothd, NULL, SNESFASGalerkinDefaultFunction, snes);CHKERRQ(ierr);
}
if (fas->smoothu && fas->level != fas->levels - 1) {
ierr = SNESSetFunction(fas->smoothu, NULL, SNESFASGalerkinDefaultFunction, snes);CHKERRQ(ierr);
}
}
/* sets the down (pre) smoother's default norm and sets it from options */
if (fas->smoothd) {
if (fas->level == 0 && fas->levels != 1) {
ierr = SNESSetNormSchedule(fas->smoothd, SNES_NORM_NONE);CHKERRQ(ierr);
} else {
ierr = SNESSetNormSchedule(fas->smoothd, SNES_NORM_FINAL_ONLY);CHKERRQ(ierr);
}
ierr = PetscObjectCopyFortranFunctionPointers((PetscObject)snes, (PetscObject)fas->smoothd);CHKERRQ(ierr);
ierr = SNESSetFromOptions(fas->smoothd);CHKERRQ(ierr);
ierr = SNESGetLineSearch(snes,&linesearch);CHKERRQ(ierr);
ierr = SNESGetLineSearch(fas->smoothd,&slinesearch);CHKERRQ(ierr);
ierr = SNESLineSearchGetPreCheck(linesearch,&precheck,&lsprectx);CHKERRQ(ierr);
ierr = SNESLineSearchGetPostCheck(linesearch,&postcheck,&lspostctx);CHKERRQ(ierr);
ierr = SNESLineSearchSetPreCheck(slinesearch,precheck,lsprectx);CHKERRQ(ierr);
ierr = SNESLineSearchSetPostCheck(slinesearch,postcheck,lspostctx);CHKERRQ(ierr);
ierr = PetscObjectCopyFortranFunctionPointers((PetscObject)linesearch, (PetscObject)slinesearch);CHKERRQ(ierr);
fas->smoothd->vec_sol = snes->vec_sol;
ierr = PetscObjectReference((PetscObject)snes->vec_sol);CHKERRQ(ierr);
fas->smoothd->vec_sol_update = snes->vec_sol_update;
ierr = PetscObjectReference((PetscObject)snes->vec_sol_update);CHKERRQ(ierr);
fas->smoothd->vec_func = snes->vec_func;
ierr = PetscObjectReference((PetscObject)snes->vec_func);CHKERRQ(ierr);
if (fas->eventsmoothsetup) {ierr = PetscLogEventBegin(fas->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = SNESSetUp(fas->smoothd);CHKERRQ(ierr);
if (fas->eventsmoothsetup) {ierr = PetscLogEventEnd(fas->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
}
/* sets the up (post) smoother's default norm and sets it from options */
if (fas->smoothu) {
if (fas->level != fas->levels - 1) {
ierr = SNESSetNormSchedule(fas->smoothu, SNES_NORM_NONE);CHKERRQ(ierr);
} else {
ierr = SNESSetNormSchedule(fas->smoothu, SNES_NORM_FINAL_ONLY);CHKERRQ(ierr);
}
ierr = PetscObjectCopyFortranFunctionPointers((PetscObject)snes, (PetscObject)fas->smoothu);CHKERRQ(ierr);
ierr = SNESSetFromOptions(fas->smoothu);CHKERRQ(ierr);
ierr = SNESGetLineSearch(snes,&linesearch);CHKERRQ(ierr);
ierr = SNESGetLineSearch(fas->smoothu,&slinesearch);CHKERRQ(ierr);
ierr = SNESLineSearchGetPreCheck(linesearch,&precheck,&lsprectx);CHKERRQ(ierr);
ierr = SNESLineSearchGetPostCheck(linesearch,&postcheck,&lspostctx);CHKERRQ(ierr);
ierr = SNESLineSearchSetPreCheck(slinesearch,precheck,lsprectx);CHKERRQ(ierr);
ierr = SNESLineSearchSetPostCheck(slinesearch,postcheck,lspostctx);CHKERRQ(ierr);
ierr = PetscObjectCopyFortranFunctionPointers((PetscObject)linesearch, (PetscObject)slinesearch);CHKERRQ(ierr);
fas->smoothu->vec_sol = snes->vec_sol;
ierr = PetscObjectReference((PetscObject)snes->vec_sol);CHKERRQ(ierr);
fas->smoothu->vec_sol_update = snes->vec_sol_update;
ierr = PetscObjectReference((PetscObject)snes->vec_sol_update);CHKERRQ(ierr);
fas->smoothu->vec_func = snes->vec_func;
ierr = PetscObjectReference((PetscObject)snes->vec_func);CHKERRQ(ierr);
if (fas->eventsmoothsetup) {ierr = PetscLogEventBegin(fas->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = SNESSetUp(fas->smoothu);CHKERRQ(ierr);
if (fas->eventsmoothsetup) {ierr = PetscLogEventEnd(fas->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
}
if (next) {
/* gotta set up the solution vector for this to work */
if (!next->vec_sol) {ierr = SNESFASCreateCoarseVec(snes,&next->vec_sol);CHKERRQ(ierr);}
if (!next->vec_rhs) {ierr = SNESFASCreateCoarseVec(snes,&next->vec_rhs);CHKERRQ(ierr);}
ierr = PetscObjectCopyFortranFunctionPointers((PetscObject)snes, (PetscObject)next);CHKERRQ(ierr);
ierr = SNESGetLineSearch(snes,&linesearch);CHKERRQ(ierr);
ierr = SNESGetLineSearch(fas->next,&slinesearch);CHKERRQ(ierr);
ierr = SNESLineSearchGetPreCheck(linesearch,&precheck,&lsprectx);CHKERRQ(ierr);
ierr = SNESLineSearchGetPostCheck(linesearch,&postcheck,&lspostctx);CHKERRQ(ierr);
ierr = SNESLineSearchSetPreCheck(slinesearch,precheck,lsprectx);CHKERRQ(ierr);
ierr = SNESLineSearchSetPostCheck(slinesearch,postcheck,lspostctx);CHKERRQ(ierr);
ierr = PetscObjectCopyFortranFunctionPointers((PetscObject)linesearch, (PetscObject)slinesearch);CHKERRQ(ierr);
ierr = SNESSetUp(next);CHKERRQ(ierr);
}
/* setup FAS work vectors */
if (fas->galerkin) {
ierr = VecDuplicate(snes->vec_sol, &fas->Xg);CHKERRQ(ierr);
ierr = VecDuplicate(snes->vec_sol, &fas->Fg);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode PCSetUp_MG(PC pc)
{
PC_MG *mg = (PC_MG*)pc->data;
PC_MG_Levels **mglevels = mg->levels;
PetscErrorCode ierr;
PetscInt i,n = mglevels[0]->levels;
PC cpc;
PetscBool dump = PETSC_FALSE,opsset,use_amat,missinginterpolate = PETSC_FALSE;
Mat dA,dB;
Vec tvec;
DM *dms;
PetscViewer viewer = 0;
PetscFunctionBegin;
/* FIX: Move this to PCSetFromOptions_MG? */
if (mg->usedmfornumberoflevels) {
PetscInt levels;
ierr = DMGetRefineLevel(pc->dm,&levels);CHKERRQ(ierr);
levels++;
if (levels > n) { /* the problem is now being solved on a finer grid */
ierr = PCMGSetLevels(pc,levels,NULL);CHKERRQ(ierr);
n = levels;
ierr = PCSetFromOptions(pc);CHKERRQ(ierr); /* it is bad to call this here, but otherwise will never be called for the new hierarchy */
mglevels = mg->levels;
}
}
ierr = KSPGetPC(mglevels[0]->smoothd,&cpc);CHKERRQ(ierr);
/* If user did not provide fine grid operators OR operator was not updated since last global KSPSetOperators() */
/* so use those from global PC */
/* Is this what we always want? What if user wants to keep old one? */
ierr = KSPGetOperatorsSet(mglevels[n-1]->smoothd,NULL,&opsset);CHKERRQ(ierr);
if (opsset) {
Mat mmat;
ierr = KSPGetOperators(mglevels[n-1]->smoothd,NULL,&mmat);CHKERRQ(ierr);
if (mmat == pc->pmat) opsset = PETSC_FALSE;
}
if (!opsset) {
ierr = PCGetUseAmat(pc,&use_amat);CHKERRQ(ierr);
if(use_amat){
ierr = PetscInfo(pc,"Using outer operators to define finest grid operator \n because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[n-1]->smoothd,pc->mat,pc->pmat);CHKERRQ(ierr);
}
else {
ierr = PetscInfo(pc,"Using matrix (pmat) operators to define finest grid operator \n because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[n-1]->smoothd,pc->pmat,pc->pmat);CHKERRQ(ierr);
}
}
for (i=n-1; i>0; i--) {
if (!(mglevels[i]->interpolate || mglevels[i]->restrct)) {
missinginterpolate = PETSC_TRUE;
continue;
}
}
/*
Skipping if user has provided all interpolation/restriction needed (since DM might not be able to produce them (when coming from SNES/TS)
Skipping for galerkin==2 (externally managed hierarchy such as ML and GAMG). Cleaner logic here would be great. Wrap ML/GAMG as DMs?
*/
if (missinginterpolate && pc->dm && mg->galerkin != 2 && !pc->setupcalled) {
/* construct the interpolation from the DMs */
Mat p;
Vec rscale;
ierr = PetscMalloc1(n,&dms);CHKERRQ(ierr);
dms[n-1] = pc->dm;
/* Separately create them so we do not get DMKSP interference between levels */
for (i=n-2; i>-1; i--) {ierr = DMCoarsen(dms[i+1],MPI_COMM_NULL,&dms[i]);CHKERRQ(ierr);}
for (i=n-2; i>-1; i--) {
DMKSP kdm;
PetscBool dmhasrestrict;
ierr = KSPSetDM(mglevels[i]->smoothd,dms[i]);CHKERRQ(ierr);
if (mg->galerkin) {ierr = KSPSetDMActive(mglevels[i]->smoothd,PETSC_FALSE);CHKERRQ(ierr);}
ierr = DMGetDMKSPWrite(dms[i],&kdm);CHKERRQ(ierr);
/* Ugly hack so that the next KSPSetUp() will use the RHS that we set. A better fix is to change dmActive to take
* a bitwise OR of computing the matrix, RHS, and initial iterate. */
kdm->ops->computerhs = NULL;
kdm->rhsctx = NULL;
if (!mglevels[i+1]->interpolate) {
ierr = DMCreateInterpolation(dms[i],dms[i+1],&p,&rscale);CHKERRQ(ierr);
ierr = PCMGSetInterpolation(pc,i+1,p);CHKERRQ(ierr);
if (rscale) {ierr = PCMGSetRScale(pc,i+1,rscale);CHKERRQ(ierr);}
ierr = VecDestroy(&rscale);CHKERRQ(ierr);
ierr = MatDestroy(&p);CHKERRQ(ierr);
}
ierr = DMHasCreateRestriction(dms[i],&dmhasrestrict);CHKERRQ(ierr);
if (dmhasrestrict && !mglevels[i+1]->restrct){
ierr = DMCreateRestriction(dms[i],dms[i+1],&p);CHKERRQ(ierr);
ierr = PCMGSetRestriction(pc,i+1,p);CHKERRQ(ierr);
ierr = MatDestroy(&p);CHKERRQ(ierr);
}
}
for (i=n-2; i>-1; i--) {ierr = DMDestroy(&dms[i]);CHKERRQ(ierr);}
ierr = PetscFree(dms);CHKERRQ(ierr);
}
if (pc->dm && !pc->setupcalled) {
/* finest smoother also gets DM but it is not active, independent of whether galerkin==2 */
ierr = KSPSetDM(mglevels[n-1]->smoothd,pc->dm);CHKERRQ(ierr);
ierr = KSPSetDMActive(mglevels[n-1]->smoothd,PETSC_FALSE);CHKERRQ(ierr);
}
if (mg->galerkin == 1) {
Mat B;
/* currently only handle case where mat and pmat are the same on coarser levels */
ierr = KSPGetOperators(mglevels[n-1]->smoothd,&dA,&dB);CHKERRQ(ierr);
if (!pc->setupcalled) {
for (i=n-2; i>-1; i--) {
if (!mglevels[i+1]->restrct && !mglevels[i+1]->interpolate) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must provide interpolation or restriction for each MG level except level 0");
if (!mglevels[i+1]->interpolate) {
ierr = PCMGSetInterpolation(pc,i+1,mglevels[i+1]->restrct);CHKERRQ(ierr);
}
if (!mglevels[i+1]->restrct) {
ierr = PCMGSetRestriction(pc,i+1,mglevels[i+1]->interpolate);CHKERRQ(ierr);
}
if (mglevels[i+1]->interpolate == mglevels[i+1]->restrct) {
ierr = MatPtAP(dB,mglevels[i+1]->interpolate,MAT_INITIAL_MATRIX,1.0,&B);CHKERRQ(ierr);
} else {
ierr = MatMatMatMult(mglevels[i+1]->restrct,dB,mglevels[i+1]->interpolate,MAT_INITIAL_MATRIX,1.0,&B);CHKERRQ(ierr);
}
ierr = KSPSetOperators(mglevels[i]->smoothd,B,B);CHKERRQ(ierr);
if (i != n-2) {ierr = PetscObjectDereference((PetscObject)dB);CHKERRQ(ierr);}
dB = B;
}
if (n > 1) {ierr = PetscObjectDereference((PetscObject)dB);CHKERRQ(ierr);}
} else {
for (i=n-2; i>-1; i--) {
if (!mglevels[i+1]->restrct && !mglevels[i+1]->interpolate) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must provide interpolation or restriction for each MG level except level 0");
if (!mglevels[i+1]->interpolate) {
ierr = PCMGSetInterpolation(pc,i+1,mglevels[i+1]->restrct);CHKERRQ(ierr);
}
if (!mglevels[i+1]->restrct) {
ierr = PCMGSetRestriction(pc,i+1,mglevels[i+1]->interpolate);CHKERRQ(ierr);
}
ierr = KSPGetOperators(mglevels[i]->smoothd,NULL,&B);CHKERRQ(ierr);
if (mglevels[i+1]->interpolate == mglevels[i+1]->restrct) {
ierr = MatPtAP(dB,mglevels[i+1]->interpolate,MAT_REUSE_MATRIX,1.0,&B);CHKERRQ(ierr);
} else {
ierr = MatMatMatMult(mglevels[i+1]->restrct,dB,mglevels[i+1]->interpolate,MAT_REUSE_MATRIX,1.0,&B);CHKERRQ(ierr);
}
ierr = KSPSetOperators(mglevels[i]->smoothd,B,B);CHKERRQ(ierr);
dB = B;
}
}
} else if (!mg->galerkin && pc->dm && pc->dm->x) {
/* need to restrict Jacobian location to coarser meshes for evaluation */
for (i=n-2; i>-1; i--) {
Mat R;
Vec rscale;
if (!mglevels[i]->smoothd->dm->x) {
Vec *vecs;
ierr = KSPCreateVecs(mglevels[i]->smoothd,1,&vecs,0,NULL);CHKERRQ(ierr);
mglevels[i]->smoothd->dm->x = vecs[0];
ierr = PetscFree(vecs);CHKERRQ(ierr);
}
ierr = PCMGGetRestriction(pc,i+1,&R);CHKERRQ(ierr);
ierr = PCMGGetRScale(pc,i+1,&rscale);CHKERRQ(ierr);
ierr = MatRestrict(R,mglevels[i+1]->smoothd->dm->x,mglevels[i]->smoothd->dm->x);CHKERRQ(ierr);
ierr = VecPointwiseMult(mglevels[i]->smoothd->dm->x,mglevels[i]->smoothd->dm->x,rscale);CHKERRQ(ierr);
}
}
if (!mg->galerkin && pc->dm) {
for (i=n-2; i>=0; i--) {
DM dmfine,dmcoarse;
Mat Restrict,Inject;
Vec rscale;
ierr = KSPGetDM(mglevels[i+1]->smoothd,&dmfine);CHKERRQ(ierr);
ierr = KSPGetDM(mglevels[i]->smoothd,&dmcoarse);CHKERRQ(ierr);
ierr = PCMGGetRestriction(pc,i+1,&Restrict);CHKERRQ(ierr);
ierr = PCMGGetRScale(pc,i+1,&rscale);CHKERRQ(ierr);
Inject = NULL; /* Callback should create it if it needs Injection */
ierr = DMRestrict(dmfine,Restrict,rscale,Inject,dmcoarse);CHKERRQ(ierr);
}
}
if (!pc->setupcalled) {
for (i=0; i<n; i++) {
ierr = KSPSetFromOptions(mglevels[i]->smoothd);CHKERRQ(ierr);
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu && (mglevels[i]->smoothu != mglevels[i]->smoothd)) {
ierr = KSPSetFromOptions(mglevels[i]->smoothu);CHKERRQ(ierr);
}
}
/* insure that if either interpolation or restriction is set the other other one is set */
for (i=1; i<n; i++) {
ierr = PCMGGetInterpolation(pc,i,NULL);CHKERRQ(ierr);
ierr = PCMGGetRestriction(pc,i,NULL);CHKERRQ(ierr);
}
for (i=0; i<n-1; i++) {
if (!mglevels[i]->b) {
Vec *vec;
ierr = KSPCreateVecs(mglevels[i]->smoothd,1,&vec,0,NULL);CHKERRQ(ierr);
ierr = PCMGSetRhs(pc,i,*vec);CHKERRQ(ierr);
ierr = VecDestroy(vec);CHKERRQ(ierr);
ierr = PetscFree(vec);CHKERRQ(ierr);
}
if (!mglevels[i]->r && i) {
ierr = VecDuplicate(mglevels[i]->b,&tvec);CHKERRQ(ierr);
ierr = PCMGSetR(pc,i,tvec);CHKERRQ(ierr);
ierr = VecDestroy(&tvec);CHKERRQ(ierr);
}
if (!mglevels[i]->x) {
ierr = VecDuplicate(mglevels[i]->b,&tvec);CHKERRQ(ierr);
ierr = PCMGSetX(pc,i,tvec);CHKERRQ(ierr);
ierr = VecDestroy(&tvec);CHKERRQ(ierr);
}
}
if (n != 1 && !mglevels[n-1]->r) {
/* PCMGSetR() on the finest level if user did not supply it */
Vec *vec;
ierr = KSPCreateVecs(mglevels[n-1]->smoothd,1,&vec,0,NULL);CHKERRQ(ierr);
ierr = PCMGSetR(pc,n-1,*vec);CHKERRQ(ierr);
ierr = VecDestroy(vec);CHKERRQ(ierr);
ierr = PetscFree(vec);CHKERRQ(ierr);
}
}
if (pc->dm) {
/* need to tell all the coarser levels to rebuild the matrix using the DM for that level */
for (i=0; i<n-1; i++) {
if (mglevels[i]->smoothd->setupstage != KSP_SETUP_NEW) mglevels[i]->smoothd->setupstage = KSP_SETUP_NEWMATRIX;
}
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu == mglevels[i]->smoothd || mg->am == PC_MG_FULL || mg->am == PC_MG_KASKADE || mg->cyclesperpcapply > 1){
/* if doing only down then initial guess is zero */
ierr = KSPSetInitialGuessNonzero(mglevels[i]->smoothd,PETSC_TRUE);CHKERRQ(ierr);
}
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[i]->smoothd);CHKERRQ(ierr);
if (mglevels[i]->smoothd->reason == KSP_DIVERGED_PCSETUP_FAILED) {
pc->failedreason = PC_SUBPC_ERROR;
}
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
if (!mglevels[i]->residual) {
Mat mat;
ierr = KSPGetOperators(mglevels[i]->smoothd,NULL,&mat);CHKERRQ(ierr);
ierr = PCMGSetResidual(pc,i,PCMGResidualDefault,mat);CHKERRQ(ierr);
}
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu && mglevels[i]->smoothu != mglevels[i]->smoothd) {
Mat downmat,downpmat;
/* check if operators have been set for up, if not use down operators to set them */
ierr = KSPGetOperatorsSet(mglevels[i]->smoothu,&opsset,NULL);CHKERRQ(ierr);
if (!opsset) {
ierr = KSPGetOperators(mglevels[i]->smoothd,&downmat,&downpmat);CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[i]->smoothu,downmat,downpmat);CHKERRQ(ierr);
}
ierr = KSPSetInitialGuessNonzero(mglevels[i]->smoothu,PETSC_TRUE);CHKERRQ(ierr);
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[i]->smoothu);CHKERRQ(ierr);
if (mglevels[i]->smoothu->reason == KSP_DIVERGED_PCSETUP_FAILED) {
pc->failedreason = PC_SUBPC_ERROR;
}
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
}
}
if (mglevels[0]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[0]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[0]->smoothd);CHKERRQ(ierr);
if (mglevels[0]->smoothd->reason == KSP_DIVERGED_PCSETUP_FAILED) {
pc->failedreason = PC_SUBPC_ERROR;
}
if (mglevels[0]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[0]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
/*
Dump the interpolation/restriction matrices plus the
Jacobian/stiffness on each level. This allows MATLAB users to
easily check if the Galerkin condition A_c = R A_f R^T is satisfied.
Only support one or the other at the same time.
*/
#if defined(PETSC_USE_SOCKET_VIEWER)
ierr = PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_mg_dump_matlab",&dump,NULL);CHKERRQ(ierr);
if (dump) viewer = PETSC_VIEWER_SOCKET_(PetscObjectComm((PetscObject)pc));
dump = PETSC_FALSE;
#endif
ierr = PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_mg_dump_binary",&dump,NULL);CHKERRQ(ierr);
if (dump) viewer = PETSC_VIEWER_BINARY_(PetscObjectComm((PetscObject)pc));
if (viewer) {
for (i=1; i<n; i++) {
ierr = MatView(mglevels[i]->restrct,viewer);CHKERRQ(ierr);
}
for (i=0; i<n; i++) {
ierr = KSPGetPC(mglevels[i]->smoothd,&pc);CHKERRQ(ierr);
ierr = MatView(pc->mat,viewer);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscInt M1 = 3,M2,dof = 1,s = 1,ratio = 2,dim = 1;
PetscErrorCode ierr;
DM da_c,da_f;
Vec v_c,v_f;
Mat Interp;
PetscScalar one = 1.0;
PetscBool pt;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dim",&dim,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-M",&M1,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-stencil_width",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-ratio",&ratio,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-periodic",(PetscBool*)&pt,NULL);CHKERRQ(ierr);
if (pt) {
if (dim > 0) bx = DMDA_BOUNDARY_PERIODIC;
if (dim > 1) by = DMDA_BOUNDARY_PERIODIC;
if (dim > 2) bz = DMDA_BOUNDARY_PERIODIC;
}
if (bx == DMDA_BOUNDARY_NONE) {
M2 = ratio*(M1-1) + 1;
} else {
M2 = ratio*M1;
}
/* Set up the array */
if (dim == 1) {
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M1,dof,s,NULL,&da_c);CHKERRQ(ierr);
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M2,dof,s,NULL,&da_f);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,DMDA_STENCIL_BOX,M1,M1,PETSC_DECIDE,PETSC_DECIDE,dof,s,NULL,NULL,&da_c);CHKERRQ(ierr);
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,DMDA_STENCIL_BOX,M2,M2,PETSC_DECIDE,PETSC_DECIDE,dof,s,NULL,NULL,&da_f);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,DMDA_STENCIL_BOX,M1,M1,M1,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,s,NULL,NULL,NULL,&da_c);CHKERRQ(ierr);
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,DMDA_STENCIL_BOX,M2,M2,M2,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,s,NULL,NULL,NULL,&da_f);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"dim must be 1,2, or 3");
ierr = DMCreateGlobalVector(da_c,&v_c);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da_f,&v_f);CHKERRQ(ierr);
ierr = VecSet(v_c,one);CHKERRQ(ierr);
ierr = DMCreateInterpolation(da_c,da_f,&Interp,NULL);CHKERRQ(ierr);
ierr = MatMult(Interp,v_c,v_f);CHKERRQ(ierr);
ierr = VecView(v_f,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatMultTranspose(Interp,v_f,v_c);CHKERRQ(ierr);
ierr = VecView(v_c,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&Interp);CHKERRQ(ierr);
ierr = VecDestroy(&v_c);CHKERRQ(ierr);
ierr = DMDestroy(&da_c);CHKERRQ(ierr);
ierr = VecDestroy(&v_f);CHKERRQ(ierr);
ierr = DMDestroy(&da_f);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}