PetscErrorCode DMDACreate2d(MPI_Comm comm,DMDABoundaryType bx,DMDABoundaryType by,DMDAStencilType stencil_type,
PetscInt M,PetscInt N,PetscInt m,PetscInt n,PetscInt dof,PetscInt s,const PetscInt lx[],const PetscInt ly[],DM *da)
{
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMDACreate(comm, da);
CHKERRQ(ierr);
ierr = DMDASetDim(*da, 2);
CHKERRQ(ierr);
ierr = DMDASetSizes(*da, M, N, 1);
CHKERRQ(ierr);
ierr = DMDASetNumProcs(*da, m, n, PETSC_DECIDE);
CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*da, bx, by, DMDA_BOUNDARY_NONE);
CHKERRQ(ierr);
ierr = DMDASetDof(*da, dof);
CHKERRQ(ierr);
ierr = DMDASetStencilType(*da, stencil_type);
CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*da, s);
CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*da, lx, ly, NULL);
CHKERRQ(ierr);
/* This violates the behavior for other classes, but right now users expect negative dimensions to be handled this way */
ierr = DMSetFromOptions(*da);
CHKERRQ(ierr);
ierr = DMSetUp(*da);
CHKERRQ(ierr);
ierr = DMViewFromOptions(*da,"-dm_view");
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMCreateSubDM_DA(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
{
DM_DA *da = (DM_DA*) dm->data;
PetscSection section;
PetscErrorCode ierr;
PetscFunctionBegin;
if (subdm) {
PetscSF sf;
Vec coords;
void *ctx;
/* Cannot use DMClone since the dof stuff is mixed in. Ugh
ierr = DMClone(dm, subdm);CHKERRQ(ierr); */
ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = DMSetPointSF(*subdm, sf);CHKERRQ(ierr);
ierr = DMGetApplicationContext(dm, &ctx);CHKERRQ(ierr);
ierr = DMSetApplicationContext(*subdm, ctx);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coords);CHKERRQ(ierr);
if (coords) {
ierr = DMSetCoordinatesLocal(*subdm, coords);CHKERRQ(ierr);
} else {
ierr = DMGetCoordinates(dm, &coords);CHKERRQ(ierr);
if (coords) {ierr = DMSetCoordinates(*subdm, coords);CHKERRQ(ierr);}
}
ierr = DMSetType(*subdm, DMDA);CHKERRQ(ierr);
ierr = DMSetDimension(*subdm, dm->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(*subdm, da->M, da->N, da->P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*subdm, da->m, da->n, da->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*subdm, da->bx, da->by, da->bz);CHKERRQ(ierr);
ierr = DMDASetDof(*subdm, numFields);CHKERRQ(ierr);
ierr = DMDASetStencilType(*subdm, da->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*subdm, da->s);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*subdm, da->lx, da->ly, da->lz);CHKERRQ(ierr);
}
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
if (section) {
ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
} else {
if (is) {
PetscInt *indices, cnt = 0, dof = da->w, i, j;
ierr = PetscMalloc1(da->Nlocal*numFields/dof, &indices);CHKERRQ(ierr);
for (i = da->base/dof; i < (da->base+da->Nlocal)/dof; ++i) {
for (j = 0; j < numFields; ++j) {
indices[cnt++] = dof*i + fields[j];
}
}
if (cnt != da->Nlocal*numFields/dof) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Count %d does not equal expected value %d", cnt, da->Nlocal*numFields/dof);
ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), cnt, indices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
PetscErrorCode DMClone_DA(DM dm, DM *newdm)
{
DM_DA *da = (DM_DA *) dm->data;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMSetType(*newdm, DMDA);CHKERRQ(ierr);
ierr = DMSetDimension(*newdm, dm->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(*newdm, da->M, da->N, da->P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*newdm, da->m, da->n, da->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*newdm, da->bx, da->by, da->bz);CHKERRQ(ierr);
ierr = DMDASetDof(*newdm, da->w);CHKERRQ(ierr);
ierr = DMDASetStencilType(*newdm, da->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*newdm, da->s);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*newdm, da->lx, da->ly, da->lz);CHKERRQ(ierr);
ierr = DMSetUp(*newdm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMDACreate1d - Creates an object that will manage the communication of one-dimensional
regular array data that is distributed across some processors.
Collective on MPI_Comm
Input Parameters:
+ comm - MPI communicator
. bx - type of ghost cells at the boundary the array should have, if any. Use
DM_BOUNDARY_NONE, DM_BOUNDARY_GHOSTED, or DM_BOUNDARY_PERIODIC.
. M - global dimension of the array (use -M to indicate that it may be set to a different value
from the command line with -da_grid_x <M>)
. dof - number of degrees of freedom per node
. s - stencil width
- lx - array containing number of nodes in the X direction on each processor,
or NULL. If non-null, must be of length as the number of processes in the MPI_Comm.
Output Parameter:
. da - the resulting distributed array object
Options Database Key:
+ -dm_view - Calls DMView() at the conclusion of DMDACreate1d()
. -da_grid_x <nx> - number of grid points in x direction; can set if M < 0
. -da_refine_x <rx> - refinement factor
- -da_refine <n> - refine the DMDA n times before creating it, if M < 0
Level: beginner
Notes:
The array data itself is NOT stored in the DMDA, it is stored in Vec objects;
The appropriate vector objects can be obtained with calls to DMCreateGlobalVector()
and DMCreateLocalVector() and calls to VecDuplicate() if more are needed.
.keywords: distributed array, create, one-dimensional
.seealso: DMDestroy(), DMView(), DMDACreate2d(), DMDACreate3d(), DMGlobalToLocalBegin(), DMDASetRefinementFactor(),
DMGlobalToLocalEnd(), DMLocalToGlobalBegin(), DMLocalToLocalBegin(), DMLocalToLocalEnd(), DMDAGetRefinementFactor(),
DMDAGetInfo(), DMCreateGlobalVector(), DMCreateLocalVector(), DMDACreateNaturalVector(), DMLoad(), DMDAGetOwnershipRanges()
@*/
PetscErrorCode DMDACreate1d(MPI_Comm comm, DMBoundaryType bx, PetscInt M, PetscInt dof, PetscInt s, const PetscInt lx[], DM *da)
{
PetscErrorCode ierr;
PetscMPIInt size;
PetscFunctionBegin;
ierr = DMDACreate(comm, da);CHKERRQ(ierr);
ierr = DMSetDimension(*da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(*da, M, 1, 1);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*da, size, PETSC_DECIDE, PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*da, bx, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(*da, dof);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*da, s);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*da, lx, NULL, NULL);CHKERRQ(ierr);
/* This violates the behavior for other classes, but right now users expect negative dimensions to be handled this way */
ierr = DMSetFromOptions(*da);CHKERRQ(ierr);
ierr = DMSetUp(*da);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
KSP ksp;
PC pc;
Vec x,b;
DM da;
Mat A;
PetscInt dof=1;
PetscBool flg;
PetscScalar zero=0.0;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD,&da);CHKERRQ(ierr);
ierr = DMDASetDim(da,3);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetStencilType(da,DMDA_STENCIL_STAR);CHKERRQ(ierr);
ierr = DMDASetSizes(da,3,3,3);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetDof(da,dof);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da,1);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&b);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,MATAIJ,&A);CHKERRQ(ierr);
ierr = VecSet(b,zero);CHKERRQ(ierr);
/* Test sbaij matrix */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-test_sbaij",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
Mat sA;
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
A = sA;
}
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetDM(pc,(DM)da);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* check final residual */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL, "-check_final_residual", &flg,PETSC_NULL);CHKERRQ(ierr);
if (flg){
Vec b1;
PetscReal norm;
ierr = KSPGetSolution(ksp,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&b1);CHKERRQ(ierr);
ierr = MatMult(A,x,b1);CHKERRQ(ierr);
ierr = VecAXPY(b1,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(b1,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Final residual %g\n",norm);CHKERRQ(ierr);
ierr = VecDestroy(&b1);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode DMCoarsen_DA(DM da, MPI_Comm comm,DM *daref)
{
PetscErrorCode ierr;
PetscInt M,N,P,i;
DM da2;
DM_DA *dd = (DM_DA*)da->data,*dd2;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidPointer(daref,3);
if (dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
M = dd->M / dd->coarsen_x;
} else {
M = 1 + (dd->M - 1) / dd->coarsen_x;
}
if (dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
if (dd->dim > 1) {
N = dd->N / dd->coarsen_y;
} else {
N = 1;
}
} else {
N = 1 + (dd->N - 1) / dd->coarsen_y;
}
if (dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
if (dd->dim > 2) {
P = dd->P / dd->coarsen_z;
} else {
P = 1;
}
} else {
P = 1 + (dd->P - 1) / dd->coarsen_z;
}
ierr = DMDACreate(((PetscObject)da)->comm,&da2);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da2,((PetscObject)da)->prefix);CHKERRQ(ierr);
ierr = DMDASetDim(da2,dd->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da2,M,N,P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da2,dd->m,dd->n,dd->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da2,dd->bx,dd->by,dd->bz);CHKERRQ(ierr);
ierr = DMDASetDof(da2,dd->w);CHKERRQ(ierr);
ierr = DMDASetStencilType(da2,dd->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da2,dd->s);CHKERRQ(ierr);
if (dd->dim == 3) {
PetscInt *lx,*ly,*lz;
ierr = PetscMalloc3(dd->m,PetscInt,&lx,dd->n,PetscInt,&ly,dd->p,PetscInt,&lz);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_z,dd->p,dd->lz,lz);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,lz);CHKERRQ(ierr);
ierr = PetscFree3(lx,ly,lz);CHKERRQ(ierr);
} else if (dd->dim == 2) {
PetscInt *lx,*ly;
ierr = PetscMalloc2(dd->m,PetscInt,&lx,dd->n,PetscInt,&ly);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree2(lx,ly);CHKERRQ(ierr);
} else if (dd->dim == 1) {
PetscInt *lx;
ierr = PetscMalloc(dd->m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
}
dd2 = (DM_DA*)da2->data;
/* allow overloaded (user replaced) operations to be inherited by refinement clones; why are only some inherited and not all? */
/* da2->ops->createinterpolation = da->ops->createinterpolation; copying this one causes trouble for DMSetVI */
da2->ops->creatematrix = da->ops->creatematrix;
da2->ops->getcoloring = da->ops->getcoloring;
dd2->interptype = dd->interptype;
/* copy fill information if given */
if (dd->dfill) {
ierr = PetscMalloc((dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt),&dd2->dfill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->dfill,dd->dfill,(dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
if (dd->ofill) {
ierr = PetscMalloc((dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt),&dd2->ofill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->ofill,dd->ofill,(dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
/* copy the refine information */
dd2->coarsen_x = dd2->refine_x = dd->coarsen_x;
dd2->coarsen_y = dd2->refine_y = dd->coarsen_y;
dd2->coarsen_z = dd2->refine_z = dd->coarsen_z;
/* copy vector type information */
ierr = PetscFree(da2->vectype);CHKERRQ(ierr);
ierr = PetscStrallocpy(da->vectype,(char**)&da2->vectype);CHKERRQ(ierr);
dd2->lf = dd->lf;
dd2->lj = dd->lj;
da2->leveldown = da->leveldown + 1;
da2->levelup = da->levelup;
ierr = DMSetFromOptions(da2);CHKERRQ(ierr);
ierr = DMSetUp(da2);CHKERRQ(ierr);
ierr = DMViewFromOptions(da2,"-dm_view");CHKERRQ(ierr);
/* inject coordinates if they are set on the fine grid */
if (da->coordinates) {
DM cdaf,cdac;
Vec coordsc,coordsf;
VecScatter inject;
ierr = DMGetCoordinateDM(da,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(da,&coordsf);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da2,&cdac);CHKERRQ(ierr);
/* force creation of the coordinate vector */
ierr = DMDASetUniformCoordinates(da2,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMGetCoordinates(da2,&coordsc);CHKERRQ(ierr);
ierr = DMCreateInjection(cdac,cdaf,&inject);CHKERRQ(ierr);
ierr = VecScatterBegin(inject,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(inject ,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&inject);CHKERRQ(ierr);
}
for (i=0; i<da->bs; i++) {
const char *fieldname;
ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr);
ierr = DMDASetFieldName(da2,i,fieldname);CHKERRQ(ierr);
}
*daref = da2;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
KSP ksp;
PC pc;
Vec x,b;
DM da;
Mat A,Atrans;
PetscInt dof=1,M=8;
PetscBool flg,trans=PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-trans",&trans,NULL);CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD,&da);CHKERRQ(ierr);
ierr = DMSetDimension(da,3);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetStencilType(da,DMDA_STENCIL_STAR);CHKERRQ(ierr);
ierr = DMDASetSizes(da,M,M,M);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetDof(da,dof);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da,1);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&b);CHKERRQ(ierr);
ierr = ComputeRHS(da,b);CHKERRQ(ierr);
ierr = DMSetMatType(da,MATBAIJ);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,&A);CHKERRQ(ierr);
ierr = ComputeMatrix(da,A);CHKERRQ(ierr);
/* A is non-symmetric. Make A = 0.5*(A + Atrans) symmetric for testing icc and cholesky */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX,&Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,1.0,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatScale(A,0.5);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* Test sbaij matrix */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL, "-test_sbaij1", &flg,NULL);CHKERRQ(ierr);
if (flg) {
Mat sA;
PetscBool issymm;
ierr = MatIsTranspose(A,A,0.0,&issymm);CHKERRQ(ierr);
if (issymm) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else {ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: A is non-symmetric\n");CHKERRQ(ierr);}
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
A = sA;
}
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetDM(pc,(DM)da);CHKERRQ(ierr);
if (trans) {
ierr = KSPSolveTranspose(ksp,b,x);CHKERRQ(ierr);
} else {
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
}
/* check final residual */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL, "-check_final_residual", &flg,NULL);CHKERRQ(ierr);
if (flg) {
Vec b1;
PetscReal norm;
ierr = KSPGetSolution(ksp,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&b1);CHKERRQ(ierr);
ierr = MatMult(A,x,b1);CHKERRQ(ierr);
ierr = VecAXPY(b1,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(b1,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Final residual %g\n",norm);CHKERRQ(ierr);
ierr = VecDestroy(&b1);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*
DMPatchZoom - Create a version of the coarse patch (identified by rank) with halo on communicator commz
Collective on DM
Input Parameters:
+ dm - the DM
. rank - the rank which holds the given patch
- commz - the new communicator for the patch
Output Parameters:
+ dmz - the patch DM
. sfz - the PetscSF mapping the patch+halo to the zoomed version
. sfzr - the PetscSF mapping the patch to the restricted zoomed version
Level: intermediate
Note: All processes in commz should have the same rank (could autosplit comm)
.seealso: DMPatchSolve()
*/
PetscErrorCode DMPatchZoom(DM dm, Vec X, MatStencil lower, MatStencil upper, MPI_Comm commz, DM *dmz, PetscSF *sfz, PetscSF *sfzr)
{
DMDAStencilType st;
MatStencil blower, bupper, loclower, locupper;
IS is;
const PetscInt *ranges, *indices;
PetscInt *localPoints = NULL;
PetscSFNode *remotePoints = NULL;
PetscInt dim, dof;
PetscInt M, N, P, rM, rN, rP, halo = 1, sxb, syb, szb, sxr, syr, szr, exr, eyr, ezr, mxb, myb, mzb, i, j, k, q;
PetscMPIInt size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
/* Create patch DM */
ierr = DMDAGetInfo(dm, &dim, &M, &N, &P, 0,0,0, &dof, 0,0,0,0, &st);CHKERRQ(ierr);
/* Get piece for rank r, expanded by halo */
bupper.i = PetscMin(M, upper.i + halo); blower.i = PetscMax(lower.i - halo, 0);
bupper.j = PetscMin(N, upper.j + halo); blower.j = PetscMax(lower.j - halo, 0);
bupper.k = PetscMin(P, upper.k + halo); blower.k = PetscMax(lower.k - halo, 0);
rM = bupper.i - blower.i;
rN = bupper.j - blower.j;
rP = bupper.k - blower.k;
if (commz != MPI_COMM_NULL) {
ierr = DMDACreate(commz, dmz);CHKERRQ(ierr);
ierr = DMSetDimension(*dmz, dim);CHKERRQ(ierr);
ierr = DMDASetSizes(*dmz, rM, rN, rP);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*dmz, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*dmz, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(*dmz, dof);CHKERRQ(ierr);
ierr = DMDASetStencilType(*dmz, st);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*dmz, 0);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*dmz, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = DMSetFromOptions(*dmz);CHKERRQ(ierr);
ierr = DMSetUp(*dmz);CHKERRQ(ierr);
ierr = DMDAGetCorners(*dmz, &sxb, &syb, &szb, &mxb, &myb, &mzb);CHKERRQ(ierr);
sxr = PetscMax(sxb, lower.i - blower.i);
syr = PetscMax(syb, lower.j - blower.j);
szr = PetscMax(szb, lower.k - blower.k);
exr = PetscMin(sxb+mxb, upper.i - blower.i);
eyr = PetscMin(syb+myb, upper.j - blower.j);
ezr = PetscMin(szb+mzb, upper.k - blower.k);
ierr = PetscMalloc2(rM*rN*rP,&localPoints,rM*rN*rP,&remotePoints);CHKERRQ(ierr);
} else {
sxr = syr = szr = exr = eyr = ezr = sxb = syb = szb = mxb = myb = mzb = 0;
}
/* Create SF for restricted map */
ierr = VecGetOwnershipRanges(X,&ranges);CHKERRQ(ierr);
loclower.i = blower.i + sxr; locupper.i = blower.i + exr;
loclower.j = blower.j + syr; locupper.j = blower.j + eyr;
loclower.k = blower.k + szr; locupper.k = blower.k + ezr;
ierr = DMDACreatePatchIS(dm, &loclower, &locupper, &is);CHKERRQ(ierr);
ierr = ISGetIndices(is, &indices);CHKERRQ(ierr);
q = 0;
for (k = szb; k < szb+mzb; ++k) {
if ((k < szr) || (k >= ezr)) continue;
for (j = syb; j < syb+myb; ++j) {
if ((j < syr) || (j >= eyr)) continue;
for (i = sxb; i < sxb+mxb; ++i) {
const PetscInt lp = ((k-szb)*rN + (j-syb))*rM + i-sxb;
PetscInt r;
if ((i < sxr) || (i >= exr)) continue;
localPoints[q] = lp;
ierr = PetscFindInt(indices[q], size+1, ranges, &r);CHKERRQ(ierr);
remotePoints[q].rank = r < 0 ? -(r+1) - 1 : r;
remotePoints[q].index = indices[q] - ranges[remotePoints[q].rank];
++q;
}
}
}
ierr = ISRestoreIndices(is, &indices);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfzr);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *sfzr, "Restricted Map");CHKERRQ(ierr);
ierr = PetscSFSetGraph(*sfzr, M*N*P, q, localPoints, PETSC_COPY_VALUES, remotePoints, PETSC_COPY_VALUES);CHKERRQ(ierr);
/* Create SF for buffered map */
loclower.i = blower.i + sxb; locupper.i = blower.i + sxb+mxb;
loclower.j = blower.j + syb; locupper.j = blower.j + syb+myb;
loclower.k = blower.k + szb; locupper.k = blower.k + szb+mzb;
ierr = DMDACreatePatchIS(dm, &loclower, &locupper, &is);CHKERRQ(ierr);
ierr = ISGetIndices(is, &indices);CHKERRQ(ierr);
q = 0;
for (k = szb; k < szb+mzb; ++k) {
for (j = syb; j < syb+myb; ++j) {
for (i = sxb; i < sxb+mxb; ++i, ++q) {
PetscInt r;
localPoints[q] = q;
ierr = PetscFindInt(indices[q], size+1, ranges, &r);CHKERRQ(ierr);
remotePoints[q].rank = r < 0 ? -(r+1) - 1 : r;
remotePoints[q].index = indices[q] - ranges[remotePoints[q].rank];
}
}
}
ierr = ISRestoreIndices(is, &indices);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfz);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *sfz, "Buffered Map");CHKERRQ(ierr);
ierr = PetscSFSetGraph(*sfz, M*N*P, q, localPoints, PETSC_COPY_VALUES, remotePoints, PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscFree2(localPoints, remotePoints);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN void PETSC_STDCALL dmdasetownershipranges_(DM da, PetscInt lx[], PetscInt ly[], PetscInt lz[], int *__ierr ){
*__ierr = DMDASetOwnershipRanges(
(DM)PetscToPointer((da) ),lx,ly,lz);
}
PetscErrorCode DMRefine_DA(DM da,MPI_Comm comm,DM *daref)
{
PetscErrorCode ierr;
PetscInt M,N,P,i;
DM da2;
DM_DA *dd = (DM_DA*)da->data,*dd2;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidPointer(daref,3);
if (dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0) {
M = dd->refine_x*dd->M;
} else {
M = 1 + dd->refine_x*(dd->M - 1);
}
if (dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0) {
if (dd->dim > 1) {
N = dd->refine_y*dd->N;
} else {
N = 1;
}
} else {
N = 1 + dd->refine_y*(dd->N - 1);
}
if (dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0) {
if (dd->dim > 2) {
P = dd->refine_z*dd->P;
} else {
P = 1;
}
} else {
P = 1 + dd->refine_z*(dd->P - 1);
}
ierr = DMDACreate(PetscObjectComm((PetscObject)da),&da2);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da2,((PetscObject)da)->prefix);CHKERRQ(ierr);
ierr = DMDASetDim(da2,dd->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da2,M,N,P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da2,dd->m,dd->n,dd->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da2,dd->bx,dd->by,dd->bz);CHKERRQ(ierr);
ierr = DMDASetDof(da2,dd->w);CHKERRQ(ierr);
ierr = DMDASetStencilType(da2,dd->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da2,dd->s);CHKERRQ(ierr);
if (dd->dim == 3) {
PetscInt *lx,*ly,*lz;
ierr = PetscMalloc3(dd->m,&lx,dd->n,&ly,dd->p,&lz);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_z,dd->p,dd->lz,lz);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,lz);CHKERRQ(ierr);
ierr = PetscFree3(lx,ly,lz);CHKERRQ(ierr);
} else if (dd->dim == 2) {
PetscInt *lx,*ly;
ierr = PetscMalloc2(dd->m,&lx,dd->n,&ly);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,NULL);CHKERRQ(ierr);
ierr = PetscFree2(lx,ly);CHKERRQ(ierr);
} else if (dd->dim == 1) {
PetscInt *lx;
ierr = PetscMalloc1(dd->m,&lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,NULL,NULL);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
}
dd2 = (DM_DA*)da2->data;
/* allow overloaded (user replaced) operations to be inherited by refinement clones */
da2->ops->creatematrix = da->ops->creatematrix;
/* da2->ops->createinterpolation = da->ops->createinterpolation; this causes problem with SNESVI */
da2->ops->getcoloring = da->ops->getcoloring;
dd2->interptype = dd->interptype;
/* copy fill information if given */
if (dd->dfill) {
ierr = PetscMalloc1((dd->dfill[dd->w]+dd->w+1),&dd2->dfill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->dfill,dd->dfill,(dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
if (dd->ofill) {
ierr = PetscMalloc1((dd->ofill[dd->w]+dd->w+1),&dd2->ofill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->ofill,dd->ofill,(dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
/* copy the refine information */
dd2->coarsen_x = dd2->refine_x = dd->refine_x;
dd2->coarsen_y = dd2->refine_y = dd->refine_y;
dd2->coarsen_z = dd2->refine_z = dd->refine_z;
/* copy vector type information */
ierr = DMSetVecType(da2,da->vectype);CHKERRQ(ierr);
dd2->lf = dd->lf;
dd2->lj = dd->lj;
da2->leveldown = da->leveldown;
da2->levelup = da->levelup + 1;
ierr = DMSetFromOptions(da2);CHKERRQ(ierr);
ierr = DMSetUp(da2);CHKERRQ(ierr);
/* interpolate coordinates if they are set on the coarse grid */
if (da->coordinates) {
DM cdaf,cdac;
Vec coordsc,coordsf;
Mat II;
ierr = DMGetCoordinateDM(da,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinates(da,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da2,&cdaf);CHKERRQ(ierr);
/* force creation of the coordinate vector */
ierr = DMDASetUniformCoordinates(da2,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMGetCoordinates(da2,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&II,NULL);CHKERRQ(ierr);
ierr = MatInterpolate(II,coordsc,coordsf);CHKERRQ(ierr);
ierr = MatDestroy(&II);CHKERRQ(ierr);
}
for (i=0; i<da->bs; i++) {
const char *fieldname;
ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr);
ierr = DMDASetFieldName(da2,i,fieldname);CHKERRQ(ierr);
}
*daref = da2;
PetscFunctionReturn(0);
}
