PetscErrorCode DMDACreate2d(MPI_Comm comm,DMDABoundaryType bx,DMDABoundaryType by,DMDAStencilType stencil_type,
PetscInt M,PetscInt N,PetscInt m,PetscInt n,PetscInt dof,PetscInt s,const PetscInt lx[],const PetscInt ly[],DM *da)
{
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMDACreate(comm, da);
CHKERRQ(ierr);
ierr = DMDASetDim(*da, 2);
CHKERRQ(ierr);
ierr = DMDASetSizes(*da, M, N, 1);
CHKERRQ(ierr);
ierr = DMDASetNumProcs(*da, m, n, PETSC_DECIDE);
CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*da, bx, by, DMDA_BOUNDARY_NONE);
CHKERRQ(ierr);
ierr = DMDASetDof(*da, dof);
CHKERRQ(ierr);
ierr = DMDASetStencilType(*da, stencil_type);
CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*da, s);
CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*da, lx, ly, NULL);
CHKERRQ(ierr);
/* This violates the behavior for other classes, but right now users expect negative dimensions to be handled this way */
ierr = DMSetFromOptions(*da);
CHKERRQ(ierr);
ierr = DMSetUp(*da);
CHKERRQ(ierr);
ierr = DMViewFromOptions(*da,"-dm_view");
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMCreateFieldDecomposition_DA(DM dm, PetscInt *len,char ***namelist, IS **islist, DM** dmlist)
{
PetscInt i;
PetscErrorCode ierr;
DM_DA *dd = (DM_DA*)dm->data;
PetscInt dof = dd->w;
PetscFunctionBegin;
if(len) *len = dof;
if (islist) {
Vec v;
PetscInt rstart,n;
ierr = DMGetGlobalVector(dm,&v);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(v,&rstart,PETSC_NULL);CHKERRQ(ierr);
ierr = VecGetLocalSize(v,&n);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dm,&v);CHKERRQ(ierr);
ierr = PetscMalloc(dof*sizeof(IS),islist);CHKERRQ(ierr);
for (i=0; i<dof; i++) {
ierr = ISCreateStride(((PetscObject)dm)->comm,n/dof,rstart+i,dof,&(*islist)[i]);CHKERRQ(ierr);
}
}
if (namelist) {
ierr = PetscMalloc(dof*sizeof(const char *), namelist);CHKERRQ(ierr);
if (dd->fieldname) {
for (i=0; i<dof; i++) {
ierr = PetscStrallocpy(dd->fieldname[i],&(*namelist)[i]);CHKERRQ(ierr);
}
} else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Currently DMDA must have fieldnames");
}
if (dmlist) {
DM da;
ierr = DMDACreate(((PetscObject)dm)->comm, &da);CHKERRQ(ierr);
ierr = DMDASetDim(da, dd->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da, dd->M, dd->N, dd->P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da, dd->m, dd->n, dd->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, dd->bx, dd->by, dd->bz);CHKERRQ(ierr);
ierr = DMDASetDof(da, 1);CHKERRQ(ierr);
ierr = DMDASetStencilType(da, dd->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, dd->s);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = PetscMalloc(dof*sizeof(DM),dmlist);CHKERRQ(ierr);
for (i=0; i<dof-1; i++) {ierr = PetscObjectReference((PetscObject)da);CHKERRQ(ierr);}
for (i=0; i<dof; i++) (*dmlist)[i] = da;
}
PetscFunctionReturn(0);
}
static PetscErrorCode CreatePoints_Grid(DM dm, PetscInt *Np, PetscReal **pcoords, PetscBool *pointsAllProcs, AppCtx *ctx)
{
DM da;
DMDALocalInfo info;
PetscInt N = 3, n = 0, spaceDim, i, j, k, *ind, d;
PetscReal *h;
PetscMPIInt rank;
PetscErrorCode ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = DMGetCoordinateDim(dm, &spaceDim);CHKERRQ(ierr);
ierr = PetscCalloc1(spaceDim,&ind);CHKERRQ(ierr);
ierr = PetscCalloc1(spaceDim,&h);CHKERRQ(ierr);
h[0] = 1.0/(N-1); h[1] = 1.0/(N-1); h[2] = 1.0/(N-1);
ierr = DMDACreate(PetscObjectComm((PetscObject) dm), &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, ctx->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da, N, N, N);CHKERRQ(ierr);
ierr = DMDASetDof(da, 1);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(da, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(da, &info);CHKERRQ(ierr);
*Np = info.xm * info.ym * info.zm;
ierr = PetscCalloc1(*Np * spaceDim, pcoords);CHKERRQ(ierr);
for (k = info.zs; k < info.zs + info.zm; ++k) {
ind[2] = k;
for (j = info.ys; j < info.ys + info.ym; ++j) {
ind[1] = j;
for (i = info.xs; i < info.xs + info.xm; ++i, ++n) {
ind[0] = i;
for (d = 0; d < spaceDim; ++d) (*pcoords)[n*spaceDim+d] = ind[d]*h[d];
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d]Point %D (", rank, n);CHKERRQ(ierr);
for (d = 0; d < spaceDim; ++d) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%g", (double)(*pcoords)[n*spaceDim+d]);CHKERRQ(ierr);
if (d < spaceDim-1) {ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, ", ");CHKERRQ(ierr);}
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, ")\n");CHKERRQ(ierr);
}
}
}
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD, NULL);CHKERRQ(ierr);
ierr = PetscFree(ind);CHKERRQ(ierr);
ierr = PetscFree(h);CHKERRQ(ierr);
*pointsAllProcs = PETSC_FALSE;
PetscFunctionReturn(0);
}
/*@C
DMDACreate1d - Creates an object that will manage the communication of one-dimensional
regular array data that is distributed across some processors.
Collective on MPI_Comm
Input Parameters:
+ comm - MPI communicator
. bx - type of ghost cells at the boundary the array should have, if any. Use
DM_BOUNDARY_NONE, DM_BOUNDARY_GHOSTED, or DM_BOUNDARY_PERIODIC.
. M - global dimension of the array (use -M to indicate that it may be set to a different value
from the command line with -da_grid_x <M>)
. dof - number of degrees of freedom per node
. s - stencil width
- lx - array containing number of nodes in the X direction on each processor,
or NULL. If non-null, must be of length as the number of processes in the MPI_Comm.
Output Parameter:
. da - the resulting distributed array object
Options Database Key:
+ -dm_view - Calls DMView() at the conclusion of DMDACreate1d()
. -da_grid_x <nx> - number of grid points in x direction; can set if M < 0
. -da_refine_x <rx> - refinement factor
- -da_refine <n> - refine the DMDA n times before creating it, if M < 0
Level: beginner
Notes:
The array data itself is NOT stored in the DMDA, it is stored in Vec objects;
The appropriate vector objects can be obtained with calls to DMCreateGlobalVector()
and DMCreateLocalVector() and calls to VecDuplicate() if more are needed.
.keywords: distributed array, create, one-dimensional
.seealso: DMDestroy(), DMView(), DMDACreate2d(), DMDACreate3d(), DMGlobalToLocalBegin(), DMDASetRefinementFactor(),
DMGlobalToLocalEnd(), DMLocalToGlobalBegin(), DMLocalToLocalBegin(), DMLocalToLocalEnd(), DMDAGetRefinementFactor(),
DMDAGetInfo(), DMCreateGlobalVector(), DMCreateLocalVector(), DMDACreateNaturalVector(), DMLoad(), DMDAGetOwnershipRanges()
@*/
PetscErrorCode DMDACreate1d(MPI_Comm comm, DMBoundaryType bx, PetscInt M, PetscInt dof, PetscInt s, const PetscInt lx[], DM *da)
{
PetscErrorCode ierr;
PetscMPIInt size;
PetscFunctionBegin;
ierr = DMDACreate(comm, da);CHKERRQ(ierr);
ierr = DMSetDimension(*da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(*da, M, 1, 1);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*da, size, PETSC_DECIDE, PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*da, bx, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(*da, dof);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*da, s);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*da, lx, NULL, NULL);CHKERRQ(ierr);
/* This violates the behavior for other classes, but right now users expect negative dimensions to be handled this way */
ierr = DMSetFromOptions(*da);CHKERRQ(ierr);
ierr = DMSetUp(*da);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
MatCreateSeqUSFFT - Creates a matrix object that provides sequential USFFT
via the external package FFTW
Collective on MPI_Comm
Input Parameter:
+ da - geometry of the domain encoded by a DMDA
Output Parameter:
. A - the matrix
Options Database Keys:
+ -mat_usfft_plannerflags - set the FFTW planner flags
Level: intermediate
@*/
PetscErrorCode MatCreateSeqUSFFT(Vec sampleCoords, DMDA freqDA, Mat *A)
{
PetscErrorCode ierr;
Mat_USFFT *usfft;
PetscInt m,n,M,N,i;
const char *p_flags[]={"FFTW_ESTIMATE","FFTW_MEASURE","FFTW_PATIENT","FFTW_EXHAUSTIVE"};
PetscBool flg;
PetscInt p_flag;
PetscInt dof, dim, freqSizes[3];
MPI_Comm comm;
PetscInt size;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)inda, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
if (size > 1) SETERRQ(comm,PETSC_ERR_USER, "Parallel DMDA (in) not yet supported by USFFT");
ierr = PetscObjectGetComm((PetscObject)outda, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
if (size > 1) SETERRQ(comm,PETSC_ERR_USER, "Parallel DMDA (out) not yet supported by USFFT");
ierr = MatCreate(comm,A);CHKERRQ(ierr);
ierr = PetscNewLog(*A,Mat_USFFT,&usfft);CHKERRQ(ierr);
(*A)->data = (void*)usfft;
usfft->inda = inda;
usfft->outda = outda;
/* inda */
ierr = DMDAGetInfo(usfft->inda, &ndim, dim+0, dim+1, dim+2, NULL, NULL, NULL, &dof, NULL, NULL, NULL);CHKERRQ(ierr);
if (ndim <= 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"ndim %d must be > 0",ndim);
if (dof <= 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"dof %d must be > 0",dof);
usfft->ndim = ndim;
usfft->dof = dof;
usfft->freqDA = freqDA;
/* NB: we reverse the freq and resample DMDA sizes, since the DMDA ordering (natural on x-y-z, with x varying the fastest)
is the order opposite of that assumed by FFTW: z varying the fastest */
ierr = PetscMalloc((usfft->ndim+1)*sizeof(PetscInt),&usfft->indim);CHKERRQ(ierr);
for (i = usfft->ndim; i > 0; --i) usfft->indim[usfft->ndim-i] = dim[i-1];
/* outda */
ierr = DMDAGetInfo(usfft->outda, &ndim, dim+0, dim+1, dim+2, NULL, NULL, NULL, &dof, NULL, NULL, NULL);CHKERRQ(ierr);
if (ndim != usfft->ndim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"in and out DMDA dimensions must match: %d != %d",usfft->ndim, ndim);
if (dof != usfft->dof) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"in and out DMDA dof must match: %d != %d",usfft->dof, dof);
/* Store output dimensions */
/* NB: we reverse the DMDA dimensions, since the DMDA ordering (natural on x-y-z, with x varying the fastest)
is the order opposite of that assumed by FFTW: z varying the fastest */
ierr = PetscMalloc((usfft->ndim+1)*sizeof(PetscInt),&usfft->outdim);CHKERRQ(ierr);
for (i = usfft->ndim; i > 0; --i) usfft->outdim[usfft->ndim-i] = dim[i-1];
/* TODO: Use the new form of DMDACreate() */
#if 0
ierr = DMDACreate(comm,usfft->dim, DMDA_NONPERIODIC, DMDA_STENCIL_STAR, usfft->freqSizes[0], usfft->freqSizes[1], usfft->freqSizes[2],
PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE, dof, 0, NULL, NULL, NULL, 0, &(usfft->resampleDA));CHKERRQ(ierr);
#endif
ierr = DMDAGetVec(usfft->resampleDA, usfft->resample);CHKERRQ(ierr);
/* CONTINUE: Need to build the connectivity "Sieve" attaching sample points to the resample points they are close to */
/* CONTINUE: recalculate matrix sizes based on the connectivity "Sieve" */
/* mat sizes */
m = 1; n = 1;
for (i=0; i<usfft->ndim; i++) {
if (usfft->indim[i] <= 0) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"indim[%d]=%d must be > 0",i,usfft->indim[i]);
if (usfft->outdim[i] <= 0) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"outdim[%d]=%d must be > 0",i,usfft->outdim[i]);
n *= usfft->indim[i];
m *= usfft->outdim[i];
}
N = n*usfft->dof;
M = m*usfft->dof;
ierr = MatSetSizes(*A,M,N,M,N);CHKERRQ(ierr); /* "in size" is the number of columns, "out size" is the number of rows" */
ierr = PetscObjectChangeTypeName((PetscObject)*A,MATSEQUSFFT);CHKERRQ(ierr);
usfft->m = m; usfft->n = n; usfft->M = M; usfft->N = N;
/* FFTW */
usfft->p_forward = 0;
usfft->p_backward = 0;
usfft->p_flag = FFTW_ESTIMATE;
/* set Mat ops */
(*A)->ops->mult = MatMult_SeqUSFFT;
(*A)->ops->multtranspose = MatMultTranspose_SeqUSFFT;
(*A)->assembled = PETSC_TRUE;
(*A)->ops->destroy = MatDestroy_SeqUSFFT;
/* get runtime options */
ierr = PetscOptionsBegin(((PetscObject)(*A))->comm,((PetscObject)(*A))->prefix,"USFFT Options","Mat");CHKERRQ(ierr);
ierr = PetscOptionsEList("-mat_usfft_fftw_plannerflags","Planner Flags","None",p_flags,4,p_flags[0],&p_flag,&flg);CHKERRQ(ierr);
if (flg) usfft->p_flag = (unsigned)p_flag;
PetscOptionsEnd();
PetscFunctionReturn(0);
} /* MatCreateSeqUSFFT() */
int main(int argc,char **argv)
{
PetscErrorCode ierr;
KSP ksp;
PC pc;
Vec x,b;
DM da;
Mat A;
PetscInt dof=1;
PetscBool flg;
PetscScalar zero=0.0;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD,&da);CHKERRQ(ierr);
ierr = DMDASetDim(da,3);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetStencilType(da,DMDA_STENCIL_STAR);CHKERRQ(ierr);
ierr = DMDASetSizes(da,3,3,3);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetDof(da,dof);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da,1);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&b);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,MATAIJ,&A);CHKERRQ(ierr);
ierr = VecSet(b,zero);CHKERRQ(ierr);
/* Test sbaij matrix */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-test_sbaij",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
Mat sA;
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
A = sA;
}
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetDM(pc,(DM)da);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* check final residual */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL, "-check_final_residual", &flg,PETSC_NULL);CHKERRQ(ierr);
if (flg){
Vec b1;
PetscReal norm;
ierr = KSPGetSolution(ksp,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&b1);CHKERRQ(ierr);
ierr = MatMult(A,x,b1);CHKERRQ(ierr);
ierr = VecAXPY(b1,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(b1,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Final residual %g\n",norm);CHKERRQ(ierr);
ierr = VecDestroy(&b1);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode DMCoarsen_DA(DM da, MPI_Comm comm,DM *daref)
{
PetscErrorCode ierr;
PetscInt M,N,P,i;
DM da2;
DM_DA *dd = (DM_DA*)da->data,*dd2;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidPointer(daref,3);
if (dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
M = dd->M / dd->coarsen_x;
} else {
M = 1 + (dd->M - 1) / dd->coarsen_x;
}
if (dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
if (dd->dim > 1) {
N = dd->N / dd->coarsen_y;
} else {
N = 1;
}
} else {
N = 1 + (dd->N - 1) / dd->coarsen_y;
}
if (dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
if (dd->dim > 2) {
P = dd->P / dd->coarsen_z;
} else {
P = 1;
}
} else {
P = 1 + (dd->P - 1) / dd->coarsen_z;
}
ierr = DMDACreate(((PetscObject)da)->comm,&da2);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da2,((PetscObject)da)->prefix);CHKERRQ(ierr);
ierr = DMDASetDim(da2,dd->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da2,M,N,P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da2,dd->m,dd->n,dd->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da2,dd->bx,dd->by,dd->bz);CHKERRQ(ierr);
ierr = DMDASetDof(da2,dd->w);CHKERRQ(ierr);
ierr = DMDASetStencilType(da2,dd->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da2,dd->s);CHKERRQ(ierr);
if (dd->dim == 3) {
PetscInt *lx,*ly,*lz;
ierr = PetscMalloc3(dd->m,PetscInt,&lx,dd->n,PetscInt,&ly,dd->p,PetscInt,&lz);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_z,dd->p,dd->lz,lz);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,lz);CHKERRQ(ierr);
ierr = PetscFree3(lx,ly,lz);CHKERRQ(ierr);
} else if (dd->dim == 2) {
PetscInt *lx,*ly;
ierr = PetscMalloc2(dd->m,PetscInt,&lx,dd->n,PetscInt,&ly);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree2(lx,ly);CHKERRQ(ierr);
} else if (dd->dim == 1) {
PetscInt *lx;
ierr = PetscMalloc(dd->m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
}
dd2 = (DM_DA*)da2->data;
/* allow overloaded (user replaced) operations to be inherited by refinement clones; why are only some inherited and not all? */
/* da2->ops->createinterpolation = da->ops->createinterpolation; copying this one causes trouble for DMSetVI */
da2->ops->creatematrix = da->ops->creatematrix;
da2->ops->getcoloring = da->ops->getcoloring;
dd2->interptype = dd->interptype;
/* copy fill information if given */
if (dd->dfill) {
ierr = PetscMalloc((dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt),&dd2->dfill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->dfill,dd->dfill,(dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
if (dd->ofill) {
ierr = PetscMalloc((dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt),&dd2->ofill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->ofill,dd->ofill,(dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
/* copy the refine information */
dd2->coarsen_x = dd2->refine_x = dd->coarsen_x;
dd2->coarsen_y = dd2->refine_y = dd->coarsen_y;
dd2->coarsen_z = dd2->refine_z = dd->coarsen_z;
/* copy vector type information */
ierr = PetscFree(da2->vectype);CHKERRQ(ierr);
ierr = PetscStrallocpy(da->vectype,(char**)&da2->vectype);CHKERRQ(ierr);
dd2->lf = dd->lf;
dd2->lj = dd->lj;
da2->leveldown = da->leveldown + 1;
da2->levelup = da->levelup;
ierr = DMSetFromOptions(da2);CHKERRQ(ierr);
ierr = DMSetUp(da2);CHKERRQ(ierr);
ierr = DMViewFromOptions(da2,"-dm_view");CHKERRQ(ierr);
/* inject coordinates if they are set on the fine grid */
if (da->coordinates) {
DM cdaf,cdac;
Vec coordsc,coordsf;
VecScatter inject;
ierr = DMGetCoordinateDM(da,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(da,&coordsf);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da2,&cdac);CHKERRQ(ierr);
/* force creation of the coordinate vector */
ierr = DMDASetUniformCoordinates(da2,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMGetCoordinates(da2,&coordsc);CHKERRQ(ierr);
ierr = DMCreateInjection(cdac,cdaf,&inject);CHKERRQ(ierr);
ierr = VecScatterBegin(inject,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(inject ,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&inject);CHKERRQ(ierr);
}
for (i=0; i<da->bs; i++) {
const char *fieldname;
ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr);
ierr = DMDASetFieldName(da2,i,fieldname);CHKERRQ(ierr);
}
*daref = da2;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
KSP ksp;
PC pc;
Vec x,b;
DM da;
Mat A,Atrans;
PetscInt dof=1,M=8;
PetscBool flg,trans=PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-trans",&trans,NULL);CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD,&da);CHKERRQ(ierr);
ierr = DMSetDimension(da,3);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetStencilType(da,DMDA_STENCIL_STAR);CHKERRQ(ierr);
ierr = DMDASetSizes(da,M,M,M);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetDof(da,dof);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da,1);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&b);CHKERRQ(ierr);
ierr = ComputeRHS(da,b);CHKERRQ(ierr);
ierr = DMSetMatType(da,MATBAIJ);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,&A);CHKERRQ(ierr);
ierr = ComputeMatrix(da,A);CHKERRQ(ierr);
/* A is non-symmetric. Make A = 0.5*(A + Atrans) symmetric for testing icc and cholesky */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX,&Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,1.0,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatScale(A,0.5);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* Test sbaij matrix */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL, "-test_sbaij1", &flg,NULL);CHKERRQ(ierr);
if (flg) {
Mat sA;
PetscBool issymm;
ierr = MatIsTranspose(A,A,0.0,&issymm);CHKERRQ(ierr);
if (issymm) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else {ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: A is non-symmetric\n");CHKERRQ(ierr);}
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
A = sA;
}
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetDM(pc,(DM)da);CHKERRQ(ierr);
if (trans) {
ierr = KSPSolveTranspose(ksp,b,x);CHKERRQ(ierr);
} else {
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
}
/* check final residual */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL, "-check_final_residual", &flg,NULL);CHKERRQ(ierr);
if (flg) {
Vec b1;
PetscReal norm;
ierr = KSPGetSolution(ksp,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&b1);CHKERRQ(ierr);
ierr = MatMult(A,x,b1);CHKERRQ(ierr);
ierr = VecAXPY(b1,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(b1,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Final residual %g\n",norm);CHKERRQ(ierr);
ierr = VecDestroy(&b1);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da;
/* Initialize the Petsc context */
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 1);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* test moving data in and out */
ierr = PetscFinalize();
return 0;
}
/*
DMPatchZoom - Create a version of the coarse patch (identified by rank) with halo on communicator commz
Collective on DM
Input Parameters:
+ dm - the DM
. rank - the rank which holds the given patch
- commz - the new communicator for the patch
Output Parameters:
+ dmz - the patch DM
. sfz - the PetscSF mapping the patch+halo to the zoomed version
. sfzr - the PetscSF mapping the patch to the restricted zoomed version
Level: intermediate
Note: All processes in commz should have the same rank (could autosplit comm)
.seealso: DMPatchSolve()
*/
PetscErrorCode DMPatchZoom(DM dm, Vec X, MatStencil lower, MatStencil upper, MPI_Comm commz, DM *dmz, PetscSF *sfz, PetscSF *sfzr)
{
DMDAStencilType st;
MatStencil blower, bupper, loclower, locupper;
IS is;
const PetscInt *ranges, *indices;
PetscInt *localPoints = NULL;
PetscSFNode *remotePoints = NULL;
PetscInt dim, dof;
PetscInt M, N, P, rM, rN, rP, halo = 1, sxb, syb, szb, sxr, syr, szr, exr, eyr, ezr, mxb, myb, mzb, i, j, k, q;
PetscMPIInt size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
/* Create patch DM */
ierr = DMDAGetInfo(dm, &dim, &M, &N, &P, 0,0,0, &dof, 0,0,0,0, &st);CHKERRQ(ierr);
/* Get piece for rank r, expanded by halo */
bupper.i = PetscMin(M, upper.i + halo); blower.i = PetscMax(lower.i - halo, 0);
bupper.j = PetscMin(N, upper.j + halo); blower.j = PetscMax(lower.j - halo, 0);
bupper.k = PetscMin(P, upper.k + halo); blower.k = PetscMax(lower.k - halo, 0);
rM = bupper.i - blower.i;
rN = bupper.j - blower.j;
rP = bupper.k - blower.k;
if (commz != MPI_COMM_NULL) {
ierr = DMDACreate(commz, dmz);CHKERRQ(ierr);
ierr = DMSetDimension(*dmz, dim);CHKERRQ(ierr);
ierr = DMDASetSizes(*dmz, rM, rN, rP);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*dmz, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*dmz, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(*dmz, dof);CHKERRQ(ierr);
ierr = DMDASetStencilType(*dmz, st);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*dmz, 0);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*dmz, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = DMSetFromOptions(*dmz);CHKERRQ(ierr);
ierr = DMSetUp(*dmz);CHKERRQ(ierr);
ierr = DMDAGetCorners(*dmz, &sxb, &syb, &szb, &mxb, &myb, &mzb);CHKERRQ(ierr);
sxr = PetscMax(sxb, lower.i - blower.i);
syr = PetscMax(syb, lower.j - blower.j);
szr = PetscMax(szb, lower.k - blower.k);
exr = PetscMin(sxb+mxb, upper.i - blower.i);
eyr = PetscMin(syb+myb, upper.j - blower.j);
ezr = PetscMin(szb+mzb, upper.k - blower.k);
ierr = PetscMalloc2(rM*rN*rP,&localPoints,rM*rN*rP,&remotePoints);CHKERRQ(ierr);
} else {
sxr = syr = szr = exr = eyr = ezr = sxb = syb = szb = mxb = myb = mzb = 0;
}
/* Create SF for restricted map */
ierr = VecGetOwnershipRanges(X,&ranges);CHKERRQ(ierr);
loclower.i = blower.i + sxr; locupper.i = blower.i + exr;
loclower.j = blower.j + syr; locupper.j = blower.j + eyr;
loclower.k = blower.k + szr; locupper.k = blower.k + ezr;
ierr = DMDACreatePatchIS(dm, &loclower, &locupper, &is);CHKERRQ(ierr);
ierr = ISGetIndices(is, &indices);CHKERRQ(ierr);
q = 0;
for (k = szb; k < szb+mzb; ++k) {
if ((k < szr) || (k >= ezr)) continue;
for (j = syb; j < syb+myb; ++j) {
if ((j < syr) || (j >= eyr)) continue;
for (i = sxb; i < sxb+mxb; ++i) {
const PetscInt lp = ((k-szb)*rN + (j-syb))*rM + i-sxb;
PetscInt r;
if ((i < sxr) || (i >= exr)) continue;
localPoints[q] = lp;
ierr = PetscFindInt(indices[q], size+1, ranges, &r);CHKERRQ(ierr);
remotePoints[q].rank = r < 0 ? -(r+1) - 1 : r;
remotePoints[q].index = indices[q] - ranges[remotePoints[q].rank];
++q;
}
}
}
ierr = ISRestoreIndices(is, &indices);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfzr);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *sfzr, "Restricted Map");CHKERRQ(ierr);
ierr = PetscSFSetGraph(*sfzr, M*N*P, q, localPoints, PETSC_COPY_VALUES, remotePoints, PETSC_COPY_VALUES);CHKERRQ(ierr);
/* Create SF for buffered map */
loclower.i = blower.i + sxb; locupper.i = blower.i + sxb+mxb;
loclower.j = blower.j + syb; locupper.j = blower.j + syb+myb;
loclower.k = blower.k + szb; locupper.k = blower.k + szb+mzb;
ierr = DMDACreatePatchIS(dm, &loclower, &locupper, &is);CHKERRQ(ierr);
ierr = ISGetIndices(is, &indices);CHKERRQ(ierr);
q = 0;
for (k = szb; k < szb+mzb; ++k) {
for (j = syb; j < syb+myb; ++j) {
for (i = sxb; i < sxb+mxb; ++i, ++q) {
PetscInt r;
localPoints[q] = q;
ierr = PetscFindInt(indices[q], size+1, ranges, &r);CHKERRQ(ierr);
remotePoints[q].rank = r < 0 ? -(r+1) - 1 : r;
remotePoints[q].index = indices[q] - ranges[remotePoints[q].rank];
}
}
}
ierr = ISRestoreIndices(is, &indices);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfz);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *sfz, "Buffered Map");CHKERRQ(ierr);
ierr = PetscSFSetGraph(*sfz, M*N*P, q, localPoints, PETSC_COPY_VALUES, remotePoints, PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscFree2(localPoints, remotePoints);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN void PETSC_STDCALL dmdacreate_(MPI_Fint * comm,DM *da, int *__ierr ){
*__ierr = DMDACreate(
MPI_Comm_f2c(*(comm)),da);
}
PetscErrorCode DMRefine_DA(DM da,MPI_Comm comm,DM *daref)
{
PetscErrorCode ierr;
PetscInt M,N,P,i;
DM da2;
DM_DA *dd = (DM_DA*)da->data,*dd2;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidPointer(daref,3);
if (dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0) {
M = dd->refine_x*dd->M;
} else {
M = 1 + dd->refine_x*(dd->M - 1);
}
if (dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0) {
if (dd->dim > 1) {
N = dd->refine_y*dd->N;
} else {
N = 1;
}
} else {
N = 1 + dd->refine_y*(dd->N - 1);
}
if (dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0) {
if (dd->dim > 2) {
P = dd->refine_z*dd->P;
} else {
P = 1;
}
} else {
P = 1 + dd->refine_z*(dd->P - 1);
}
ierr = DMDACreate(PetscObjectComm((PetscObject)da),&da2);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da2,((PetscObject)da)->prefix);CHKERRQ(ierr);
ierr = DMDASetDim(da2,dd->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da2,M,N,P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da2,dd->m,dd->n,dd->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da2,dd->bx,dd->by,dd->bz);CHKERRQ(ierr);
ierr = DMDASetDof(da2,dd->w);CHKERRQ(ierr);
ierr = DMDASetStencilType(da2,dd->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da2,dd->s);CHKERRQ(ierr);
if (dd->dim == 3) {
PetscInt *lx,*ly,*lz;
ierr = PetscMalloc3(dd->m,&lx,dd->n,&ly,dd->p,&lz);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_z,dd->p,dd->lz,lz);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,lz);CHKERRQ(ierr);
ierr = PetscFree3(lx,ly,lz);CHKERRQ(ierr);
} else if (dd->dim == 2) {
PetscInt *lx,*ly;
ierr = PetscMalloc2(dd->m,&lx,dd->n,&ly);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,NULL);CHKERRQ(ierr);
ierr = PetscFree2(lx,ly);CHKERRQ(ierr);
} else if (dd->dim == 1) {
PetscInt *lx;
ierr = PetscMalloc1(dd->m,&lx);CHKERRQ(ierr);
ierr = DMDARefineOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->refine_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,NULL,NULL);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
}
dd2 = (DM_DA*)da2->data;
/* allow overloaded (user replaced) operations to be inherited by refinement clones */
da2->ops->creatematrix = da->ops->creatematrix;
/* da2->ops->createinterpolation = da->ops->createinterpolation; this causes problem with SNESVI */
da2->ops->getcoloring = da->ops->getcoloring;
dd2->interptype = dd->interptype;
/* copy fill information if given */
if (dd->dfill) {
ierr = PetscMalloc1((dd->dfill[dd->w]+dd->w+1),&dd2->dfill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->dfill,dd->dfill,(dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
if (dd->ofill) {
ierr = PetscMalloc1((dd->ofill[dd->w]+dd->w+1),&dd2->ofill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->ofill,dd->ofill,(dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
/* copy the refine information */
dd2->coarsen_x = dd2->refine_x = dd->refine_x;
dd2->coarsen_y = dd2->refine_y = dd->refine_y;
dd2->coarsen_z = dd2->refine_z = dd->refine_z;
/* copy vector type information */
ierr = DMSetVecType(da2,da->vectype);CHKERRQ(ierr);
dd2->lf = dd->lf;
dd2->lj = dd->lj;
da2->leveldown = da->leveldown;
da2->levelup = da->levelup + 1;
ierr = DMSetFromOptions(da2);CHKERRQ(ierr);
ierr = DMSetUp(da2);CHKERRQ(ierr);
/* interpolate coordinates if they are set on the coarse grid */
if (da->coordinates) {
DM cdaf,cdac;
Vec coordsc,coordsf;
Mat II;
ierr = DMGetCoordinateDM(da,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinates(da,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da2,&cdaf);CHKERRQ(ierr);
/* force creation of the coordinate vector */
ierr = DMDASetUniformCoordinates(da2,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMGetCoordinates(da2,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&II,NULL);CHKERRQ(ierr);
ierr = MatInterpolate(II,coordsc,coordsf);CHKERRQ(ierr);
ierr = MatDestroy(&II);CHKERRQ(ierr);
}
for (i=0; i<da->bs; i++) {
const char *fieldname;
ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr);
ierr = DMDASetFieldName(da2,i,fieldname);CHKERRQ(ierr);
}
*daref = da2;
PetscFunctionReturn(0);
}
PetscErrorCode DMDASubDomainDA_Private(DM dm, PetscInt *nlocal, DM **sdm)
{
DM *da;
PetscInt dim,size,i,j,k,idx;
PetscErrorCode ierr;
DMDALocalInfo info;
PetscInt xsize,ysize,zsize;
PetscInt xo,yo,zo;
PetscInt xs,ys,zs;
PetscInt xm=1,ym=1,zm=1;
PetscInt xol,yol,zol;
PetscInt m=1,n=1,p=1;
PetscInt M,N,P;
PetscInt pm,mtmp;
PetscFunctionBegin;
ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr);
ierr = DMDAGetOverlap(dm,&xol,&yol,&zol);CHKERRQ(ierr);
ierr = DMDAGetNumLocalSubDomains(dm,&size);CHKERRQ(ierr);
ierr = PetscMalloc1(size,&da);CHKERRQ(ierr);
if (nlocal) *nlocal = size;
dim = info.dim;
M = info.xm;
N = info.ym;
P = info.zm;
if (dim == 1) {
m = size;
} else if (dim == 2) {
m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)N)));
while (m > 0) {
n = size/m;
if (m*n*p == size) break;
m--;
}
} else if (dim == 3) {
n = (PetscInt)(0.5 + PetscPowReal(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.))); if (!n) n = 1;
while (n > 0) {
pm = size/n;
if (n*pm == size) break;
n--;
}
if (!n) n = 1;
m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
if (!m) m = 1;
while (m > 0) {
p = size/(m*n);
if (m*n*p == size) break;
m--;
}
if (M > P && m < p) {mtmp = m; m = p; p = mtmp;}
}
zs = info.zs;
idx = 0;
for (k = 0; k < p; k++) {
ys = info.ys;
for (j = 0; j < n; j++) {
xs = info.xs;
for (i = 0; i < m; i++) {
if (dim == 1) {
xm = M/m + ((M % m) > i);
} else if (dim == 2) {
xm = M/m + ((M % m) > i);
ym = N/n + ((N % n) > j);
} else if (dim == 3) {
xm = M/m + ((M % m) > i);
ym = N/n + ((N % n) > j);
zm = P/p + ((P % p) > k);
}
xsize = xm;
ysize = ym;
zsize = zm;
xo = xs;
yo = ys;
zo = zs;
ierr = DMDACreate(PETSC_COMM_SELF,&(da[idx]));CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da[idx],"sub_");CHKERRQ(ierr);
ierr = DMSetDimension(da[idx], info.dim);CHKERRQ(ierr);
ierr = DMDASetDof(da[idx], info.dof);CHKERRQ(ierr);
ierr = DMDASetStencilType(da[idx],info.st);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da[idx],info.sw);CHKERRQ(ierr);
if (info.bx == DM_BOUNDARY_PERIODIC || (xs != 0)) {
xsize += xol;
xo -= xol;
}
if (info.by == DM_BOUNDARY_PERIODIC || (ys != 0)) {
ysize += yol;
yo -= yol;
}
if (info.bz == DM_BOUNDARY_PERIODIC || (zs != 0)) {
zsize += zol;
zo -= zol;
}
if (info.bx == DM_BOUNDARY_PERIODIC || (xs+xm != info.mx)) xsize += xol;
if (info.by == DM_BOUNDARY_PERIODIC || (ys+ym != info.my)) ysize += yol;
if (info.bz == DM_BOUNDARY_PERIODIC || (zs+zm != info.mz)) zsize += zol;
if (info.bx != DM_BOUNDARY_PERIODIC) {
if (xo < 0) {
xsize += xo;
xo = 0;
}
if (xo+xsize > info.mx-1) {
xsize -= xo+xsize - info.mx;
}
}
if (info.by != DM_BOUNDARY_PERIODIC) {
if (yo < 0) {
ysize += yo;
yo = 0;
}
if (yo+ysize > info.my-1) {
ysize -= yo+ysize - info.my;
}
}
if (info.bz != DM_BOUNDARY_PERIODIC) {
if (zo < 0) {
zsize += zo;
zo = 0;
}
if (zo+zsize > info.mz-1) {
zsize -= zo+zsize - info.mz;
}
}
ierr = DMDASetSizes(da[idx], xsize, ysize, zsize);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da[idx], 1, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da[idx], DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
/* set up as a block instead */
ierr = DMSetUp(da[idx]);CHKERRQ(ierr);
/* nonoverlapping region */
ierr = DMDASetNonOverlappingRegion(da[idx],xs,ys,zs,xm,ym,zm);CHKERRQ(ierr);
/* this alters the behavior of DMDAGetInfo, DMDAGetLocalInfo, DMDAGetCorners, and DMDAGetGhostedCorners and should be used with care */
ierr = DMDASetOffset(da[idx],xo,yo,zo,info.mx,info.my,info.mz);CHKERRQ(ierr);
xs += xm;
idx++;
}
ys += ym;
}
zs += zm;
}
*sdm = da;
PetscFunctionReturn(0);
}