PetscErrorCode TcSolver::generateBodyInfo()
{
PetscErrorCode ierr;
PetscInt m,n;
const PetscInt *lxp, *lyp;
PetscInt numProcs;
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &numProcs); CHKERRQ(ierr);
/* set arrays to store boundaryPoint by number of procs */
boundaryPointIndices.resize(numProcs);
numBoundaryPointsOnProcess.resize(numProcs);
numPhiOnProcess.resize(numProcs);
globalIndexMapping.resize(x.size());
ierr = DMDAGetOwnershipRanges(pda,&lxp,&lyp,NULL);CHKERRQ(ierr);
ierr = DMDAGetInfo(pda, NULL, NULL, NULL, NULL, &m, &n, NULL, NULL,NULL,NULL, NULL, NULL, NULL); CHKERRQ(ierr);
PetscInt xStart, yStart, xEnd, yEnd,
procIdx = 0;
yStart=0;
for(PetscInt j=0; j<n; j++)
{
yEnd = yStart + lyp[j];
xStart = 0;
for(PetscInt i=0; i<m; i++)
{
procIdx = j*m + i;
xEnd = xStart + lxp[i];
numPhiOnProcess[procIdx] = lxp[i] * lyp[j];
for(size_t l=0; l<x.size(); l++)
{
if(x[l] >= fluid.x[xStart] && x[l] < fluid.x[xEnd] && y[l] >= fluid.y[yStart] && y[l] < fluid.y[yEnd])
{
numBoundaryPointsOnProcess[procIdx]++;
}
}
boundaryPointIndices[procIdx].reserve(numBoundaryPointsOnProcess[procIdx]);
for(size_t l=0; l<x.size(); l++)
{
if(x[l]>=fluid.x[xStart] && x[l]<fluid.x[xEnd] && y[l]>= fluid.y[yStart] && y[l]< fluid.y[yEnd])
{
/* */
boundaryPointIndices[procIdx].push_back(l);
}
}
xStart = xEnd;
}
yStart = yEnd;
}
return 0;
}
/*@
DMDAGetReducedDMDA - Gets the DMDA with the same layout but with fewer or more fields
Collective on DMDA
Input Parameters:
+ da - the distributed array
- nfields - number of fields in new DMDA
Output Parameter:
. nda - the new DMDA
Level: intermediate
.keywords: distributed array, get, corners, nodes, local indices, coordinates
.seealso: DMDAGetGhostCorners(), DMSetCoordinates(), DMDASetUniformCoordinates(), DMGetCoordinates(), DMDAGetGhostedCoordinates()
@*/
PetscErrorCode DMDAGetReducedDMDA(DM da,PetscInt nfields,DM *nda)
{
PetscErrorCode ierr;
DM_DA *dd = (DM_DA*)da->data;
PetscInt s,m,n,p,M,N,P,dim,Mo,No,Po;
const PetscInt *lx,*ly,*lz;
DMBoundaryType bx,by,bz;
DMDAStencilType stencil_type;
PetscInt ox,oy,oz;
PetscInt cl,rl;
PetscFunctionBegin;
dim = da->dim;
M = dd->M;
N = dd->N;
P = dd->P;
m = dd->m;
n = dd->n;
p = dd->p;
s = dd->s;
bx = dd->bx;
by = dd->by;
bz = dd->bz;
stencil_type = dd->stencil_type;
ierr = DMDAGetOwnershipRanges(da,&lx,&ly,&lz);CHKERRQ(ierr);
if (dim == 1) {
ierr = DMDACreate1d(PetscObjectComm((PetscObject)da),bx,M,nfields,s,dd->lx,nda);CHKERRQ(ierr);
} else if (dim == 2) {
ierr = DMDACreate2d(PetscObjectComm((PetscObject)da),bx,by,stencil_type,M,N,m,n,nfields,s,lx,ly,nda);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PetscObjectComm((PetscObject)da),bx,by,bz,stencil_type,M,N,P,m,n,p,nfields,s,lx,ly,lz,nda);CHKERRQ(ierr);
}
ierr = DMSetUp(*nda);CHKERRQ(ierr);
if (da->coordinates) {
ierr = PetscObjectReference((PetscObject)da->coordinates);CHKERRQ(ierr);
(*nda)->coordinates = da->coordinates;
}
/* allow for getting a reduced DA corresponding to a domain decomposition */
ierr = DMDAGetOffset(da,&ox,&oy,&oz,&Mo,&No,&Po);CHKERRQ(ierr);
ierr = DMDASetOffset(*nda,ox,oy,oz,Mo,No,Po);CHKERRQ(ierr);
/* allow for getting a reduced DA corresponding to a coarsened DA */
ierr = DMGetCoarsenLevel(da,&cl);CHKERRQ(ierr);
ierr = DMGetRefineLevel(da,&rl);CHKERRQ(ierr);
(*nda)->levelup = rl;
(*nda)->leveldown = cl;
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
DM da, daX, daY;
DMDALocalInfo info;
MPI_Comm commX, commY;
Vec basisX, basisY;
PetscScalar **arrayX, **arrayY;
const PetscInt *lx, *ly;
PetscInt M = 3, N = 3;
PetscInt p = 1;
PetscInt numGP = 3;
PetscInt dof = 2*(p+1)*numGP;
PetscMPIInt rank, subsize, subrank;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&argv,0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
/* Create 2D DMDA */
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DMDA_STENCIL_STAR, M, N, PETSC_DECIDE, PETSC_DECIDE, 1, 1, NULL, NULL, &da);CHKERRQ(ierr);
/* Create 1D DMDAs along two directions */
ierr = DMDAGetOwnershipRanges(da, &lx, &ly, NULL);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(da, &info);CHKERRQ(ierr);
ierr = DMDAGetProcessorSubsets(da, DMDA_X, &commX);CHKERRQ(ierr);
ierr = DMDAGetProcessorSubsets(da, DMDA_Y, &commY);CHKERRQ(ierr);
ierr = MPI_Comm_size(commX, &subsize);CHKERRQ(ierr);
ierr = MPI_Comm_rank(commX, &subrank);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "[%d]X subrank: %d subsize: %d\n", rank, subrank, subsize);
ierr = MPI_Comm_size(commY, &subsize);CHKERRQ(ierr);
ierr = MPI_Comm_rank(commY, &subrank);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "[%d]Y subrank: %d subsize: %d\n", rank, subrank, subsize);
ierr = DMDACreate1d(commX, DM_BOUNDARY_NONE, M, dof, 1, lx, &daX);CHKERRQ(ierr);
ierr = DMDACreate1d(commY, DM_BOUNDARY_NONE, N, dof, 1, ly, &daY);CHKERRQ(ierr);
/* Create 1D vectors for basis functions */
ierr = DMGetGlobalVector(daX, &basisX);CHKERRQ(ierr);
ierr = DMGetGlobalVector(daY, &basisY);CHKERRQ(ierr);
/* Extract basis functions */
ierr = DMDAVecGetArrayDOF(daX, basisX, &arrayX);CHKERRQ(ierr);
ierr = DMDAVecGetArrayDOF(daY, basisY, &arrayY);CHKERRQ(ierr);
/*arrayX[i][ndof]; */
/*arrayY[j][ndof]; */
ierr = DMDAVecRestoreArrayDOF(daX, basisX, &arrayX);CHKERRQ(ierr);
ierr = DMDAVecRestoreArrayDOF(daY, basisY, &arrayY);CHKERRQ(ierr);
/* Return basis vectors */
ierr = DMRestoreGlobalVector(daX, &basisX);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(daY, &basisY);CHKERRQ(ierr);
/* Cleanup */
ierr = DMDestroy(&daX);CHKERRQ(ierr);
ierr = DMDestroy(&daY);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*
Sets up a monitor that will display He as a function of space and cluster size for each time step
*/
PetscErrorCode MyMonitorSetUp(TS ts)
{
DM da;
PetscErrorCode ierr;
PetscInt xi,xs,xm,*idx,M,xj,cnt = 0,dof = 3*N + N*N;
const PetscInt *lx;
Vec C;
MyMonitorCtx *ctx;
PetscBool flg;
IS is;
char ycoor[32];
PetscReal valuebounds[4] = {0, 1.2, 0, 1.2};
PetscFunctionBeginUser;
ierr = PetscOptionsHasName(PETSC_NULL,"-mymonitor",&flg);CHKERRQ(ierr);
if (!flg) PetscFunctionReturn(0);
ierr = TSGetDM(ts,&da);CHKERRQ(ierr);
ierr = PetscNew(MyMonitorCtx,&ctx);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(((PetscObject)da)->comm,PETSC_NULL,"",PETSC_DECIDE,PETSC_DECIDE,600,400,&ctx->viewer);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,PETSC_IGNORE,&M,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);CHKERRQ(ierr);
ierr = DMDAGetOwnershipRanges(da,&lx,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DMDACreate2d(((PetscObject)da)->comm,DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_STAR,M,N,PETSC_DETERMINE,1,2,1,lx,PETSC_NULL,&ctx->da);CHKERRQ(ierr);
ierr = DMDASetFieldName(ctx->da,0,"He");CHKERRQ(ierr);
ierr = DMDASetFieldName(ctx->da,1,"V");CHKERRQ(ierr);
ierr = DMDASetCoordinateName(ctx->da,0,"X coordinate direction");CHKERRQ(ierr);
ierr = PetscSNPrintf(ycoor,32,"%D ... Cluster size ... 1",N);CHKERRQ(ierr);
ierr = DMDASetCoordinateName(ctx->da,1,ycoor);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(ctx->da,&ctx->He);CHKERRQ(ierr);
ierr = PetscMalloc(2*N*xm*sizeof(PetscInt),&idx);CHKERRQ(ierr);
cnt = 0;
for (xj=0; xj<N; xj++) {
for (xi=xs; xi<xs+xm; xi++) {
idx[cnt++] = dof*xi + xj;
idx[cnt++] = dof*xi + xj + N;
}
}
ierr = ISCreateGeneral(((PetscObject)ts)->comm,2*N*xm,idx,PETSC_OWN_POINTER,&is);CHKERRQ(ierr);
ierr = TSGetSolution(ts,&C);CHKERRQ(ierr);
ierr = VecScatterCreate(C,is,ctx->He,PETSC_NULL,&ctx->scatter);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
/* sets the bounds on the contour plot values so the colors mean the same thing for different timesteps */
ierr = PetscViewerDrawSetBounds(ctx->viewer,2,valuebounds);CHKERRQ(ierr);
ierr = TSMonitorSet(ts,MyMonitorMonitor,ctx,MyMonitorDestroy);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN void PETSC_STDCALL dmdagetownershipranges_(DM *da,PetscInt lx[],PetscInt ly[],PetscInt lz[],PetscErrorCode *ierr)
{
const PetscInt *gx,*gy,*gz;
PetscInt M,N,P,i;
CHKFORTRANNULLINTEGER(lx);
CHKFORTRANNULLINTEGER(ly);
CHKFORTRANNULLINTEGER(lz);
*ierr = DMDAGetInfo(*da,0,0,0,0,&M,&N,&P,0,0,0,0,0,0);if (*ierr) return;
*ierr = DMDAGetOwnershipRanges(*da,&gx,&gy,&gz);if (*ierr) return;
if (lx) {
for (i=0; i<M; i++) lx[i] = gx[i];
}
if (ly) {
for (i=0; i<N; i++) ly[i] = gy[i];
}
if (lz) {
for (i=0; i<P; i++) lz[i] = gz[i];
}
}
void createArrays(simInfo *data)
{
PetscErrorCode ierr;
PetscInt *lx, *ly, m, n;
const PetscInt *lxu, *lyu;
// Create distributed array and get vectors
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nCreate staggered U, V and P vectors [DMDACreate2d, DMDAGetInfo, DMDAGetOwnershipRanges, DMCreateGlobalVector]"); CHKERRV(ierr); hline();
ierr = DMCompositeCreate(PETSC_COMM_WORLD, &(data->pack)); CHKERRV(ierr);
// create u DA
ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_GHOSTED, DMDA_BOUNDARY_GHOSTED, DMDA_STENCIL_BOX, data->nx-1, data->ny, PETSC_DECIDE, PETSC_DECIDE, 1, 1, NULL, NULL, &(data->uda)); CHKERRV(ierr);
ierr = DMCompositeAddDM(data->pack, data->uda); CHKERRV(ierr);
// determine process distribution of v
ierr = DMDAGetInfo(data->uda, NULL, NULL, NULL, NULL, &m, &n, NULL, NULL, NULL, NULL, NULL, NULL, NULL); CHKERRV(ierr);
ierr = DMDAGetOwnershipRanges(data->uda, &lxu, &lyu, NULL); CHKERRV(ierr);
ierr = PetscMalloc(m*sizeof(*lx), &lx); CHKERRV(ierr);
ierr = PetscMalloc(n*sizeof(*ly), &ly); CHKERRV(ierr);
ierr = PetscMemcpy(lx ,lxu, m*sizeof(*lx)); CHKERRV(ierr);
ierr = PetscMemcpy(ly ,lyu, n*sizeof(*ly)); CHKERRV(ierr);
lx[m-1]++;
// create v DA
ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_GHOSTED, DMDA_BOUNDARY_GHOSTED, DMDA_STENCIL_STAR, data->nx, data->ny, m, n, 1, 1, lx, ly, &(data->pda)); CHKERRV(ierr);
// create vectors p and bc2
ierr = DMCreateGlobalVector(data->pda, &(data->pGlobal)); CHKERRV(ierr);
ierr = VecDuplicate(data->pGlobal, &(data->bc2)); CHKERRV(ierr);
ly[n-1]--;
// create p DA
ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_GHOSTED, DMDA_BOUNDARY_GHOSTED, DMDA_STENCIL_BOX, data->nx, data->ny-1, m, n, 1, 1, lx, ly, &(data->vda)); CHKERRV(ierr);
ierr = DMCompositeAddDM(data->pack, data->vda); CHKERRV(ierr);
PetscFree(lx);
PetscFree(ly);
// create vector uPacked
ierr = DMCreateGlobalVector(data->pack, &(data->uPacked)); CHKERRV(ierr);
}
int main(int argc,char *argv[])
{
PetscErrorCode ierr;
DM da;
PetscInt dim = 2,m,n,p,i;
const PetscInt *lx,*ly,*lz;
PetscMPIInt rank,size;
ierr = PetscInitialize(&argc,&argv,0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,0,"-dim",&dim,0);CHKERRQ(ierr);
switch (dim) {
case 2:
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR, 3,5,PETSC_DECIDE,PETSC_DECIDE,2,1,NULL,NULL,&da);CHKERRQ(ierr);
break;
case 3:
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR, 3,5,7,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,2,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
break;
default: SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"No support for %D dimensions",dim);
}
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da, 0, 0,0,0, &m,&n,&p, 0,0, 0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetOwnershipRanges(da,&lx,&ly,&lz);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
for (i=0; i<size; i++) {
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
if (i == rank) {
ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_SELF,"[%d] lx ly%s\n",rank,dim>2 ? " lz" : "");CHKERRQ(ierr);
ierr = PetscIntView(m,lx,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscIntView(n,ly,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
if (dim > 2) {ierr = PetscIntView(n,lz,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);}
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscMPIInt rank, M, N, m, n, i;
PetscInt nx = 21, ny = 11;
const PetscInt *lxu, *lyu;
PetscInt *lxv, *lyv;
PetscInt *lxp, *lyp;
PetscErrorCode ierr;
DM uda, vda, pda;
Vec u;
ierr = PetscInitialize(&argc, &argv, NULL, NULL); CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
// Read options
ierr = PetscOptionsGetInt(NULL, "-nx", &nx, NULL); CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL, "-ny", &ny, NULL); CHKERRQ(ierr);
// Create distributed array and get vectors
ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_GHOSTED, DMDA_BOUNDARY_GHOSTED, DMDA_STENCIL_BOX, nx-1, ny, PETSC_DECIDE, PETSC_DECIDE, 1, 1, NULL, NULL, &uda); CHKERRQ(ierr);
ierr = DMCreateLocalVector(uda, &u); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nCreated local distributed array U [DMDACreate2d, DMCreateLocalVector]"); CHKERRQ(ierr); hline();
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nPrinting DA Info for U [DMDAGetInfo]"); CHKERRQ(ierr); hline();
ierr = DMDAGetInfo(uda, NULL, &M, &N, NULL, &m, &n, NULL, NULL, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Total size of array : M, N: %d, %d\n", M, N); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Number of processors: m, n: %d, %d\n", m, n); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nU-Node distribution along each direction [DMDAGetOwnershipRanges]"); CHKERRQ(ierr); hline();
ierr = DMDAGetOwnershipRanges(uda, &lxu, &lyu, NULL); CHKERRQ(ierr);
for(i=0; i<m; i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t", lxu[i]); CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "\n"); CHKERRQ(ierr);
for(i=0; i<n; i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t", lyu[i]); CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "\n"); CHKERRQ(ierr);
ierr = PetscMalloc(m*sizeof(*lxv), &lxv); CHKERRQ(ierr);
ierr = PetscMalloc(n*sizeof(*lyv), &lyv); CHKERRQ(ierr);
ierr = PetscMemcpy(lxv, lxu, m*sizeof(*lxv)); CHKERRQ(ierr);
ierr = PetscMemcpy(lyv, lyu, n*sizeof(*lyv)); CHKERRQ(ierr);
lxv[m-1]++;
lyv[n-1]--;
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nV-Node distributed along each direction [Modified from U's info]"); CHKERRQ(ierr); hline();
for(i=0; i<m; i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t", lxv[i]); CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "\n"); CHKERRQ(ierr);
for(i=0; i<n; i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t", lyv[i]); CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "\n"); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nCreated and printing DA Info for V [DMDACreate2d, DMDAGetInfo]"); CHKERRQ(ierr); hline();
ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_GHOSTED, DMDA_BOUNDARY_GHOSTED, DMDA_STENCIL_BOX, nx, ny-1, m, n, 1, 1, lxv, lyv, &vda); CHKERRQ(ierr);
ierr = DMDAGetInfo(vda, NULL, &M, &N, NULL, &m, &n, NULL, NULL, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "M, N: %d, %d\n", M, N); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "m, n: %d, %d\n", m, n); CHKERRQ(ierr);
ierr = PetscMalloc(m*sizeof(*lxp), &lxp); CHKERRQ(ierr);
ierr = PetscMalloc(n*sizeof(*lyp), &lyp); CHKERRQ(ierr);
ierr = PetscMemcpy(lxp, lxu, m*sizeof(*lxp)); CHKERRQ(ierr);
ierr = PetscMemcpy(lyp, lyu, n*sizeof(*lyp)); CHKERRQ(ierr);
lxp[m-1]++;
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nCreated and printing DA Info for P [DMDACreate2d, DMDAGetInfo]"); CHKERRQ(ierr); hline();
ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_GHOSTED, DMDA_BOUNDARY_GHOSTED, DMDA_STENCIL_STAR, nx, ny, m, n, 1, 1, lxp, lyp, &pda); CHKERRQ(ierr);
ierr = DMDAGetInfo(pda, NULL, &M, &N, NULL, &m, &n, NULL, NULL, NULL, NULL, NULL, NULL, NULL); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "M, N: %d, %d\n", M, N); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "m, n: %d, %d\n", m, n); CHKERRQ(ierr);
ierr = PetscFree(lxv); CHKERRQ(ierr);
ierr = PetscFree(lyv); CHKERRQ(ierr);
ierr = PetscFree(lxp); CHKERRQ(ierr);
ierr = PetscFree(lyp); CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = DMDestroy(&uda);CHKERRQ(ierr);
ierr = DMDestroy(&vda);CHKERRQ(ierr);
ierr = DMDestroy(&pda);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode VTKIO_VTI_PieceExtend(FILE *vtk_fp,PetscInt indent_level,DM da,const char local_file_prefix[])
{
PetscMPIInt size,rank;
MPI_Comm comm;
const PetscInt *lx,*ly,*lz;
PetscInt M,N,P,pM,pN,pP,sum,*olx,*oly,*olz;
PetscInt *osx,*osy,*osz,*oex,*oey,*oez;
PetscInt i,j,k,II,stencil;
PetscErrorCode ierr;
PetscFunctionBeginUser;
/* create file name */
PetscObjectGetComm((PetscObject)da,&comm);
MPI_Comm_size(comm,&size);
MPI_Comm_rank(comm,&rank);
ierr = DMDAGetInfo(da,0,&M,&N,&P,&pM,&pN,&pP,0,&stencil,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetOwnershipRanges(da,&lx,&ly,&lz);CHKERRQ(ierr);
/* generate start,end list */
ierr = PetscMalloc1(pM+1,&olx);CHKERRQ(ierr);
ierr = PetscMalloc1(pN+1,&oly);CHKERRQ(ierr);
ierr = PetscMalloc1(pP+1,&olz);CHKERRQ(ierr);
sum = 0;
for (i=0; i<pM; i++) {
olx[i] = sum;
sum = sum + lx[i];
}
olx[pM] = sum;
sum = 0;
for (i=0; i<pN; i++) {
oly[i] = sum;
sum = sum + ly[i];
}
oly[pN] = sum;
sum = 0;
for (i=0; i<pP; i++) {
olz[i] = sum;
sum = sum + lz[i];
}
olz[pP] = sum;
ierr = PetscMalloc1(pM,&osx);CHKERRQ(ierr);
ierr = PetscMalloc1(pN,&osy);CHKERRQ(ierr);
ierr = PetscMalloc1(pP,&osz);CHKERRQ(ierr);
ierr = PetscMalloc1(pM,&oex);CHKERRQ(ierr);
ierr = PetscMalloc1(pN,&oey);CHKERRQ(ierr);
ierr = PetscMalloc1(pP,&oez);CHKERRQ(ierr);
for (i=0; i<pM; i++) {
osx[i] = olx[i] - stencil;
oex[i] = olx[i] + lx[i] + stencil;
if (osx[i]<0) osx[i]=0;
if (oex[i]>M) oex[i]=M;
}
for (i=0; i<pN; i++) {
osy[i] = oly[i] - stencil;
oey[i] = oly[i] + ly[i] + stencil;
if (osy[i]<0)osy[i]=0;
if (oey[i]>M)oey[i]=N;
}
for (i=0; i<pP; i++) {
osz[i] = olz[i] - stencil;
oez[i] = olz[i] + lz[i] + stencil;
if (osz[i]<0) osz[i]=0;
if (oez[i]>P) oez[i]=P;
}
for (k=0; k<pP; k++) {
for (j=0; j<pN; j++) {
for (i=0; i<pM; i++) {
char name[PETSC_MAX_PATH_LEN];
PetscInt procid = i + j*pM + k*pM*pN; /* convert proc(i,j,k) to pid */
ierr = PetscSNPrintf(name,sizeof(name),"subdomain-%s-p%1.4d.vti",local_file_prefix,procid);CHKERRQ(ierr);
for (II=0; II<indent_level; II++) PetscFPrintf(PETSC_COMM_SELF,vtk_fp," ");
PetscFPrintf(PETSC_COMM_SELF,vtk_fp,"<Piece Extent=\"%d %d %d %d %d %d\" Source=\"%s\"/>\n",
osx[i],oex[i]-1,
osy[j],oey[j]-1,
osz[k],oez[k]-1,name);
}
}
}
ierr = PetscFree(olx);CHKERRQ(ierr);
ierr = PetscFree(oly);CHKERRQ(ierr);
ierr = PetscFree(olz);CHKERRQ(ierr);
ierr = PetscFree(osx);CHKERRQ(ierr);
ierr = PetscFree(osy);CHKERRQ(ierr);
ierr = PetscFree(osz);CHKERRQ(ierr);
ierr = PetscFree(oex);CHKERRQ(ierr);
ierr = PetscFree(oey);CHKERRQ(ierr);
ierr = PetscFree(oez);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
DM dau,dak,pack;
const PetscInt *lxu;
PetscInt *lxk,m,sizes;
User user;
SNES snes;
Vec X,F,Xu,Xk,Fu,Fk;
Mat B;
IS *isg;
PetscBool view_draw,pass_dm;
PetscInitialize(&argc,&argv,0,help);
ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,-10,1,1,NULL,&dau);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(dau,"u_");CHKERRQ(ierr);
ierr = DMSetFromOptions(dau);CHKERRQ(ierr);
ierr = DMDAGetOwnershipRanges(dau,&lxu,0,0);CHKERRQ(ierr);
ierr = DMDAGetInfo(dau,0, &m,0,0, &sizes,0,0, 0,0,0,0,0,0);CHKERRQ(ierr);
ierr = PetscMalloc1(sizes,&lxk);CHKERRQ(ierr);
ierr = PetscMemcpy(lxk,lxu,sizes*sizeof(*lxk));CHKERRQ(ierr);
lxk[0]--;
ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,m-1,1,1,lxk,&dak);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(dak,"k_");CHKERRQ(ierr);
ierr = DMSetFromOptions(dak);CHKERRQ(ierr);
ierr = PetscFree(lxk);CHKERRQ(ierr);
ierr = DMCompositeCreate(PETSC_COMM_WORLD,&pack);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(pack,"pack_");CHKERRQ(ierr);
ierr = DMCompositeAddDM(pack,dau);CHKERRQ(ierr);
ierr = DMCompositeAddDM(pack,dak);CHKERRQ(ierr);
ierr = DMDASetFieldName(dau,0,"u");CHKERRQ(ierr);
ierr = DMDASetFieldName(dak,0,"k");CHKERRQ(ierr);
ierr = DMSetFromOptions(pack);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(pack,&X);CHKERRQ(ierr);
ierr = VecDuplicate(X,&F);CHKERRQ(ierr);
ierr = PetscNew(&user);CHKERRQ(ierr);
user->pack = pack;
ierr = DMCompositeGetGlobalISs(pack,&isg);CHKERRQ(ierr);
ierr = DMCompositeGetLocalVectors(pack,&user->Uloc,&user->Kloc);CHKERRQ(ierr);
ierr = DMCompositeScatter(pack,X,user->Uloc,user->Kloc);CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"Coupled problem options","SNES");CHKERRQ(ierr);
{
user->ptype = 0; view_draw = PETSC_FALSE; pass_dm = PETSC_TRUE;
ierr = PetscOptionsInt("-problem_type","0: solve for u only, 1: solve for k only, 2: solve for both",0,user->ptype,&user->ptype,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-view_draw","Draw the final coupled solution regardless of whether only one physics was solved",0,view_draw,&view_draw,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-pass_dm","Pass the packed DM to SNES to use when determining splits and forward into splits",0,pass_dm,&pass_dm,NULL);CHKERRQ(ierr);
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = FormInitial_Coupled(user,X);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
switch (user->ptype) {
case 0:
ierr = DMCompositeGetAccess(pack,X,&Xu,0);CHKERRQ(ierr);
ierr = DMCompositeGetAccess(pack,F,&Fu,0);CHKERRQ(ierr);
ierr = DMCreateMatrix(dau,&B);CHKERRQ(ierr);
ierr = SNESSetFunction(snes,Fu,FormFunction_All,user);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes,B,B,FormJacobian_All,user);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = SNESSetDM(snes,dau);CHKERRQ(ierr);
ierr = SNESSolve(snes,NULL,Xu);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(pack,X,&Xu,0);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(pack,F,&Fu,0);CHKERRQ(ierr);
break;
case 1:
ierr = DMCompositeGetAccess(pack,X,0,&Xk);CHKERRQ(ierr);
ierr = DMCompositeGetAccess(pack,F,0,&Fk);CHKERRQ(ierr);
ierr = DMCreateMatrix(dak,&B);CHKERRQ(ierr);
ierr = SNESSetFunction(snes,Fk,FormFunction_All,user);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes,B,B,FormJacobian_All,user);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = SNESSetDM(snes,dak);CHKERRQ(ierr);
ierr = SNESSolve(snes,NULL,Xk);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(pack,X,0,&Xk);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(pack,F,0,&Fk);CHKERRQ(ierr);
break;
case 2:
ierr = DMCreateMatrix(pack,&B);CHKERRQ(ierr);
/* This example does not correctly allocate off-diagonal blocks. These options allows new nonzeros (slow). */
ierr = MatSetOption(B,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = SNESSetFunction(snes,F,FormFunction_All,user);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes,B,B,FormJacobian_All,user);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
if (!pass_dm) { /* Manually provide index sets and names for the splits */
KSP ksp;
PC pc;
ierr = SNESGetKSP(snes,&ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc,"u",isg[0]);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc,"k",isg[1]);CHKERRQ(ierr);
} else {
/* The same names come from the options prefix for dau and dak. This option can support geometric multigrid inside
* of splits, but it requires using a DM (perhaps your own implementation). */
ierr = SNESSetDM(snes,pack);CHKERRQ(ierr);
}
ierr = SNESSolve(snes,NULL,X);CHKERRQ(ierr);
break;
}
if (view_draw) {ierr = VecView(X,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);}
if (0) {
PetscInt col = 0;
PetscBool mult_dup = PETSC_FALSE,view_dup = PETSC_FALSE;
Mat D;
Vec Y;
ierr = PetscOptionsGetInt(0,"-col",&col,0);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(0,"-mult_dup",&mult_dup,0);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(0,"-view_dup",&view_dup,0);CHKERRQ(ierr);
ierr = VecDuplicate(X,&Y);CHKERRQ(ierr);
/* ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); */
/* ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); */
ierr = MatConvert(B,MATAIJ,MAT_INITIAL_MATRIX,&D);CHKERRQ(ierr);
ierr = VecZeroEntries(X);CHKERRQ(ierr);
ierr = VecSetValue(X,col,1.0,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
ierr = MatMult(mult_dup ? D : B,X,Y);CHKERRQ(ierr);
ierr = MatView(view_dup ? D : B,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* ierr = VecView(X,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = VecView(Y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&D);CHKERRQ(ierr);
ierr = VecDestroy(&Y);CHKERRQ(ierr);
}
ierr = DMCompositeRestoreLocalVectors(pack,&user->Uloc,&user->Kloc);CHKERRQ(ierr);
ierr = PetscFree(user);CHKERRQ(ierr);
ierr = ISDestroy(&isg[0]);CHKERRQ(ierr);
ierr = ISDestroy(&isg[1]);CHKERRQ(ierr);
ierr = PetscFree(isg);CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&F);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = DMDestroy(&dau);CHKERRQ(ierr);
ierr = DMDestroy(&dak);CHKERRQ(ierr);
ierr = DMDestroy(&pack);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
/*@
DMDACreatePatchIS - Creates an index set corresponding to a patch of the DA.
Not Collective
Input Parameters:
+ da - the DMDA
. lower - a matstencil with i, j and k corresponding to the lower corner of the patch
- upper - a matstencil with i, j and k corresponding to the upper corner of the patch
Output Parameters:
. is - the IS corresponding to the patch
Level: developer
.seealso: DMDACreateDomainDecomposition(), DMDACreateDomainDecompositionScatters()
@*/
PetscErrorCode DMDACreatePatchIS(DM da,MatStencil *lower,MatStencil *upper,IS *is)
{
PetscInt ms=0,ns=0,ps=0;
PetscInt me=1,ne=1,pe=1;
PetscInt mr=0,nr=0,pr=0;
PetscInt ii,jj,kk;
PetscInt si,sj,sk;
PetscInt i,j,k,l,idx;
PetscInt base;
PetscInt xm=1,ym=1,zm=1;
const PetscInt *lx,*ly,*lz;
PetscInt ox,oy,oz;
PetscInt m,n,p,M,N,P,dof;
PetscInt nindices;
PetscInt *indices;
DM_DA *dd = (DM_DA*)da->data;
PetscErrorCode ierr;
PetscFunctionBegin;
/* need to get the sizes of the actual DM rather than the "global" space of a subdomain DM */
M = dd->M;N = dd->N;P=dd->P;
m = dd->m;n = dd->n;p=dd->p;
dof = dd->w;
ierr = DMDAGetOffset(da,&ox,&oy,&oz,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = DMDAGetOwnershipRanges(da,&lx,&ly,&lz);CHKERRQ(ierr);
nindices = (upper->i - lower->i)*(upper->j - lower->j)*(upper->k - lower->k)*dof;
ierr = PetscMalloc(sizeof(PetscInt)*nindices,&indices);CHKERRQ(ierr);
/* start at index 0 on processor 0 */
mr = 0;
nr = 0;
pr = 0;
ms = 0;
ns = 0;
ps = 0;
if (lx) me = lx[0];
if (ly) ne = ly[0];
if (lz) pe = lz[0];
idx = 0;
for (k=lower->k-oz;k<upper->k-oz;k++) {
for (j=lower->j-oy;j < upper->j-oy;j++) {
for (i=lower->i-ox;i < upper->i-ox;i++) {
/* "actual" indices rather than ones outside of the domain */
ii = i;
jj = j;
kk = k;
if (ii < 0) ii = M + ii;
if (jj < 0) jj = N + jj;
if (kk < 0) kk = P + kk;
if (ii > M-1) ii = ii - M;
if (jj > N-1) jj = jj - N;
if (kk > P-1) kk = kk - P;
/* gone out of processor range on x axis */
while(ii > me-1 || ii < ms) {
if (mr == m-1) {
ms = 0;
me = lx[0];
mr = 0;
} else {
mr++;
ms = me;
me += lx[mr];
}
}
/* gone out of processor range on y axis */
while(jj > ne-1 || jj < ns) {
if (nr == n-1) {
ns = 0;
ne = ly[0];
nr = 0;
} else {
nr++;
ns = ne;
ne += ly[nr];
}
}
/* gone out of processor range on z axis */
while(kk > pe-1 || kk < ps) {
if (pr == p-1) {
ps = 0;
pe = lz[0];
pr = 0;
} else {
pr++;
ps = pe;
pe += lz[pr];
}
}
/* compute the vector base on owning processor */
xm = me - ms;
ym = ne - ns;
zm = pe - ps;
base = ms*ym*zm + ns*M*zm + ps*M*N;
/* compute the local coordinates on owning processor */
si = ii - ms;
sj = jj - ns;
sk = kk - ps;
for (l=0;l<dof;l++) {
indices[idx] = l + dof*(base + si + xm*sj + xm*ym*sk);
idx++;
}
}
}
}
ISCreateGeneral(PETSC_COMM_SELF,idx,indices,PETSC_OWN_POINTER,is);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
