/*@C
DMPlexInterpolate - Take in a cell-vertex mesh and return one with all intermediate faces, edges, etc.
Collective on DM
Input Parameters:
+ dm - The DMPlex object with only cells and vertices
- dmInt - If NULL a new DM is created, otherwise the interpolated DM is put into the given DM
Output Parameter:
. dmInt - The complete DMPlex object
Level: intermediate
.keywords: mesh
.seealso: DMPlexUninterpolate(), DMPlexCreateFromCellList()
@*/
PetscErrorCode DMPlexInterpolate(DM dm, DM *dmInt)
{
DM idm, odm = dm;
PetscSF sfPoint;
PetscInt depth, dim, d;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogEventBegin(DMPLEX_Interpolate,dm,0,0,0);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
if (dim <= 1) {
ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr);
idm = dm;
}
for (d = 1; d < dim; ++d) {
/* Create interpolated mesh */
if ((d == dim-1) && *dmInt) {idm = *dmInt;}
else {ierr = DMCreate(PetscObjectComm((PetscObject)dm), &idm);CHKERRQ(ierr);}
ierr = DMSetType(idm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(idm, dim);CHKERRQ(ierr);
if (depth > 0) {
ierr = DMPlexInterpolateFaces_Internal(odm, 1, idm);CHKERRQ(ierr);
ierr = DMGetPointSF(odm, &sfPoint);CHKERRQ(ierr);
ierr = DMPlexInterpolatePointSF(idm, sfPoint, depth);CHKERRQ(ierr);
}
if (odm != dm) {ierr = DMDestroy(&odm);CHKERRQ(ierr);}
odm = idm;
}
*dmInt = idm;
ierr = PetscLogEventEnd(DMPLEX_Interpolate,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPatchCreateGrid(MPI_Comm comm, PetscInt dim, MatStencil patchSize, MatStencil commSize, MatStencil gridSize, DM *dm)
{
DM_Patch *mesh;
DM da;
PetscInt dof = 1, width = 1;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPatchCreate(comm, dm);CHKERRQ(ierr);
mesh = (DM_Patch*) (*dm)->data;
if (dim < 2) {
gridSize.j = 1;
patchSize.j = 1;
}
if (dim < 3) {
gridSize.k = 1;
patchSize.k = 1;
}
ierr = DMCreate(comm, &da);CHKERRQ(ierr);
ierr = DMSetType(da, DMDA);CHKERRQ(ierr);
ierr = DMSetDimension(da, dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da, gridSize.i, gridSize.j, gridSize.k);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, dof);CHKERRQ(ierr);
ierr = DMDASetStencilType(da, DMDA_STENCIL_BOX);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, width);CHKERRQ(ierr);
mesh->dmCoarse = da;
ierr = DMPatchSetPatchSize(*dm, patchSize);CHKERRQ(ierr);
ierr = DMPatchSetCommSize(*dm, commSize);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode CreateHex_3D(MPI_Comm comm, DM *newdm)
{
DM dm;
PetscInt numPoints[2] = {12, 2};
PetscInt coneSize[14] = {8, 8, 0,0,0,0,0,0,0,0,0,0,0,0};
PetscInt cones[16] = {2,5,4,3,6,7,8,9, 3,4,11,10,7,12,13,8};
PetscInt coneOrientations[16] = {0,0,0,0,0,0,0,0, 0,0, 0, 0,0, 0, 0,0};
PetscScalar vertexCoords[36] = {-0.5,0.0,0.0, 0.0,0.0,0.0, 0.0,1.0,0.0, -0.5,1.0,0.0,
-0.5,0.0,1.0, 0.0,0.0,1.0, 0.0,1.0,1.0, -0.5,1.0,1.0,
0.5,0.0,0.0, 0.5,1.0,0.0, 0.5,0.0,1.0, 0.5,1.0,1.0};
PetscInt markerPoints[24] = {2,1,3,1,4,1,5,1,6,1,7,1,8,1,9,1,10,1,11,1,12,1,13,1};
PetscInt dim = 3, depth = 1, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);
ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
ierr = DMPlexCreateFromDAG(dm, depth, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
for(p = 0; p < 12; ++p) {
ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);
}
*newdm = dm;
PetscFunctionReturn(0);
}
PetscErrorCode DMCreateSubDM_DA(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
{
DM_DA *da = (DM_DA*) dm->data;
PetscSection section;
PetscErrorCode ierr;
PetscFunctionBegin;
if (subdm) {
PetscSF sf;
Vec coords;
void *ctx;
/* Cannot use DMClone since the dof stuff is mixed in. Ugh
ierr = DMClone(dm, subdm);CHKERRQ(ierr); */
ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = DMSetPointSF(*subdm, sf);CHKERRQ(ierr);
ierr = DMGetApplicationContext(dm, &ctx);CHKERRQ(ierr);
ierr = DMSetApplicationContext(*subdm, ctx);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coords);CHKERRQ(ierr);
if (coords) {
ierr = DMSetCoordinatesLocal(*subdm, coords);CHKERRQ(ierr);
} else {
ierr = DMGetCoordinates(dm, &coords);CHKERRQ(ierr);
if (coords) {ierr = DMSetCoordinates(*subdm, coords);CHKERRQ(ierr);}
}
ierr = DMSetType(*subdm, DMDA);CHKERRQ(ierr);
ierr = DMSetDimension(*subdm, dm->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(*subdm, da->M, da->N, da->P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*subdm, da->m, da->n, da->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*subdm, da->bx, da->by, da->bz);CHKERRQ(ierr);
ierr = DMDASetDof(*subdm, numFields);CHKERRQ(ierr);
ierr = DMDASetStencilType(*subdm, da->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*subdm, da->s);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*subdm, da->lx, da->ly, da->lz);CHKERRQ(ierr);
}
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
if (section) {
ierr = DMCreateSubDM_Section_Private(dm, numFields, fields, is, subdm);CHKERRQ(ierr);
} else {
if (is) {
PetscInt *indices, cnt = 0, dof = da->w, i, j;
ierr = PetscMalloc1(da->Nlocal*numFields/dof, &indices);CHKERRQ(ierr);
for (i = da->base/dof; i < (da->base+da->Nlocal)/dof; ++i) {
for (j = 0; j < numFields; ++j) {
indices[cnt++] = dof*i + fields[j];
}
}
if (cnt != da->Nlocal*numFields/dof) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Count %d does not equal expected value %d", cnt, da->Nlocal*numFields/dof);
ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), cnt, indices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
PetscErrorCode DMCreateFieldDecomposition_DA(DM dm, PetscInt *len,char ***namelist, IS **islist, DM **dmlist)
{
PetscInt i;
PetscErrorCode ierr;
DM_DA *dd = (DM_DA*)dm->data;
PetscInt dof = dd->w;
PetscFunctionBegin;
if (len) *len = dof;
if (islist) {
Vec v;
PetscInt rstart,n;
ierr = DMGetGlobalVector(dm,&v);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(v,&rstart,NULL);CHKERRQ(ierr);
ierr = VecGetLocalSize(v,&n);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dm,&v);CHKERRQ(ierr);
ierr = PetscMalloc1(dof,islist);CHKERRQ(ierr);
for (i=0; i<dof; i++) {
ierr = ISCreateStride(PetscObjectComm((PetscObject)dm),n/dof,rstart+i,dof,&(*islist)[i]);CHKERRQ(ierr);
}
}
if (namelist) {
ierr = PetscMalloc1(dof, namelist);CHKERRQ(ierr);
if (dd->fieldname) {
for (i=0; i<dof; i++) {
ierr = PetscStrallocpy(dd->fieldname[i],&(*namelist)[i]);CHKERRQ(ierr);
}
} else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Currently DMDA must have fieldnames");
}
if (dmlist) {
DM da;
ierr = DMDACreate(PetscObjectComm((PetscObject)dm), &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, dm->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da, dd->M, dd->N, dd->P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da, dd->m, dd->n, dd->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, dd->bx, dd->by, dd->bz);CHKERRQ(ierr);
ierr = DMDASetDof(da, 1);CHKERRQ(ierr);
ierr = DMDASetStencilType(da, dd->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, dd->s);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = PetscMalloc1(dof,dmlist);CHKERRQ(ierr);
for (i=0; i<dof-1; i++) {ierr = PetscObjectReference((PetscObject)da);CHKERRQ(ierr);}
for (i=0; i<dof; i++) (*dmlist)[i] = da;
}
PetscFunctionReturn(0);
}
static PetscErrorCode CreatePoints_Grid(DM dm, PetscInt *Np, PetscReal **pcoords, PetscBool *pointsAllProcs, AppCtx *ctx)
{
DM da;
DMDALocalInfo info;
PetscInt N = 3, n = 0, spaceDim, i, j, k, *ind, d;
PetscReal *h;
PetscMPIInt rank;
PetscErrorCode ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = DMGetCoordinateDim(dm, &spaceDim);CHKERRQ(ierr);
ierr = PetscCalloc1(spaceDim,&ind);CHKERRQ(ierr);
ierr = PetscCalloc1(spaceDim,&h);CHKERRQ(ierr);
h[0] = 1.0/(N-1); h[1] = 1.0/(N-1); h[2] = 1.0/(N-1);
ierr = DMDACreate(PetscObjectComm((PetscObject) dm), &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, ctx->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da, N, N, N);CHKERRQ(ierr);
ierr = DMDASetDof(da, 1);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(da, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(da, &info);CHKERRQ(ierr);
*Np = info.xm * info.ym * info.zm;
ierr = PetscCalloc1(*Np * spaceDim, pcoords);CHKERRQ(ierr);
for (k = info.zs; k < info.zs + info.zm; ++k) {
ind[2] = k;
for (j = info.ys; j < info.ys + info.ym; ++j) {
ind[1] = j;
for (i = info.xs; i < info.xs + info.xm; ++i, ++n) {
ind[0] = i;
for (d = 0; d < spaceDim; ++d) (*pcoords)[n*spaceDim+d] = ind[d]*h[d];
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d]Point %D (", rank, n);CHKERRQ(ierr);
for (d = 0; d < spaceDim; ++d) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%g", (double)(*pcoords)[n*spaceDim+d]);CHKERRQ(ierr);
if (d < spaceDim-1) {ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, ", ");CHKERRQ(ierr);}
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, ")\n");CHKERRQ(ierr);
}
}
}
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD, NULL);CHKERRQ(ierr);
ierr = PetscFree(ind);CHKERRQ(ierr);
ierr = PetscFree(h);CHKERRQ(ierr);
*pointsAllProcs = PETSC_FALSE;
PetscFunctionReturn(0);
}
PetscErrorCode DMClone_DA(DM dm, DM *newdm)
{
DM_DA *da = (DM_DA *) dm->data;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMSetType(*newdm, DMDA);CHKERRQ(ierr);
ierr = DMSetDimension(*newdm, dm->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(*newdm, da->M, da->N, da->P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*newdm, da->m, da->n, da->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*newdm, da->bx, da->by, da->bz);CHKERRQ(ierr);
ierr = DMDASetDof(*newdm, da->w);CHKERRQ(ierr);
ierr = DMDASetStencilType(*newdm, da->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*newdm, da->s);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*newdm, da->lx, da->ly, da->lz);CHKERRQ(ierr);
ierr = DMSetUp(*newdm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMDACreate1d - Creates an object that will manage the communication of one-dimensional
regular array data that is distributed across some processors.
Collective on MPI_Comm
Input Parameters:
+ comm - MPI communicator
. bx - type of ghost cells at the boundary the array should have, if any. Use
DM_BOUNDARY_NONE, DM_BOUNDARY_GHOSTED, or DM_BOUNDARY_PERIODIC.
. M - global dimension of the array (use -M to indicate that it may be set to a different value
from the command line with -da_grid_x <M>)
. dof - number of degrees of freedom per node
. s - stencil width
- lx - array containing number of nodes in the X direction on each processor,
or NULL. If non-null, must be of length as the number of processes in the MPI_Comm.
Output Parameter:
. da - the resulting distributed array object
Options Database Key:
+ -dm_view - Calls DMView() at the conclusion of DMDACreate1d()
. -da_grid_x <nx> - number of grid points in x direction; can set if M < 0
. -da_refine_x <rx> - refinement factor
- -da_refine <n> - refine the DMDA n times before creating it, if M < 0
Level: beginner
Notes:
The array data itself is NOT stored in the DMDA, it is stored in Vec objects;
The appropriate vector objects can be obtained with calls to DMCreateGlobalVector()
and DMCreateLocalVector() and calls to VecDuplicate() if more are needed.
.keywords: distributed array, create, one-dimensional
.seealso: DMDestroy(), DMView(), DMDACreate2d(), DMDACreate3d(), DMGlobalToLocalBegin(), DMDASetRefinementFactor(),
DMGlobalToLocalEnd(), DMLocalToGlobalBegin(), DMLocalToLocalBegin(), DMLocalToLocalEnd(), DMDAGetRefinementFactor(),
DMDAGetInfo(), DMCreateGlobalVector(), DMCreateLocalVector(), DMDACreateNaturalVector(), DMLoad(), DMDAGetOwnershipRanges()
@*/
PetscErrorCode DMDACreate1d(MPI_Comm comm, DMBoundaryType bx, PetscInt M, PetscInt dof, PetscInt s, const PetscInt lx[], DM *da)
{
PetscErrorCode ierr;
PetscMPIInt size;
PetscFunctionBegin;
ierr = DMDACreate(comm, da);CHKERRQ(ierr);
ierr = DMSetDimension(*da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(*da, M, 1, 1);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*da, size, PETSC_DECIDE, PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*da, bx, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(*da, dof);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*da, s);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*da, lx, NULL, NULL);CHKERRQ(ierr);
/* This violates the behavior for other classes, but right now users expect negative dimensions to be handled this way */
ierr = DMSetFromOptions(*da);CHKERRQ(ierr);
ierr = DMSetUp(*da);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMLoad_DA(DM da,PetscViewer viewer)
{
PetscErrorCode ierr;
PetscInt dim,m,n,p,dof,swidth;
DMDAStencilType stencil;
DMBoundaryType bx,by,bz;
PetscBool coors;
DM dac;
Vec c;
PetscFunctionBegin;
ierr = PetscViewerBinaryRead(viewer,&dim,1,NULL,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&m,1,NULL,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&n,1,NULL,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&p,1,NULL,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&dof,1,NULL,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&swidth,1,NULL,PETSC_INT);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&bx,1,NULL,PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&by,1,NULL,PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&bz,1,NULL,PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&stencil,1,NULL,PETSC_ENUM);CHKERRQ(ierr);
ierr = DMSetDimension(da, dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da, m,n,p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, bx, by, bz);CHKERRQ(ierr);
ierr = DMDASetDof(da, dof);CHKERRQ(ierr);
ierr = DMDASetStencilType(da, stencil);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, swidth);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = PetscViewerBinaryRead(viewer,&coors,1,NULL,PETSC_ENUM);CHKERRQ(ierr);
if (coors) {
ierr = DMGetCoordinateDM(da,&dac);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dac,&c);CHKERRQ(ierr);
ierr = VecLoad(c,viewer);CHKERRQ(ierr);
ierr = DMSetCoordinates(da,c);CHKERRQ(ierr);
ierr = VecDestroy(&c);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@C
DMStagCreate1d - Create an object to manage data living on the faces, edges, and vertices of a parallelized regular 1D grid.
Collective
Input Parameters:
+ comm - MPI communicator
. bndx - boundary type: DM_BOUNDARY_NONE, DM_BOUNDARY_PERIODIC, or DM_BOUNDARY_GHOSTED
. M - global number of grid points
. dof0 - number of degrees of freedom per vertex/point/node/0-cell
. dof1 - number of degrees of freedom per element/edge/1-cell
. stencilType - ghost/halo region type: DMSTAG_STENCIL_BOX or DMSTAG_STENCIL_NONE
. stencilWidth - width, in elements, of halo/ghost region
- lx - array of local sizes, of length equal to the comm size, summing to M
Output Parameter:
. dm - the new DMStag object
Options Database Keys:
+ -dm_view - calls DMViewFromOptions() a the conclusion of DMSetUp()
. -stag_grid_x <nx> - number of elements in the x direction
. -stag_ghost_stencil_width - width of ghost region, in elements
- -stag_boundary_type_x <none,ghosted,periodic> - DMBoundaryType value
Notes:
You must call DMSetUp() after this call before using the DM.
If you wish to use the options database (see the keys above) to change values in the DMStag, you must call
DMSetFromOptions() after this function but before DMSetUp().
Level: beginner
.seealso: DMSTAG, DMStagCreate2d(), DMStagCreate3d(), DMDestroy(), DMView(), DMCreateGlobalVector(), DMCreateLocalVector(), DMLocalToGlobalBegin()
@*/
PETSC_EXTERN PetscErrorCode DMStagCreate1d(MPI_Comm comm,DMBoundaryType bndx,PetscInt M,PetscInt dof0,PetscInt dof1,DMStagStencilType stencilType,PetscInt stencilWidth,const PetscInt lx[],DM* dm)
{
PetscErrorCode ierr;
DM_Stag *stag;
PetscMPIInt size;
PetscFunctionBegin;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = DMCreate(comm,dm);CHKERRQ(ierr);
ierr = DMSetDimension(*dm,1);CHKERRQ(ierr); /* Must precede DMSetType */
ierr = DMSetType(*dm,DMSTAG);CHKERRQ(ierr);
stag = (DM_Stag*)(*dm)->data;
/* Global sizes and flags (derived quantities set in DMSetUp_Stag) */
stag->boundaryType[0] = bndx;
stag->N[0] = M;
stag->nRanks[0] = size;
stag->stencilType = stencilType;
stag->stencilWidth = stencilWidth;
ierr = DMStagSetDOF(*dm,dof0,dof1,0,0);CHKERRQ(ierr);
ierr = DMStagSetOwnershipRanges(*dm,lx,NULL,NULL);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode CreateSimplex_2D(MPI_Comm comm, DM *newdm)
{
DM dm;
PetscInt numPoints[2] = {4, 2};
PetscInt coneSize[6] = {3, 3, 0, 0, 0, 0};
PetscInt cones[6] = {2, 3, 4, 5, 4, 3};
PetscInt coneOrientations[6] = {0, 0, 0, 0, 0, 0};
PetscScalar vertexCoords[8] = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
PetscInt markerPoints[8] = {2, 1, 3, 1, 4, 1, 5, 1};
PetscInt dim = 2, depth = 1, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);
ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(dm, dim);CHKERRQ(ierr);
ierr = DMPlexCreateFromDAG(dm, depth, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
for (p = 0; p < 4; ++p) {
ierr = DMSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);
}
*newdm = dm;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
KSP ksp;
PC pc;
Vec x,b;
DM da;
Mat A,Atrans;
PetscInt dof=1,M=8;
PetscBool flg,trans=PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-trans",&trans,NULL);CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD,&da);CHKERRQ(ierr);
ierr = DMSetDimension(da,3);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetStencilType(da,DMDA_STENCIL_STAR);CHKERRQ(ierr);
ierr = DMDASetSizes(da,M,M,M);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetDof(da,dof);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da,1);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&x);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&b);CHKERRQ(ierr);
ierr = ComputeRHS(da,b);CHKERRQ(ierr);
ierr = DMSetMatType(da,MATBAIJ);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,&A);CHKERRQ(ierr);
ierr = ComputeMatrix(da,A);CHKERRQ(ierr);
/* A is non-symmetric. Make A = 0.5*(A + Atrans) symmetric for testing icc and cholesky */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX,&Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,1.0,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatScale(A,0.5);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* Test sbaij matrix */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL, "-test_sbaij1", &flg,NULL);CHKERRQ(ierr);
if (flg) {
Mat sA;
PetscBool issymm;
ierr = MatIsTranspose(A,A,0.0,&issymm);CHKERRQ(ierr);
if (issymm) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else {ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: A is non-symmetric\n");CHKERRQ(ierr);}
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
A = sA;
}
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetDM(pc,(DM)da);CHKERRQ(ierr);
if (trans) {
ierr = KSPSolveTranspose(ksp,b,x);CHKERRQ(ierr);
} else {
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
}
/* check final residual */
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL, "-check_final_residual", &flg,NULL);CHKERRQ(ierr);
if (flg) {
Vec b1;
PetscReal norm;
ierr = KSPGetSolution(ksp,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&b1);CHKERRQ(ierr);
ierr = MatMult(A,x,b1);CHKERRQ(ierr);
ierr = VecAXPY(b1,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(b1,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Final residual %g\n",norm);CHKERRQ(ierr);
ierr = VecDestroy(&b1);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscBool cellHybrid = user->cellHybrid;
PetscBool cellSimplex = user->cellSimplex;
PetscMPIInt rank, size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
switch (dim) {
case 1:
if (cellHybrid) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make hybrid meshes for dimension %d", dim);
ierr = CreateSimplex_1D(comm, dm);CHKERRQ(ierr);
break;
case 2:
if (cellSimplex) {
if (cellHybrid) {
ierr = CreateSimplexHybrid_2D(comm, user->testNum, dm);CHKERRQ(ierr);
} else {
ierr = CreateSimplex_2D(comm, dm);CHKERRQ(ierr);
}
} else {
if (cellHybrid) {
ierr = CreateTensorProductHybrid_2D(comm, user->testNum, dm);CHKERRQ(ierr);
} else {
ierr = CreateTensorProduct_2D(comm, user->testNum, dm);CHKERRQ(ierr);
}
}
break;
case 3:
if (cellSimplex) {
if (cellHybrid) {
ierr = CreateSimplexHybrid_3D(comm, user->testNum, dm);CHKERRQ(ierr);
} else {
ierr = CreateSimplex_3D(comm, user->testNum, dm);CHKERRQ(ierr);
}
} else {
if (cellHybrid) {
ierr = CreateTensorProductHybrid_3D(comm, user->testNum, dm);CHKERRQ(ierr);
} else {
ierr = CreateTensorProduct_3D(comm, user->testNum, dm);CHKERRQ(ierr);
}
}
break;
default:
SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
}
if (user->testPartition && size > 1) {
PetscPartitioner part;
PetscInt *sizes = NULL;
PetscInt *points = NULL;
if (!rank) {
if (dim == 2 && cellSimplex && !cellHybrid && size == 2) {
switch (user->testNum) {
case 0: {
PetscInt triSizes_p2[2] = {1, 1};
PetscInt triPoints_p2[2] = {0, 1};
ierr = PetscMalloc2(2, &sizes, 2, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, triSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, triPoints_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);break;}
default:
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test number %d for triangular mesh on 2 procs", user->testNum);
}
} else if (dim == 2 && cellSimplex && cellHybrid && size == 2) {
switch (user->testNum) {
case 0: {
PetscInt triSizes_p2[2] = {1, 2};
PetscInt triPoints_p2[3] = {0, 1, 2};
ierr = PetscMalloc2(2, &sizes, 3, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, triSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, triPoints_p2, 3 * sizeof(PetscInt));CHKERRQ(ierr);break;}
default:
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test number %d for triangular hybrid mesh on 2 procs", user->testNum);
}
} else if (dim == 2 && !cellSimplex && !cellHybrid && size == 2) {
switch (user->testNum) {
case 0: {
PetscInt quadSizes_p2[2] = {1, 1};
PetscInt quadPoints_p2[2] = {0, 1};
ierr = PetscMalloc2(2, &sizes, 2, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, quadSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, quadPoints_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);break;}
default:
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test number %d for quadrilateral mesh on 2 procs", user->testNum);
}
} else if (dim == 2 && !cellSimplex && cellHybrid && size == 2) {
switch (user->testNum) {
case 0: {
PetscInt quadSizes_p2[2] = {1, 2};
PetscInt quadPoints_p2[3] = {0, 1, 2};
ierr = PetscMalloc2(2, &sizes, 3, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, quadSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, quadPoints_p2, 3 * sizeof(PetscInt));CHKERRQ(ierr);break;}
default:
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test number %d for quadrilateral hybrid mesh on 2 procs", user->testNum);
}
} else if (dim == 3 && cellSimplex && !cellHybrid && size == 2) {
switch (user->testNum) {
case 0: {
PetscInt tetSizes_p2[2] = {1, 1};
PetscInt tetPoints_p2[2] = {0, 1};
ierr = PetscMalloc2(2, &sizes, 2, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, tetSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, tetPoints_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);break;}
case 1: {
PetscInt tetSizes_p2[2] = {1, 1};
PetscInt tetPoints_p2[2] = {0, 1};
ierr = PetscMalloc2(2, &sizes, 2, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, tetSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, tetPoints_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);break;}
default:
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test number %d for tetrahedral mesh on 2 procs", user->testNum);
}
} else if (dim == 3 && cellSimplex && cellHybrid && size == 2) {
switch (user->testNum) {
case 0: {
PetscInt tetSizes_p2[2] = {1, 2};
PetscInt tetPoints_p2[3] = {0, 1, 2};
ierr = PetscMalloc2(2, &sizes, 3, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, tetSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, tetPoints_p2, 3 * sizeof(PetscInt));CHKERRQ(ierr);break;}
case 1: {
PetscInt tetSizes_p2[2] = {3, 4};
PetscInt tetPoints_p2[7] = {0, 3, 5, 1, 2, 4, 6};
ierr = PetscMalloc2(2, &sizes, 7, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, tetSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, tetPoints_p2, 7 * sizeof(PetscInt));CHKERRQ(ierr);break;}
default:
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test number %d for tetrahedral hybrid mesh on 2 procs", user->testNum);
}
} else if (dim == 3 && !cellSimplex && !cellHybrid && size == 2) {
switch (user->testNum) {
case 0: {
PetscInt hexSizes_p2[2] = {1, 1};
PetscInt hexPoints_p2[2] = {0, 1};
ierr = PetscMalloc2(2, &sizes, 2, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, hexSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, hexPoints_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);break;}
default:
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test number %d for hexahedral mesh on 2 procs", user->testNum);
}
} else if (dim == 3 && !cellSimplex && cellHybrid && size == 2) {
switch (user->testNum) {
case 0: {
PetscInt hexSizes_p2[2] = {1, 1};
PetscInt hexPoints_p2[2] = {0, 1};
ierr = PetscMalloc2(2, &sizes, 2, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, hexSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, hexPoints_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);break;}
case 1: {
PetscInt hexSizes_p2[2] = {5, 4};
PetscInt hexPoints_p2[9] = {3, 4, 5, 7, 8, 0, 1, 2, 6};
ierr = PetscMalloc2(2, &sizes, 9, &points);CHKERRQ(ierr);
ierr = PetscMemcpy(sizes, hexSizes_p2, 2 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(points, hexPoints_p2, 9 * sizeof(PetscInt));CHKERRQ(ierr);break;}
default:
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test number %d for hexahedral hybrid mesh on 2 procs", user->testNum);
}
} else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Could not find matching test partition");
}
ierr = DMPlexGetPartitioner(*dm, &part);CHKERRQ(ierr);
ierr = PetscPartitionerSetType(part, PETSCPARTITIONERSHELL);CHKERRQ(ierr);
ierr = PetscPartitionerShellSetPartition(part, size, sizes, points);CHKERRQ(ierr);
ierr = PetscFree2(sizes, points);CHKERRQ(ierr);
} else {
PetscPartitioner part;
ierr = DMPlexGetPartitioner(*dm,&part);CHKERRQ(ierr);
ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
}
{
DM pdm = NULL;
ierr = DMPlexDistribute(*dm, 0, NULL, &pdm);CHKERRQ(ierr);
if (pdm) {
ierr = DMViewFromOptions(pdm, NULL, "-dm_view");CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = pdm;
}
}
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
if (user->simplex2tensor) {
DM rdm = NULL;
ierr = DMPlexSetRefinementUniform(*dm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMPlexRefineSimplexToTensor(*dm, &rdm);CHKERRQ(ierr);
if (rdm) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = rdm;
}
user->cellSimplex = PETSC_FALSE;
}
if (user->uninterpolate || user->reinterpolate) {
DM udm = NULL;
ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
ierr = DMPlexCopyCoordinates(*dm, udm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = udm;
}
if (user->reinterpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMPlexCopyCoordinates(*dm, idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
ierr = PetscObjectSetName((PetscObject) *dm, "Hybrid Mesh");CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
ierr = PetscObjectSetOptionsPrefix((PetscObject) *dm, "hyb_");CHKERRQ(ierr);
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da;
/* Initialize the Petsc context */
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 1);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 1D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"1D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 1);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p1d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 2D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"2D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 2);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p2d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_none */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_NONE\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"n3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_ghosted */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_GHOSTED\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"g3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* Build of the DMDA -- 3D -- boundary_periodic */
ierr = PetscPrintf(PETSC_COMM_WORLD,"3D -- DM_BOUNDARY_PERIODIC\n");CHKERRQ(ierr);
ierr = DMDACreate(PETSC_COMM_WORLD, &da);CHKERRQ(ierr);
ierr = DMSetDimension(da, 3);CHKERRQ(ierr);
ierr = DMDASetSizes(da, -8, -8, -8);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC, DM_BOUNDARY_PERIODIC);CHKERRQ(ierr);
ierr = DMDASetDof(da, 2);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da, 1);CHKERRQ(ierr);
ierr = DMDASetOverlap(da,1,1,1);CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da,"p3d_");CHKERRQ(ierr);
ierr = DMSetFromOptions(da);CHKERRQ(ierr);
ierr = DMSetUp(da);CHKERRQ(ierr);
ierr = DMView(da,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
/* test moving data in and out */
ierr = PetscFinalize();
return 0;
}
/*@
DMPlexUninterpolate - Take in a mesh with all intermediate faces, edges, etc. and return a cell-vertex mesh
Collective on DM
Input Parameter:
. dm - The complete DMPlex object
Output Parameter:
. dmUnint - The DMPlex object with only cells and vertices
Level: intermediate
.keywords: mesh
.seealso: DMPlexInterpolate(), DMPlexCreateFromCellList()
@*/
PetscErrorCode DMPlexUninterpolate(DM dm, DM *dmUnint)
{
DM udm;
PetscInt dim, vStart, vEnd, cStart, cEnd, c, maxConeSize = 0, *cone;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
if (dim <= 1) {
ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr);
*dmUnint = dm;
PetscFunctionReturn(0);
}
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMCreate(PetscObjectComm((PetscObject) dm), &udm);CHKERRQ(ierr);
ierr = DMSetType(udm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(udm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(udm, cStart, vEnd);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {
PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
const PetscInt p = closure[cl];
if ((p >= vStart) && (p < vEnd)) ++coneSize;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
ierr = DMPlexSetConeSize(udm, c, coneSize);CHKERRQ(ierr);
maxConeSize = PetscMax(maxConeSize, coneSize);
}
ierr = DMSetUp(udm);CHKERRQ(ierr);
ierr = PetscMalloc1(maxConeSize, &cone);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {
PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
const PetscInt p = closure[cl];
if ((p >= vStart) && (p < vEnd)) cone[coneSize++] = p;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
ierr = DMPlexSetCone(udm, c, cone);CHKERRQ(ierr);
}
ierr = PetscFree(cone);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(udm);CHKERRQ(ierr);
ierr = DMPlexStratify(udm);CHKERRQ(ierr);
/* Reduce SF */
{
PetscSF sfPoint, sfPointUn;
const PetscSFNode *remotePoints;
const PetscInt *localPoints;
PetscSFNode *remotePointsUn;
PetscInt *localPointsUn;
PetscInt vEnd, numRoots, numLeaves, l;
PetscInt numLeavesUn = 0, n = 0;
PetscErrorCode ierr;
/* Get original SF information */
ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
ierr = DMGetPointSF(udm, &sfPointUn);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, NULL, &vEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
/* Allocate space for cells and vertices */
for (l = 0; l < numLeaves; ++l) if (localPoints[l] < vEnd) numLeavesUn++;
/* Fill in leaves */
if (vEnd >= 0) {
ierr = PetscMalloc1(numLeavesUn, &remotePointsUn);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeavesUn, &localPointsUn);CHKERRQ(ierr);
for (l = 0; l < numLeaves; l++) {
if (localPoints[l] < vEnd) {
localPointsUn[n] = localPoints[l];
remotePointsUn[n].rank = remotePoints[l].rank;
remotePointsUn[n].index = remotePoints[l].index;
++n;
}
}
if (n != numLeavesUn) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Inconsistent number of leaves %d != %d", n, numLeavesUn);
ierr = PetscSFSetGraph(sfPointUn, vEnd, numLeavesUn, localPointsUn, PETSC_OWN_POINTER, remotePointsUn, PETSC_OWN_POINTER);CHKERRQ(ierr);
}
}
*dmUnint = udm;
PetscFunctionReturn(0);
}
int
main (int argc, char **argv)
{
sc_MPI_Comm mpicomm;
int mpiret;
int mpisize, mpirank;
p4est_t *p4est;
p4est_connectivity_t *conn;
sc_array_t *points_per_dim, *cone_sizes, *cones,
*cone_orientations, *coords,
*children, *parents, *childids, *leaves, *remotes;
p4est_locidx_t first_local_quad = -1;
/* initialize MPI */
mpiret = sc_MPI_Init (&argc, &argv);
SC_CHECK_MPI (mpiret);
mpicomm = sc_MPI_COMM_WORLD;
mpiret = sc_MPI_Comm_size (mpicomm, &mpisize);
SC_CHECK_MPI (mpiret);
mpiret = sc_MPI_Comm_rank (mpicomm, &mpirank);
SC_CHECK_MPI (mpiret);
sc_init (mpicomm, 1, 1, NULL, SC_LP_DEFAULT);
p4est_init (NULL, SC_LP_DEFAULT);
#ifndef P4_TO_P8
conn = p4est_connectivity_new_moebius ();
#else
conn = p8est_connectivity_new_rotcubes ();
#endif
p4est = p4est_new_ext (mpicomm, conn, 0, 1, 1, 0, NULL, NULL);
p4est_refine (p4est, 1, refine_fn, NULL);
p4est_balance (p4est, P4EST_CONNECT_FULL, NULL);
p4est_partition (p4est, 0, NULL);
points_per_dim = sc_array_new (sizeof (p4est_locidx_t));
cone_sizes = sc_array_new (sizeof (p4est_locidx_t));
cones = sc_array_new (sizeof (p4est_locidx_t));
cone_orientations = sc_array_new (sizeof (p4est_locidx_t));
coords = sc_array_new (3 * sizeof (double));
children = sc_array_new (sizeof (p4est_locidx_t));
parents = sc_array_new (sizeof (p4est_locidx_t));
childids = sc_array_new (sizeof (p4est_locidx_t));
leaves = sc_array_new (sizeof (p4est_locidx_t));
remotes = sc_array_new (2 * sizeof (p4est_locidx_t));
p4est_get_plex_data (p4est, P4EST_CONNECT_FULL, (mpisize > 1) ? 2 : 0,
&first_local_quad, points_per_dim, cone_sizes, cones,
cone_orientations, coords, children, parents, childids,
leaves, remotes);
#ifdef P4EST_WITH_PETSC
{
PetscErrorCode ierr;
DM plex, refTree;
PetscInt pStart, pEnd;
PetscSection parentSection;
PetscSF pointSF;
size_t zz, count;
locidx_to_PetscInt (points_per_dim);
locidx_to_PetscInt (cone_sizes);
locidx_to_PetscInt (cones);
locidx_to_PetscInt (cone_orientations);
coords_double_to_PetscScalar (coords);
locidx_to_PetscInt (children);
locidx_to_PetscInt (parents);
locidx_to_PetscInt (childids);
locidx_to_PetscInt (leaves);
locidx_pair_to_PetscSFNode (remotes);
P4EST_GLOBAL_PRODUCTION ("Begin PETSc routines\n");
ierr = PetscInitialize (&argc, &argv, 0, help);
CHKERRQ (ierr);
ierr = DMPlexCreate (mpicomm, &plex);
CHKERRQ (ierr);
ierr = DMSetDimension (plex, P4EST_DIM);
CHKERRQ (ierr);
ierr = DMSetCoordinateDim (plex, 3);
CHKERRQ (ierr);
ierr = DMPlexCreateFromDAG (plex, P4EST_DIM,
(PetscInt *) points_per_dim->array,
(PetscInt *) cone_sizes->array,
(PetscInt *) cones->array,
(PetscInt *) cone_orientations->array,
(PetscScalar *) coords->array);
CHKERRQ (ierr);
ierr = PetscSFCreate (mpicomm, &pointSF);
CHKERRQ (ierr);
ierr =
DMPlexCreateDefaultReferenceTree (mpicomm, P4EST_DIM, PETSC_FALSE,
&refTree);
CHKERRQ (ierr);
ierr = DMPlexSetReferenceTree (plex, refTree);
CHKERRQ (ierr);
ierr = DMDestroy (&refTree);
CHKERRQ (ierr);
ierr = PetscSectionCreate (mpicomm, &parentSection);
CHKERRQ (ierr);
ierr = DMPlexGetChart (plex, &pStart, &pEnd);
CHKERRQ (ierr);
ierr = PetscSectionSetChart (parentSection, pStart, pEnd);
CHKERRQ (ierr);
count = children->elem_count;
for (zz = 0; zz < count; zz++) {
PetscInt child =
*((PetscInt *) sc_array_index (children, zz));
ierr = PetscSectionSetDof (parentSection, child, 1);
CHKERRQ (ierr);
}
ierr = PetscSectionSetUp (parentSection);
CHKERRQ (ierr);
ierr =
DMPlexSetTree (plex, parentSection, (PetscInt *) parents->array,
(PetscInt *) childids->array);
CHKERRQ (ierr);
ierr = PetscSectionDestroy (&parentSection);
CHKERRQ (ierr);
ierr =
PetscSFSetGraph (pointSF, pEnd - pStart, (PetscInt) leaves->elem_count,
(PetscInt *) leaves->array, PETSC_COPY_VALUES,
(PetscSFNode *) remotes->array, PETSC_COPY_VALUES);
CHKERRQ (ierr);
ierr = DMViewFromOptions (plex, NULL, "-dm_view");
CHKERRQ (ierr);
/* TODO: test with rigid body modes as in plex ex3 */
ierr = DMDestroy (&plex);
CHKERRQ (ierr);
ierr = PetscFinalize ();
P4EST_GLOBAL_PRODUCTION ("End PETSc routines\n");
}
#endif
sc_array_destroy (points_per_dim);
sc_array_destroy (cone_sizes);
sc_array_destroy (cones);
sc_array_destroy (cone_orientations);
sc_array_destroy (coords);
sc_array_destroy (children);
sc_array_destroy (parents);
sc_array_destroy (childids);
sc_array_destroy (leaves);
sc_array_destroy (remotes);
p4est_destroy (p4est);
p4est_connectivity_destroy (conn);
sc_finalize ();
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
/*
DMPatchZoom - Create a version of the coarse patch (identified by rank) with halo on communicator commz
Collective on DM
Input Parameters:
+ dm - the DM
. rank - the rank which holds the given patch
- commz - the new communicator for the patch
Output Parameters:
+ dmz - the patch DM
. sfz - the PetscSF mapping the patch+halo to the zoomed version
. sfzr - the PetscSF mapping the patch to the restricted zoomed version
Level: intermediate
Note: All processes in commz should have the same rank (could autosplit comm)
.seealso: DMPatchSolve()
*/
PetscErrorCode DMPatchZoom(DM dm, Vec X, MatStencil lower, MatStencil upper, MPI_Comm commz, DM *dmz, PetscSF *sfz, PetscSF *sfzr)
{
DMDAStencilType st;
MatStencil blower, bupper, loclower, locupper;
IS is;
const PetscInt *ranges, *indices;
PetscInt *localPoints = NULL;
PetscSFNode *remotePoints = NULL;
PetscInt dim, dof;
PetscInt M, N, P, rM, rN, rP, halo = 1, sxb, syb, szb, sxr, syr, szr, exr, eyr, ezr, mxb, myb, mzb, i, j, k, q;
PetscMPIInt size;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size);CHKERRQ(ierr);
/* Create patch DM */
ierr = DMDAGetInfo(dm, &dim, &M, &N, &P, 0,0,0, &dof, 0,0,0,0, &st);CHKERRQ(ierr);
/* Get piece for rank r, expanded by halo */
bupper.i = PetscMin(M, upper.i + halo); blower.i = PetscMax(lower.i - halo, 0);
bupper.j = PetscMin(N, upper.j + halo); blower.j = PetscMax(lower.j - halo, 0);
bupper.k = PetscMin(P, upper.k + halo); blower.k = PetscMax(lower.k - halo, 0);
rM = bupper.i - blower.i;
rN = bupper.j - blower.j;
rP = bupper.k - blower.k;
if (commz != MPI_COMM_NULL) {
ierr = DMDACreate(commz, dmz);CHKERRQ(ierr);
ierr = DMSetDimension(*dmz, dim);CHKERRQ(ierr);
ierr = DMDASetSizes(*dmz, rM, rN, rP);CHKERRQ(ierr);
ierr = DMDASetNumProcs(*dmz, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(*dmz, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE);CHKERRQ(ierr);
ierr = DMDASetDof(*dmz, dof);CHKERRQ(ierr);
ierr = DMDASetStencilType(*dmz, st);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(*dmz, 0);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(*dmz, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = DMSetFromOptions(*dmz);CHKERRQ(ierr);
ierr = DMSetUp(*dmz);CHKERRQ(ierr);
ierr = DMDAGetCorners(*dmz, &sxb, &syb, &szb, &mxb, &myb, &mzb);CHKERRQ(ierr);
sxr = PetscMax(sxb, lower.i - blower.i);
syr = PetscMax(syb, lower.j - blower.j);
szr = PetscMax(szb, lower.k - blower.k);
exr = PetscMin(sxb+mxb, upper.i - blower.i);
eyr = PetscMin(syb+myb, upper.j - blower.j);
ezr = PetscMin(szb+mzb, upper.k - blower.k);
ierr = PetscMalloc2(rM*rN*rP,&localPoints,rM*rN*rP,&remotePoints);CHKERRQ(ierr);
} else {
sxr = syr = szr = exr = eyr = ezr = sxb = syb = szb = mxb = myb = mzb = 0;
}
/* Create SF for restricted map */
ierr = VecGetOwnershipRanges(X,&ranges);CHKERRQ(ierr);
loclower.i = blower.i + sxr; locupper.i = blower.i + exr;
loclower.j = blower.j + syr; locupper.j = blower.j + eyr;
loclower.k = blower.k + szr; locupper.k = blower.k + ezr;
ierr = DMDACreatePatchIS(dm, &loclower, &locupper, &is);CHKERRQ(ierr);
ierr = ISGetIndices(is, &indices);CHKERRQ(ierr);
q = 0;
for (k = szb; k < szb+mzb; ++k) {
if ((k < szr) || (k >= ezr)) continue;
for (j = syb; j < syb+myb; ++j) {
if ((j < syr) || (j >= eyr)) continue;
for (i = sxb; i < sxb+mxb; ++i) {
const PetscInt lp = ((k-szb)*rN + (j-syb))*rM + i-sxb;
PetscInt r;
if ((i < sxr) || (i >= exr)) continue;
localPoints[q] = lp;
ierr = PetscFindInt(indices[q], size+1, ranges, &r);CHKERRQ(ierr);
remotePoints[q].rank = r < 0 ? -(r+1) - 1 : r;
remotePoints[q].index = indices[q] - ranges[remotePoints[q].rank];
++q;
}
}
}
ierr = ISRestoreIndices(is, &indices);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfzr);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *sfzr, "Restricted Map");CHKERRQ(ierr);
ierr = PetscSFSetGraph(*sfzr, M*N*P, q, localPoints, PETSC_COPY_VALUES, remotePoints, PETSC_COPY_VALUES);CHKERRQ(ierr);
/* Create SF for buffered map */
loclower.i = blower.i + sxb; locupper.i = blower.i + sxb+mxb;
loclower.j = blower.j + syb; locupper.j = blower.j + syb+myb;
loclower.k = blower.k + szb; locupper.k = blower.k + szb+mzb;
ierr = DMDACreatePatchIS(dm, &loclower, &locupper, &is);CHKERRQ(ierr);
ierr = ISGetIndices(is, &indices);CHKERRQ(ierr);
q = 0;
for (k = szb; k < szb+mzb; ++k) {
for (j = syb; j < syb+myb; ++j) {
for (i = sxb; i < sxb+mxb; ++i, ++q) {
PetscInt r;
localPoints[q] = q;
ierr = PetscFindInt(indices[q], size+1, ranges, &r);CHKERRQ(ierr);
remotePoints[q].rank = r < 0 ? -(r+1) - 1 : r;
remotePoints[q].index = indices[q] - ranges[remotePoints[q].rank];
}
}
}
ierr = ISRestoreIndices(is, &indices);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)dm), sfz);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *sfz, "Buffered Map");CHKERRQ(ierr);
ierr = PetscSFSetGraph(*sfz, M*N*P, q, localPoints, PETSC_COPY_VALUES, remotePoints, PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscFree2(localPoints, remotePoints);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode CreateMesh(MPI_Comm comm, PetscInt testNum, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscBool cellSimplex = user->cellSimplex;
PetscBool useGenerator = user->useGenerator;
const char *filename = user->filename;
size_t len;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
if (len) {
ierr = DMPlexCreateFromFile(comm, filename, PETSC_FALSE, dm);CHKERRQ(ierr);
ierr = DMGetDimension(*dm, &dim);CHKERRQ(ierr);
} else if (useGenerator) {
if (cellSimplex) {
ierr = DMPlexCreateBoxMesh(comm, dim, dim == 2 ? 2 : 1, PETSC_FALSE, dm);CHKERRQ(ierr);
} else {
const PetscInt cells[3] = {2, 2, 2};
ierr = DMPlexCreateHexBoxMesh(comm, dim, cells, PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, dm);CHKERRQ(ierr);
}
} else {
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
switch (dim) {
case 2:
if (cellSimplex) {
ierr = CreateSimplex_2D(comm, *dm);CHKERRQ(ierr);
} else {
ierr = CreateQuad_2D(comm, testNum, *dm);CHKERRQ(ierr);
}
break;
case 3:
if (cellSimplex) {
ierr = CreateSimplex_3D(comm, *dm);CHKERRQ(ierr);
} else {
ierr = CreateHex_3D(comm, *dm);CHKERRQ(ierr);
}
break;
default:
SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
}
}
{
DM interpolatedMesh = NULL;
ierr = CheckMesh(*dm, user);CHKERRQ(ierr);
ierr = DMPlexInterpolate(*dm, &interpolatedMesh);CHKERRQ(ierr);
ierr = DMPlexCopyCoordinates(*dm, interpolatedMesh);CHKERRQ(ierr);
ierr = CompareCones(*dm, interpolatedMesh);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = interpolatedMesh;
}
{
DM distributedMesh = NULL;
/* Distribute mesh over processes */
ierr = DMPlexDistribute(*dm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
if (distributedMesh) {
ierr = DMViewFromOptions(distributedMesh, NULL, "-dm_view");CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = distributedMesh;
}
}
ierr = PetscObjectSetName((PetscObject) *dm, "Interpolated Mesh");CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
user->dm = *dm;
PetscFunctionReturn(0);
}
/*@
DMPlexCreateExodus - Create a DMPlex mesh from an ExodusII file ID.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. exoid - The ExodusII id associated with a exodus file and obtained using ex_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Level: beginner
.keywords: mesh,ExodusII
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateExodus(MPI_Comm comm, PetscInt exoid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_EXODUSII)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from ex_get_init() */
char title[PETSC_MAX_PATH_LEN+1];
int dim = 0, numVertices = 0, numCells = 0;
int num_cs = 0, num_vs = 0, num_fs = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_EXODUSII)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open EXODUS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
ierr = PetscMemzero(title,(PETSC_MAX_PATH_LEN+1)*sizeof(char));CHKERRQ(ierr);
ierr = ex_get_init(exoid, title, &dim, &numVertices, &numCells, &num_cs, &num_vs, &num_fs);
if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"ExodusII ex_get_init() failed with error code %D\n",ierr);
if (!num_cs) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Exodus file does not contain any cell set\n");
}
ierr = MPI_Bcast(title, PETSC_MAX_PATH_LEN+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, title);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read cell sets information */
if (!rank) {
PetscInt *cone;
int c, cs, c_loc, v, v_loc;
/* Read from ex_get_elem_blk_ids() */
int *cs_id;
/* Read from ex_get_elem_block() */
char buffer[PETSC_MAX_PATH_LEN+1];
int num_cell_in_set, num_vertex_per_cell, num_attr;
/* Read from ex_get_elem_conn() */
int *cs_connect;
/* Get cell sets IDs */
ierr = PetscMalloc1(num_cs, &cs_id);CHKERRQ(ierr);
ierr = ex_get_elem_blk_ids(exoid, cs_id);CHKERRQ(ierr);
/* Read the cell set connectivity table and build mesh topology
EXO standard requires that cells in cell sets be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
for (c_loc = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, num_vertex_per_cell);CHKERRQ(ierr);
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc2(num_vertex_per_cell*num_cell_in_set,&cs_connect,num_vertex_per_cell,&cone);CHKERRQ(ierr);
ierr = ex_get_elem_conn(exoid, cs_id[cs], cs_connect);CHKERRQ(ierr);
/* EXO uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
for (v_loc = 0; v_loc < num_vertex_per_cell; ++v_loc, ++v) {
cone[v_loc] = cs_connect[v]+numCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (num_vertex_per_cell == 4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (num_vertex_per_cell == 8) {
PetscInt tmp = cone[1];
cone[1] = cone[3];
cone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMSetLabelValue(*dm, "Cell Sets", c, cs_id[cs]);CHKERRQ(ierr);
}
ierr = PetscFree2(cs_connect,cone);CHKERRQ(ierr);
}
ierr = PetscFree(cs_id);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
/* Maintain Cell Sets label */
{
DMLabel label;
ierr = DMRemoveLabel(*dm, "Cell Sets", &label);CHKERRQ(ierr);
if (label) {ierr = DMAddLabel(idm, label);CHKERRQ(ierr);}
}
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Create vertex set label */
if (!rank && (num_vs > 0)) {
int vs, v;
/* Read from ex_get_node_set_ids() */
int *vs_id;
/* Read from ex_get_node_set_param() */
int num_vertex_in_set, num_attr;
/* Read from ex_get_node_set() */
int *vs_vertex_list;
/* Get vertex set ids */
ierr = PetscMalloc1(num_vs, &vs_id);CHKERRQ(ierr);
ierr = ex_get_node_set_ids(exoid, vs_id);CHKERRQ(ierr);
for (vs = 0; vs < num_vs; ++vs) {
ierr = ex_get_node_set_param(exoid, vs_id[vs], &num_vertex_in_set, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc1(num_vertex_in_set, &vs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_node_set(exoid, vs_id[vs], vs_vertex_list);CHKERRQ(ierr);
for (v = 0; v < num_vertex_in_set; ++v) {
ierr = DMSetLabelValue(*dm, "Vertex Sets", vs_vertex_list[v]+numCells-1, vs_id[vs]);CHKERRQ(ierr);
}
ierr = PetscFree(vs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(vs_id);CHKERRQ(ierr);
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
float *x, *y, *z;
ierr = PetscMalloc3(numVertices,&x,numVertices,&y,numVertices,&z);CHKERRQ(ierr);
ierr = ex_get_coord(exoid, x, y, z);CHKERRQ(ierr);
if (dim > 0) {
for (v = 0; v < numVertices; ++v) coords[v*dim+0] = x[v];
}
if (dim > 1) {
for (v = 0; v < numVertices; ++v) coords[v*dim+1] = y[v];
}
if (dim > 2) {
for (v = 0; v < numVertices; ++v) coords[v*dim+2] = z[v];
}
ierr = PetscFree3(x,y,z);CHKERRQ(ierr);
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Create side set label */
if (!rank && interpolate && (num_fs > 0)) {
int fs, f, voff;
/* Read from ex_get_side_set_ids() */
int *fs_id;
/* Read from ex_get_side_set_param() */
int num_side_in_set, num_dist_fact_in_set;
/* Read from ex_get_side_set_node_list() */
int *fs_vertex_count_list, *fs_vertex_list;
/* Get side set ids */
ierr = PetscMalloc1(num_fs, &fs_id);CHKERRQ(ierr);
ierr = ex_get_side_set_ids(exoid, fs_id);CHKERRQ(ierr);
for (fs = 0; fs < num_fs; ++fs) {
ierr = ex_get_side_set_param(exoid, fs_id[fs], &num_side_in_set, &num_dist_fact_in_set);CHKERRQ(ierr);
ierr = PetscMalloc2(num_side_in_set,&fs_vertex_count_list,num_side_in_set*4,&fs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_side_set_node_list(exoid, fs_id[fs], fs_vertex_count_list, fs_vertex_list);CHKERRQ(ierr);
for (f = 0, voff = 0; f < num_side_in_set; ++f) {
const PetscInt *faces = NULL;
PetscInt faceSize = fs_vertex_count_list[f], numFaces;
PetscInt faceVertices[4], v;
if (faceSize > 4) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "ExodusII side cannot have %d > 4 vertices", faceSize);
for (v = 0; v < faceSize; ++v, ++voff) {
faceVertices[v] = fs_vertex_list[voff]+numCells-1;
}
ierr = DMPlexGetFullJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Invalid ExodusII side %d in set %d maps to %d faces", f, fs, numFaces);
ierr = DMSetLabelValue(*dm, "Face Sets", faces[0], fs_id[fs]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
}
ierr = PetscFree2(fs_vertex_count_list,fs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(fs_id);CHKERRQ(ierr);
}
#else
SETERRQ(comm, PETSC_ERR_SUP, "This method requires ExodusII support. Reconfigure using --download-exodusii");
#endif
PetscFunctionReturn(0);
}
PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscInt numRefinements = user->numRefinements;
PetscBool cellHybrid = user->cellHybrid;
PetscBool cellSimplex = user->cellSimplex;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
switch (dim) {
case 1:
if (cellHybrid) SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make hybrid meshes for dimension %d", dim);
ierr = CreateSimplex_1D(comm, dm);CHKERRQ(ierr);
break;
case 2:
if (cellSimplex) {
if (cellHybrid) {
ierr = CreateSimplexHybrid_2D(comm, user->testNum, dm);CHKERRQ(ierr);
} else {
ierr = CreateSimplex_2D(comm, dm);CHKERRQ(ierr);
}
} else {
if (cellHybrid) {
ierr = CreateTensorProductHybrid_2D(comm, user->testNum, dm);CHKERRQ(ierr);
} else {
ierr = CreateTensorProduct_2D(comm, user->testNum, dm);CHKERRQ(ierr);
}
}
break;
case 3:
if (cellSimplex) {
if (cellHybrid) {
ierr = CreateSimplexHybrid_3D(comm, user->testNum, dm);CHKERRQ(ierr);
} else {
ierr = CreateSimplex_3D(comm, user->testNum, dm);CHKERRQ(ierr);
}
} else {
if (cellHybrid) {
ierr = CreateTensorProductHybrid_3D(comm, user->testNum, dm);CHKERRQ(ierr);
} else {
ierr = CreateTensorProduct_3D(comm, user->testNum, dm);CHKERRQ(ierr);
}
}
break;
default:
SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
}
{
DM refinedMesh = NULL;
DM distributedMesh = NULL;
PetscInt r;
/* Distribute mesh over processes */
ierr = DMPlexDistribute(*dm, 0, NULL, &distributedMesh);CHKERRQ(ierr);
if (distributedMesh) {
ierr = DMViewFromOptions(distributedMesh, NULL, "-dm_view");CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = distributedMesh;
}
for (r = 0; r < numRefinements; ++r) {
ierr = DMViewFromOptions(*dm, NULL, "-orig_dm_view");CHKERRQ(ierr);
ierr = DMPlexCheckSymmetry(*dm);CHKERRQ(ierr);
ierr = DMPlexCheckSkeleton(*dm, user->cellSimplex, 0);CHKERRQ(ierr);
ierr = DMPlexCheckFaces(*dm, user->cellSimplex, 0);CHKERRQ(ierr);
ierr = DMPlexSetRefinementUniform(*dm, PETSC_TRUE);CHKERRQ(ierr);
ierr = DMRefine(*dm, comm, &refinedMesh);CHKERRQ(ierr);
if (refinedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = refinedMesh;
}
}
}
if (user->uninterpolate) {
DM udm = NULL;
ierr = DMPlexUninterpolate(*dm, &udm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = udm;
}
ierr = PetscObjectSetName((PetscObject) *dm, "Hybrid Mesh");CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexCreateGmsh - Create a DMPlex mesh from a Gmsh file viewer
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Gmsh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.geuz.org/gmsh/doc/texinfo/#MSH-ASCII-file-format
and http://www.geuz.org/gmsh/doc/texinfo/#MSH-binary-file-format
Level: beginner
.keywords: mesh,Gmsh
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateGmsh(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
PetscViewerType vtype;
GmshElement *gmsh_elem;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords, *coordsIn = NULL;
PetscInt dim = 0, coordSize, c, v, d, r, cell;
int i, numVertices = 0, numCells = 0, trueNumCells = 0, numRegions = 0, snum;
PetscMPIInt num_proc, rank;
char line[PETSC_MAX_PATH_LEN];
PetscBool match, binary, bswap = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
ierr = PetscViewerGetType(viewer, &vtype);CHKERRQ(ierr);
ierr = PetscStrcmp(vtype, PETSCVIEWERBINARY, &binary);CHKERRQ(ierr);
if (!rank || binary) {
PetscBool match;
int fileType, dataSize;
float version;
/* Read header */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$MeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 3, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%f %d %d", &version, &fileType, &dataSize);
if (snum != 3) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Gmsh file header: %s", line);
if (version < 2.0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Gmsh file must be at least version 2.0");
if (dataSize != sizeof(double)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Data size %d is not valid for a Gmsh file", dataSize);
if (binary) {
int checkInt;
ierr = PetscViewerRead(viewer, &checkInt, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
if (checkInt != 1) {
ierr = PetscByteSwap(&checkInt, PETSC_ENUM, 1);CHKERRQ(ierr);
if (checkInt == 1) bswap = PETSC_TRUE;
else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh binary file", fileType);
}
} else if (fileType) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh ASCII file", fileType);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndMeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* OPTIONAL Read physical names */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$PhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (match) {
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numRegions);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
for (r = 0; r < numRegions; ++r) {
PetscInt rdim, tag;
ierr = PetscViewerRead(viewer, &rdim, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &tag, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndPhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Initial read for vertex section */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
/* Read vertices */
ierr = PetscStrncmp(line, "$Nodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numVertices);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscMalloc1(numVertices*3, &coordsIn);CHKERRQ(ierr);
if (binary) {
size_t doubleSize, intSize;
PetscInt elementSize;
char *buffer;
PetscScalar *baseptr;
ierr = PetscDataTypeGetSize(PETSC_ENUM, &intSize);CHKERRQ(ierr);
ierr = PetscDataTypeGetSize(PETSC_DOUBLE, &doubleSize);CHKERRQ(ierr);
elementSize = (intSize + 3*doubleSize);
ierr = PetscMalloc1(elementSize*numVertices, &buffer);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, buffer, elementSize*numVertices, NULL, PETSC_CHAR);CHKERRQ(ierr);
if (bswap) ierr = PetscByteSwap(buffer, PETSC_CHAR, elementSize*numVertices);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
baseptr = ((PetscScalar*)(buffer+v*elementSize+intSize));
coordsIn[v*3+0] = baseptr[0];
coordsIn[v*3+1] = baseptr[1];
coordsIn[v*3+2] = baseptr[2];
}
ierr = PetscFree(buffer);CHKERRQ(ierr);
} else {
for (v = 0; v < numVertices; ++v) {
ierr = PetscViewerRead(viewer, &i, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &(coordsIn[v*3]), 3, NULL, PETSC_DOUBLE);CHKERRQ(ierr);
if (i != (int)v+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid node number %d should be %d", i, v+1);
}
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndNodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read cells */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$Elements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
snum = sscanf(line, "%d", &numCells);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
if (!rank || binary) {
/* Gmsh elements can be of any dimension/co-dimension, so we need to traverse the
file contents multiple times to figure out the true number of cells and facets
in the given mesh. To make this more efficient we read the file contents only
once and store them in memory, while determining the true number of cells. */
ierr = DMPlexCreateGmsh_ReadElement(viewer, numCells, binary, bswap, &gmsh_elem);CHKERRQ(ierr);
for (trueNumCells=0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim > dim) {dim = gmsh_elem[c].dim; trueNumCells = 0;}
if (gmsh_elem[c].dim == dim) trueNumCells++;
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndElements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
/* For binary we read on all ranks, but only build the plex on rank 0 */
if (binary && rank) {trueNumCells = 0; numVertices = 0;};
/* Allocate the cell-vertex mesh */
ierr = DMPlexSetChart(*dm, 0, trueNumCells+numVertices);CHKERRQ(ierr);
if (!rank) {
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
ierr = DMPlexSetConeSize(*dm, cell, gmsh_elem[c].numNodes);CHKERRQ(ierr);
cell++;
}
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
/* Add cell-vertex connections */
if (!rank) {
PetscInt pcone[8], corner;
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + trueNumCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (gmsh_elem[c].numNodes == 4) {
PetscInt tmp = pcone[0];
pcone[0] = pcone[1];
pcone[1] = tmp;
}
/* Hexahedra are inverted */
if (gmsh_elem[c].numNodes == 8) {
PetscInt tmp = pcone[1];
pcone[1] = pcone[3];
pcone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, cell, pcone);CHKERRQ(ierr);
cell++;
}
}
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
if (!rank) {
/* Apply boundary IDs by finding the relevant facets with vertex joins */
PetscInt pcone[8], corner, vStart, vEnd;
ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim-1) {
PetscInt joinSize;
const PetscInt *join;
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + vStart - 1;
}
ierr = DMPlexGetFullJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for element %d", gmsh_elem[c].id);
ierr = DMSetLabelValue(*dm, "Face Sets", join[0], gmsh_elem[c].tags[0]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
}
}
/* Create cell sets */
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
if (gmsh_elem[c].numTags > 0) {
ierr = DMSetLabelValue(*dm, "Cell Sets", cell, gmsh_elem[c].tags[0]);CHKERRQ(ierr);
cell++;
}
}
}
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, trueNumCells, trueNumCells + numVertices);CHKERRQ(ierr);
for (v = trueNumCells; v < trueNumCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*3+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Clean up intermediate storage */
if (!rank || binary) ierr = PetscFree(gmsh_elem);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexCreateFluent - Create a DMPlex mesh from a Fluent mesh file.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Fluent mesh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://aerojet.engr.ucdavis.edu/fluenthelp/html/ug/node1490.htm
Level: beginner
.keywords: mesh, fluent, case
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateFluent(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
PetscMPIInt rank;
PetscInt c, f, v, dim = PETSC_DETERMINE, numCells = 0, numVertices = 0, numCellVertices = PETSC_DETERMINE;
PetscInt numFaces = PETSC_DETERMINE, numFaceEntries = PETSC_DETERMINE, numFaceVertices = PETSC_DETERMINE;
PetscInt *faces = NULL, *cellVertices, *faceZoneIDs = NULL;
PetscInt d, coordSize;
PetscScalar *coords, *coordsIn = NULL;
PetscSection coordSection;
Vec coordinates;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
if (!rank) {
FluentSection s;
do {
ierr = DMPlexCreateFluent_ReadSection(viewer, &s);CHKERRQ(ierr);
if (s.index == 2) { /* Dimension */
dim = s.nd;
} else if (s.index == 10 || s.index == 2010) { /* Vertices */
if (s.zoneID == 0) numVertices = s.last;
else {
if (coordsIn) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Currently no support for multiple coordinate sets in Fluent files");
coordsIn = (PetscScalar *) s.data;
}
} else if (s.index == 12 || s.index == 2012) { /* Cells */
if (s.zoneID == 0) numCells = s.last;
else {
switch (s.nd) {
case 0: numCellVertices = PETSC_DETERMINE; break;
case 1: numCellVertices = 3; break; /* triangular */
case 2: numCellVertices = 4; break; /* tetrahedral */
case 3: numCellVertices = 4; break; /* quadrilateral */
case 4: numCellVertices = 8; break; /* hexahedral */
case 5: numCellVertices = 5; break; /* pyramid */
case 6: numCellVertices = 6; break; /* wedge */
default: numCellVertices = PETSC_DETERMINE;
}
}
} else if (s.index == 13 || s.index == 2013) { /* Facets */
if (s.zoneID == 0) { /* Header section */
numFaces = s.last - s.first + 1;
if (s.nd == 0 || s.nd == 5) numFaceVertices = PETSC_DETERMINE;
else numFaceVertices = s.nd;
} else { /* Data section */
if (numFaceVertices != PETSC_DETERMINE && s.nd != numFaceVertices) {
SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mixed facets in Fluent files are not supported");
}
if (numFaces < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No header section for facets in Fluent file");
if (numFaceVertices == PETSC_DETERMINE) numFaceVertices = s.nd;
numFaceEntries = numFaceVertices + 2;
if (!faces) {ierr = PetscMalloc1(numFaces*numFaceEntries, &faces);CHKERRQ(ierr);}
if (!faceZoneIDs) {ierr = PetscMalloc1(numFaces, &faceZoneIDs);CHKERRQ(ierr);}
ierr = PetscMemcpy(&(faces[(s.first-1)*numFaceEntries]), s.data, (s.last-s.first+1)*numFaceEntries*sizeof(PetscInt));CHKERRQ(ierr);
/* Record the zoneID for each face set */
for (f = s.first -1; f < s.last; f++) faceZoneIDs[f] = s.zoneID;
ierr = PetscFree(s.data);CHKERRQ(ierr);
}
}
} while (s.index >= 0);
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
if (dim < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Fluent file does not include dimension");
/* Allocate cell-vertex mesh */
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells + numVertices);CHKERRQ(ierr);
if (!rank) {
if (numCells < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown number of cells in Fluent file");
/* If no cell type was given we assume simplices */
if (numCellVertices == PETSC_DETERMINE) numCellVertices = numFaceVertices + 1;
for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(*dm, c, numCellVertices);CHKERRQ(ierr);}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
if (!rank) {
/* Derive cell-vertex list from face-vertex and face-cell maps */
ierr = PetscMalloc1(numCells*numCellVertices, &cellVertices);CHKERRQ(ierr);
for (c = 0; c < numCells*numCellVertices; c++) cellVertices[c] = -1;
for (f = 0; f < numFaces; f++) {
PetscInt *cell;
const PetscInt cl = faces[f*numFaceEntries + numFaceVertices];
const PetscInt cr = faces[f*numFaceEntries + numFaceVertices + 1];
const PetscInt *face = &(faces[f*numFaceEntries]);
if (cl > 0) {
cell = &(cellVertices[(cl-1) * numCellVertices]);
for (v = 0; v < numFaceVertices; v++) {
PetscBool found = PETSC_FALSE;
for (c = 0; c < numCellVertices; c++) {
if (cell[c] < 0) break;
if (cell[c] == face[v]-1 + numCells) {found = PETSC_TRUE; break;}
}
if (!found) cell[c] = face[v]-1 + numCells;
}
}
if (cr > 0) {
cell = &(cellVertices[(cr-1) * numCellVertices]);
for (v = 0; v < numFaceVertices; v++) {
PetscBool found = PETSC_FALSE;
for (c = 0; c < numCellVertices; c++) {
if (cell[c] < 0) break;
if (cell[c] == face[v]-1 + numCells) {found = PETSC_TRUE; break;}
}
if (!found) cell[c] = face[v]-1 + numCells;
}
}
}
for (c = 0; c < numCells; c++) {
ierr = DMPlexSetCone(*dm, c, &(cellVertices[c*numCellVertices]));CHKERRQ(ierr);
}
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
if (!rank) {
PetscInt fi, joinSize, meetSize, *fverts, cells[2];
const PetscInt *join, *meet;
ierr = PetscMalloc1(numFaceVertices, &fverts);CHKERRQ(ierr);
/* Mark facets by finding the full join of all adjacent vertices */
for (f = 0; f < numFaces; f++) {
const PetscInt cl = faces[f*numFaceEntries + numFaceVertices] - 1;
const PetscInt cr = faces[f*numFaceEntries + numFaceVertices + 1] - 1;
if (cl > 0 && cr > 0) {
/* If we know both adjoining cells we can use a single-level meet */
cells[0] = cl; cells[1] = cr;
ierr = DMPlexGetMeet(*dm, 2, cells, &meetSize, &meet);CHKERRQ(ierr);
if (meetSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for Fluent face %d", f);
ierr = DMPlexSetLabelValue(*dm, "Face Sets", meet[0], faceZoneIDs[f]);CHKERRQ(ierr);
ierr = DMPlexRestoreMeet(*dm, numFaceVertices, fverts, &meetSize, &meet);CHKERRQ(ierr);
} else {
for (fi = 0; fi < numFaceVertices; fi++) fverts[fi] = faces[f*numFaceEntries + fi] + numCells - 1;
ierr = DMPlexGetFullJoin(*dm, numFaceVertices, fverts, &joinSize, &join);CHKERRQ(ierr);
if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for Fluent face %d", f);
ierr = DMPlexSetLabelValue(*dm, "Face Sets", join[0], faceZoneIDs[f]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, numFaceVertices, fverts, &joinSize, &join);CHKERRQ(ierr);
}
}
ierr = PetscFree(fverts);CHKERRQ(ierr);
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(comm, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*dim+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
if (!rank) {
ierr = PetscFree(cellVertices);CHKERRQ(ierr);
ierr = PetscFree(faces);CHKERRQ(ierr);
ierr = PetscFree(faceZoneIDs);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@
DMPlexPermute - Reorder the mesh according to the input permutation
Collective on DM
Input Parameter:
+ dm - The DMPlex object
- perm - The point permutation, perm[old point number] = new point number
Output Parameter:
. pdm - The permuted DM
Level: intermediate
.keywords: mesh
.seealso: MatPermute()
@*/
PetscErrorCode DMPlexPermute(DM dm, IS perm, DM *pdm)
{
DM_Plex *plex = (DM_Plex *) dm->data, *plexNew;
PetscSection section, sectionNew;
PetscInt dim;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(perm, IS_CLASSID, 2);
PetscValidPointer(pdm, 3);
ierr = DMCreate(PetscObjectComm((PetscObject) dm), pdm);CHKERRQ(ierr);
ierr = DMSetType(*pdm, DMPLEX);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMSetDimension(*pdm, dim);CHKERRQ(ierr);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
if (section) {
ierr = PetscSectionPermute(section, perm, §ionNew);CHKERRQ(ierr);
ierr = DMSetDefaultSection(*pdm, sectionNew);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionNew);CHKERRQ(ierr);
}
plexNew = (DM_Plex *) (*pdm)->data;
/* Ignore ltogmap, ltogmapb */
/* Ignore sf, defaultSF */
/* Ignore globalVertexNumbers, globalCellNumbers */
/* Remap coordinates */
{
DM cdm, cdmNew;
PetscSection csection, csectionNew;
Vec coordinates, coordinatesNew;
PetscScalar *coords, *coordsNew;
const PetscInt *pperm;
PetscInt pStart, pEnd, p;
const char *name;
ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
ierr = DMGetDefaultSection(cdm, &csection);CHKERRQ(ierr);
ierr = PetscSectionPermute(csection, perm, &csectionNew);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = VecDuplicate(coordinates, &coordinatesNew);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)coordinatesNew,name);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
ierr = VecGetArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
ierr = PetscSectionGetChart(csectionNew, &pStart, &pEnd);CHKERRQ(ierr);
ierr = ISGetIndices(perm, &pperm);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(csectionNew, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csectionNew, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) coordsNew[offNew+d] = coords[off+d];
}
ierr = ISRestoreIndices(perm, &pperm);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(*pdm, &cdmNew);CHKERRQ(ierr);
ierr = DMSetDefaultSection(cdmNew, csectionNew);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*pdm, coordinatesNew);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&csectionNew);CHKERRQ(ierr);
ierr = VecDestroy(&coordinatesNew);CHKERRQ(ierr);
}
/* Reorder labels */
{
PetscInt numLabels, l;
DMLabel label, labelNew;
ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
for (l = numLabels-1; l >= 0; --l) {
ierr = DMGetLabelByNum(dm, l, &label);CHKERRQ(ierr);
ierr = DMLabelPermute(label, perm, &labelNew);CHKERRQ(ierr);
ierr = DMAddLabel(*pdm, labelNew);CHKERRQ(ierr);
}
if (plex->subpointMap) {ierr = DMLabelPermute(plex->subpointMap, perm, &plexNew->subpointMap);CHKERRQ(ierr);}
}
/* Reorder topology */
{
const PetscInt *pperm;
PetscInt maxConeSize, maxSupportSize, n, pStart, pEnd, p;
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
plexNew->maxConeSize = maxConeSize;
plexNew->maxSupportSize = maxSupportSize;
ierr = PetscSectionDestroy(&plexNew->coneSection);CHKERRQ(ierr);
ierr = PetscSectionPermute(plex->coneSection, perm, &plexNew->coneSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(plexNew->coneSection, &n);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->cones);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->coneOrientations);CHKERRQ(ierr);
ierr = ISGetIndices(perm, &pperm);CHKERRQ(ierr);
ierr = PetscSectionGetChart(plex->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(plexNew->coneSection, pperm[p], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plex->coneSection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plexNew->coneSection, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
plexNew->cones[offNew+d] = pperm[plex->cones[off+d]];
plexNew->coneOrientations[offNew+d] = plex->coneOrientations[off+d];
}
}
ierr = PetscSectionDestroy(&plexNew->supportSection);CHKERRQ(ierr);
ierr = PetscSectionPermute(plex->supportSection, perm, &plexNew->supportSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(plexNew->supportSection, &n);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->supports);CHKERRQ(ierr);
ierr = PetscSectionGetChart(plex->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(plexNew->supportSection, pperm[p], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plex->supportSection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plexNew->supportSection, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
plexNew->supports[offNew+d] = pperm[plex->supports[off+d]];
}
}
ierr = ISRestoreIndices(perm, &pperm);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
pmesh->useAnchors = mesh->useAnchors;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
const PetscReal *maxCell, *L;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
ierr = DMGetPeriodicity(dm, &maxCell, &L);CHKERRQ(ierr);
if (L) {ierr = DMSetPeriodicity(*dmParallel, maxCell, L);CHKERRQ(ierr);}
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Set up tree */
{
DM refTree;
PetscSection origParentSection, newParentSection;
PetscInt *origParents, *origChildIDs;
ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
ierr = DMPlexSetReferenceTree(*dmParallel,refTree);CHKERRQ(ierr);
ierr = DMPlexGetTree(dm,&origParentSection,&origParents,&origChildIDs,NULL,NULL);CHKERRQ(ierr);
if (origParentSection) {
PetscInt pStart, pEnd;
PetscInt *newParents, *newChildIDs;
PetscInt *remoteOffsetsParents, newParentSize;
PetscSF parentSF;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)*dmParallel),&newParentSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newParentSection,pStart,pEnd);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, origParentSection, &remoteOffsetsParents, newParentSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(pointSF, origParentSection, remoteOffsetsParents, newParentSection, &parentSF);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newParentSection,&newParentSize);CHKERRQ(ierr);
ierr = PetscMalloc2(newParentSize,&newParents,newParentSize,&newChildIDs);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(parentSF, MPIU_INT, origParents, newParents);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(parentSF, MPIU_INT, origParents, newParents);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(parentSF, MPIU_INT, origChildIDs, newChildIDs);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(parentSF, MPIU_INT, origChildIDs, newChildIDs);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering,IS_GTOLM_MASK, newParentSize, newParents, NULL, newParents);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-parents_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Parent Section: \n");CHKERRQ(ierr);
ierr = PetscSectionView(origParentSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Parent Section: \n");CHKERRQ(ierr);
ierr = PetscSectionView(newParentSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(parentSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexSetTree(*dmParallel,newParentSection,newParents,newChildIDs);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&newParentSection);CHKERRQ(ierr);
ierr = PetscFree2(newParents,newChildIDs);CHKERRQ(ierr);
ierr = PetscSFDestroy(&parentSF);CHKERRQ(ierr);
}
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMDASubDomainDA_Private(DM dm, PetscInt *nlocal, DM **sdm)
{
DM *da;
PetscInt dim,size,i,j,k,idx;
PetscErrorCode ierr;
DMDALocalInfo info;
PetscInt xsize,ysize,zsize;
PetscInt xo,yo,zo;
PetscInt xs,ys,zs;
PetscInt xm=1,ym=1,zm=1;
PetscInt xol,yol,zol;
PetscInt m=1,n=1,p=1;
PetscInt M,N,P;
PetscInt pm,mtmp;
PetscFunctionBegin;
ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr);
ierr = DMDAGetOverlap(dm,&xol,&yol,&zol);CHKERRQ(ierr);
ierr = DMDAGetNumLocalSubDomains(dm,&size);CHKERRQ(ierr);
ierr = PetscMalloc1(size,&da);CHKERRQ(ierr);
if (nlocal) *nlocal = size;
dim = info.dim;
M = info.xm;
N = info.ym;
P = info.zm;
if (dim == 1) {
m = size;
} else if (dim == 2) {
m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)N)));
while (m > 0) {
n = size/m;
if (m*n*p == size) break;
m--;
}
} else if (dim == 3) {
n = (PetscInt)(0.5 + PetscPowReal(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.))); if (!n) n = 1;
while (n > 0) {
pm = size/n;
if (n*pm == size) break;
n--;
}
if (!n) n = 1;
m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
if (!m) m = 1;
while (m > 0) {
p = size/(m*n);
if (m*n*p == size) break;
m--;
}
if (M > P && m < p) {mtmp = m; m = p; p = mtmp;}
}
zs = info.zs;
idx = 0;
for (k = 0; k < p; k++) {
ys = info.ys;
for (j = 0; j < n; j++) {
xs = info.xs;
for (i = 0; i < m; i++) {
if (dim == 1) {
xm = M/m + ((M % m) > i);
} else if (dim == 2) {
xm = M/m + ((M % m) > i);
ym = N/n + ((N % n) > j);
} else if (dim == 3) {
xm = M/m + ((M % m) > i);
ym = N/n + ((N % n) > j);
zm = P/p + ((P % p) > k);
}
xsize = xm;
ysize = ym;
zsize = zm;
xo = xs;
yo = ys;
zo = zs;
ierr = DMDACreate(PETSC_COMM_SELF,&(da[idx]));CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da[idx],"sub_");CHKERRQ(ierr);
ierr = DMSetDimension(da[idx], info.dim);CHKERRQ(ierr);
ierr = DMDASetDof(da[idx], info.dof);CHKERRQ(ierr);
ierr = DMDASetStencilType(da[idx],info.st);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da[idx],info.sw);CHKERRQ(ierr);
if (info.bx == DM_BOUNDARY_PERIODIC || (xs != 0)) {
xsize += xol;
xo -= xol;
}
if (info.by == DM_BOUNDARY_PERIODIC || (ys != 0)) {
ysize += yol;
yo -= yol;
}
if (info.bz == DM_BOUNDARY_PERIODIC || (zs != 0)) {
zsize += zol;
zo -= zol;
}
if (info.bx == DM_BOUNDARY_PERIODIC || (xs+xm != info.mx)) xsize += xol;
if (info.by == DM_BOUNDARY_PERIODIC || (ys+ym != info.my)) ysize += yol;
if (info.bz == DM_BOUNDARY_PERIODIC || (zs+zm != info.mz)) zsize += zol;
if (info.bx != DM_BOUNDARY_PERIODIC) {
if (xo < 0) {
xsize += xo;
xo = 0;
}
if (xo+xsize > info.mx-1) {
xsize -= xo+xsize - info.mx;
}
}
if (info.by != DM_BOUNDARY_PERIODIC) {
if (yo < 0) {
ysize += yo;
yo = 0;
}
if (yo+ysize > info.my-1) {
ysize -= yo+ysize - info.my;
}
}
if (info.bz != DM_BOUNDARY_PERIODIC) {
if (zo < 0) {
zsize += zo;
zo = 0;
}
if (zo+zsize > info.mz-1) {
zsize -= zo+zsize - info.mz;
}
}
ierr = DMDASetSizes(da[idx], xsize, ysize, zsize);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da[idx], 1, 1, 1);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da[idx], DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr);
/* set up as a block instead */
ierr = DMSetUp(da[idx]);CHKERRQ(ierr);
/* nonoverlapping region */
ierr = DMDASetNonOverlappingRegion(da[idx],xs,ys,zs,xm,ym,zm);CHKERRQ(ierr);
/* this alters the behavior of DMDAGetInfo, DMDAGetLocalInfo, DMDAGetCorners, and DMDAGetGhostedCorners and should be used with care */
ierr = DMDASetOffset(da[idx],xo,yo,zo,info.mx,info.my,info.mz);CHKERRQ(ierr);
xs += xm;
idx++;
}
ys += ym;
}
zs += zm;
}
*sdm = da;
PetscFunctionReturn(0);
}
/*@
DMPlexCreateCGNS - Create a DMPlex mesh from a CGNS file ID.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. cgid - The CG id associated with a file and obtained using cg_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.grc.nasa.gov/WWW/cgns/CGNS_docs_current/index.html
Level: beginner
.keywords: mesh,CGNS
.seealso: DMPlexCreate(), DMPlexCreateExodus()
@*/
PetscErrorCode DMPlexCreateCGNS(MPI_Comm comm, PetscInt cgid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_CGNS)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt *cellStart, *vertStart;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from file */
char basename[CGIO_MAX_NAME_LENGTH+1];
char buffer[CGIO_MAX_NAME_LENGTH+1];
int dim = 0, physDim = 0, numVertices = 0, numCells = 0;
int nzones = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_CGNS)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open CGNS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
int nbases, z;
ierr = cg_nbases(cgid, &nbases);CHKERRQ(ierr);
if (nbases > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single base, not %d\n",nbases);
ierr = cg_base_read(cgid, 1, basename, &dim, &physDim);CHKERRQ(ierr);
ierr = cg_nzones(cgid, 1, &nzones);CHKERRQ(ierr);
ierr = PetscCalloc2(nzones+1, &cellStart, nzones+1, &vertStart);CHKERRQ(ierr);
for (z = 1; z <= nzones; ++z) {
cgsize_t sizes[3]; /* Number of vertices, number of cells, number of boundary vertices */
ierr = cg_zone_read(cgid, 1, z, buffer, sizes);CHKERRQ(ierr);
numVertices += sizes[0];
numCells += sizes[1];
cellStart[z] += sizes[1] + cellStart[z-1];
vertStart[z] += sizes[0] + vertStart[z-1];
}
for (z = 1; z <= nzones; ++z) {
vertStart[z] += numCells;
}
}
ierr = MPI_Bcast(basename, CGIO_MAX_NAME_LENGTH+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&nzones, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, basename);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read zone information */
if (!rank) {
int z, c, c_loc, v, v_loc;
/* Read the cell set connectivity table and build mesh topology
CGNS standard requires that cells in a zone be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (z = 1, c = 0; z <= nzones; ++z) {
ZoneType_t zonetype;
int nsections;
ElementType_t cellType;
cgsize_t start, end;
int nbndry, parentFlag;
PetscInt numCorners;
ierr = cg_zone_type(cgid, 1, z, &zonetype);CHKERRQ(ierr);
if (zonetype == Structured) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Can only handle Unstructured zones for CGNS");
ierr = cg_nsections(cgid, 1, z, &nsections);CHKERRQ(ierr);
if (nsections > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single section, not %d\n",nsections);
ierr = cg_section_read(cgid, 1, z, 1, buffer, &cellType, &start, &end, &nbndry, &parentFlag);CHKERRQ(ierr);
/* This alone is reason enough to bludgeon every single CGNDS developer, this must be what they describe as the "idiocy of crowds" */
if (cellType == MIXED) {
cgsize_t elementDataSize, *elements;
PetscInt off;
ierr = cg_ElementDataSize(cgid, 1, z, 1, &elementDataSize);CHKERRQ(ierr);
ierr = PetscMalloc1(elementDataSize, &elements);CHKERRQ(ierr);
ierr = cg_elements_read(cgid, 1, z, 1, elements, NULL);CHKERRQ(ierr);
for (c_loc = start, off = 0; c_loc <= end; ++c_loc, ++c) {
switch (elements[off]) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) elements[off]);
}
ierr = DMPlexSetConeSize(*dm, c, numCorners);CHKERRQ(ierr);
off += numCorners+1;
}
ierr = PetscFree(elements);CHKERRQ(ierr);
} else {
switch (cellType) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) cellType);
}
for (c_loc = start; c_loc <= end; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, numCorners);CHKERRQ(ierr);
}
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (z = 1, c = 0; z <= nzones; ++z) {
ElementType_t cellType;
cgsize_t *elements, elementDataSize, start, end;
int nbndry, parentFlag;
PetscInt *cone, numc, numCorners, maxCorners = 27;
ierr = cg_section_read(cgid, 1, z, 1, buffer, &cellType, &start, &end, &nbndry, &parentFlag);CHKERRQ(ierr);
numc = end - start;
/* This alone is reason enough to bludgeon every single CGNDS developer, this must be what they describe as the "idiocy of crowds" */
ierr = cg_ElementDataSize(cgid, 1, z, 1, &elementDataSize);CHKERRQ(ierr);
ierr = PetscMalloc2(elementDataSize,&elements,maxCorners,&cone);CHKERRQ(ierr);
ierr = cg_elements_read(cgid, 1, z, 1, elements, NULL);CHKERRQ(ierr);
if (cellType == MIXED) {
/* CGNS uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc <= numc; ++c_loc, ++c) {
switch (elements[v]) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) elements[v]);
}
++v;
for (v_loc = 0; v_loc < numCorners; ++v_loc, ++v) {
cone[v_loc] = elements[v]+numCells-1;
}
/* Tetrahedra are inverted */
if (elements[v] == TETRA_4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (elements[v] == HEXA_8) {
PetscInt tmp = cone[5];
cone[5] = cone[7];
cone[7] = tmp;
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(*dm, "zone", c, z);CHKERRQ(ierr);
}
} else {
switch (cellType) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) cellType);
}
/* CGNS uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc <= numc; ++c_loc, ++c) {
for (v_loc = 0; v_loc < numCorners; ++v_loc, ++v) {
cone[v_loc] = elements[v]+numCells-1;
}
/* Tetrahedra are inverted */
if (cellType == TETRA_4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted, and they give the top first */
if (cellType == HEXA_8) {
PetscInt tmp = cone[5];
cone[5] = cone[7];
cone[7] = tmp;
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(*dm, "zone", c, z);CHKERRQ(ierr);
}
}
ierr = PetscFree2(elements,cone);CHKERRQ(ierr);
}
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
/* Maintain zone label */
{
DMLabel label;
ierr = DMPlexRemoveLabel(*dm, "zone", &label);CHKERRQ(ierr);
if (label) {ierr = DMPlexAddLabel(idm, label);CHKERRQ(ierr);}
}
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(comm, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
PetscInt off = 0;
float *x[3];
int z, d;
ierr = PetscMalloc3(numVertices,&x[0],numVertices,&x[1],numVertices,&x[2]);CHKERRQ(ierr);
for (z = 1; z <= nzones; ++z) {
DataType_t datatype;
cgsize_t sizes[3]; /* Number of vertices, number of cells, number of boundary vertices */
cgsize_t range_min[3] = {1, 1, 1};
cgsize_t range_max[3] = {1, 1, 1};
int ngrids, ncoords;
ierr = cg_zone_read(cgid, 1, z, buffer, sizes);CHKERRQ(ierr);
range_max[0] = sizes[0];
ierr = cg_ngrids(cgid, 1, z, &ngrids);CHKERRQ(ierr);
if (ngrids > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single grid, not %d\n",ngrids);
ierr = cg_ncoords(cgid, 1, z, &ncoords);CHKERRQ(ierr);
if (ncoords != dim) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a coordinate array for each dimension, not %d\n",ncoords);
for (d = 0; d < dim; ++d) {
ierr = cg_coord_info(cgid, 1, z, 1+d, &datatype, buffer);CHKERRQ(ierr);
ierr = cg_coord_read(cgid, 1, z, buffer, RealSingle, range_min, range_max, x[d]);CHKERRQ(ierr);
}
if (dim > 0) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+0] = x[0][v];
}
if (dim > 1) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+1] = x[1][v];
}
if (dim > 2) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+2] = x[2][v];
}
off += sizes[0];
}
ierr = PetscFree3(x[0],x[1],x[2]);CHKERRQ(ierr);
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Read boundary conditions */
if (!rank) {
DMLabel label;
BCType_t bctype;
DataType_t datatype;
PointSetType_t pointtype;
cgsize_t *points;
PetscReal *normals;
int normal[3];
char *bcname = buffer;
cgsize_t npoints, nnormals;
int z, nbc, bc, c, ndatasets;
for (z = 1; z <= nzones; ++z) {
ierr = cg_nbocos(cgid, 1, z, &nbc);CHKERRQ(ierr);
for (bc = 1; bc <= nbc; ++bc) {
ierr = cg_boco_info(cgid, 1, z, bc, bcname, &bctype, &pointtype, &npoints, normal, &nnormals, &datatype, &ndatasets);CHKERRQ(ierr);
ierr = DMPlexCreateLabel(*dm, bcname);CHKERRQ(ierr);
ierr = DMPlexGetLabel(*dm, bcname, &label);CHKERRQ(ierr);
ierr = PetscMalloc2(npoints, &points, nnormals, &normals);CHKERRQ(ierr);
ierr = cg_boco_read(cgid, 1, z, bc, points, (void *) normals);CHKERRQ(ierr);
if (pointtype == ElementRange) {
/* Range of cells: assuming half-open interval since the documentation sucks */
for (c = points[0]; c < points[1]; ++c) {
ierr = DMLabelSetValue(label, c - cellStart[z-1], 1);CHKERRQ(ierr);
}
} else if (pointtype == ElementList) {
/* List of cells */
for (c = 0; c < npoints; ++c) {
ierr = DMLabelSetValue(label, points[c] - cellStart[z-1], 1);CHKERRQ(ierr);
}
} else if (pointtype == PointRange) {
GridLocation_t gridloc;
/* List of points: Oh please, someone get the CGNS developers away from a computer. This is unconscionable. */
ierr = cg_goto(cgid, 1, "Zone_t", z, "BC_t", bc, "end");CHKERRQ(ierr);
ierr = cg_gridlocation_read(&gridloc);CHKERRQ(ierr);
/* Range of points: assuming half-open interval since the documentation sucks */
for (c = points[0]; c < points[1]; ++c) {
if (gridloc == Vertex) {ierr = DMLabelSetValue(label, c - vertStart[z-1], 1);CHKERRQ(ierr);}
else {ierr = DMLabelSetValue(label, c - cellStart[z-1], 1);CHKERRQ(ierr);}
}
} else if (pointtype == PointList) {
GridLocation_t gridloc;
/* List of points: Oh please, someone get the CGNS developers away from a computer. This is unconscionable. */
ierr = cg_goto(cgid, 1, "Zone_t", z, "BC_t", bc, "end");
ierr = cg_gridlocation_read(&gridloc);
for (c = 0; c < npoints; ++c) {
if (gridloc == Vertex) {ierr = DMLabelSetValue(label, points[c] - vertStart[z-1], 1);CHKERRQ(ierr);}
else {ierr = DMLabelSetValue(label, points[c] - cellStart[z-1], 1);CHKERRQ(ierr);}
}
} else SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported point set type %d", (int) pointtype);
ierr = PetscFree2(points, normals);CHKERRQ(ierr);
}
}
ierr = PetscFree2(cellStart, vertStart);CHKERRQ(ierr);
}
#else
SETERRQ(comm, PETSC_ERR_SUP, "This method requires CGNS support. Reconfigure using --with-cgns-dir");
#endif
PetscFunctionReturn(0);
}
