/*
DMPlexGetFaces_Internal - Gets groups of vertices that correspond to faces for the given cell
*/
PetscErrorCode DMPlexGetFaces_Internal(DM dm, PetscInt dim, PetscInt p, PetscInt *numFaces, PetscInt *faceSize, const PetscInt *faces[])
{
const PetscInt *cone = NULL;
PetscInt maxConeSize, maxSupportSize, coneSize;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
ierr = DMPlexGetRawFaces_Internal(dm, dim, coneSize, cone, numFaces, faceSize, faces);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexCreateOrderingClosure_Static(DM dm, PetscInt numPoints, const PetscInt pperm[], PetscInt **clperm, PetscInt **invclperm)
{
PetscInt *perm, *iperm;
PetscInt depth, d, pStart, pEnd, fStart, fMax, fEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscMalloc1(pEnd-pStart,&perm);CHKERRQ(ierr);
ierr = PetscMalloc1(pEnd-pStart,&iperm);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) iperm[p] = -1;
for (d = depth; d > 0; --d) {
ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, d-1, &fStart, &fEnd);CHKERRQ(ierr);
fMax = fStart;
for (p = pStart; p < pEnd; ++p) {
const PetscInt *cone;
PetscInt point, coneSize, c;
if (d == depth) {
perm[p] = pperm[p];
iperm[pperm[p]] = p;
}
point = perm[p];
ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
const PetscInt oldc = cone[c];
const PetscInt newc = iperm[oldc];
if (newc < 0) {
perm[fMax] = oldc;
iperm[oldc] = fMax++;
}
}
}
if (fMax != fEnd) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of depth %d faces %d does not match permuted nubmer %d", d, fEnd-fStart, fMax-fStart);
}
*clperm = perm;
*invclperm = iperm;
PetscFunctionReturn(0);
}
#define dmplexgetmeet_ dmplexgetmeet
#define dmplexgetfullmeet_ dmplexgetfullmeet
#define dmplexrestoremeet_ dmplexrestoremeet
#define dmplexcreatesection_ dmplexcreatesection
#define dmplexcomputecellgeometry_ dmplexcomputecellgeometry
#endif
/* Definitions of Fortran Wrapper routines */
EXTERN_C_BEGIN
void PETSC_STDCALL dmplexgetcone_(DM *dm, PetscInt *p, F90Array1d *ptr, int *__ierr PETSC_F90_2PTR_PROTO(ptrd))
{
const PetscInt *v;
PetscInt n;
*__ierr = DMPlexGetConeSize(*dm, *p, &n);if (*__ierr) return;
*__ierr = DMPlexGetCone(*dm, *p, &v);if (*__ierr) return;
*__ierr = F90Array1dCreate((void *) v, PETSC_INT, 1, n, ptr PETSC_F90_2PTR_PARAM(ptrd));
}
void PETSC_STDCALL dmplexrestorecone_(DM *dm, PetscInt *p, F90Array1d *ptr, int *__ierr PETSC_F90_2PTR_PROTO(ptrd))
{
*__ierr = F90Array1dDestroy(ptr, PETSC_INT PETSC_F90_2PTR_PARAM(ptrd));if (*__ierr) return;
}
void PETSC_STDCALL dmplexgetconeorientation_(DM *dm, PetscInt *p, F90Array1d *ptr, int *__ierr PETSC_F90_2PTR_PROTO(ptrd))
{
const PetscInt *v;
PetscInt n;
*__ierr = DMPlexGetConeSize(*dm, *p, &n);if (*__ierr) return;
/*
DMPlexGetFaces_Internal - Gets groups of vertices that correspond to faces for the given cell
*/
static PetscErrorCode DMPlexGetFaces_Internal(DM dm, PetscInt dim, PetscInt p, PetscInt *numFaces, PetscInt *faceSize, const PetscInt *faces[])
{
const PetscInt *cone = NULL;
PetscInt *facesTmp;
PetscInt maxConeSize, maxSupportSize, coneSize;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
ierr = DMGetWorkArray(dm, PetscSqr(PetscMax(maxConeSize, maxSupportSize)), PETSC_INT, &facesTmp);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
switch (dim) {
case 2:
switch (coneSize) {
case 3:
if (faces) {
facesTmp[0] = cone[0]; facesTmp[1] = cone[1];
facesTmp[2] = cone[1]; facesTmp[3] = cone[2];
facesTmp[4] = cone[2]; facesTmp[5] = cone[0];
*faces = facesTmp;
}
if (numFaces) *numFaces = 3;
if (faceSize) *faceSize = 2;
break;
case 4:
/* Vertices follow right hand rule */
if (faces) {
facesTmp[0] = cone[0]; facesTmp[1] = cone[1];
facesTmp[2] = cone[1]; facesTmp[3] = cone[2];
facesTmp[4] = cone[2]; facesTmp[5] = cone[3];
facesTmp[6] = cone[3]; facesTmp[7] = cone[0];
*faces = facesTmp;
}
if (numFaces) *numFaces = 4;
if (faceSize) *faceSize = 2;
if (faces) *faces = facesTmp;
break;
default:
SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone size %D not supported for dimension %D", coneSize, dim);
}
break;
case 3:
switch (coneSize) {
case 3:
if (faces) {
facesTmp[0] = cone[0]; facesTmp[1] = cone[1];
facesTmp[2] = cone[1]; facesTmp[3] = cone[2];
facesTmp[4] = cone[2]; facesTmp[5] = cone[0];
*faces = facesTmp;
}
if (numFaces) *numFaces = 3;
if (faceSize) *faceSize = 2;
if (faces) *faces = facesTmp;
break;
case 4:
/* Vertices of first face follow right hand rule and normal points towards last vertex */
if (faces) {
facesTmp[0] = cone[0]; facesTmp[1] = cone[2]; facesTmp[2] = cone[1];
facesTmp[3] = cone[0]; facesTmp[4] = cone[1]; facesTmp[5] = cone[3];
facesTmp[6] = cone[0]; facesTmp[7] = cone[3]; facesTmp[8] = cone[2];
facesTmp[9] = cone[1]; facesTmp[10] = cone[2]; facesTmp[11] = cone[3];
*faces = facesTmp;
}
if (numFaces) *numFaces = 4;
if (faceSize) *faceSize = 3;
if (faces) *faces = facesTmp;
break;
case 8:
if (faces) {
facesTmp[0] = cone[0]; facesTmp[1] = cone[3]; facesTmp[2] = cone[2]; facesTmp[3] = cone[1];
facesTmp[4] = cone[4]; facesTmp[5] = cone[5]; facesTmp[6] = cone[6]; facesTmp[7] = cone[7];
facesTmp[8] = cone[0]; facesTmp[9] = cone[1]; facesTmp[10] = cone[5]; facesTmp[11] = cone[4];
facesTmp[12] = cone[2]; facesTmp[13] = cone[3]; facesTmp[14] = cone[7]; facesTmp[15] = cone[6];
facesTmp[16] = cone[1]; facesTmp[17] = cone[2]; facesTmp[18] = cone[6]; facesTmp[19] = cone[5];
facesTmp[20] = cone[0]; facesTmp[21] = cone[4]; facesTmp[22] = cone[7]; facesTmp[23] = cone[3];
*faces = facesTmp;
}
if (numFaces) *numFaces = 6;
if (faceSize) *faceSize = 4;
break;
default:
SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone size %D not supported for dimension %D", coneSize, dim);
}
break;
default:
SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %D not supported", dim);
}
ierr = DMRestoreWorkArray(dm, 0, PETSC_INT, &facesTmp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* This interpolates faces for cells at some stratum */
static PetscErrorCode DMPlexInterpolateFaces_Internal(DM dm, PetscInt cellDepth, DM idm)
{
PetscHashIJKL faceTable;
PetscInt *pStart, *pEnd;
PetscInt cellDim, depth, faceDepth = cellDepth, numPoints = 0, faceSizeAll = 0, face, c, d;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDimension(dm, &cellDim);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
++depth;
++cellDepth;
cellDim -= depth - cellDepth;
ierr = PetscMalloc2(depth+1,PetscInt,&pStart,depth+1,PetscInt,&pEnd);CHKERRQ(ierr);
for (d = depth-1; d >= faceDepth; --d) {
ierr = DMPlexGetDepthStratum(dm, d, &pStart[d+1], &pEnd[d+1]);CHKERRQ(ierr);
}
ierr = DMPlexGetDepthStratum(dm, -1, NULL, &pStart[faceDepth]);CHKERRQ(ierr);
pEnd[faceDepth] = pStart[faceDepth];
for (d = faceDepth-1; d >= 0; --d) {
ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
}
if (pEnd[cellDepth] > pStart[cellDepth]) {ierr = DMPlexGetFaces_Internal(dm, cellDim, pStart[cellDepth], NULL, &faceSizeAll, NULL);CHKERRQ(ierr);}
if (faceSizeAll > 4) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Do not support interpolation of meshes with faces of %D vertices", faceSizeAll);
ierr = PetscHashIJKLCreate(&faceTable);CHKERRQ(ierr);
ierr = PetscHashIJKLSetMultivalued(faceTable, PETSC_FALSE);CHKERRQ(ierr);
for (c = pStart[cellDepth], face = pStart[faceDepth]; c < pEnd[cellDepth]; ++c) {
const PetscInt *cellFaces;
PetscInt numCellFaces, faceSize, cf, f;
ierr = DMPlexGetFaces_Internal(dm, cellDim, c, &numCellFaces, &faceSize, &cellFaces);CHKERRQ(ierr);
if (faceSize != faceSizeAll) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Inconsistent face for cell %D of size %D != %D", c, faceSize, faceSizeAll);
for (cf = 0; cf < numCellFaces; ++cf) {
const PetscInt *cellFace = &cellFaces[cf*faceSize];
PetscHashIJKLKey key;
if (faceSize == 2) {
key.i = PetscMin(cellFace[0], cellFace[1]);
key.j = PetscMax(cellFace[0], cellFace[1]);
} else {
key.i = cellFace[0]; key.j = cellFace[1]; key.k = cellFace[2]; key.l = faceSize > 3 ? cellFace[3] : 0;
ierr = PetscSortInt(faceSize, (PetscInt *) &key);
}
ierr = PetscHashIJKLGet(faceTable, key, &f);CHKERRQ(ierr);
if (f < 0) {
ierr = PetscHashIJKLAdd(faceTable, key, face);CHKERRQ(ierr);
f = face++;
}
}
}
pEnd[faceDepth] = face;
ierr = PetscHashIJKLDestroy(&faceTable);CHKERRQ(ierr);
/* Count new points */
for (d = 0; d <= depth; ++d) {
numPoints += pEnd[d]-pStart[d];
}
ierr = DMPlexSetChart(idm, 0, numPoints);CHKERRQ(ierr);
/* Set cone sizes */
for (d = 0; d <= depth; ++d) {
PetscInt coneSize, p;
if (d == faceDepth) {
for (p = pStart[d]; p < pEnd[d]; ++p) {
/* I see no way to do this if we admit faces of different shapes */
ierr = DMPlexSetConeSize(idm, p, faceSizeAll);CHKERRQ(ierr);
}
} else if (d == cellDepth) {
for (p = pStart[d]; p < pEnd[d]; ++p) {
/* Number of cell faces may be different from number of cell vertices*/
ierr = DMPlexGetFaces_Internal(dm, cellDim, p, &coneSize, NULL, NULL);CHKERRQ(ierr);
ierr = DMPlexSetConeSize(idm, p, coneSize);CHKERRQ(ierr);
}
} else {
for (p = pStart[d]; p < pEnd[d]; ++p) {
ierr = DMPlexGetConeSize(dm, p, &coneSize);CHKERRQ(ierr);
ierr = DMPlexSetConeSize(idm, p, coneSize);CHKERRQ(ierr);
}
}
}
ierr = DMSetUp(idm);CHKERRQ(ierr);
/* Get face cones from subsets of cell vertices */
if (faceSizeAll > 4) SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Do not support interpolation of meshes with faces of %D vertices", faceSizeAll);
ierr = PetscHashIJKLCreate(&faceTable);CHKERRQ(ierr);
ierr = PetscHashIJKLSetMultivalued(faceTable, PETSC_FALSE);CHKERRQ(ierr);
for (d = depth; d > cellDepth; --d) {
const PetscInt *cone;
PetscInt p;
for (p = pStart[d]; p < pEnd[d]; ++p) {
ierr = DMPlexGetCone(dm, p, &cone);CHKERRQ(ierr);
ierr = DMPlexSetCone(idm, p, cone);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, p, &cone);CHKERRQ(ierr);
ierr = DMPlexSetConeOrientation(idm, p, cone);CHKERRQ(ierr);
}
}
for (c = pStart[cellDepth], face = pStart[faceDepth]; c < pEnd[cellDepth]; ++c) {
const PetscInt *cellFaces;
PetscInt numCellFaces, faceSize, cf, f;
ierr = DMPlexGetFaces_Internal(dm, cellDim, c, &numCellFaces, &faceSize, &cellFaces);CHKERRQ(ierr);
if (faceSize != faceSizeAll) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Inconsistent face for cell %D of size %D != %D", c, faceSize, faceSizeAll);
for (cf = 0; cf < numCellFaces; ++cf) {
const PetscInt *cellFace = &cellFaces[cf*faceSize];
PetscHashIJKLKey key;
if (faceSize == 2) {
key.i = PetscMin(cellFace[0], cellFace[1]);
key.j = PetscMax(cellFace[0], cellFace[1]);
} else {
key.i = cellFace[0]; key.j = cellFace[1]; key.k = cellFace[2]; key.l = faceSize > 3 ? cellFace[3] : 0;
ierr = PetscSortInt(faceSize, (PetscInt *) &key);
}
ierr = PetscHashIJKLGet(faceTable, key, &f);CHKERRQ(ierr);
if (f < 0) {
ierr = DMPlexSetCone(idm, face, cellFace);CHKERRQ(ierr);
ierr = PetscHashIJKLAdd(faceTable, key, face);CHKERRQ(ierr);
f = face++;
ierr = DMPlexInsertCone(idm, c, cf, f);CHKERRQ(ierr);
} else {
const PetscInt *cone;
PetscInt coneSize, ornt, i, j;
ierr = DMPlexInsertCone(idm, c, cf, f);CHKERRQ(ierr);
/* Orient face */
ierr = DMPlexGetConeSize(idm, f, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(idm, f, &cone);CHKERRQ(ierr);
if (coneSize != faceSize) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of face vertices %D for face %D should be %D", coneSize, f, faceSize);
/* - First find the initial vertex */
for (i = 0; i < faceSize; ++i) if (cellFace[0] == cone[i]) break;
/* - Try forward comparison */
for (j = 0; j < faceSize; ++j) if (cellFace[j] != cone[(i+j)%faceSize]) break;
if (j == faceSize) {
if ((faceSize == 2) && (i == 1)) ornt = -2;
else ornt = i;
} else {
/* - Try backward comparison */
for (j = 0; j < faceSize; ++j) if (cellFace[j] != cone[(i+faceSize-j)%faceSize]) break;
if (j == faceSize) ornt = -(i+1);
else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not determine face orientation");
}
ierr = DMPlexInsertConeOrientation(idm, c, cf, ornt);CHKERRQ(ierr);
}
}
}
if (face != pEnd[faceDepth]) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid number of faces %D should be %D", face-pStart[faceDepth], pEnd[faceDepth]-pStart[faceDepth]);
ierr = PetscFree2(pStart,pEnd);CHKERRQ(ierr);
ierr = PetscHashIJKLDestroy(&faceTable);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(idm);CHKERRQ(ierr);
ierr = DMPlexStratify(idm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeJacobian_LS(DM dm, Vec locX, PetscInt cell, PetscScalar CellValues[], void *ctx)
{
User user = (User) ctx;
Physics phys = user->model->physics;
PetscInt dof = phys->dof;
const PetscScalar *facegeom, *cellgeom,*x;
PetscErrorCode ierr;
DM dmFace, dmCell;
DM dmGrad = user->dmGrad;
PetscInt fStart, fEnd, face, cStart;
Vec locGrad, locGradLimiter, Grad;
/*here the localGradLimiter refers to the gradient that has been multiplied by the limiter function.
The locGradLimiter is used to construct the uL and uR, and the locGrad is used to caculate the diffusion term*/
Vec TempVec; /*a temperal vec for the vector restore*/
PetscFunctionBeginUser;
ierr = VecGetDM(user->facegeom,&dmFace);CHKERRQ(ierr);
ierr = VecGetDM(user->cellgeom,&dmCell);CHKERRQ(ierr);
ierr = DMGetGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecDuplicate(Grad, &TempVec);CHKERRQ(ierr);
ierr = VecCopy(Grad, TempVec);CHKERRQ(ierr);
/*Backup the original vector and use it to restore the value of dmGrad,
because I do not want to change the values of the cell gradient*/
ierr = VecGetArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
{
PetscScalar *grad;
ierr = VecGetArray(Grad,&grad);CHKERRQ(ierr);
/* Limit interior gradients. Using cell-based loop because it generalizes better to vector limiters. */
const PetscInt *faces;
PetscInt numFaces,f;
PetscReal *cellPhi; /* Scalar limiter applied to each component separately */
const PetscScalar *cx;
const CellGeom *cg;
PetscScalar *cgrad;
PetscInt i;
ierr = PetscMalloc(phys->dof*sizeof(PetscScalar),&cellPhi);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm,cell,&numFaces);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm,cell,&faces);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm,cell,x,&cx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell,cell,cellgeom,&cg);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmGrad,cell,grad,&cgrad);CHKERRQ(ierr);
/* Limiter will be minimum value over all neighbors */
for (i=0; i<dof; i++) {
cellPhi[i] = PETSC_MAX_REAL;
}
for (f=0; f<numFaces; f++) {
const PetscScalar *ncx;
const CellGeom *ncg;
const PetscInt *fcells;
PetscInt face = faces[f],ncell;
PetscScalar v[DIM];
PetscBool ghost;
ierr = IsExteriorGhostFace(dm,face,&ghost);CHKERRQ(ierr);
if (ghost) continue;
ierr = DMPlexGetSupport(dm,face,&fcells);CHKERRQ(ierr);
ncell = cell == fcells[0] ? fcells[1] : fcells[0]; /*The expression (x ? y : z) has the value of y if x is nonzero, z otherwise */
ierr = DMPlexPointLocalRead(dm,ncell,x,&ncx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell,ncell,cellgeom,&ncg);CHKERRQ(ierr);
Waxpy2(-1, cg->centroid, ncg->centroid, v);
for (i=0; i<dof; i++) {
/* We use the symmetric slope limited form of Berger, Aftosmis, and Murman 2005 */
PetscScalar phi,flim = 0.5 * (ncx[i] - cx[i]) / Dot2(&cgrad[i*DIM],v);
phi = (*user->LimitGrad)(flim);
cellPhi[i] = PetscMin(cellPhi[i],phi);
}
}
/* Apply limiter to gradient */
for (i=0; i<dof; i++) Scale2(cellPhi[i],&cgrad[i*DIM],&cgrad[i*DIM]);
ierr = PetscFree(cellPhi);CHKERRQ(ierr);
ierr = VecRestoreArray(Grad,&grad);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(dmGrad,&locGradLimiter);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad,Grad,INSERT_VALUES,locGradLimiter);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad,Grad,INSERT_VALUES,locGradLimiter);CHKERRQ(ierr);
ierr = VecCopy(TempVec, Grad);CHKERRQ(ierr);/*Restore the vector*/
ierr = DMGetLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad,Grad,INSERT_VALUES,locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad,Grad,INSERT_VALUES,locGrad);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecDestroy(&TempVec);CHKERRQ(ierr);
{
const PetscScalar *grad, *gradlimiter;
ierr = VecGetArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = VecGetArrayRead(locGradLimiter,&gradlimiter);CHKERRQ(ierr);
for (face=fStart; face<fEnd; face++) {
const PetscInt *cells;
PetscInt ghost,i,j;
PetscScalar *fluxcon, *fluxdiff, *fx[2];
const FaceGeom *fg;
const CellGeom *cg[2];
const PetscScalar *cx[2],*cgrad[2], *cgradlimiter[2];
PetscScalar *uL, *uR;
PetscReal FaceArea;
ierr = PetscMalloc(phys->dof * phys->dof * sizeof(PetscScalar), &fluxcon);CHKERRQ(ierr); /*For the convection terms*/
ierr = PetscMalloc(phys->dof * phys->dof * sizeof(PetscScalar), &fluxdiff);CHKERRQ(ierr); /*For the diffusion terms*/
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &uL);CHKERRQ(ierr);
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &uR);CHKERRQ(ierr);
fx[0] = uL; fx[1] = uR;
ierr = DMPlexGetLabelValue(dm, "ghost", face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace,face,facegeom,&fg);CHKERRQ(ierr);
for (i=0; i<2; i++) {
PetscScalar dx[DIM];
ierr = DMPlexPointLocalRead(dmCell,cells[i],cellgeom,&cg[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm,cells[i],x,&cx[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad,cells[i],gradlimiter,&cgradlimiter[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad,cells[i],grad,&cgrad[i]);CHKERRQ(ierr);
Waxpy2(-1,cg[i]->centroid,fg->centroid,dx);
for (j=0; j<dof; j++) {
fx[i][j] = cx[i][j] + Dot2(cgradlimiter[i],dx);
}
/*fx[0] and fx[1] are the value of the variables on the left and right
side of the face, respectively, that is u_L and u_R.*/
}
ierr = RiemannSolver_Rusanov_Jacobian(user, cgrad[0], cgrad[1], fg->centroid, cg[0]->centroid, cg[1]->centroid, fg->normal,
fx[0], fx[1], fluxcon, fluxdiff);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, face, &FaceArea, NULL, NULL);CHKERRQ(ierr);
/*Compute the face area*/
for (i=0; i<phys->dof; i++) {
for (j=0; j<phys->dof; j++) {
if(cells[0]<user->cEndInterior) CellValues[cells[0]*dof*dof + i*dof + j] -= cells[0]*1.0;
if(cells[1]<user->cEndInterior) CellValues[cells[1]*dof*dof + i*dof + j] += cells[1]*1.2;
}
}
// ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
ierr = PetscFree(fluxcon);CHKERRQ(ierr);
ierr = PetscFree(fluxdiff);CHKERRQ(ierr);
ierr = PetscFree(uL);CHKERRQ(ierr);
ierr = PetscFree(uR);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locGradLimiter,&gradlimiter);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGradLimiter);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Compute the gradient of the variables at the center of the cell
by the least-square reconstruction method based on the function
BuildLeastSquares.
*/
PetscErrorCode ConstructCellCentriodGradientJacobian(DM dm,DM dmFace,DM dmCell,PetscReal time,Vec locX, PetscInt cell, PetscScalar CellValues[],User user)
{
DM dmGrad = user->dmGrad;
Model mod = user->model;
Physics phys = mod->physics;
const PetscInt dof = phys->dof;
PetscErrorCode ierr;
const PetscScalar *facegeom, *cellgeom, *x;
PetscInt fStart, fEnd, face, cStart;
PetscInt numFaces;
const PetscInt *faces;
Vec Grad;
PetscFunctionBeginUser;
ierr = DMGetGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecZeroEntries(Grad);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm,cell,&numFaces);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm,cell,&faces);CHKERRQ(ierr);
{
PetscScalar *grad;
ierr = VecGetArray(Grad,&grad);CHKERRQ(ierr);
/* Reconstruct gradients */
for (face=0; face<numFaces; face++) {
const PetscInt *cells;
const PetscScalar *cx[2];
const FaceGeom *fg;
PetscScalar *cgrad[2];
PetscInt i,j;
PetscBool ghost;
ierr = IsExteriorGhostFace(dm,faces[face],&ghost);CHKERRQ(ierr);
if (ghost) continue;
ierr = DMPlexGetSupport(dm,faces[face],&cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace,faces[face],facegeom,&fg);CHKERRQ(ierr);
for (i=0; i<2; i++) {
ierr = DMPlexPointLocalRead(dm,cells[i],x,&cx[i]);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmGrad,cells[i],grad,&cgrad[i]);CHKERRQ(ierr);
}
for (i=0; i<dof; i++) {
PetscScalar delta = cx[1][i] - cx[0][i];
for (j=0; j<DIM; j++) {
if (cgrad[0]) cgrad[0][i*DIM+j] += fg->grad[0][j] * delta;
if (cgrad[1]) cgrad[1][i*DIM+j] -= fg->grad[1][j] * delta;
}
}
}
ierr = VecRestoreArray(Grad,&grad);CHKERRQ(ierr);
}
ierr = DMRestoreGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locX,&x);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/**
Caculate the flux using the Monotonic Upstream-Centered Scheme for Conservation Laws (van Leer, 1979)
*/
PetscErrorCode CaculateLocalFunction_LS(DM dm,DM dmFace,DM dmCell,PetscReal time,Vec locX,Vec F,User user)
{
DM dmGrad = user->dmGrad;
Model mod = user->model;
Physics phys = mod->physics;
const PetscInt dof = phys->dof;
PetscErrorCode ierr;
const PetscScalar *facegeom, *cellgeom, *x;
PetscScalar *f;
PetscInt fStart, fEnd, face, cStart, cell;
Vec locGrad, locGradLimiter, Grad;
/*
Here the localGradLimiter refers to the gradient that
has been multiplied by the limiter function.
The locGradLimiter is used to construct the uL and uR,
and the locGrad is used to caculate the diffusion term
*/
Vec TempVec; /*a temperal vec for the vector restore*/
PetscFunctionBeginUser;
ierr = DMGetGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecDuplicate(Grad, &TempVec);CHKERRQ(ierr);
ierr = VecCopy(Grad, TempVec);CHKERRQ(ierr);
/*
Backup the original vector and use it to restore the value of dmGrad,
because I do not want to change the values of the cell gradient.
*/
ierr = VecGetArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
{
PetscScalar *grad;
ierr = VecGetArray(Grad,&grad);CHKERRQ(ierr);
const PetscInt *faces;
PetscInt numFaces,f;
PetscReal *cellPhi; // Scalar limiter applied to each component separately
const PetscScalar *cx;
const CellGeom *cg;
PetscScalar *cgrad;
PetscInt i;
ierr = PetscMalloc(phys->dof*sizeof(PetscScalar),&cellPhi);CHKERRQ(ierr);
// Limit interior gradients. Using cell-based loop because it generalizes better to vector limiters.
for (cell=cStart; cell<user->cEndInterior; cell++) {
ierr = DMPlexGetConeSize(dm,cell,&numFaces);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm,cell,&faces);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm,cell,x,&cx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell,cell,cellgeom,&cg);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmGrad,cell,grad,&cgrad);CHKERRQ(ierr);
if (!cgrad) continue; // ghost cell, we don't compute
// Limiter will be minimum value over all neighbors
for (i=0; i<dof; i++) cellPhi[i] = PETSC_MAX_REAL;
for (f=0; f<numFaces; f++) {
const PetscScalar *ncx;
const CellGeom *ncg;
const PetscInt *fcells;
PetscInt face = faces[f],ncell;
PetscScalar v[DIM];
PetscBool ghost;
ierr = IsExteriorGhostFace(dm,face,&ghost);CHKERRQ(ierr);
if (ghost) continue;
ierr = DMPlexGetSupport(dm,face,&fcells);CHKERRQ(ierr);
ncell = cell == fcells[0] ? fcells[1] : fcells[0];
// The expression (x ? y : z) has the value of y if x is nonzero, z otherwise
ierr = DMPlexPointLocalRead(dm,ncell,x,&ncx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell,ncell,cellgeom,&ncg);CHKERRQ(ierr);
Waxpy2(-1, cg->centroid, ncg->centroid, v);
for (i=0; i<dof; i++) {
// We use the symmetric slope limited form of Berger, Aftosmis, and Murman 2005
PetscScalar phi,flim = 0.5 * (ncx[i] - cx[i]) / Dot2(&cgrad[i*DIM],v);
phi = (*user->Limit)(flim);
cellPhi[i] = PetscMin(cellPhi[i],phi);
}
}
// Apply limiter to gradient
for (i=0; i<dof; i++) Scale2(cellPhi[i],&cgrad[i*DIM],&cgrad[i*DIM]);
}
ierr = PetscFree(cellPhi);CHKERRQ(ierr);
ierr = VecRestoreArray(Grad,&grad);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(dmGrad,&locGradLimiter);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad,Grad,INSERT_VALUES,locGradLimiter);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad,Grad,INSERT_VALUES,locGradLimiter);CHKERRQ(ierr);
ierr = VecCopy(TempVec, Grad);CHKERRQ(ierr);//Restore the vector
ierr = DMGetLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad,Grad,INSERT_VALUES,locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad,Grad,INSERT_VALUES,locGrad);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecDestroy(&TempVec);CHKERRQ(ierr);
{
const PetscScalar *grad, *gradlimiter;
const PetscInt *cells;
PetscInt ghost,i,j;
PetscScalar *fluxcon, *fluxdiff, *fx[2],*cf[2];
const FaceGeom *fg;
const CellGeom *cg[2];
const PetscScalar *cx[2],*cgrad[2], *cgradlimiter[2];
PetscScalar *uL, *uR;
PetscReal FaceArea;
ierr = VecGetArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = VecGetArrayRead(locGradLimiter,&gradlimiter);CHKERRQ(ierr);
ierr = VecGetArray(F,&f);CHKERRQ(ierr);
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &fluxcon);CHKERRQ(ierr); // For the convection terms
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &fluxdiff);CHKERRQ(ierr); // For the diffusion terms
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &uL);CHKERRQ(ierr);
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &uR);CHKERRQ(ierr);// Please do not put the Malloc function into a for loop!!!!
for (face=fStart; face<fEnd; face++) {
fx[0] = uL; fx[1] = uR;
ierr = DMPlexGetLabelValue(dm, "ghost", face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace,face,facegeom,&fg);CHKERRQ(ierr);
for (i=0; i<2; i++) {
PetscScalar dx[DIM];
ierr = DMPlexPointLocalRead(dmCell,cells[i],cellgeom,&cg[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm,cells[i],x,&cx[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad,cells[i],gradlimiter,&cgradlimiter[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad,cells[i],grad,&cgrad[i]);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm,cells[i],f,&cf[i]);CHKERRQ(ierr);
Waxpy2(-1,cg[i]->centroid,fg->centroid,dx);
for (j=0; j<dof; j++) {
fx[i][j] = cx[i][j] + Dot2(cgradlimiter[i],dx);
}
// fx[0] and fx[1] are the value of the variables on the left and right
// side of the face, respectively, that is u_L and u_R.
}
ierr = RiemannSolver_Rusanov(user, cgrad[0], cgrad[1], fg->centroid, cg[0]->centroid, cg[1]->centroid, fg->normal, fx[0], fx[1], fluxcon, fluxdiff);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, face, &FaceArea, NULL, NULL);CHKERRQ(ierr);
// Compute the face area
for (i=0; i<phys->dof; i++) {
if (cf[0]) cf[0][i] -= FaceArea*(fluxcon[i] + fluxdiff[i])/cg[0]->volume;
if (cf[1]) cf[1][i] += FaceArea*(fluxcon[i] + fluxdiff[i])/cg[1]->volume;
// The flux on the interface, for the cell[0], it is an outcome flux and for the cell[1], it is
// an income flux.
}
// ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locGradLimiter,&gradlimiter);CHKERRQ(ierr);
ierr = VecRestoreArray(F,&f);CHKERRQ(ierr);
ierr = PetscFree(fluxcon);CHKERRQ(ierr);
ierr = PetscFree(fluxdiff);CHKERRQ(ierr);
ierr = PetscFree(uL);CHKERRQ(ierr);
ierr = PetscFree(uR);CHKERRQ(ierr);
// VecView(F,PETSC_VIEWER_STDOUT_WORLD);
}
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGradLimiter);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexTSSetupGeometry(DM dm, PetscFV fvm, DMTS_Plex *dmplexts)
{
DM dmFace, dmCell;
DMLabel ghostLabel;
PetscSection sectionFace, sectionCell;
PetscSection coordSection;
Vec coordinates;
PetscReal minradius;
PetscScalar *fgeom, *cgeom;
PetscInt dim, cStart, cEnd, cEndInterior, c, fStart, fEnd, f;
PetscErrorCode ierr;
PetscFunctionBegin;
if (dmplexts->setupGeom) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
/* Make cell centroids and volumes */
ierr = DMClone(dm, &dmCell);CHKERRQ(ierr);
ierr = DMSetCoordinateSection(dmCell, coordSection);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dmCell, coordinates);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionCell);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionCell, cStart, cEnd);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {ierr = PetscSectionSetDof(sectionCell, c, sizeof(CellGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionCell);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmCell, sectionCell);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionCell);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmCell, &dmplexts->cellgeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
for (c = cStart; c < cEndInterior; ++c) {
CellGeom *cg;
ierr = DMPlexPointLocalRef(dmCell, c, cgeom, &cg);CHKERRQ(ierr);
ierr = PetscMemzero(cg, sizeof(*cg));CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dmCell, c, &cg->volume, cg->centroid, NULL);CHKERRQ(ierr);
}
/* Compute face normals and minimum cell radius */
ierr = DMClone(dm, &dmFace);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionFace);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionFace, fStart, fEnd);CHKERRQ(ierr);
for (f = fStart; f < fEnd; ++f) {ierr = PetscSectionSetDof(sectionFace, f, sizeof(FaceGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionFace);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmFace, sectionFace);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionFace);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmFace, &dmplexts->facegeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = DMPlexGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
minradius = PETSC_MAX_REAL;
for (f = fStart; f < fEnd; ++f) {
FaceGeom *fg;
PetscReal area;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, f, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexPointLocalRef(dmFace, f, fgeom, &fg);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, f, &area, fg->centroid, fg->normal);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) fg->normal[d] *= area;
/* Flip face orientation if necessary to match ordering in support, and Update minimum radius */
{
CellGeom *cL, *cR;
const PetscInt *cells;
PetscReal *lcentroid, *rcentroid;
PetscReal v[3];
ierr = DMPlexGetSupport(dm, f, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[0], cgeom, &cL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[1], cgeom, &cR);CHKERRQ(ierr);
lcentroid = cells[0] >= cEndInterior ? fg->centroid : cL->centroid;
rcentroid = cells[1] >= cEndInterior ? fg->centroid : cR->centroid;
WaxpyD(dim, -1, lcentroid, rcentroid, v);
if (DotRealD(dim, fg->normal, v) < 0) {
for (d = 0; d < dim; ++d) fg->normal[d] = -fg->normal[d];
}
if (DotRealD(dim, fg->normal, v) <= 0) {
if (dim == 2) SETERRQ5(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g) v (%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) v[0], (double) v[1]);
if (dim == 3) SETERRQ7(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g,%g) v (%g,%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) fg->normal[2], (double) v[0], (double) v[1], (double) v[2]);
SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed", f);
}
if (cells[0] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cL->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
if (cells[1] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cR->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
}
}
ierr = MPI_Allreduce(&minradius, &dmplexts->minradius, 1, MPIU_REAL, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
/* Compute centroids of ghost cells */
for (c = cEndInterior; c < cEnd; ++c) {
FaceGeom *fg;
const PetscInt *cone, *support;
PetscInt coneSize, supportSize, s;
ierr = DMPlexGetConeSize(dmCell, c, &coneSize);CHKERRQ(ierr);
if (coneSize != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ghost cell %d has cone size %d != 1", c, coneSize);
ierr = DMPlexGetCone(dmCell, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(dmCell, cone[0], &supportSize);CHKERRQ(ierr);
if (supportSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d has support size %d != 1", cone[0], supportSize);
ierr = DMPlexGetSupport(dmCell, cone[0], &support);CHKERRQ(ierr);
ierr = DMPlexPointLocalRef(dmFace, cone[0], fgeom, &fg);CHKERRQ(ierr);
for (s = 0; s < 2; ++s) {
/* Reflect ghost centroid across plane of face */
if (support[s] == c) {
const CellGeom *ci;
CellGeom *cg;
PetscReal c2f[3], a;
ierr = DMPlexPointLocalRead(dmCell, support[(s+1)%2], cgeom, &ci);CHKERRQ(ierr);
WaxpyD(dim, -1, ci->centroid, fg->centroid, c2f); /* cell to face centroid */
a = DotRealD(dim, c2f, fg->normal)/DotRealD(dim, fg->normal, fg->normal);
ierr = DMPlexPointLocalRef(dmCell, support[s], cgeom, &cg);CHKERRQ(ierr);
WaxpyD(dim, 2*a, fg->normal, ci->centroid, cg->centroid);
cg->volume = ci->volume;
}
}
}
ierr = VecRestoreArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
ierr = DMDestroy(&dmCell);CHKERRQ(ierr);
ierr = DMDestroy(&dmFace);CHKERRQ(ierr);
dmplexts->setupGeom = PETSC_TRUE;
PetscFunctionReturn(0);
}
static PetscErrorCode TSComputeRHSFunction_DMPlex(TS ts, PetscReal time, Vec X, Vec F, void *ctx)
{
DM dm;
DMTS_Plex *dmplexts = (DMTS_Plex *) ctx;
void (*riemann)(const PetscReal[], const PetscReal[], const PetscScalar[], const PetscScalar[], PetscScalar[], void *) = dmplexts->riemann;
PetscFV fvm;
PetscLimiter lim;
Vec faceGeometry = dmplexts->facegeom;
Vec cellGeometry = dmplexts->cellgeom;
Vec Grad = NULL, locGrad, locX;
DM dmFace, dmCell;
DMLabel ghostLabel;
PetscCellGeometry fgeom, cgeom;
const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
PetscScalar *grad, *f, *uL, *uR, *fluxL, *fluxR;
PetscReal *centroid, *normal, *vol, *cellPhi;
PetscBool computeGradients;
PetscInt Nf, dim, pdim, fStart, fEnd, numFaces = 0, face, iface, cell, cStart, cEnd, cEndInterior;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(ts,TS_CLASSID,1);
PetscValidHeaderSpecific(X,VEC_CLASSID,3);
PetscValidHeaderSpecific(F,VEC_CLASSID,5);
ierr = TSGetDM(ts, &dm);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &locX);CHKERRQ(ierr);
ierr = VecZeroEntries(locX);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX);CHKERRQ(ierr);
ierr = VecZeroEntries(F);CHKERRQ(ierr);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
ierr = DMGetField(dm, 0, (PetscObject *) &fvm);CHKERRQ(ierr);
ierr = PetscFVGetLimiter(fvm, &lim);CHKERRQ(ierr);
ierr = PetscFVGetNumComponents(fvm, &pdim);CHKERRQ(ierr);
ierr = PetscFVGetComputeGradients(fvm, &computeGradients);CHKERRQ(ierr);
if (computeGradients) {
ierr = DMGetGlobalVector(dmplexts->dmGrad, &Grad);CHKERRQ(ierr);
ierr = VecZeroEntries(Grad);CHKERRQ(ierr);
ierr = VecGetArray(Grad, &grad);CHKERRQ(ierr);
}
ierr = DMPlexGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = VecGetDM(faceGeometry, &dmFace);CHKERRQ(ierr);
ierr = VecGetDM(cellGeometry, &dmCell);CHKERRQ(ierr);
ierr = VecGetArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX, &x);CHKERRQ(ierr);
/* Count faces and reconstruct gradients */
for (face = fStart; face < fEnd; ++face) {
const PetscInt *cells;
const FaceGeom *fg;
const PetscScalar *cx[2];
PetscScalar *cgrad[2];
PetscBool boundary;
PetscInt ghost, c, pd, d;
ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
++numFaces;
if (!computeGradients) continue;
ierr = DMPlexIsBoundaryPoint(dm, face, &boundary);CHKERRQ(ierr);
if (boundary) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
for (c = 0; c < 2; ++c) {
ierr = DMPlexPointLocalRead(dm, cells[c], x, &cx[c]);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmplexts->dmGrad, cells[c], grad, &cgrad[c]);CHKERRQ(ierr);
}
for (pd = 0; pd < pdim; ++pd) {
PetscScalar delta = cx[1][pd] - cx[0][pd];
for (d = 0; d < dim; ++d) {
if (cgrad[0]) cgrad[0][pd*dim+d] += fg->grad[0][d] * delta;
if (cgrad[1]) cgrad[1][pd*dim+d] -= fg->grad[1][d] * delta;
}
}
}
/* Limit interior gradients (using cell-based loop because it generalizes better to vector limiters) */
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = DMGetWorkArray(dm, pdim, PETSC_REAL, &cellPhi);CHKERRQ(ierr);
for (cell = computeGradients && lim ? cStart : cEnd; cell < cEndInterior; ++cell) {
const PetscInt *faces;
const PetscScalar *cx;
const CellGeom *cg;
PetscScalar *cgrad;
PetscInt coneSize, f, pd, d;
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &faces);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm, cell, x, &cx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cell, cellgeom, &cg);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmplexts->dmGrad, cell, grad, &cgrad);CHKERRQ(ierr);
if (!cgrad) continue; /* Unowned overlap cell, we do not compute */
/* Limiter will be minimum value over all neighbors */
for (d = 0; d < pdim; ++d) cellPhi[d] = PETSC_MAX_REAL;
for (f = 0; f < coneSize; ++f) {
const PetscScalar *ncx;
const CellGeom *ncg;
const PetscInt *fcells;
PetscInt face = faces[f], ncell, ghost;
PetscReal v[3];
PetscBool boundary;
ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
ierr = DMPlexIsBoundaryPoint(dm, face, &boundary);CHKERRQ(ierr);
if ((ghost >= 0) || boundary) continue;
ierr = DMPlexGetSupport(dm, face, &fcells);CHKERRQ(ierr);
ncell = cell == fcells[0] ? fcells[1] : fcells[0];
ierr = DMPlexPointLocalRead(dm, ncell, x, &ncx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, ncell, cellgeom, &ncg);CHKERRQ(ierr);
WaxpyD(dim, -1, cg->centroid, ncg->centroid, v);
for (d = 0; d < pdim; ++d) {
/* We use the symmetric slope limited form of Berger, Aftosmis, and Murman 2005 */
PetscReal phi, flim = 0.5 * PetscRealPart(ncx[d] - cx[d]) / DotD(dim, &cgrad[d*dim], v);
ierr = PetscLimiterLimit(lim, flim, &phi);CHKERRQ(ierr);
cellPhi[d] = PetscMin(cellPhi[d], phi);
}
}
/* Apply limiter to gradient */
for (pd = 0; pd < pdim; ++pd)
/* Scalar limiter applied to each component separately */
for (d = 0; d < dim; ++d) cgrad[pd*dim+d] *= cellPhi[pd];
}
ierr = DMRestoreWorkArray(dm, pdim, PETSC_REAL, &cellPhi);CHKERRQ(ierr);
ierr = DMPlexInsertBoundaryValuesFVM_Static(dm, fvm, time, locX, Grad, dmplexts);CHKERRQ(ierr);
if (computeGradients) {
ierr = VecRestoreArray(Grad, &grad);CHKERRQ(ierr);
ierr = DMGetLocalVector(dmplexts->dmGrad, &locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmplexts->dmGrad, Grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmplexts->dmGrad, Grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dmplexts->dmGrad, &Grad);CHKERRQ(ierr);
ierr = VecGetArrayRead(locGrad, &lgrad);CHKERRQ(ierr);
}
ierr = PetscMalloc7(numFaces*dim,¢roid,numFaces*dim,&normal,numFaces*2,&vol,numFaces*pdim,&uL,numFaces*pdim,&uR,numFaces*pdim,&fluxL,numFaces*pdim,&fluxR);CHKERRQ(ierr);
/* Read out values */
for (face = fStart, iface = 0; face < fEnd; ++face) {
const PetscInt *cells;
const FaceGeom *fg;
const CellGeom *cgL, *cgR;
const PetscScalar *xL, *xR, *gL, *gR;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm, cells[0], x, &xL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm, cells[1], x, &xR);CHKERRQ(ierr);
if (computeGradients) {
PetscReal dxL[3], dxR[3];
ierr = DMPlexPointLocalRead(dmplexts->dmGrad, cells[0], lgrad, &gL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmplexts->dmGrad, cells[1], lgrad, &gR);CHKERRQ(ierr);
WaxpyD(dim, -1, cgL->centroid, fg->centroid, dxL);
WaxpyD(dim, -1, cgR->centroid, fg->centroid, dxR);
for (d = 0; d < pdim; ++d) {
uL[iface*pdim+d] = xL[d] + DotD(dim, &gL[d*dim], dxL);
uR[iface*pdim+d] = xR[d] + DotD(dim, &gR[d*dim], dxR);
}
} else {
for (d = 0; d < pdim; ++d) {
uL[iface*pdim+d] = xL[d];
uR[iface*pdim+d] = xR[d];
}
}
for (d = 0; d < dim; ++d) {
centroid[iface*dim+d] = fg->centroid[d];
normal[iface*dim+d] = fg->normal[d];
}
vol[iface*2+0] = cgL->volume;
vol[iface*2+1] = cgR->volume;
++iface;
}
if (computeGradients) {
ierr = VecRestoreArrayRead(locGrad,&lgrad);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmplexts->dmGrad, &locGrad);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(locX, &x);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
fgeom.v0 = centroid;
fgeom.n = normal;
cgeom.vol = vol;
/* Riemann solve */
ierr = PetscFVIntegrateRHSFunction(fvm, numFaces, Nf, &fvm, 0, fgeom, cgeom, uL, uR, riemann, fluxL, fluxR, dmplexts->rhsfunctionlocalctx);CHKERRQ(ierr);
/* Insert fluxes */
ierr = VecGetArray(F, &f);CHKERRQ(ierr);
for (face = fStart, iface = 0; face < fEnd; ++face) {
const PetscInt *cells;
PetscScalar *fL, *fR;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm, cells[0], f, &fL);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm, cells[1], f, &fR);CHKERRQ(ierr);
for (d = 0; d < pdim; ++d) {
if (fL) fL[d] -= fluxL[iface*pdim+d];
if (fR) fR[d] += fluxR[iface*pdim+d];
}
++iface;
}
ierr = VecRestoreArray(F, &f);CHKERRQ(ierr);
ierr = PetscFree7(centroid,normal,vol,uL,uR,fluxL,fluxR);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &locX);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETSC_STDCALL dmplexgetconesize_(DM dm,PetscInt *p,PetscInt *size, int *__ierr ){
*__ierr = DMPlexGetConeSize(
(DM)PetscToPointer((dm) ),*p,size);
}
PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
{
DM_Plex *mesh = (DM_Plex*) dm->data;
#ifdef PETSC_HAVE_PRAGMATIC
DM udm, coordDM;
DMLabel bd;
Mat A;
Vec coordinates, mb, mx;
PetscSection coordSection;
const PetscScalar *coords;
double *coarseCoords;
IS bdIS;
PetscReal *x, *y, *z, *eqns, *metric;
PetscReal coarseRatio = PetscSqr(0.5);
const PetscInt *faces;
PetscInt *cells, *bdFaces, *bdFaceIds;
PetscInt dim, numCorners, cStart, cEnd, numCells, numCoarseCells, c, vStart, vEnd, numVertices, numCoarseVertices, v, numBdFaces, f, maxConeSize, size, bdSize, coff;
#endif
PetscErrorCode ierr;
PetscFunctionBegin;
#ifdef PETSC_HAVE_PRAGMATIC
if (!mesh->coarseMesh) {
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm, &coordDM);CHKERRQ(ierr);
ierr = DMGetDefaultSection(coordDM, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(udm, &maxConeSize, NULL);CHKERRQ(ierr);
numCells = cEnd - cStart;
numVertices = vEnd - vStart;
ierr = PetscCalloc5(numVertices, &x, numVertices, &y, numVertices, &z, numVertices*PetscSqr(dim), &metric, numCells*maxConeSize, &cells);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
PetscInt off;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
x[v-vStart] = coords[off+0];
y[v-vStart] = coords[off+1];
if (dim > 2) z[v-vStart] = coords[off+2];
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (c = 0, coff = 0; c < numCells; ++c) {
const PetscInt *cone;
PetscInt coneSize, cl;
ierr = DMPlexGetConeSize(udm, c, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(udm, c, &cone);CHKERRQ(ierr);
for (cl = 0; cl < coneSize; ++cl) cells[coff++] = cone[cl] - vStart;
}
switch (dim) {
case 2:
pragmatic_2d_init(&numVertices, &numCells, cells, x, y);
break;
case 3:
pragmatic_3d_init(&numVertices, &numCells, cells, x, y, z);
break;
default:
SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No Pragmatic coarsening defined for dimension %d", dim);
}
/* Create boundary mesh */
ierr = DMLabelCreate("boundary", &bd);CHKERRQ(ierr);
ierr = DMPlexMarkBoundaryFaces(dm, bd);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(bd, 1, &bdIS);CHKERRQ(ierr);
ierr = DMLabelGetStratumSize(bd, 1, &numBdFaces);CHKERRQ(ierr);
ierr = ISGetIndices(bdIS, &faces);CHKERRQ(ierr);
for (f = 0, bdSize = 0; f < numBdFaces; ++f) {
PetscInt *closure = NULL;
PetscInt closureSize, cl;
ierr = DMPlexGetTransitiveClosure(dm, faces[f], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) ++bdSize;
}
ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
ierr = PetscMalloc2(bdSize, &bdFaces, numBdFaces, &bdFaceIds);CHKERRQ(ierr);
for (f = 0, bdSize = 0; f < numBdFaces; ++f) {
PetscInt *closure = NULL;
PetscInt closureSize, cl;
ierr = DMPlexGetTransitiveClosure(dm, faces[f], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) bdFaces[bdSize++] = closure[cl] - vStart;
}
/* TODO Fix */
bdFaceIds[f] = 1;
ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
ierr = ISDestroy(&bdIS);CHKERRQ(ierr);
ierr = DMLabelDestroy(&bd);CHKERRQ(ierr);
pragmatic_set_boundary(&numBdFaces, bdFaces, bdFaceIds);
/* Create metric */
size = (dim*(dim+1))/2;
ierr = PetscMalloc1(PetscSqr(size), &eqns);CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF, size, size, eqns, &A);CHKERRQ(ierr);
ierr = MatCreateVecs(A, &mx, &mb);CHKERRQ(ierr);
ierr = VecSet(mb, 1.0);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
const PetscScalar *sol;
PetscScalar *cellCoords = NULL;
PetscReal e[3], vol;
const PetscInt *cone;
PetscInt coneSize, cl, i, j, d, r;
ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, NULL, &cellCoords);CHKERRQ(ierr);
/* Only works for simplices */
for (i = 0, r = 0; i < dim+1; ++i) {
for (j = 0; j < i; ++j, ++r) {
for (d = 0; d < dim; ++d) e[d] = cellCoords[i*dim+d] - cellCoords[j*dim+d];
/* FORTRAN ORDERING */
if (dim == 2) {
eqns[0*size+r] = PetscSqr(e[0]);
eqns[1*size+r] = 2.0*e[0]*e[1];
eqns[2*size+r] = PetscSqr(e[1]);
} else {
eqns[0*size+r] = PetscSqr(e[0]);
eqns[1*size+r] = 2.0*e[0]*e[1];
eqns[2*size+r] = 2.0*e[0]*e[2];
eqns[3*size+r] = PetscSqr(e[1]);
eqns[4*size+r] = 2.0*e[1]*e[2];
eqns[5*size+r] = PetscSqr(e[2]);
}
}
}
ierr = MatSetUnfactored(A);CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, NULL, &cellCoords);CHKERRQ(ierr);
ierr = MatLUFactor(A, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = MatSolve(A, mb, mx);CHKERRQ(ierr);
ierr = VecGetArrayRead(mx, &sol);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
ierr = DMPlexGetCone(udm, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(udm, c, &coneSize);CHKERRQ(ierr);
for (cl = 0; cl < coneSize; ++cl) {
const PetscInt v = cone[cl] - vStart;
if (dim == 2) {
metric[v*4+0] += vol*coarseRatio*sol[0];
metric[v*4+1] += vol*coarseRatio*sol[1];
metric[v*4+2] += vol*coarseRatio*sol[1];
metric[v*4+3] += vol*coarseRatio*sol[2];
} else {
metric[v*9+0] += vol*coarseRatio*sol[0];
metric[v*9+1] += vol*coarseRatio*sol[1];
metric[v*9+3] += vol*coarseRatio*sol[1];
metric[v*9+2] += vol*coarseRatio*sol[2];
metric[v*9+6] += vol*coarseRatio*sol[2];
metric[v*9+4] += vol*coarseRatio*sol[3];
metric[v*9+5] += vol*coarseRatio*sol[4];
metric[v*9+7] += vol*coarseRatio*sol[4];
metric[v*9+8] += vol*coarseRatio*sol[5];
}
}
ierr = VecRestoreArrayRead(mx, &sol);CHKERRQ(ierr);
}
for (v = 0; v < numVertices; ++v) {
const PetscInt *support;
PetscInt supportSize, s;
PetscReal vol, totVol = 0.0;
ierr = DMPlexGetSupport(udm, v+vStart, &support);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(udm, v+vStart, &supportSize);CHKERRQ(ierr);
for (s = 0; s < supportSize; ++s) {ierr = DMPlexComputeCellGeometryFVM(dm, support[s], &vol, NULL, NULL);CHKERRQ(ierr); totVol += vol;}
for (s = 0; s < PetscSqr(dim); ++s) metric[v*PetscSqr(dim)+s] /= totVol;
}
ierr = VecDestroy(&mx);CHKERRQ(ierr);
ierr = VecDestroy(&mb);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&udm);CHKERRQ(ierr);
ierr = PetscFree(eqns);CHKERRQ(ierr);
pragmatic_set_metric(metric);
pragmatic_adapt();
/* Read out mesh */
pragmatic_get_info(&numCoarseVertices, &numCoarseCells);
ierr = PetscMalloc1(numCoarseVertices*dim, &coarseCoords);CHKERRQ(ierr);
switch (dim) {
case 2:
pragmatic_get_coords_2d(x, y);
numCorners = 3;
for (v = 0; v < numCoarseVertices; ++v) {coarseCoords[v*2+0] = x[v]; coarseCoords[v*2+1] = y[v];}
break;
case 3:
pragmatic_get_coords_3d(x, y, z);
numCorners = 4;
for (v = 0; v < numCoarseVertices; ++v) {coarseCoords[v*3+0] = x[v]; coarseCoords[v*3+1] = y[v]; coarseCoords[v*3+2] = z[v];}
break;
default:
SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No Pragmatic coarsening defined for dimension %d", dim);
}
pragmatic_get_elements(cells);
/* TODO Read out markers for boundary */
ierr = DMPlexCreateFromCellList(PetscObjectComm((PetscObject) dm), dim, numCoarseCells, numCoarseVertices, numCorners, PETSC_TRUE, cells, dim, coarseCoords, &mesh->coarseMesh);CHKERRQ(ierr);
pragmatic_finalize();
ierr = PetscFree5(x, y, z, metric, cells);CHKERRQ(ierr);
ierr = PetscFree2(bdFaces, bdFaceIds);CHKERRQ(ierr);
ierr = PetscFree(coarseCoords);CHKERRQ(ierr);
}
#endif
ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
*dmCoarsened = mesh->coarseMesh;
PetscFunctionReturn(0);
}
/*
- Checks face match
- Flips non-matching
- Inserts faces of support cells in FIFO
*/
static PetscErrorCode DMPlexCheckFace_Internal(DM dm, PetscInt *faceFIFO, PetscInt *fTop, PetscInt *fBottom, PetscInt cStart, PetscInt fStart, PetscInt fEnd, PetscBT seenCells, PetscBT flippedCells, PetscBT seenFaces)
{
const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
PetscInt supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
PetscInt face, dim, seenA, flippedA, seenB, flippedB, mismatch, c;
PetscErrorCode ierr;
PetscFunctionBegin;
face = faceFIFO[(*fTop)++];
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
if (supportSize < 2) PetscFunctionReturn(0);
if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
seenA = PetscBTLookup(seenCells, support[0]-cStart);
flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
seenB = PetscBTLookup(seenCells, support[1]-cStart);
flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
for (c = 0; c < coneSizeA; ++c) {
if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
faceFIFO[(*fBottom)++] = coneA[c];
ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
}
if (coneA[c] == face) posA = c;
if (*fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], *fBottom, fEnd-fStart);
}
if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
for (c = 0; c < coneSizeB; ++c) {
if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
faceFIFO[(*fBottom)++] = coneB[c];
ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
}
if (coneB[c] == face) posB = c;
if (*fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], *fBottom, fEnd-fStart);
}
if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
if (dim == 1) {
mismatch = posA == posB;
} else {
mismatch = coneOA[posA] == coneOB[posB];
}
if (mismatch ^ (flippedA ^ flippedB)) {
if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
if (!seenA && !flippedA) {
ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
} else if (!seenB && !flippedB) {
ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
} else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexOrient - Give a consistent orientation to the input mesh
Input Parameters:
. dm - The DM
Note: The orientation data for the DM are change in-place.
$ This routine will fail for non-orientable surfaces, such as the Moebius strip.
Level: advanced
.seealso: DMCreate(), DMPLEX
@*/
PetscErrorCode DMPlexOrient(DM dm)
{
MPI_Comm comm;
PetscSF sf;
const PetscInt *lpoints;
const PetscSFNode *rpoints;
PetscSFNode *rorntComp = NULL, *lorntComp = NULL;
PetscInt *numNeighbors, **neighbors;
PetscSFNode *nrankComp;
PetscBool *match, *flipped;
PetscBT seenCells, flippedCells, seenFaces;
PetscInt *faceFIFO, fTop, fBottom, *cellComp, *faceComp;
PetscInt numLeaves, numRoots, dim, h, cStart, cEnd, c, cell, fStart, fEnd, face, off, totNeighbors = 0;
PetscMPIInt rank, size, numComponents, comp = 0;
PetscBool flg, flg2;
PetscViewer viewer = NULL, selfviewer = NULL;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view", &flg);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view_synchronized", &flg2);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
/* Truth Table
mismatch flips do action mismatch flipA ^ flipB action
F 0 flips no F F F
F 1 flip yes F T T
F 2 flips no T F T
T 0 flips yes T T F
T 1 flip no
T 2 flips yes
*/
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscCalloc3(fEnd - fStart, &faceFIFO, cEnd-cStart, &cellComp, fEnd-fStart, &faceComp);CHKERRQ(ierr);
/*
OLD STYLE
- Add an integer array over cells and faces (component) for connected component number
Foreach component
- Mark the initial cell as seen
- Process component as usual
- Set component for all seenCells
- Wipe seenCells and seenFaces (flippedCells can stay)
- Generate parallel adjacency for component using SF and seenFaces
- Collect numComponents adj data from each proc to 0
- Build same serial graph
- Use same solver
- Use Scatterv to to send back flipped flags for each component
- Negate flippedCells by component
NEW STYLE
- Create the adj on each process
- Bootstrap to complete graph on proc 0
*/
/* Loop over components */
for (cell = cStart; cell < cEnd; ++cell) cellComp[cell-cStart] = -1;
do {
/* Look for first unmarked cell */
for (cell = cStart; cell < cEnd; ++cell) if (cellComp[cell-cStart] < 0) break;
if (cell >= cEnd) break;
/* Initialize FIFO with first cell in component */
{
const PetscInt *cone;
PetscInt coneSize;
fTop = fBottom = 0;
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
faceFIFO[fBottom++] = cone[c];
ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
}
ierr = PetscBTSet(seenCells, cell-cStart);CHKERRQ(ierr);
}
/* Consider each face in FIFO */
while (fTop < fBottom) {
ierr = DMPlexCheckFace_Internal(dm, faceFIFO, &fTop, &fBottom, cStart, fStart, fEnd, seenCells, flippedCells, seenFaces);CHKERRQ(ierr);
}
/* Set component for cells and faces */
for (cell = 0; cell < cEnd-cStart; ++cell) {
if (PetscBTLookup(seenCells, cell)) cellComp[cell] = comp;
}
for (face = 0; face < fEnd-fStart; ++face) {
if (PetscBTLookup(seenFaces, face)) faceComp[face] = comp;
}
/* Wipe seenCells and seenFaces for next component */
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
++comp;
} while (1);
numComponents = comp;
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for serial flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Now all subdomains are oriented, but we need a consistent parallel orientation */
if (numLeaves >= 0) {
/* Store orientations of boundary faces*/
ierr = PetscCalloc2(numRoots,&rorntComp,numRoots,&lorntComp);CHKERRQ(ierr);
for (face = fStart; face < fEnd; ++face) {
const PetscInt *cone, *support, *ornt;
PetscInt coneSize, supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) continue;
ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
if (dim == 1) {
/* Use cone position instead, shifted to -1 or 1 */
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = 1-c*2;
else rorntComp[face].rank = c*2-1;
} else {
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = ornt[c] < 0 ? -1 : 1;
else rorntComp[face].rank = ornt[c] < 0 ? 1 : -1;
}
rorntComp[face].index = faceComp[face-fStart];
}
/* Communicate boundary edge orientations */
ierr = PetscSFBcastBegin(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
}
/* Get process adjacency */
ierr = PetscMalloc2(numComponents, &numNeighbors, numComponents, &neighbors);CHKERRQ(ierr);
viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)dm));
if (flg2) {ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
for (comp = 0; comp < numComponents; ++comp) {
PetscInt l, n;
numNeighbors[comp] = 0;
ierr = PetscMalloc1(PetscMax(numLeaves, 0), &neighbors[comp]);CHKERRQ(ierr);
/* I know this is p^2 time in general, but for bounded degree its alright */
for (l = 0; l < numLeaves; ++l) {
const PetscInt face = lpoints[l];
/* Find a representative face (edge) separating pairs of procs */
if ((face >= fStart) && (face < fEnd) && (faceComp[face-fStart] == comp)) {
const PetscInt rrank = rpoints[l].rank;
const PetscInt rcomp = lorntComp[face].index;
for (n = 0; n < numNeighbors[comp]; ++n) if ((rrank == rpoints[neighbors[comp][n]].rank) && (rcomp == lorntComp[lpoints[neighbors[comp][n]]].index)) break;
if (n >= numNeighbors[comp]) {
PetscInt supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
if (flg) {ierr = PetscViewerASCIIPrintf(selfviewer, "[%d]: component %d, Found representative leaf %d (face %d) connecting to face %d on (%d, %d) with orientation %d\n", rank, comp, l, face, rpoints[l].index, rrank, rcomp, lorntComp[face].rank);CHKERRQ(ierr);}
neighbors[comp][numNeighbors[comp]++] = l;
}
}
}
totNeighbors += numNeighbors[comp];
}
ierr = PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (flg2) {ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscMalloc2(totNeighbors, &nrankComp, totNeighbors, &match);CHKERRQ(ierr);
for (comp = 0, off = 0; comp < numComponents; ++comp) {
PetscInt n;
for (n = 0; n < numNeighbors[comp]; ++n, ++off) {
const PetscInt face = lpoints[neighbors[comp][n]];
const PetscInt o = rorntComp[face].rank*lorntComp[face].rank;
if (o < 0) match[off] = PETSC_TRUE;
else if (o > 0) match[off] = PETSC_FALSE;
else SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid face %d (%d, %d) neighbor: %d comp: %d", face, rorntComp[face], lorntComp[face], neighbors[comp][n], comp);
nrankComp[off].rank = rpoints[neighbors[comp][n]].rank;
nrankComp[off].index = lorntComp[lpoints[neighbors[comp][n]]].index;
}
ierr = PetscFree(neighbors[comp]);CHKERRQ(ierr);
}
/* Collect the graph on 0 */
if (numLeaves >= 0) {
Mat G;
PetscBT seenProcs, flippedProcs;
PetscInt *procFIFO, pTop, pBottom;
PetscInt *N = NULL, *Noff;
PetscSFNode *adj = NULL;
PetscBool *val = NULL;
PetscMPIInt *recvcounts = NULL, *displs = NULL, *Nc, p, o;
PetscMPIInt size = 0;
ierr = PetscCalloc1(numComponents, &flipped);CHKERRQ(ierr);
if (!rank) {ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);}
ierr = PetscCalloc4(size, &recvcounts, size+1, &displs, size, &Nc, size+1, &Noff);CHKERRQ(ierr);
ierr = MPI_Gather(&numComponents, 1, MPI_INT, Nc, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
if (!rank) {ierr = PetscMalloc1(displs[size],&N);CHKERRQ(ierr);}
ierr = MPI_Gatherv(numNeighbors, numComponents, MPIU_INT, N, Nc, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0, o = 0; p < size; ++p) {
recvcounts[p] = 0;
for (c = 0; c < Nc[p]; ++c, ++o) recvcounts[p] += N[o];
displs[p+1] = displs[p] + recvcounts[p];
}
if (!rank) {ierr = PetscMalloc2(displs[size], &adj, displs[size], &val);CHKERRQ(ierr);}
ierr = MPI_Gatherv(nrankComp, totNeighbors, MPIU_2INT, adj, recvcounts, displs, MPIU_2INT, 0, comm);CHKERRQ(ierr);
ierr = MPI_Gatherv(match, totNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
if (!rank) {
for (p = 1; p <= size; ++p) {Noff[p] = Noff[p-1] + Nc[p-1];}
if (flg) {
PetscInt n;
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
ierr = PetscPrintf(PETSC_COMM_SELF, "Proc %d Comp %d:\n", p, c);CHKERRQ(ierr);
for (n = 0; n < N[Noff[p]+c]; ++n, ++off) {
ierr = PetscPrintf(PETSC_COMM_SELF, " edge (%d, %d) (%d):\n", adj[off].rank, adj[off].index, val[off]);CHKERRQ(ierr);
}
}
}
}
/* Symmetrize the graph */
ierr = MatCreate(PETSC_COMM_SELF, &G);CHKERRQ(ierr);
ierr = MatSetSizes(G, Noff[size], Noff[size], Noff[size], Noff[size]);CHKERRQ(ierr);
ierr = MatSetUp(G);CHKERRQ(ierr);
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
const PetscInt r = Noff[p]+c;
PetscInt n;
for (n = 0; n < N[r]; ++n, ++off) {
const PetscInt q = Noff[adj[off].rank] + adj[off].index;
const PetscScalar o = val[off] ? 1.0 : 0.0;
ierr = MatSetValues(G, 1, &r, 1, &q, &o, INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(G, 1, &q, 1, &r, &o, INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &seenProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], seenProcs);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &flippedProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], flippedProcs);CHKERRQ(ierr);
ierr = PetscMalloc1(Noff[size], &procFIFO);CHKERRQ(ierr);
pTop = pBottom = 0;
for (p = 0; p < Noff[size]; ++p) {
if (PetscBTLookup(seenProcs, p)) continue;
/* Initialize FIFO with next proc */
procFIFO[pBottom++] = p;
ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
/* Consider each proc in FIFO */
while (pTop < pBottom) {
const PetscScalar *ornt;
const PetscInt *neighbors;
PetscInt proc, nproc, seen, flippedA, flippedB, mismatch, numNeighbors, n;
proc = procFIFO[pTop++];
flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
ierr = MatGetRow(G, proc, &numNeighbors, &neighbors, &ornt);CHKERRQ(ierr);
/* Loop over neighboring procs */
for (n = 0; n < numNeighbors; ++n) {
nproc = neighbors[n];
mismatch = PetscRealPart(ornt[n]) > 0.5 ? 0 : 1;
seen = PetscBTLookup(seenProcs, nproc);
flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
if (mismatch ^ (flippedA ^ flippedB)) {
if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
if (!flippedB) {
ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
} else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
if (!seen) {
procFIFO[pBottom++] = nproc;
ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
}
}
}
}
ierr = PetscFree(procFIFO);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
ierr = PetscFree2(adj, val);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
}
/* Scatter flip flags */
{
PetscBool *flips = NULL;
if (!rank) {
ierr = PetscMalloc1(Noff[size], &flips);CHKERRQ(ierr);
for (p = 0; p < Noff[size]; ++p) {
flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
if (flg && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc+Comp %d:\n", p);CHKERRQ(ierr);}
}
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
}
ierr = MPI_Scatterv(flips, Nc, displs, MPIU_BOOL, flipped, numComponents, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree(flips);CHKERRQ(ierr);
}
if (!rank) {ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);}
ierr = PetscFree(N);CHKERRQ(ierr);
ierr = PetscFree4(recvcounts, displs, Nc, Noff);CHKERRQ(ierr);
ierr = PetscFree2(nrankComp, match);CHKERRQ(ierr);
/* Decide whether to flip cells in each component */
for (c = 0; c < cEnd-cStart; ++c) {if (flipped[cellComp[c]]) {ierr = PetscBTNegate(flippedCells, c);CHKERRQ(ierr);}}
ierr = PetscFree(flipped);CHKERRQ(ierr);
}
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for parallel flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Reverse flipped cells in the mesh */
for (c = cStart; c < cEnd; ++c) {
if (PetscBTLookup(flippedCells, c-cStart)) {
ierr = DMPlexReverseCell(dm, c);CHKERRQ(ierr);
}
}
ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
ierr = PetscFree2(rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscFree3(faceFIFO, cellComp, faceComp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
