static PetscErrorCode DMPlexGetAdjacency_Support_Internal(DM dm, PetscInt p, PetscInt *adjSize, PetscInt adj[])
{
const PetscInt *support = NULL;
PetscInt numAdj = 0, maxAdjSize = *adjSize, supportSize, s;
PetscErrorCode ierr;
PetscFunctionBeginHot;
ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
for (s = 0; s < supportSize; ++s) {
const PetscInt *cone = NULL;
PetscInt coneSize, c, q;
ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
for (q = 0; q < numAdj || (adj[numAdj++] = cone[c],0); ++q) {
if (cone[c] == adj[q]) break;
}
if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
}
}
*adjSize = numAdj;
PetscFunctionReturn(0);
}
/*
Compute the gradient of the variables at the center of the cell
by the least-square reconstruction method based on the function
BuildLeastSquares.
*/
PetscErrorCode ConstructCellCentriodGradient(DM dm,DM dmFace,DM dmCell,PetscReal time,Vec locX,Vec F,User user)
{
DM dmGrad = user->dmGrad;
Model mod = user->model;
Physics phys = mod->physics;
const PetscInt dof = phys->dof;
PetscErrorCode ierr;
const PetscScalar *facegeom, *cellgeom, *x;
PetscInt fStart, fEnd, face, cStart;
Vec Grad;
PetscFunctionBeginUser;
ierr = DMGetGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecZeroEntries(Grad);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
{
PetscScalar *grad;
ierr = VecGetArray(Grad,&grad);CHKERRQ(ierr);
/* Reconstruct gradients */
for (face=fStart; face<fEnd; face++) {
const PetscInt *cells;
const PetscScalar *cx[2];
const FaceGeom *fg;
PetscScalar *cgrad[2];
PetscInt i,j;
PetscBool ghost;
ierr = IsExteriorGhostFace(dm,face,&ghost);CHKERRQ(ierr);
if (ghost) continue;
ierr = DMPlexGetSupport(dm,face,&cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace,face,facegeom,&fg);CHKERRQ(ierr);
for (i=0; i<2; i++) {
ierr = DMPlexPointLocalRead(dm,cells[i],x,&cx[i]);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmGrad,cells[i],grad,&cgrad[i]);CHKERRQ(ierr);
}
for (i=0; i<dof; i++) {
PetscScalar delta = cx[1][i] - cx[0][i];
for (j=0; j<DIM; j++) {
if (cgrad[0]) cgrad[0][i*DIM+j] += fg->grad[0][j] * delta;
if (cgrad[1]) cgrad[1][i*DIM+j] -= fg->grad[1][j] * delta;
}
}
}
ierr = VecRestoreArray(Grad,&grad);CHKERRQ(ierr);
}
ierr = DMRestoreGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locX,&x);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode BuildGradientReconstruction(DM dm, PetscFV fvm, DM dmFace, PetscScalar *fgeom, DM dmCell, PetscScalar *cgeom)
{
DMLabel ghostLabel;
PetscScalar *dx, *grad, **gref;
PetscInt dim, cStart, cEnd, c, cEndInterior, maxNumFaces;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(dm, &maxNumFaces, NULL);CHKERRQ(ierr);
ierr = PetscFVLeastSquaresSetMaxFaces(fvm, maxNumFaces);CHKERRQ(ierr);
ierr = DMPlexGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
ierr = PetscMalloc3(maxNumFaces*dim, &dx, maxNumFaces*dim, &grad, maxNumFaces, &gref);CHKERRQ(ierr);
for (c = cStart; c < cEndInterior; c++) {
const PetscInt *faces;
PetscInt numFaces, usedFaces, f, d;
const CellGeom *cg;
PetscBool boundary;
PetscInt ghost;
ierr = DMPlexPointLocalRead(dmCell, c, cgeom, &cg);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, c, &numFaces);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, c, &faces);CHKERRQ(ierr);
if (numFaces < dim) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Cell %D has only %D faces, not enough for gradient reconstruction", c, numFaces);
for (f = 0, usedFaces = 0; f < numFaces; ++f) {
const CellGeom *cg1;
FaceGeom *fg;
const PetscInt *fcells;
PetscInt ncell, side;
ierr = DMLabelGetValue(ghostLabel, faces[f], &ghost);CHKERRQ(ierr);
ierr = DMPlexIsBoundaryPoint(dm, faces[f], &boundary);CHKERRQ(ierr);
if ((ghost >= 0) || boundary) continue;
ierr = DMPlexGetSupport(dm, faces[f], &fcells);CHKERRQ(ierr);
side = (c != fcells[0]); /* c is on left=0 or right=1 of face */
ncell = fcells[!side]; /* the neighbor */
ierr = DMPlexPointLocalRef(dmFace, faces[f], fgeom, &fg);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, ncell, cgeom, &cg1);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) dx[usedFaces*dim+d] = cg1->centroid[d] - cg->centroid[d];
gref[usedFaces++] = fg->grad[side]; /* Gradient reconstruction term will go here */
}
if (!usedFaces) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_USER, "Mesh contains isolated cell (no neighbors). Is it intentional?");
ierr = PetscFVComputeGradient(fvm, usedFaces, dx, grad);CHKERRQ(ierr);
for (f = 0, usedFaces = 0; f < numFaces; ++f) {
ierr = DMLabelGetValue(ghostLabel, faces[f], &ghost);CHKERRQ(ierr);
ierr = DMPlexIsBoundaryPoint(dm, faces[f], &boundary);CHKERRQ(ierr);
if ((ghost >= 0) || boundary) continue;
for (d = 0; d < dim; ++d) gref[usedFaces][d] = grad[usedFaces*dim+d];
++usedFaces;
}
}
ierr = PetscFree3(dx, grad, gref);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMNetworkGetSupportingEdges - Return the supporting edges for this vertex point
Not Collective
Input Parameters:
+ dm - The DMNetwork object
- p - the vertex point
Output Paramters:
+ nedges - number of edges connected to this vertex point
- edges - List of edge points
Level: intermediate
Fortran Notes:
Since it returns an array, this routine is only available in Fortran 90, and you must
include petsc.h90 in your code.
.seealso: DMNetworkCreate, DMNetworkGetConnectedNodes
@*/
PetscErrorCode DMNetworkGetSupportingEdges(DM dm,PetscInt vertex,PetscInt *nedges,const PetscInt *edges[])
{
PetscErrorCode ierr;
DM_Network *network = (DM_Network*)dm->data;
PetscFunctionBegin;
ierr = DMPlexGetSupportSize(network->plex,vertex,nedges);CHKERRQ(ierr);
ierr = DMPlexGetSupport(network->plex,vertex,edges);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexReverseCell - Give a mesh cell the opposite orientation
Input Parameters:
+ dm - The DM
- cell - The cell number
Note: The modification of the DM is done in-place.
Level: advanced
.seealso: DMPlexOrient(), DMCreate(), DMPLEX
@*/
PetscErrorCode DMPlexReverseCell(DM dm, PetscInt cell)
{
/* Note that the reverse orientation ro of a face with orientation o is:
ro = o >= 0 ? -(faceSize - o) : faceSize + o
where faceSize is the size of the cone for the face.
*/
const PetscInt *cone, *coneO, *support;
PetscInt *revcone, *revconeO;
PetscInt maxConeSize, coneSize, supportSize, faceSize, cp, sp;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
ierr = DMGetWorkArray(dm, maxConeSize, MPIU_INT, &revcone);CHKERRQ(ierr);
ierr = DMGetWorkArray(dm, maxConeSize, MPIU_INT, &revconeO);CHKERRQ(ierr);
/* Reverse cone, and reverse orientations of faces */
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, cell, &coneO);CHKERRQ(ierr);
for (cp = 0; cp < coneSize; ++cp) {
const PetscInt rcp = coneSize-cp-1;
ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
revcone[cp] = cone[rcp];
revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
}
ierr = DMPlexSetCone(dm, cell, revcone);CHKERRQ(ierr);
ierr = DMPlexSetConeOrientation(dm, cell, revconeO);CHKERRQ(ierr);
/* Reverse orientation of this cell in the support hypercells */
faceSize = coneSize;
ierr = DMPlexGetSupportSize(dm, cell, &supportSize);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, cell, &support);CHKERRQ(ierr);
for (sp = 0; sp < supportSize; ++sp) {
ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
for (cp = 0; cp < coneSize; ++cp) {
if (cone[cp] != cell) continue;
ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
}
}
ierr = DMRestoreWorkArray(dm, maxConeSize, MPIU_INT, &revcone);CHKERRQ(ierr);
ierr = DMRestoreWorkArray(dm, maxConeSize, MPIU_INT, &revconeO);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeJacobian_LS(DM dm, Vec locX, PetscInt cell, PetscScalar CellValues[], void *ctx)
{
User user = (User) ctx;
Physics phys = user->model->physics;
PetscInt dof = phys->dof;
const PetscScalar *facegeom, *cellgeom,*x;
PetscErrorCode ierr;
DM dmFace, dmCell;
DM dmGrad = user->dmGrad;
PetscInt fStart, fEnd, face, cStart;
Vec locGrad, locGradLimiter, Grad;
/*here the localGradLimiter refers to the gradient that has been multiplied by the limiter function.
The locGradLimiter is used to construct the uL and uR, and the locGrad is used to caculate the diffusion term*/
Vec TempVec; /*a temperal vec for the vector restore*/
PetscFunctionBeginUser;
ierr = VecGetDM(user->facegeom,&dmFace);CHKERRQ(ierr);
ierr = VecGetDM(user->cellgeom,&dmCell);CHKERRQ(ierr);
ierr = DMGetGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecDuplicate(Grad, &TempVec);CHKERRQ(ierr);
ierr = VecCopy(Grad, TempVec);CHKERRQ(ierr);
/*Backup the original vector and use it to restore the value of dmGrad,
because I do not want to change the values of the cell gradient*/
ierr = VecGetArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
{
PetscScalar *grad;
ierr = VecGetArray(Grad,&grad);CHKERRQ(ierr);
/* Limit interior gradients. Using cell-based loop because it generalizes better to vector limiters. */
const PetscInt *faces;
PetscInt numFaces,f;
PetscReal *cellPhi; /* Scalar limiter applied to each component separately */
const PetscScalar *cx;
const CellGeom *cg;
PetscScalar *cgrad;
PetscInt i;
ierr = PetscMalloc(phys->dof*sizeof(PetscScalar),&cellPhi);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm,cell,&numFaces);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm,cell,&faces);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm,cell,x,&cx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell,cell,cellgeom,&cg);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmGrad,cell,grad,&cgrad);CHKERRQ(ierr);
/* Limiter will be minimum value over all neighbors */
for (i=0; i<dof; i++) {
cellPhi[i] = PETSC_MAX_REAL;
}
for (f=0; f<numFaces; f++) {
const PetscScalar *ncx;
const CellGeom *ncg;
const PetscInt *fcells;
PetscInt face = faces[f],ncell;
PetscScalar v[DIM];
PetscBool ghost;
ierr = IsExteriorGhostFace(dm,face,&ghost);CHKERRQ(ierr);
if (ghost) continue;
ierr = DMPlexGetSupport(dm,face,&fcells);CHKERRQ(ierr);
ncell = cell == fcells[0] ? fcells[1] : fcells[0]; /*The expression (x ? y : z) has the value of y if x is nonzero, z otherwise */
ierr = DMPlexPointLocalRead(dm,ncell,x,&ncx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell,ncell,cellgeom,&ncg);CHKERRQ(ierr);
Waxpy2(-1, cg->centroid, ncg->centroid, v);
for (i=0; i<dof; i++) {
/* We use the symmetric slope limited form of Berger, Aftosmis, and Murman 2005 */
PetscScalar phi,flim = 0.5 * (ncx[i] - cx[i]) / Dot2(&cgrad[i*DIM],v);
phi = (*user->LimitGrad)(flim);
cellPhi[i] = PetscMin(cellPhi[i],phi);
}
}
/* Apply limiter to gradient */
for (i=0; i<dof; i++) Scale2(cellPhi[i],&cgrad[i*DIM],&cgrad[i*DIM]);
ierr = PetscFree(cellPhi);CHKERRQ(ierr);
ierr = VecRestoreArray(Grad,&grad);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(dmGrad,&locGradLimiter);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad,Grad,INSERT_VALUES,locGradLimiter);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad,Grad,INSERT_VALUES,locGradLimiter);CHKERRQ(ierr);
ierr = VecCopy(TempVec, Grad);CHKERRQ(ierr);/*Restore the vector*/
ierr = DMGetLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad,Grad,INSERT_VALUES,locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad,Grad,INSERT_VALUES,locGrad);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecDestroy(&TempVec);CHKERRQ(ierr);
{
const PetscScalar *grad, *gradlimiter;
ierr = VecGetArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = VecGetArrayRead(locGradLimiter,&gradlimiter);CHKERRQ(ierr);
for (face=fStart; face<fEnd; face++) {
const PetscInt *cells;
PetscInt ghost,i,j;
PetscScalar *fluxcon, *fluxdiff, *fx[2];
const FaceGeom *fg;
const CellGeom *cg[2];
const PetscScalar *cx[2],*cgrad[2], *cgradlimiter[2];
PetscScalar *uL, *uR;
PetscReal FaceArea;
ierr = PetscMalloc(phys->dof * phys->dof * sizeof(PetscScalar), &fluxcon);CHKERRQ(ierr); /*For the convection terms*/
ierr = PetscMalloc(phys->dof * phys->dof * sizeof(PetscScalar), &fluxdiff);CHKERRQ(ierr); /*For the diffusion terms*/
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &uL);CHKERRQ(ierr);
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &uR);CHKERRQ(ierr);
fx[0] = uL; fx[1] = uR;
ierr = DMPlexGetLabelValue(dm, "ghost", face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace,face,facegeom,&fg);CHKERRQ(ierr);
for (i=0; i<2; i++) {
PetscScalar dx[DIM];
ierr = DMPlexPointLocalRead(dmCell,cells[i],cellgeom,&cg[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm,cells[i],x,&cx[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad,cells[i],gradlimiter,&cgradlimiter[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad,cells[i],grad,&cgrad[i]);CHKERRQ(ierr);
Waxpy2(-1,cg[i]->centroid,fg->centroid,dx);
for (j=0; j<dof; j++) {
fx[i][j] = cx[i][j] + Dot2(cgradlimiter[i],dx);
}
/*fx[0] and fx[1] are the value of the variables on the left and right
side of the face, respectively, that is u_L and u_R.*/
}
ierr = RiemannSolver_Rusanov_Jacobian(user, cgrad[0], cgrad[1], fg->centroid, cg[0]->centroid, cg[1]->centroid, fg->normal,
fx[0], fx[1], fluxcon, fluxdiff);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, face, &FaceArea, NULL, NULL);CHKERRQ(ierr);
/*Compute the face area*/
for (i=0; i<phys->dof; i++) {
for (j=0; j<phys->dof; j++) {
if(cells[0]<user->cEndInterior) CellValues[cells[0]*dof*dof + i*dof + j] -= cells[0]*1.0;
if(cells[1]<user->cEndInterior) CellValues[cells[1]*dof*dof + i*dof + j] += cells[1]*1.2;
}
}
// ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
ierr = PetscFree(fluxcon);CHKERRQ(ierr);
ierr = PetscFree(fluxdiff);CHKERRQ(ierr);
ierr = PetscFree(uL);CHKERRQ(ierr);
ierr = PetscFree(uR);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locGradLimiter,&gradlimiter);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGradLimiter);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/**
Caculate the flux using the Monotonic Upstream-Centered Scheme for Conservation Laws (van Leer, 1979)
*/
PetscErrorCode CaculateLocalFunction_LS(DM dm,DM dmFace,DM dmCell,PetscReal time,Vec locX,Vec F,User user)
{
DM dmGrad = user->dmGrad;
Model mod = user->model;
Physics phys = mod->physics;
const PetscInt dof = phys->dof;
PetscErrorCode ierr;
const PetscScalar *facegeom, *cellgeom, *x;
PetscScalar *f;
PetscInt fStart, fEnd, face, cStart, cell;
Vec locGrad, locGradLimiter, Grad;
/*
Here the localGradLimiter refers to the gradient that
has been multiplied by the limiter function.
The locGradLimiter is used to construct the uL and uR,
and the locGrad is used to caculate the diffusion term
*/
Vec TempVec; /*a temperal vec for the vector restore*/
PetscFunctionBeginUser;
ierr = DMGetGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecDuplicate(Grad, &TempVec);CHKERRQ(ierr);
ierr = VecCopy(Grad, TempVec);CHKERRQ(ierr);
/*
Backup the original vector and use it to restore the value of dmGrad,
because I do not want to change the values of the cell gradient.
*/
ierr = VecGetArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
{
PetscScalar *grad;
ierr = VecGetArray(Grad,&grad);CHKERRQ(ierr);
const PetscInt *faces;
PetscInt numFaces,f;
PetscReal *cellPhi; // Scalar limiter applied to each component separately
const PetscScalar *cx;
const CellGeom *cg;
PetscScalar *cgrad;
PetscInt i;
ierr = PetscMalloc(phys->dof*sizeof(PetscScalar),&cellPhi);CHKERRQ(ierr);
// Limit interior gradients. Using cell-based loop because it generalizes better to vector limiters.
for (cell=cStart; cell<user->cEndInterior; cell++) {
ierr = DMPlexGetConeSize(dm,cell,&numFaces);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm,cell,&faces);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm,cell,x,&cx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell,cell,cellgeom,&cg);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmGrad,cell,grad,&cgrad);CHKERRQ(ierr);
if (!cgrad) continue; // ghost cell, we don't compute
// Limiter will be minimum value over all neighbors
for (i=0; i<dof; i++) cellPhi[i] = PETSC_MAX_REAL;
for (f=0; f<numFaces; f++) {
const PetscScalar *ncx;
const CellGeom *ncg;
const PetscInt *fcells;
PetscInt face = faces[f],ncell;
PetscScalar v[DIM];
PetscBool ghost;
ierr = IsExteriorGhostFace(dm,face,&ghost);CHKERRQ(ierr);
if (ghost) continue;
ierr = DMPlexGetSupport(dm,face,&fcells);CHKERRQ(ierr);
ncell = cell == fcells[0] ? fcells[1] : fcells[0];
// The expression (x ? y : z) has the value of y if x is nonzero, z otherwise
ierr = DMPlexPointLocalRead(dm,ncell,x,&ncx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell,ncell,cellgeom,&ncg);CHKERRQ(ierr);
Waxpy2(-1, cg->centroid, ncg->centroid, v);
for (i=0; i<dof; i++) {
// We use the symmetric slope limited form of Berger, Aftosmis, and Murman 2005
PetscScalar phi,flim = 0.5 * (ncx[i] - cx[i]) / Dot2(&cgrad[i*DIM],v);
phi = (*user->Limit)(flim);
cellPhi[i] = PetscMin(cellPhi[i],phi);
}
}
// Apply limiter to gradient
for (i=0; i<dof; i++) Scale2(cellPhi[i],&cgrad[i*DIM],&cgrad[i*DIM]);
}
ierr = PetscFree(cellPhi);CHKERRQ(ierr);
ierr = VecRestoreArray(Grad,&grad);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(dmGrad,&locGradLimiter);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad,Grad,INSERT_VALUES,locGradLimiter);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad,Grad,INSERT_VALUES,locGradLimiter);CHKERRQ(ierr);
ierr = VecCopy(TempVec, Grad);CHKERRQ(ierr);//Restore the vector
ierr = DMGetLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad,Grad,INSERT_VALUES,locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad,Grad,INSERT_VALUES,locGrad);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecDestroy(&TempVec);CHKERRQ(ierr);
{
const PetscScalar *grad, *gradlimiter;
const PetscInt *cells;
PetscInt ghost,i,j;
PetscScalar *fluxcon, *fluxdiff, *fx[2],*cf[2];
const FaceGeom *fg;
const CellGeom *cg[2];
const PetscScalar *cx[2],*cgrad[2], *cgradlimiter[2];
PetscScalar *uL, *uR;
PetscReal FaceArea;
ierr = VecGetArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = VecGetArrayRead(locGradLimiter,&gradlimiter);CHKERRQ(ierr);
ierr = VecGetArray(F,&f);CHKERRQ(ierr);
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &fluxcon);CHKERRQ(ierr); // For the convection terms
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &fluxdiff);CHKERRQ(ierr); // For the diffusion terms
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &uL);CHKERRQ(ierr);
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &uR);CHKERRQ(ierr);// Please do not put the Malloc function into a for loop!!!!
for (face=fStart; face<fEnd; face++) {
fx[0] = uL; fx[1] = uR;
ierr = DMPlexGetLabelValue(dm, "ghost", face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace,face,facegeom,&fg);CHKERRQ(ierr);
for (i=0; i<2; i++) {
PetscScalar dx[DIM];
ierr = DMPlexPointLocalRead(dmCell,cells[i],cellgeom,&cg[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm,cells[i],x,&cx[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad,cells[i],gradlimiter,&cgradlimiter[i]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad,cells[i],grad,&cgrad[i]);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm,cells[i],f,&cf[i]);CHKERRQ(ierr);
Waxpy2(-1,cg[i]->centroid,fg->centroid,dx);
for (j=0; j<dof; j++) {
fx[i][j] = cx[i][j] + Dot2(cgradlimiter[i],dx);
}
// fx[0] and fx[1] are the value of the variables on the left and right
// side of the face, respectively, that is u_L and u_R.
}
ierr = RiemannSolver_Rusanov(user, cgrad[0], cgrad[1], fg->centroid, cg[0]->centroid, cg[1]->centroid, fg->normal, fx[0], fx[1], fluxcon, fluxdiff);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, face, &FaceArea, NULL, NULL);CHKERRQ(ierr);
// Compute the face area
for (i=0; i<phys->dof; i++) {
if (cf[0]) cf[0][i] -= FaceArea*(fluxcon[i] + fluxdiff[i])/cg[0]->volume;
if (cf[1]) cf[1][i] += FaceArea*(fluxcon[i] + fluxdiff[i])/cg[1]->volume;
// The flux on the interface, for the cell[0], it is an outcome flux and for the cell[1], it is
// an income flux.
}
// ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locGradLimiter,&gradlimiter);CHKERRQ(ierr);
ierr = VecRestoreArray(F,&f);CHKERRQ(ierr);
ierr = PetscFree(fluxcon);CHKERRQ(ierr);
ierr = PetscFree(fluxdiff);CHKERRQ(ierr);
ierr = PetscFree(uL);CHKERRQ(ierr);
ierr = PetscFree(uR);CHKERRQ(ierr);
// VecView(F,PETSC_VIEWER_STDOUT_WORLD);
}
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGradLimiter);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Use the first order upwind scheme to compute the flux
*/
PetscErrorCode CaculateLocalFunction_Upwind(DM dm,DM dmFace,DM dmCell,PetscReal time,Vec locX,Vec F,User user)
{
Physics phys = user->model->physics;
DM dmGrad = user->dmGrad;
PetscErrorCode ierr;
const PetscScalar *facegeom, *cellgeom, *x;
PetscScalar *f;
PetscInt fStart, fEnd, face;
Vec locGrad, Grad;
PetscFunctionBeginUser;
ierr = VecGetArrayRead(user->facegeom, &facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom, &cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX, &x);CHKERRQ(ierr);
ierr = VecGetArray(F, &f);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMGetGlobalVector(dmGrad, &Grad);CHKERRQ(ierr);
ierr = DMGetLocalVector(dmGrad, &locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmGrad, Grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmGrad, Grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dmGrad, &Grad);CHKERRQ(ierr);
const PetscScalar *grad;
ierr = VecGetArrayRead(locGrad, &grad);CHKERRQ(ierr);
{
const PetscInt *cells;
PetscInt i,ghost;
PetscScalar *fluxcon, *fluxdiff, *fL,*fR;
const FaceGeom *fg;
const CellGeom *cgL,*cgR;
const PetscScalar *xL,*xR;
const PetscScalar *cgrad[2];
PetscReal FaceArea;
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &fluxcon);CHKERRQ(ierr); /*For the convection terms*/
ierr = PetscMalloc(phys->dof * sizeof(PetscScalar), &fluxdiff);CHKERRQ(ierr); /*For the diffusion terms*/
for (face = fStart; face < fEnd; ++face) {
ierr = DMPlexGetLabelValue(dm, "ghost", face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);/*The support of a face is the cells (two cells)*/
ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);/*Read the data from "facegeom" for the point "face"*/
ierr = DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR);CHKERRQ(ierr);
/*
PetscPrintf(PETSC_COMM_SELF, "face[%d]:(%f, %f, %f), cgL:(%f, %f, %f), cgR:(%f, %f, %f), fnorm:(%f, %f, %f)\n",
face, fg->centroid[0], fg->centroid[1], fg->centroid[2],
cgL->centroid[0], cgL->centroid[1], cgL->centroid[2], cgR->centroid[0], cgR->centroid[1], cgR->centroid[2],
fg->normal[0], fg->normal[1], fg->normal[2]);
*/
ierr = DMPlexPointLocalRead(dm, cells[0], x, &xL);CHKERRQ(ierr); /*For the unkown variables*/
ierr = DMPlexPointLocalRead(dm, cells[1], x, &xR);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm, cells[0], f, &fL);CHKERRQ(ierr); /*For the functions*/
ierr = DMPlexPointGlobalRef(dm, cells[1], f, &fR);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad[0]);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmGrad, cells[1], grad, &cgrad[1]);CHKERRQ(ierr);
ierr = RiemannSolver_Rusanov(user, cgrad[0], cgrad[1], fg->centroid, cgL->centroid, cgR->centroid, fg->normal, xL, xR, fluxcon, fluxdiff);CHKERRQ(ierr);
/*Caculate the flux*/
ierr = DMPlexComputeCellGeometryFVM(dm, face, &FaceArea, NULL, NULL);CHKERRQ(ierr);
//PetscPrintf(PETSC_COMM_SELF, "FaceArea=%f, Volume=%f\n",FaceArea,cgL->volume);
/*Compute the face area*/
// FaceArea = 0.0;
for (i=0; i<phys->dof; i++) {
/*
PetscPrintf(PETSC_COMM_SELF, "face[%d]:(%f, %f, %f), GradL[%d] = (%f, %f, %f), GradR[%d] = (%f, %f, %f), fluxcon[%d] = %f\n",
face, fg->centroid[0], fg->centroid[1], fg->centroid[2],
i, cgrad[0][DIM*i], cgrad[0][DIM*i + 1], cgrad[0][DIM*i + 2],
i, cgrad[1][DIM*i], cgrad[1][DIM*i + 1], cgrad[1][DIM*i + 2], i, fluxcon[i]);
*/
if (fL) {
fL[i] -= FaceArea*(fluxcon[i] + fluxdiff[i])/cgL->volume;
// if (fL) PetscPrintf(PETSC_COMM_SELF, "x=%g, y=%g, z=%g,fluxcon[i]=%g, FaceArea=%g, volume=%g, fLc=%g, fL=%g\n",fg->centroid[0], fg->centroid[1], fg->centroid[2],fluxcon[i],FaceArea,cgL->volume,FaceArea*(fluxcon[i] + fluxdiff[i])/cgL->volume,fL[i]);
}
if (fR) fR[i] += FaceArea*(fluxcon[i] + fluxdiff[i])/cgR->volume;
}
}
ierr = PetscFree(fluxcon);CHKERRQ(ierr);
ierr = PetscFree(fluxdiff);CHKERRQ(ierr);
}
// VecView(F,PETSC_VIEWER_STDOUT_WORLD);
ierr = VecRestoreArrayRead(locGrad,&grad);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmGrad,&locGrad);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locX,&x);CHKERRQ(ierr);
ierr = VecRestoreArray(F,&f);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode TSComputeRHSFunction_DMPlex(TS ts, PetscReal time, Vec X, Vec F, void *ctx)
{
DM dm;
DMTS_Plex *dmplexts = (DMTS_Plex *) ctx;
void (*riemann)(const PetscReal[], const PetscReal[], const PetscScalar[], const PetscScalar[], PetscScalar[], void *) = dmplexts->riemann;
PetscFV fvm;
PetscLimiter lim;
Vec faceGeometry = dmplexts->facegeom;
Vec cellGeometry = dmplexts->cellgeom;
Vec Grad = NULL, locGrad, locX;
DM dmFace, dmCell;
DMLabel ghostLabel;
PetscCellGeometry fgeom, cgeom;
const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
PetscScalar *grad, *f, *uL, *uR, *fluxL, *fluxR;
PetscReal *centroid, *normal, *vol, *cellPhi;
PetscBool computeGradients;
PetscInt Nf, dim, pdim, fStart, fEnd, numFaces = 0, face, iface, cell, cStart, cEnd, cEndInterior;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(ts,TS_CLASSID,1);
PetscValidHeaderSpecific(X,VEC_CLASSID,3);
PetscValidHeaderSpecific(F,VEC_CLASSID,5);
ierr = TSGetDM(ts, &dm);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &locX);CHKERRQ(ierr);
ierr = VecZeroEntries(locX);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX);CHKERRQ(ierr);
ierr = VecZeroEntries(F);CHKERRQ(ierr);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetNumFields(dm, &Nf);CHKERRQ(ierr);
ierr = DMGetField(dm, 0, (PetscObject *) &fvm);CHKERRQ(ierr);
ierr = PetscFVGetLimiter(fvm, &lim);CHKERRQ(ierr);
ierr = PetscFVGetNumComponents(fvm, &pdim);CHKERRQ(ierr);
ierr = PetscFVGetComputeGradients(fvm, &computeGradients);CHKERRQ(ierr);
if (computeGradients) {
ierr = DMGetGlobalVector(dmplexts->dmGrad, &Grad);CHKERRQ(ierr);
ierr = VecZeroEntries(Grad);CHKERRQ(ierr);
ierr = VecGetArray(Grad, &grad);CHKERRQ(ierr);
}
ierr = DMPlexGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = VecGetDM(faceGeometry, &dmFace);CHKERRQ(ierr);
ierr = VecGetDM(cellGeometry, &dmCell);CHKERRQ(ierr);
ierr = VecGetArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX, &x);CHKERRQ(ierr);
/* Count faces and reconstruct gradients */
for (face = fStart; face < fEnd; ++face) {
const PetscInt *cells;
const FaceGeom *fg;
const PetscScalar *cx[2];
PetscScalar *cgrad[2];
PetscBool boundary;
PetscInt ghost, c, pd, d;
ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
++numFaces;
if (!computeGradients) continue;
ierr = DMPlexIsBoundaryPoint(dm, face, &boundary);CHKERRQ(ierr);
if (boundary) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
for (c = 0; c < 2; ++c) {
ierr = DMPlexPointLocalRead(dm, cells[c], x, &cx[c]);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmplexts->dmGrad, cells[c], grad, &cgrad[c]);CHKERRQ(ierr);
}
for (pd = 0; pd < pdim; ++pd) {
PetscScalar delta = cx[1][pd] - cx[0][pd];
for (d = 0; d < dim; ++d) {
if (cgrad[0]) cgrad[0][pd*dim+d] += fg->grad[0][d] * delta;
if (cgrad[1]) cgrad[1][pd*dim+d] -= fg->grad[1][d] * delta;
}
}
}
/* Limit interior gradients (using cell-based loop because it generalizes better to vector limiters) */
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = DMGetWorkArray(dm, pdim, PETSC_REAL, &cellPhi);CHKERRQ(ierr);
for (cell = computeGradients && lim ? cStart : cEnd; cell < cEndInterior; ++cell) {
const PetscInt *faces;
const PetscScalar *cx;
const CellGeom *cg;
PetscScalar *cgrad;
PetscInt coneSize, f, pd, d;
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &faces);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm, cell, x, &cx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cell, cellgeom, &cg);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmplexts->dmGrad, cell, grad, &cgrad);CHKERRQ(ierr);
if (!cgrad) continue; /* Unowned overlap cell, we do not compute */
/* Limiter will be minimum value over all neighbors */
for (d = 0; d < pdim; ++d) cellPhi[d] = PETSC_MAX_REAL;
for (f = 0; f < coneSize; ++f) {
const PetscScalar *ncx;
const CellGeom *ncg;
const PetscInt *fcells;
PetscInt face = faces[f], ncell, ghost;
PetscReal v[3];
PetscBool boundary;
ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
ierr = DMPlexIsBoundaryPoint(dm, face, &boundary);CHKERRQ(ierr);
if ((ghost >= 0) || boundary) continue;
ierr = DMPlexGetSupport(dm, face, &fcells);CHKERRQ(ierr);
ncell = cell == fcells[0] ? fcells[1] : fcells[0];
ierr = DMPlexPointLocalRead(dm, ncell, x, &ncx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, ncell, cellgeom, &ncg);CHKERRQ(ierr);
WaxpyD(dim, -1, cg->centroid, ncg->centroid, v);
for (d = 0; d < pdim; ++d) {
/* We use the symmetric slope limited form of Berger, Aftosmis, and Murman 2005 */
PetscReal phi, flim = 0.5 * PetscRealPart(ncx[d] - cx[d]) / DotD(dim, &cgrad[d*dim], v);
ierr = PetscLimiterLimit(lim, flim, &phi);CHKERRQ(ierr);
cellPhi[d] = PetscMin(cellPhi[d], phi);
}
}
/* Apply limiter to gradient */
for (pd = 0; pd < pdim; ++pd)
/* Scalar limiter applied to each component separately */
for (d = 0; d < dim; ++d) cgrad[pd*dim+d] *= cellPhi[pd];
}
ierr = DMRestoreWorkArray(dm, pdim, PETSC_REAL, &cellPhi);CHKERRQ(ierr);
ierr = DMPlexInsertBoundaryValuesFVM_Static(dm, fvm, time, locX, Grad, dmplexts);CHKERRQ(ierr);
if (computeGradients) {
ierr = VecRestoreArray(Grad, &grad);CHKERRQ(ierr);
ierr = DMGetLocalVector(dmplexts->dmGrad, &locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dmplexts->dmGrad, Grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dmplexts->dmGrad, Grad, INSERT_VALUES, locGrad);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dmplexts->dmGrad, &Grad);CHKERRQ(ierr);
ierr = VecGetArrayRead(locGrad, &lgrad);CHKERRQ(ierr);
}
ierr = PetscMalloc7(numFaces*dim,¢roid,numFaces*dim,&normal,numFaces*2,&vol,numFaces*pdim,&uL,numFaces*pdim,&uR,numFaces*pdim,&fluxL,numFaces*pdim,&fluxR);CHKERRQ(ierr);
/* Read out values */
for (face = fStart, iface = 0; face < fEnd; ++face) {
const PetscInt *cells;
const FaceGeom *fg;
const CellGeom *cgL, *cgR;
const PetscScalar *xL, *xR, *gL, *gR;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm, cells[0], x, &xL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm, cells[1], x, &xR);CHKERRQ(ierr);
if (computeGradients) {
PetscReal dxL[3], dxR[3];
ierr = DMPlexPointLocalRead(dmplexts->dmGrad, cells[0], lgrad, &gL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmplexts->dmGrad, cells[1], lgrad, &gR);CHKERRQ(ierr);
WaxpyD(dim, -1, cgL->centroid, fg->centroid, dxL);
WaxpyD(dim, -1, cgR->centroid, fg->centroid, dxR);
for (d = 0; d < pdim; ++d) {
uL[iface*pdim+d] = xL[d] + DotD(dim, &gL[d*dim], dxL);
uR[iface*pdim+d] = xR[d] + DotD(dim, &gR[d*dim], dxR);
}
} else {
for (d = 0; d < pdim; ++d) {
uL[iface*pdim+d] = xL[d];
uR[iface*pdim+d] = xR[d];
}
}
for (d = 0; d < dim; ++d) {
centroid[iface*dim+d] = fg->centroid[d];
normal[iface*dim+d] = fg->normal[d];
}
vol[iface*2+0] = cgL->volume;
vol[iface*2+1] = cgR->volume;
++iface;
}
if (computeGradients) {
ierr = VecRestoreArrayRead(locGrad,&lgrad);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dmplexts->dmGrad, &locGrad);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(locX, &x);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
fgeom.v0 = centroid;
fgeom.n = normal;
cgeom.vol = vol;
/* Riemann solve */
ierr = PetscFVIntegrateRHSFunction(fvm, numFaces, Nf, &fvm, 0, fgeom, cgeom, uL, uR, riemann, fluxL, fluxR, dmplexts->rhsfunctionlocalctx);CHKERRQ(ierr);
/* Insert fluxes */
ierr = VecGetArray(F, &f);CHKERRQ(ierr);
for (face = fStart, iface = 0; face < fEnd; ++face) {
const PetscInt *cells;
PetscScalar *fL, *fR;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, face, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm, cells[0], f, &fL);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dm, cells[1], f, &fR);CHKERRQ(ierr);
for (d = 0; d < pdim; ++d) {
if (fL) fL[d] -= fluxL[iface*pdim+d];
if (fR) fR[d] += fluxR[iface*pdim+d];
}
++iface;
}
ierr = VecRestoreArray(F, &f);CHKERRQ(ierr);
ierr = PetscFree7(centroid,normal,vol,uL,uR,fluxL,fluxR);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &locX);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexInsertBoundaryValuesFVM_Static(DM dm, PetscFV fvm, PetscReal time, Vec locX, Vec Grad, DMTS_Plex *dmplexts)
{
Vec faceGeometry = dmplexts->facegeom;
Vec cellGeometry = dmplexts->cellgeom;
DM dmFace, dmCell;
const PetscScalar *facegeom, *cellgeom, *grad;
PetscScalar *x, *fx;
PetscInt numBd, b, dim, pdim, fStart, fEnd;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = PetscFVGetNumComponents(fvm, &pdim);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMPlexGetNumBoundary(dm, &numBd);CHKERRQ(ierr);
if (Grad) {
ierr = VecGetDM(cellGeometry, &dmCell);CHKERRQ(ierr);
ierr = VecGetArrayRead(cellGeometry, &cellgeom);CHKERRQ(ierr);
ierr = DMGetWorkArray(dm, pdim, PETSC_SCALAR, &fx);CHKERRQ(ierr);
ierr = VecGetArrayRead(Grad, &grad);CHKERRQ(ierr);
}
ierr = VecGetDM(faceGeometry, &dmFace);CHKERRQ(ierr);
ierr = VecGetArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
ierr = VecGetArray(locX, &x);CHKERRQ(ierr);
for (b = 0; b < numBd; ++b) {
PetscErrorCode (*func)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*);
DMLabel label;
const char *labelname;
const PetscInt *ids;
PetscInt numids, i;
void *ctx;
ierr = DMPlexGetBoundary(dm, b, NULL, NULL, &labelname, NULL, (void (**)()) &func, &numids, &ids, &ctx);CHKERRQ(ierr);
ierr = DMPlexGetLabel(dm, labelname, &label);CHKERRQ(ierr);
for (i = 0; i < numids; ++i) {
IS faceIS;
const PetscInt *faces;
PetscInt numFaces, f;
ierr = DMLabelGetStratumIS(label, ids[i], &faceIS);CHKERRQ(ierr);
if (!faceIS) continue; /* No points with that id on this process */
ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
for (f = 0; f < numFaces; ++f) {
const PetscInt face = faces[f], *cells;
const FaceGeom *fg;
if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */
ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
if (Grad) {
const CellGeom *cg;
const PetscScalar *cx, *cgrad;
PetscScalar *xG;
PetscReal dx[3];
PetscInt d;
ierr = DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm, cells[0], x, &cx);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmplexts->dmGrad, cells[0], grad, &cgrad);CHKERRQ(ierr);
ierr = DMPlexPointLocalRef(dm, cells[1], x, &xG);CHKERRQ(ierr);
WaxpyD(dim, -1, cg->centroid, fg->centroid, dx);
for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DotD(dim, &cgrad[d*dim], dx);
ierr = (*func)(time, fg->centroid, fg->normal, fx, xG, ctx);CHKERRQ(ierr);
} else {
const PetscScalar *xI;
PetscScalar *xG;
ierr = DMPlexPointLocalRead(dm, cells[0], x, &xI);CHKERRQ(ierr);
ierr = DMPlexPointLocalRef(dm, cells[1], x, &xG);CHKERRQ(ierr);
ierr = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx);CHKERRQ(ierr);
}
}
ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
}
}
ierr = VecRestoreArrayRead(faceGeometry, &facegeom);CHKERRQ(ierr);
ierr = VecRestoreArray(locX, &x);CHKERRQ(ierr);
if (Grad) {
ierr = DMRestoreWorkArray(dm, pdim, PETSC_SCALAR, &fx);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(Grad, &grad);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
DM dm;
PetscBool interpolate = PETSC_TRUE;
PetscInt dim;
PetscInt pStart, pEnd;
PetscInt vStart, vEnd;
PetscInt eStart, eEnd;
PetscInt cStart, cEnd;
PetscSection s;
ierr = PetscInitialize(&argc, &argv, (char*)0, help); CHKERRQ(ierr);
ierr = DMPlexCreateGmshFromFile(MPI_COMM_WORLD, argv[1], interpolate, &dm); CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr);
printf("dim = %d\n", dim);
ierr = DMPlexGetChart(dm, &pStart, &pEnd); CHKERRQ(ierr);
printf("chart: pstart, pend = %d %d\n", pStart, pEnd);
// vertices
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd); CHKERRQ(ierr);
printf("Depth 0: vstart, vend, len = %d %d %d\n", vStart, vEnd, vEnd-vStart);
// edges
ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd); CHKERRQ(ierr);
printf("Depth 1: estart, eend, len = %d %d %d\n", eStart, eEnd, eEnd-eStart);
// cells
ierr = DMPlexGetDepthStratum(dm, 2, &cStart, &cEnd); CHKERRQ(ierr);
printf("Depth 2: cstart, cend, len = %d %d %d\n", cStart, cEnd, cEnd-cStart);
// cells adjacent to face
{
FILE * fid = fopen("face_nbr.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
PetscInt nbr;
ierr = DMPlexGetSupportSize(dm, e, &nbr); CHKERRQ(ierr);
const PetscInt *nbcells;
ierr = DMPlexGetSupport(dm, e, &nbcells); CHKERRQ(ierr);
if(nbr == 1) // boundary face
fprintf(fid, "%d %d\n",e-eStart+1,nbcells[0]-cStart+1);
else if(nbr == 2) // interior face
fprintf(fid, "%d %d %d\n",e-eStart+1,nbcells[0]-cStart+1,nbcells[1]-cStart+1);
else
{
printf("nbr is not 1 or 2\n");
exit(0);
}
}
fclose(fid);
}
// point coordinates
{
Vec coordinates;
ierr = DMGetCoordinatesLocal(dm, &coordinates); CHKERRQ(ierr);
const PetscScalar *coords;
ierr = VecGetArrayRead(coordinates, &coords); CHKERRQ(ierr);
DM dmCoord;
ierr = DMGetCoordinateDM(dm, &dmCoord); CHKERRQ(ierr);
FILE * fid = fopen("vertices.txt","w");
for(PetscInt v=vStart; v<vEnd; ++v)
{
PetscScalar *vertex;
ierr = DMPlexPointLocalRead(dmCoord, v, coords, &vertex); CHKERRQ(ierr);
fprintf(fid, "%f %f\n", vertex[0], vertex[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(coordinates, &coords); CHKERRQ(ierr);
}
// compute cell and face geometry
{
Vec cellgeom, facegeom;
ierr = DMPlexComputeGeometryFVM(dm, &cellgeom, &facegeom); CHKERRQ(ierr);
// cell information
DM dmCell;
ierr = VecGetDM(cellgeom, &dmCell); CHKERRQ(ierr);
const PetscScalar *cgeom;
ierr = VecGetArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
FILE * fid = fopen("cells.txt","w");
for(PetscInt c=cStart; c<cEnd; ++c)
{
// cell properties like volume, centroid
PetscFVCellGeom *cg;
ierr = DMPlexPointLocalRead(dmCell, c, cgeom, &cg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f\n", c-cStart+1, cg->volume,
cg->centroid[0], cg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
// face information
DM dmFace;
ierr = VecGetDM(facegeom, &dmFace); CHKERRQ(ierr);
const PetscScalar *fgeom;
ierr = VecGetArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
fid = fopen("faces.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
// face properties like area normal, centroid
PetscFVFaceGeom *fg;
ierr = DMPlexPointLocalRead(dmFace, e, fgeom, &fg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f %f\n", e-eStart+1, fg->normal[0], fg->normal[1],
fg->centroid[0], fg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
}
// create section with one variable in each cell
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &s); CHKERRQ(ierr);
ierr = PetscSectionSetChart(s, pStart, pEnd); CHKERRQ(ierr);
for(PetscInt c=cStart; c<cEnd; ++c)
{
ierr = PetscSectionSetDof(s, c, 1); CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(s); CHKERRQ(ierr);
// create vector to store solution
Vec lv, gv;
ierr = DMSetDefaultSection(dm, s); CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &lv); CHKERRQ(ierr);
ierr = DMGetGlobalVector(dm, &gv); CHKERRQ(ierr);
}
PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
{
DM_Plex *mesh = (DM_Plex*) dm->data;
#ifdef PETSC_HAVE_PRAGMATIC
DM udm, coordDM;
DMLabel bd;
Mat A;
Vec coordinates, mb, mx;
PetscSection coordSection;
const PetscScalar *coords;
double *coarseCoords;
IS bdIS;
PetscReal *x, *y, *z, *eqns, *metric;
PetscReal coarseRatio = PetscSqr(0.5);
const PetscInt *faces;
PetscInt *cells, *bdFaces, *bdFaceIds;
PetscInt dim, numCorners, cStart, cEnd, numCells, numCoarseCells, c, vStart, vEnd, numVertices, numCoarseVertices, v, numBdFaces, f, maxConeSize, size, bdSize, coff;
#endif
PetscErrorCode ierr;
PetscFunctionBegin;
#ifdef PETSC_HAVE_PRAGMATIC
if (!mesh->coarseMesh) {
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm, &coordDM);CHKERRQ(ierr);
ierr = DMGetDefaultSection(coordDM, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(udm, &maxConeSize, NULL);CHKERRQ(ierr);
numCells = cEnd - cStart;
numVertices = vEnd - vStart;
ierr = PetscCalloc5(numVertices, &x, numVertices, &y, numVertices, &z, numVertices*PetscSqr(dim), &metric, numCells*maxConeSize, &cells);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
PetscInt off;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
x[v-vStart] = coords[off+0];
y[v-vStart] = coords[off+1];
if (dim > 2) z[v-vStart] = coords[off+2];
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (c = 0, coff = 0; c < numCells; ++c) {
const PetscInt *cone;
PetscInt coneSize, cl;
ierr = DMPlexGetConeSize(udm, c, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(udm, c, &cone);CHKERRQ(ierr);
for (cl = 0; cl < coneSize; ++cl) cells[coff++] = cone[cl] - vStart;
}
switch (dim) {
case 2:
pragmatic_2d_init(&numVertices, &numCells, cells, x, y);
break;
case 3:
pragmatic_3d_init(&numVertices, &numCells, cells, x, y, z);
break;
default:
SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No Pragmatic coarsening defined for dimension %d", dim);
}
/* Create boundary mesh */
ierr = DMLabelCreate("boundary", &bd);CHKERRQ(ierr);
ierr = DMPlexMarkBoundaryFaces(dm, bd);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(bd, 1, &bdIS);CHKERRQ(ierr);
ierr = DMLabelGetStratumSize(bd, 1, &numBdFaces);CHKERRQ(ierr);
ierr = ISGetIndices(bdIS, &faces);CHKERRQ(ierr);
for (f = 0, bdSize = 0; f < numBdFaces; ++f) {
PetscInt *closure = NULL;
PetscInt closureSize, cl;
ierr = DMPlexGetTransitiveClosure(dm, faces[f], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) ++bdSize;
}
ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
ierr = PetscMalloc2(bdSize, &bdFaces, numBdFaces, &bdFaceIds);CHKERRQ(ierr);
for (f = 0, bdSize = 0; f < numBdFaces; ++f) {
PetscInt *closure = NULL;
PetscInt closureSize, cl;
ierr = DMPlexGetTransitiveClosure(dm, faces[f], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) bdFaces[bdSize++] = closure[cl] - vStart;
}
/* TODO Fix */
bdFaceIds[f] = 1;
ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
ierr = ISDestroy(&bdIS);CHKERRQ(ierr);
ierr = DMLabelDestroy(&bd);CHKERRQ(ierr);
pragmatic_set_boundary(&numBdFaces, bdFaces, bdFaceIds);
/* Create metric */
size = (dim*(dim+1))/2;
ierr = PetscMalloc1(PetscSqr(size), &eqns);CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF, size, size, eqns, &A);CHKERRQ(ierr);
ierr = MatCreateVecs(A, &mx, &mb);CHKERRQ(ierr);
ierr = VecSet(mb, 1.0);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
const PetscScalar *sol;
PetscScalar *cellCoords = NULL;
PetscReal e[3], vol;
const PetscInt *cone;
PetscInt coneSize, cl, i, j, d, r;
ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, NULL, &cellCoords);CHKERRQ(ierr);
/* Only works for simplices */
for (i = 0, r = 0; i < dim+1; ++i) {
for (j = 0; j < i; ++j, ++r) {
for (d = 0; d < dim; ++d) e[d] = cellCoords[i*dim+d] - cellCoords[j*dim+d];
/* FORTRAN ORDERING */
if (dim == 2) {
eqns[0*size+r] = PetscSqr(e[0]);
eqns[1*size+r] = 2.0*e[0]*e[1];
eqns[2*size+r] = PetscSqr(e[1]);
} else {
eqns[0*size+r] = PetscSqr(e[0]);
eqns[1*size+r] = 2.0*e[0]*e[1];
eqns[2*size+r] = 2.0*e[0]*e[2];
eqns[3*size+r] = PetscSqr(e[1]);
eqns[4*size+r] = 2.0*e[1]*e[2];
eqns[5*size+r] = PetscSqr(e[2]);
}
}
}
ierr = MatSetUnfactored(A);CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, NULL, &cellCoords);CHKERRQ(ierr);
ierr = MatLUFactor(A, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = MatSolve(A, mb, mx);CHKERRQ(ierr);
ierr = VecGetArrayRead(mx, &sol);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
ierr = DMPlexGetCone(udm, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(udm, c, &coneSize);CHKERRQ(ierr);
for (cl = 0; cl < coneSize; ++cl) {
const PetscInt v = cone[cl] - vStart;
if (dim == 2) {
metric[v*4+0] += vol*coarseRatio*sol[0];
metric[v*4+1] += vol*coarseRatio*sol[1];
metric[v*4+2] += vol*coarseRatio*sol[1];
metric[v*4+3] += vol*coarseRatio*sol[2];
} else {
metric[v*9+0] += vol*coarseRatio*sol[0];
metric[v*9+1] += vol*coarseRatio*sol[1];
metric[v*9+3] += vol*coarseRatio*sol[1];
metric[v*9+2] += vol*coarseRatio*sol[2];
metric[v*9+6] += vol*coarseRatio*sol[2];
metric[v*9+4] += vol*coarseRatio*sol[3];
metric[v*9+5] += vol*coarseRatio*sol[4];
metric[v*9+7] += vol*coarseRatio*sol[4];
metric[v*9+8] += vol*coarseRatio*sol[5];
}
}
ierr = VecRestoreArrayRead(mx, &sol);CHKERRQ(ierr);
}
for (v = 0; v < numVertices; ++v) {
const PetscInt *support;
PetscInt supportSize, s;
PetscReal vol, totVol = 0.0;
ierr = DMPlexGetSupport(udm, v+vStart, &support);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(udm, v+vStart, &supportSize);CHKERRQ(ierr);
for (s = 0; s < supportSize; ++s) {ierr = DMPlexComputeCellGeometryFVM(dm, support[s], &vol, NULL, NULL);CHKERRQ(ierr); totVol += vol;}
for (s = 0; s < PetscSqr(dim); ++s) metric[v*PetscSqr(dim)+s] /= totVol;
}
ierr = VecDestroy(&mx);CHKERRQ(ierr);
ierr = VecDestroy(&mb);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&udm);CHKERRQ(ierr);
ierr = PetscFree(eqns);CHKERRQ(ierr);
pragmatic_set_metric(metric);
pragmatic_adapt();
/* Read out mesh */
pragmatic_get_info(&numCoarseVertices, &numCoarseCells);
ierr = PetscMalloc1(numCoarseVertices*dim, &coarseCoords);CHKERRQ(ierr);
switch (dim) {
case 2:
pragmatic_get_coords_2d(x, y);
numCorners = 3;
for (v = 0; v < numCoarseVertices; ++v) {coarseCoords[v*2+0] = x[v]; coarseCoords[v*2+1] = y[v];}
break;
case 3:
pragmatic_get_coords_3d(x, y, z);
numCorners = 4;
for (v = 0; v < numCoarseVertices; ++v) {coarseCoords[v*3+0] = x[v]; coarseCoords[v*3+1] = y[v]; coarseCoords[v*3+2] = z[v];}
break;
default:
SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No Pragmatic coarsening defined for dimension %d", dim);
}
pragmatic_get_elements(cells);
/* TODO Read out markers for boundary */
ierr = DMPlexCreateFromCellList(PetscObjectComm((PetscObject) dm), dim, numCoarseCells, numCoarseVertices, numCorners, PETSC_TRUE, cells, dim, coarseCoords, &mesh->coarseMesh);CHKERRQ(ierr);
pragmatic_finalize();
ierr = PetscFree5(x, y, z, metric, cells);CHKERRQ(ierr);
ierr = PetscFree2(bdFaces, bdFaceIds);CHKERRQ(ierr);
ierr = PetscFree(coarseCoords);CHKERRQ(ierr);
}
#endif
ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
*dmCoarsened = mesh->coarseMesh;
PetscFunctionReturn(0);
}
/*
- Checks face match
- Flips non-matching
- Inserts faces of support cells in FIFO
*/
static PetscErrorCode DMPlexCheckFace_Internal(DM dm, PetscInt *faceFIFO, PetscInt *fTop, PetscInt *fBottom, PetscInt cStart, PetscInt fStart, PetscInt fEnd, PetscBT seenCells, PetscBT flippedCells, PetscBT seenFaces)
{
const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
PetscInt supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
PetscInt face, dim, seenA, flippedA, seenB, flippedB, mismatch, c;
PetscErrorCode ierr;
PetscFunctionBegin;
face = faceFIFO[(*fTop)++];
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
if (supportSize < 2) PetscFunctionReturn(0);
if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
seenA = PetscBTLookup(seenCells, support[0]-cStart);
flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
seenB = PetscBTLookup(seenCells, support[1]-cStart);
flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
for (c = 0; c < coneSizeA; ++c) {
if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
faceFIFO[(*fBottom)++] = coneA[c];
ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
}
if (coneA[c] == face) posA = c;
if (*fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], *fBottom, fEnd-fStart);
}
if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
for (c = 0; c < coneSizeB; ++c) {
if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
faceFIFO[(*fBottom)++] = coneB[c];
ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
}
if (coneB[c] == face) posB = c;
if (*fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], *fBottom, fEnd-fStart);
}
if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
if (dim == 1) {
mismatch = posA == posB;
} else {
mismatch = coneOA[posA] == coneOB[posB];
}
if (mismatch ^ (flippedA ^ flippedB)) {
if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
if (!seenA && !flippedA) {
ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
} else if (!seenB && !flippedB) {
ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
} else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/**
Compute the gadient of the cell center gradient obtained by the least-square method
*/
PetscErrorCode GradientGradientJacobian(DM dm, Vec locX, PetscScalar elemMat[], void *ctx)
{
User user = (User) ctx;
Physics phys = user->model->physics;
PetscInt dof = phys->dof;
const PetscScalar *facegeom, *cellgeom,*x;
PetscErrorCode ierr;
DM dmFace, dmCell;
DM dmGrad = user->dmGrad;
PetscInt fStart, fEnd, face, cStart;
Vec Grad;
/*here the localGradLimiter refers to the gradient that has been multiplied by the limiter function.
The locGradLimiter is used to construct the uL and uR, and the locGrad is used to caculate the diffusion term*/
Vec TempVec; /*a temperal vec for the vector restore*/
PetscFunctionBeginUser;
ierr = VecGetDM(user->facegeom,&dmFace);CHKERRQ(ierr);
ierr = VecGetDM(user->cellgeom,&dmCell);CHKERRQ(ierr);
ierr = DMGetGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecZeroEntries(Grad);CHKERRQ(ierr);
ierr = VecDuplicate(Grad, &TempVec);CHKERRQ(ierr);
ierr = VecCopy(Grad, TempVec);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecGetArrayRead(locX,&x);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, NULL);CHKERRQ(ierr);
{
PetscScalar *grad;
ierr = VecGetArray(TempVec,&grad);CHKERRQ(ierr);
/* Reconstruct gradients */
for (face=fStart; face<fEnd; face++) {
const PetscInt *cells;
const PetscScalar *cx[2];
const FaceGeom *fg;
PetscScalar *cgrad[2];
PetscInt i,j;
PetscBool ghost;
ierr = IsExteriorGhostFace(dm,face,&ghost);CHKERRQ(ierr);
if (ghost) continue;
ierr = DMPlexGetSupport(dm,face,&cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmFace,face,facegeom,&fg);CHKERRQ(ierr);
for (i=0; i<2; i++) {
ierr = DMPlexPointLocalRead(dm,cells[i],x,&cx[i]);CHKERRQ(ierr);
ierr = DMPlexPointGlobalRef(dmGrad,cells[i],grad,&cgrad[i]);CHKERRQ(ierr);
}
for (i=0; i<dof; i++) {
PetscScalar delta = cx[1][i] - cx[0][i];
for (j=0; j<DIM; j++) {
if (cgrad[0]) cgrad[0][i*DIM+j] += fg->grad[0][j] * delta;
if (cgrad[1]) cgrad[1][i*DIM+j] -= fg->grad[1][j] * delta;
}
}
for (i=0; i<phys->dof; i++) {
for (j=0; j<phys->dof; j++) {
if(cells[0]<user->cEndInterior) elemMat[cells[0]*dof*dof + i*dof + j] -= cells[0]*1.0;
if(cells[1]<user->cEndInterior) elemMat[cells[1]*dof*dof + i*dof + j] += cells[1]*1.2;
}
}
}
ierr = VecRestoreArray(TempVec,&grad);CHKERRQ(ierr);
}
ierr = DMRestoreGlobalVector(dmGrad,&Grad);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->cellgeom,&cellgeom);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(locX,&x);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexTSSetupGeometry(DM dm, PetscFV fvm, DMTS_Plex *dmplexts)
{
DM dmFace, dmCell;
DMLabel ghostLabel;
PetscSection sectionFace, sectionCell;
PetscSection coordSection;
Vec coordinates;
PetscReal minradius;
PetscScalar *fgeom, *cgeom;
PetscInt dim, cStart, cEnd, cEndInterior, c, fStart, fEnd, f;
PetscErrorCode ierr;
PetscFunctionBegin;
if (dmplexts->setupGeom) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
/* Make cell centroids and volumes */
ierr = DMClone(dm, &dmCell);CHKERRQ(ierr);
ierr = DMSetCoordinateSection(dmCell, coordSection);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dmCell, coordinates);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionCell);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionCell, cStart, cEnd);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {ierr = PetscSectionSetDof(sectionCell, c, sizeof(CellGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionCell);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmCell, sectionCell);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionCell);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmCell, &dmplexts->cellgeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
for (c = cStart; c < cEndInterior; ++c) {
CellGeom *cg;
ierr = DMPlexPointLocalRef(dmCell, c, cgeom, &cg);CHKERRQ(ierr);
ierr = PetscMemzero(cg, sizeof(*cg));CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dmCell, c, &cg->volume, cg->centroid, NULL);CHKERRQ(ierr);
}
/* Compute face normals and minimum cell radius */
ierr = DMClone(dm, &dmFace);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionFace);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionFace, fStart, fEnd);CHKERRQ(ierr);
for (f = fStart; f < fEnd; ++f) {ierr = PetscSectionSetDof(sectionFace, f, sizeof(FaceGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionFace);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmFace, sectionFace);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionFace);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmFace, &dmplexts->facegeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = DMPlexGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
minradius = PETSC_MAX_REAL;
for (f = fStart; f < fEnd; ++f) {
FaceGeom *fg;
PetscReal area;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, f, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexPointLocalRef(dmFace, f, fgeom, &fg);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, f, &area, fg->centroid, fg->normal);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) fg->normal[d] *= area;
/* Flip face orientation if necessary to match ordering in support, and Update minimum radius */
{
CellGeom *cL, *cR;
const PetscInt *cells;
PetscReal *lcentroid, *rcentroid;
PetscReal v[3];
ierr = DMPlexGetSupport(dm, f, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[0], cgeom, &cL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[1], cgeom, &cR);CHKERRQ(ierr);
lcentroid = cells[0] >= cEndInterior ? fg->centroid : cL->centroid;
rcentroid = cells[1] >= cEndInterior ? fg->centroid : cR->centroid;
WaxpyD(dim, -1, lcentroid, rcentroid, v);
if (DotRealD(dim, fg->normal, v) < 0) {
for (d = 0; d < dim; ++d) fg->normal[d] = -fg->normal[d];
}
if (DotRealD(dim, fg->normal, v) <= 0) {
if (dim == 2) SETERRQ5(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g) v (%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) v[0], (double) v[1]);
if (dim == 3) SETERRQ7(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g,%g) v (%g,%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) fg->normal[2], (double) v[0], (double) v[1], (double) v[2]);
SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed", f);
}
if (cells[0] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cL->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
if (cells[1] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cR->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
}
}
ierr = MPI_Allreduce(&minradius, &dmplexts->minradius, 1, MPIU_REAL, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
/* Compute centroids of ghost cells */
for (c = cEndInterior; c < cEnd; ++c) {
FaceGeom *fg;
const PetscInt *cone, *support;
PetscInt coneSize, supportSize, s;
ierr = DMPlexGetConeSize(dmCell, c, &coneSize);CHKERRQ(ierr);
if (coneSize != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ghost cell %d has cone size %d != 1", c, coneSize);
ierr = DMPlexGetCone(dmCell, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(dmCell, cone[0], &supportSize);CHKERRQ(ierr);
if (supportSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d has support size %d != 1", cone[0], supportSize);
ierr = DMPlexGetSupport(dmCell, cone[0], &support);CHKERRQ(ierr);
ierr = DMPlexPointLocalRef(dmFace, cone[0], fgeom, &fg);CHKERRQ(ierr);
for (s = 0; s < 2; ++s) {
/* Reflect ghost centroid across plane of face */
if (support[s] == c) {
const CellGeom *ci;
CellGeom *cg;
PetscReal c2f[3], a;
ierr = DMPlexPointLocalRead(dmCell, support[(s+1)%2], cgeom, &ci);CHKERRQ(ierr);
WaxpyD(dim, -1, ci->centroid, fg->centroid, c2f); /* cell to face centroid */
a = DotRealD(dim, c2f, fg->normal)/DotRealD(dim, fg->normal, fg->normal);
ierr = DMPlexPointLocalRef(dmCell, support[s], cgeom, &cg);CHKERRQ(ierr);
WaxpyD(dim, 2*a, fg->normal, ci->centroid, cg->centroid);
cg->volume = ci->volume;
}
}
}
ierr = VecRestoreArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
ierr = DMDestroy(&dmCell);CHKERRQ(ierr);
ierr = DMDestroy(&dmFace);CHKERRQ(ierr);
dmplexts->setupGeom = PETSC_TRUE;
PetscFunctionReturn(0);
}
PetscErrorCode ApplyBC(DM dm, PetscReal time, Vec locX, User user)
{
const char *name = "Face Sets"; /*Set up in the function DMPlexCreateExodus. is the side set*/
DM dmFace;
IS idIS;
const PetscInt *ids;
PetscScalar *x;
const PetscScalar *facegeom;
PetscInt numFS, fs;
PetscErrorCode ierr;
PetscMPIInt rank;
PetscFunctionBeginUser;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = VecGetDM(user->facegeom,&dmFace);CHKERRQ(ierr);
ierr = DMPlexGetLabelIdIS(dm, name, &idIS);CHKERRQ(ierr);
// ISView(idIS, PETSC_VIEWER_STDOUT_SELF);
if (!idIS) PetscFunctionReturn(0);
ierr = ISGetLocalSize(idIS, &numFS);CHKERRQ(ierr);
ierr = ISGetIndices(idIS, &ids);CHKERRQ(ierr);
ierr = VecGetArrayRead(user->facegeom, &facegeom);CHKERRQ(ierr);
ierr = VecGetArray(locX, &x);CHKERRQ(ierr);
for (fs = 0; fs < numFS; ++fs) {
IS faceIS;
const PetscInt *faces;
PetscInt numFaces, f;
ierr = DMPlexGetStratumIS(dm, name, ids[fs], &faceIS);CHKERRQ(ierr);
ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
for (f = 0; f < numFaces; ++f) {
// PetscPrintf(PETSC_COMM_SELF, "rank[%d]: ids[%d] = %d, faceIS[%d] = %d, numFaces = %d\n", rank, fs, ids[fs], f, faces[f], numFaces);
const PetscInt face = faces[f], *cells;
const PetscScalar *xI; /*Inner point*/
PetscScalar *xG; /*Ghost point*/
const FaceGeom *fg;
ierr = DMPlexPointLocalRead(dmFace, face, facegeom, &fg);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, face, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dm, cells[0], x, &xI);CHKERRQ(ierr);
ierr = DMPlexPointLocalRef(dm, cells[1], x, &xG);CHKERRQ(ierr);
if (ids[fs]==1){
//PetscPrintf(PETSC_COMM_SELF, "Set Inlfow Boundary Condition! \n");
ierr = BoundaryInflow(time, fg->centroid, fg->normal, xI, xG, user);CHKERRQ(ierr);
// DM dmCell;
// const PetscScalar *cellgeom;
// const CellGeom *cgL, *cgR;
// ierr = VecGetDM(user->cellgeom,&dmCell);CHKERRQ(ierr);
// ierr = VecGetArrayRead(user->cellgeom, &cellgeom);CHKERRQ(ierr);
// ierr = DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL);CHKERRQ(ierr);
// ierr = DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR);CHKERRQ(ierr);
// ierr = PetscPrintf(PETSC_COMM_WORLD,"cells[0] = (%f, %f, %f), cells[1] = (%f, %f, %f)\n",cgL->centroid[0], cgL->centroid[1], cgL->centroid[2],cgR->centroid[0], cgR->centroid[1], cgR->centroid[2]);CHKERRQ(ierr);
}else if (ids[fs]==2){
//PetscPrintf(PETSC_COMM_SELF, "Set Outlfow Boundary Condition! \n");
ierr = BoundaryOutflow(time, fg->centroid, fg->normal, xI, xG, user);CHKERRQ(ierr);
}else if (ids[fs]==3){
//PetscPrintf(PETSC_COMM_SELF, "Set Wall Boundary Condition! \n");
ierr = BoundaryWallflow(time, fg->centroid, fg->normal, xI, xG, user);CHKERRQ(ierr);
}else {
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Wrong type of boundary condition setup!!! \n The set up of the boundary should be: 1 for the inflow, 2 for the outflow, and 3 for the wallflow");
}
}
// PetscPrintf(PETSC_COMM_SELF, " \n");
ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
}
ierr = VecRestoreArray(locX, &x);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(user->facegeom,&facegeom);CHKERRQ(ierr);
ierr = ISRestoreIndices(idIS, &ids);CHKERRQ(ierr);
ierr = ISDestroy(&idIS);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexOrient - Give a consistent orientation to the input mesh
Input Parameters:
. dm - The DM
Note: The orientation data for the DM are change in-place.
$ This routine will fail for non-orientable surfaces, such as the Moebius strip.
Level: advanced
.seealso: DMCreate(), DMPLEX
@*/
PetscErrorCode DMPlexOrient(DM dm)
{
MPI_Comm comm;
PetscSF sf;
const PetscInt *lpoints;
const PetscSFNode *rpoints;
PetscSFNode *rorntComp = NULL, *lorntComp = NULL;
PetscInt *numNeighbors, **neighbors;
PetscSFNode *nrankComp;
PetscBool *match, *flipped;
PetscBT seenCells, flippedCells, seenFaces;
PetscInt *faceFIFO, fTop, fBottom, *cellComp, *faceComp;
PetscInt numLeaves, numRoots, dim, h, cStart, cEnd, c, cell, fStart, fEnd, face, off, totNeighbors = 0;
PetscMPIInt rank, size, numComponents, comp = 0;
PetscBool flg, flg2;
PetscViewer viewer = NULL, selfviewer = NULL;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view", &flg);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view_synchronized", &flg2);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
/* Truth Table
mismatch flips do action mismatch flipA ^ flipB action
F 0 flips no F F F
F 1 flip yes F T T
F 2 flips no T F T
T 0 flips yes T T F
T 1 flip no
T 2 flips yes
*/
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscCalloc3(fEnd - fStart, &faceFIFO, cEnd-cStart, &cellComp, fEnd-fStart, &faceComp);CHKERRQ(ierr);
/*
OLD STYLE
- Add an integer array over cells and faces (component) for connected component number
Foreach component
- Mark the initial cell as seen
- Process component as usual
- Set component for all seenCells
- Wipe seenCells and seenFaces (flippedCells can stay)
- Generate parallel adjacency for component using SF and seenFaces
- Collect numComponents adj data from each proc to 0
- Build same serial graph
- Use same solver
- Use Scatterv to to send back flipped flags for each component
- Negate flippedCells by component
NEW STYLE
- Create the adj on each process
- Bootstrap to complete graph on proc 0
*/
/* Loop over components */
for (cell = cStart; cell < cEnd; ++cell) cellComp[cell-cStart] = -1;
do {
/* Look for first unmarked cell */
for (cell = cStart; cell < cEnd; ++cell) if (cellComp[cell-cStart] < 0) break;
if (cell >= cEnd) break;
/* Initialize FIFO with first cell in component */
{
const PetscInt *cone;
PetscInt coneSize;
fTop = fBottom = 0;
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
faceFIFO[fBottom++] = cone[c];
ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
}
ierr = PetscBTSet(seenCells, cell-cStart);CHKERRQ(ierr);
}
/* Consider each face in FIFO */
while (fTop < fBottom) {
ierr = DMPlexCheckFace_Internal(dm, faceFIFO, &fTop, &fBottom, cStart, fStart, fEnd, seenCells, flippedCells, seenFaces);CHKERRQ(ierr);
}
/* Set component for cells and faces */
for (cell = 0; cell < cEnd-cStart; ++cell) {
if (PetscBTLookup(seenCells, cell)) cellComp[cell] = comp;
}
for (face = 0; face < fEnd-fStart; ++face) {
if (PetscBTLookup(seenFaces, face)) faceComp[face] = comp;
}
/* Wipe seenCells and seenFaces for next component */
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
++comp;
} while (1);
numComponents = comp;
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for serial flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Now all subdomains are oriented, but we need a consistent parallel orientation */
if (numLeaves >= 0) {
/* Store orientations of boundary faces*/
ierr = PetscCalloc2(numRoots,&rorntComp,numRoots,&lorntComp);CHKERRQ(ierr);
for (face = fStart; face < fEnd; ++face) {
const PetscInt *cone, *support, *ornt;
PetscInt coneSize, supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) continue;
ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
if (dim == 1) {
/* Use cone position instead, shifted to -1 or 1 */
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = 1-c*2;
else rorntComp[face].rank = c*2-1;
} else {
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = ornt[c] < 0 ? -1 : 1;
else rorntComp[face].rank = ornt[c] < 0 ? 1 : -1;
}
rorntComp[face].index = faceComp[face-fStart];
}
/* Communicate boundary edge orientations */
ierr = PetscSFBcastBegin(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
}
/* Get process adjacency */
ierr = PetscMalloc2(numComponents, &numNeighbors, numComponents, &neighbors);CHKERRQ(ierr);
viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)dm));
if (flg2) {ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
for (comp = 0; comp < numComponents; ++comp) {
PetscInt l, n;
numNeighbors[comp] = 0;
ierr = PetscMalloc1(PetscMax(numLeaves, 0), &neighbors[comp]);CHKERRQ(ierr);
/* I know this is p^2 time in general, but for bounded degree its alright */
for (l = 0; l < numLeaves; ++l) {
const PetscInt face = lpoints[l];
/* Find a representative face (edge) separating pairs of procs */
if ((face >= fStart) && (face < fEnd) && (faceComp[face-fStart] == comp)) {
const PetscInt rrank = rpoints[l].rank;
const PetscInt rcomp = lorntComp[face].index;
for (n = 0; n < numNeighbors[comp]; ++n) if ((rrank == rpoints[neighbors[comp][n]].rank) && (rcomp == lorntComp[lpoints[neighbors[comp][n]]].index)) break;
if (n >= numNeighbors[comp]) {
PetscInt supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
if (flg) {ierr = PetscViewerASCIIPrintf(selfviewer, "[%d]: component %d, Found representative leaf %d (face %d) connecting to face %d on (%d, %d) with orientation %d\n", rank, comp, l, face, rpoints[l].index, rrank, rcomp, lorntComp[face].rank);CHKERRQ(ierr);}
neighbors[comp][numNeighbors[comp]++] = l;
}
}
}
totNeighbors += numNeighbors[comp];
}
ierr = PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (flg2) {ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscMalloc2(totNeighbors, &nrankComp, totNeighbors, &match);CHKERRQ(ierr);
for (comp = 0, off = 0; comp < numComponents; ++comp) {
PetscInt n;
for (n = 0; n < numNeighbors[comp]; ++n, ++off) {
const PetscInt face = lpoints[neighbors[comp][n]];
const PetscInt o = rorntComp[face].rank*lorntComp[face].rank;
if (o < 0) match[off] = PETSC_TRUE;
else if (o > 0) match[off] = PETSC_FALSE;
else SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid face %d (%d, %d) neighbor: %d comp: %d", face, rorntComp[face], lorntComp[face], neighbors[comp][n], comp);
nrankComp[off].rank = rpoints[neighbors[comp][n]].rank;
nrankComp[off].index = lorntComp[lpoints[neighbors[comp][n]]].index;
}
ierr = PetscFree(neighbors[comp]);CHKERRQ(ierr);
}
/* Collect the graph on 0 */
if (numLeaves >= 0) {
Mat G;
PetscBT seenProcs, flippedProcs;
PetscInt *procFIFO, pTop, pBottom;
PetscInt *N = NULL, *Noff;
PetscSFNode *adj = NULL;
PetscBool *val = NULL;
PetscMPIInt *recvcounts = NULL, *displs = NULL, *Nc, p, o;
PetscMPIInt size = 0;
ierr = PetscCalloc1(numComponents, &flipped);CHKERRQ(ierr);
if (!rank) {ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);}
ierr = PetscCalloc4(size, &recvcounts, size+1, &displs, size, &Nc, size+1, &Noff);CHKERRQ(ierr);
ierr = MPI_Gather(&numComponents, 1, MPI_INT, Nc, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
if (!rank) {ierr = PetscMalloc1(displs[size],&N);CHKERRQ(ierr);}
ierr = MPI_Gatherv(numNeighbors, numComponents, MPIU_INT, N, Nc, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0, o = 0; p < size; ++p) {
recvcounts[p] = 0;
for (c = 0; c < Nc[p]; ++c, ++o) recvcounts[p] += N[o];
displs[p+1] = displs[p] + recvcounts[p];
}
if (!rank) {ierr = PetscMalloc2(displs[size], &adj, displs[size], &val);CHKERRQ(ierr);}
ierr = MPI_Gatherv(nrankComp, totNeighbors, MPIU_2INT, adj, recvcounts, displs, MPIU_2INT, 0, comm);CHKERRQ(ierr);
ierr = MPI_Gatherv(match, totNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
if (!rank) {
for (p = 1; p <= size; ++p) {Noff[p] = Noff[p-1] + Nc[p-1];}
if (flg) {
PetscInt n;
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
ierr = PetscPrintf(PETSC_COMM_SELF, "Proc %d Comp %d:\n", p, c);CHKERRQ(ierr);
for (n = 0; n < N[Noff[p]+c]; ++n, ++off) {
ierr = PetscPrintf(PETSC_COMM_SELF, " edge (%d, %d) (%d):\n", adj[off].rank, adj[off].index, val[off]);CHKERRQ(ierr);
}
}
}
}
/* Symmetrize the graph */
ierr = MatCreate(PETSC_COMM_SELF, &G);CHKERRQ(ierr);
ierr = MatSetSizes(G, Noff[size], Noff[size], Noff[size], Noff[size]);CHKERRQ(ierr);
ierr = MatSetUp(G);CHKERRQ(ierr);
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
const PetscInt r = Noff[p]+c;
PetscInt n;
for (n = 0; n < N[r]; ++n, ++off) {
const PetscInt q = Noff[adj[off].rank] + adj[off].index;
const PetscScalar o = val[off] ? 1.0 : 0.0;
ierr = MatSetValues(G, 1, &r, 1, &q, &o, INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(G, 1, &q, 1, &r, &o, INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &seenProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], seenProcs);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &flippedProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], flippedProcs);CHKERRQ(ierr);
ierr = PetscMalloc1(Noff[size], &procFIFO);CHKERRQ(ierr);
pTop = pBottom = 0;
for (p = 0; p < Noff[size]; ++p) {
if (PetscBTLookup(seenProcs, p)) continue;
/* Initialize FIFO with next proc */
procFIFO[pBottom++] = p;
ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
/* Consider each proc in FIFO */
while (pTop < pBottom) {
const PetscScalar *ornt;
const PetscInt *neighbors;
PetscInt proc, nproc, seen, flippedA, flippedB, mismatch, numNeighbors, n;
proc = procFIFO[pTop++];
flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
ierr = MatGetRow(G, proc, &numNeighbors, &neighbors, &ornt);CHKERRQ(ierr);
/* Loop over neighboring procs */
for (n = 0; n < numNeighbors; ++n) {
nproc = neighbors[n];
mismatch = PetscRealPart(ornt[n]) > 0.5 ? 0 : 1;
seen = PetscBTLookup(seenProcs, nproc);
flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
if (mismatch ^ (flippedA ^ flippedB)) {
if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
if (!flippedB) {
ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
} else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
if (!seen) {
procFIFO[pBottom++] = nproc;
ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
}
}
}
}
ierr = PetscFree(procFIFO);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
ierr = PetscFree2(adj, val);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
}
/* Scatter flip flags */
{
PetscBool *flips = NULL;
if (!rank) {
ierr = PetscMalloc1(Noff[size], &flips);CHKERRQ(ierr);
for (p = 0; p < Noff[size]; ++p) {
flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
if (flg && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc+Comp %d:\n", p);CHKERRQ(ierr);}
}
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
}
ierr = MPI_Scatterv(flips, Nc, displs, MPIU_BOOL, flipped, numComponents, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree(flips);CHKERRQ(ierr);
}
if (!rank) {ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);}
ierr = PetscFree(N);CHKERRQ(ierr);
ierr = PetscFree4(recvcounts, displs, Nc, Noff);CHKERRQ(ierr);
ierr = PetscFree2(nrankComp, match);CHKERRQ(ierr);
/* Decide whether to flip cells in each component */
for (c = 0; c < cEnd-cStart; ++c) {if (flipped[cellComp[c]]) {ierr = PetscBTNegate(flippedCells, c);CHKERRQ(ierr);}}
ierr = PetscFree(flipped);CHKERRQ(ierr);
}
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for parallel flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Reverse flipped cells in the mesh */
for (c = cStart; c < cEnd; ++c) {
if (PetscBTLookup(flippedCells, c-cStart)) {
ierr = DMPlexReverseCell(dm, c);CHKERRQ(ierr);
}
}
ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
ierr = PetscFree2(rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscFree3(faceFIFO, cellComp, faceComp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
