static PetscErrorCode DMPlexGetAdjacency_Support_Internal(DM dm, PetscInt p, PetscInt *adjSize, PetscInt adj[])
{
const PetscInt *support = NULL;
PetscInt numAdj = 0, maxAdjSize = *adjSize, supportSize, s;
PetscErrorCode ierr;
PetscFunctionBeginHot;
ierr = DMPlexGetSupportSize(dm, p, &supportSize);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, p, &support);CHKERRQ(ierr);
for (s = 0; s < supportSize; ++s) {
const PetscInt *cone = NULL;
PetscInt coneSize, c, q;
ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
for (q = 0; q < numAdj || (adj[numAdj++] = cone[c],0); ++q) {
if (cone[c] == adj[q]) break;
}
if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
}
}
*adjSize = numAdj;
PetscFunctionReturn(0);
}
/*@C
DMNetworkGetSupportingEdges - Return the supporting edges for this vertex point
Not Collective
Input Parameters:
+ dm - The DMNetwork object
- p - the vertex point
Output Paramters:
+ nedges - number of edges connected to this vertex point
- edges - List of edge points
Level: intermediate
Fortran Notes:
Since it returns an array, this routine is only available in Fortran 90, and you must
include petsc.h90 in your code.
.seealso: DMNetworkCreate, DMNetworkGetConnectedNodes
@*/
PetscErrorCode DMNetworkGetSupportingEdges(DM dm,PetscInt vertex,PetscInt *nedges,const PetscInt *edges[])
{
PetscErrorCode ierr;
DM_Network *network = (DM_Network*)dm->data;
PetscFunctionBegin;
ierr = DMPlexGetSupportSize(network->plex,vertex,nedges);CHKERRQ(ierr);
ierr = DMPlexGetSupport(network->plex,vertex,edges);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexReverseCell - Give a mesh cell the opposite orientation
Input Parameters:
+ dm - The DM
- cell - The cell number
Note: The modification of the DM is done in-place.
Level: advanced
.seealso: DMPlexOrient(), DMCreate(), DMPLEX
@*/
PetscErrorCode DMPlexReverseCell(DM dm, PetscInt cell)
{
/* Note that the reverse orientation ro of a face with orientation o is:
ro = o >= 0 ? -(faceSize - o) : faceSize + o
where faceSize is the size of the cone for the face.
*/
const PetscInt *cone, *coneO, *support;
PetscInt *revcone, *revconeO;
PetscInt maxConeSize, coneSize, supportSize, faceSize, cp, sp;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
ierr = DMGetWorkArray(dm, maxConeSize, MPIU_INT, &revcone);CHKERRQ(ierr);
ierr = DMGetWorkArray(dm, maxConeSize, MPIU_INT, &revconeO);CHKERRQ(ierr);
/* Reverse cone, and reverse orientations of faces */
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, cell, &coneO);CHKERRQ(ierr);
for (cp = 0; cp < coneSize; ++cp) {
const PetscInt rcp = coneSize-cp-1;
ierr = DMPlexGetConeSize(dm, cone[rcp], &faceSize);CHKERRQ(ierr);
revcone[cp] = cone[rcp];
revconeO[cp] = coneO[rcp] >= 0 ? -(faceSize-coneO[rcp]) : faceSize+coneO[rcp];
}
ierr = DMPlexSetCone(dm, cell, revcone);CHKERRQ(ierr);
ierr = DMPlexSetConeOrientation(dm, cell, revconeO);CHKERRQ(ierr);
/* Reverse orientation of this cell in the support hypercells */
faceSize = coneSize;
ierr = DMPlexGetSupportSize(dm, cell, &supportSize);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, cell, &support);CHKERRQ(ierr);
for (sp = 0; sp < supportSize; ++sp) {
ierr = DMPlexGetConeSize(dm, support[sp], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[sp], &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[sp], &coneO);CHKERRQ(ierr);
for (cp = 0; cp < coneSize; ++cp) {
if (cone[cp] != cell) continue;
ierr = DMPlexInsertConeOrientation(dm, support[sp], cp, coneO[cp] >= 0 ? -(faceSize-coneO[cp]) : faceSize+coneO[cp]);CHKERRQ(ierr);
}
}
ierr = DMRestoreWorkArray(dm, maxConeSize, MPIU_INT, &revcone);CHKERRQ(ierr);
ierr = DMRestoreWorkArray(dm, maxConeSize, MPIU_INT, &revconeO);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexTSSetupGeometry(DM dm, PetscFV fvm, DMTS_Plex *dmplexts)
{
DM dmFace, dmCell;
DMLabel ghostLabel;
PetscSection sectionFace, sectionCell;
PetscSection coordSection;
Vec coordinates;
PetscReal minradius;
PetscScalar *fgeom, *cgeom;
PetscInt dim, cStart, cEnd, cEndInterior, c, fStart, fEnd, f;
PetscErrorCode ierr;
PetscFunctionBegin;
if (dmplexts->setupGeom) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
/* Make cell centroids and volumes */
ierr = DMClone(dm, &dmCell);CHKERRQ(ierr);
ierr = DMSetCoordinateSection(dmCell, coordSection);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dmCell, coordinates);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionCell);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionCell, cStart, cEnd);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {ierr = PetscSectionSetDof(sectionCell, c, sizeof(CellGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionCell);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmCell, sectionCell);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionCell);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmCell, &dmplexts->cellgeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
for (c = cStart; c < cEndInterior; ++c) {
CellGeom *cg;
ierr = DMPlexPointLocalRef(dmCell, c, cgeom, &cg);CHKERRQ(ierr);
ierr = PetscMemzero(cg, sizeof(*cg));CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dmCell, c, &cg->volume, cg->centroid, NULL);CHKERRQ(ierr);
}
/* Compute face normals and minimum cell radius */
ierr = DMClone(dm, &dmFace);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionFace);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionFace, fStart, fEnd);CHKERRQ(ierr);
for (f = fStart; f < fEnd; ++f) {ierr = PetscSectionSetDof(sectionFace, f, sizeof(FaceGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionFace);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmFace, sectionFace);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionFace);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmFace, &dmplexts->facegeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = DMPlexGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
minradius = PETSC_MAX_REAL;
for (f = fStart; f < fEnd; ++f) {
FaceGeom *fg;
PetscReal area;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, f, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexPointLocalRef(dmFace, f, fgeom, &fg);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, f, &area, fg->centroid, fg->normal);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) fg->normal[d] *= area;
/* Flip face orientation if necessary to match ordering in support, and Update minimum radius */
{
CellGeom *cL, *cR;
const PetscInt *cells;
PetscReal *lcentroid, *rcentroid;
PetscReal v[3];
ierr = DMPlexGetSupport(dm, f, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[0], cgeom, &cL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[1], cgeom, &cR);CHKERRQ(ierr);
lcentroid = cells[0] >= cEndInterior ? fg->centroid : cL->centroid;
rcentroid = cells[1] >= cEndInterior ? fg->centroid : cR->centroid;
WaxpyD(dim, -1, lcentroid, rcentroid, v);
if (DotRealD(dim, fg->normal, v) < 0) {
for (d = 0; d < dim; ++d) fg->normal[d] = -fg->normal[d];
}
if (DotRealD(dim, fg->normal, v) <= 0) {
if (dim == 2) SETERRQ5(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g) v (%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) v[0], (double) v[1]);
if (dim == 3) SETERRQ7(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g,%g) v (%g,%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) fg->normal[2], (double) v[0], (double) v[1], (double) v[2]);
SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed", f);
}
if (cells[0] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cL->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
if (cells[1] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cR->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
}
}
ierr = MPI_Allreduce(&minradius, &dmplexts->minradius, 1, MPIU_REAL, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
/* Compute centroids of ghost cells */
for (c = cEndInterior; c < cEnd; ++c) {
FaceGeom *fg;
const PetscInt *cone, *support;
PetscInt coneSize, supportSize, s;
ierr = DMPlexGetConeSize(dmCell, c, &coneSize);CHKERRQ(ierr);
if (coneSize != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ghost cell %d has cone size %d != 1", c, coneSize);
ierr = DMPlexGetCone(dmCell, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(dmCell, cone[0], &supportSize);CHKERRQ(ierr);
if (supportSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d has support size %d != 1", cone[0], supportSize);
ierr = DMPlexGetSupport(dmCell, cone[0], &support);CHKERRQ(ierr);
ierr = DMPlexPointLocalRef(dmFace, cone[0], fgeom, &fg);CHKERRQ(ierr);
for (s = 0; s < 2; ++s) {
/* Reflect ghost centroid across plane of face */
if (support[s] == c) {
const CellGeom *ci;
CellGeom *cg;
PetscReal c2f[3], a;
ierr = DMPlexPointLocalRead(dmCell, support[(s+1)%2], cgeom, &ci);CHKERRQ(ierr);
WaxpyD(dim, -1, ci->centroid, fg->centroid, c2f); /* cell to face centroid */
a = DotRealD(dim, c2f, fg->normal)/DotRealD(dim, fg->normal, fg->normal);
ierr = DMPlexPointLocalRef(dmCell, support[s], cgeom, &cg);CHKERRQ(ierr);
WaxpyD(dim, 2*a, fg->normal, ci->centroid, cg->centroid);
cg->volume = ci->volume;
}
}
}
ierr = VecRestoreArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
ierr = DMDestroy(&dmCell);CHKERRQ(ierr);
ierr = DMDestroy(&dmFace);CHKERRQ(ierr);
dmplexts->setupGeom = PETSC_TRUE;
PetscFunctionReturn(0);
}
void PETSC_STDCALL dmplexgetsupportsize_(DM dm,PetscInt *p,PetscInt *size, int *__ierr ){
*__ierr = DMPlexGetSupportSize(
(DM)PetscToPointer((dm) ),*p,size);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
DM dm;
PetscBool interpolate = PETSC_TRUE;
PetscInt dim;
PetscInt pStart, pEnd;
PetscInt vStart, vEnd;
PetscInt eStart, eEnd;
PetscInt cStart, cEnd;
PetscSection s;
ierr = PetscInitialize(&argc, &argv, (char*)0, help); CHKERRQ(ierr);
ierr = DMPlexCreateGmshFromFile(MPI_COMM_WORLD, argv[1], interpolate, &dm); CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr);
printf("dim = %d\n", dim);
ierr = DMPlexGetChart(dm, &pStart, &pEnd); CHKERRQ(ierr);
printf("chart: pstart, pend = %d %d\n", pStart, pEnd);
// vertices
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd); CHKERRQ(ierr);
printf("Depth 0: vstart, vend, len = %d %d %d\n", vStart, vEnd, vEnd-vStart);
// edges
ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd); CHKERRQ(ierr);
printf("Depth 1: estart, eend, len = %d %d %d\n", eStart, eEnd, eEnd-eStart);
// cells
ierr = DMPlexGetDepthStratum(dm, 2, &cStart, &cEnd); CHKERRQ(ierr);
printf("Depth 2: cstart, cend, len = %d %d %d\n", cStart, cEnd, cEnd-cStart);
// cells adjacent to face
{
FILE * fid = fopen("face_nbr.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
PetscInt nbr;
ierr = DMPlexGetSupportSize(dm, e, &nbr); CHKERRQ(ierr);
const PetscInt *nbcells;
ierr = DMPlexGetSupport(dm, e, &nbcells); CHKERRQ(ierr);
if(nbr == 1) // boundary face
fprintf(fid, "%d %d\n",e-eStart+1,nbcells[0]-cStart+1);
else if(nbr == 2) // interior face
fprintf(fid, "%d %d %d\n",e-eStart+1,nbcells[0]-cStart+1,nbcells[1]-cStart+1);
else
{
printf("nbr is not 1 or 2\n");
exit(0);
}
}
fclose(fid);
}
// point coordinates
{
Vec coordinates;
ierr = DMGetCoordinatesLocal(dm, &coordinates); CHKERRQ(ierr);
const PetscScalar *coords;
ierr = VecGetArrayRead(coordinates, &coords); CHKERRQ(ierr);
DM dmCoord;
ierr = DMGetCoordinateDM(dm, &dmCoord); CHKERRQ(ierr);
FILE * fid = fopen("vertices.txt","w");
for(PetscInt v=vStart; v<vEnd; ++v)
{
PetscScalar *vertex;
ierr = DMPlexPointLocalRead(dmCoord, v, coords, &vertex); CHKERRQ(ierr);
fprintf(fid, "%f %f\n", vertex[0], vertex[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(coordinates, &coords); CHKERRQ(ierr);
}
// compute cell and face geometry
{
Vec cellgeom, facegeom;
ierr = DMPlexComputeGeometryFVM(dm, &cellgeom, &facegeom); CHKERRQ(ierr);
// cell information
DM dmCell;
ierr = VecGetDM(cellgeom, &dmCell); CHKERRQ(ierr);
const PetscScalar *cgeom;
ierr = VecGetArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
FILE * fid = fopen("cells.txt","w");
for(PetscInt c=cStart; c<cEnd; ++c)
{
// cell properties like volume, centroid
PetscFVCellGeom *cg;
ierr = DMPlexPointLocalRead(dmCell, c, cgeom, &cg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f\n", c-cStart+1, cg->volume,
cg->centroid[0], cg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
// face information
DM dmFace;
ierr = VecGetDM(facegeom, &dmFace); CHKERRQ(ierr);
const PetscScalar *fgeom;
ierr = VecGetArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
fid = fopen("faces.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
// face properties like area normal, centroid
PetscFVFaceGeom *fg;
ierr = DMPlexPointLocalRead(dmFace, e, fgeom, &fg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f %f\n", e-eStart+1, fg->normal[0], fg->normal[1],
fg->centroid[0], fg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
}
// create section with one variable in each cell
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &s); CHKERRQ(ierr);
ierr = PetscSectionSetChart(s, pStart, pEnd); CHKERRQ(ierr);
for(PetscInt c=cStart; c<cEnd; ++c)
{
ierr = PetscSectionSetDof(s, c, 1); CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(s); CHKERRQ(ierr);
// create vector to store solution
Vec lv, gv;
ierr = DMSetDefaultSection(dm, s); CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &lv); CHKERRQ(ierr);
ierr = DMGetGlobalVector(dm, &gv); CHKERRQ(ierr);
}
PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
{
DM_Plex *mesh = (DM_Plex*) dm->data;
#ifdef PETSC_HAVE_PRAGMATIC
DM udm, coordDM;
DMLabel bd;
Mat A;
Vec coordinates, mb, mx;
PetscSection coordSection;
const PetscScalar *coords;
double *coarseCoords;
IS bdIS;
PetscReal *x, *y, *z, *eqns, *metric;
PetscReal coarseRatio = PetscSqr(0.5);
const PetscInt *faces;
PetscInt *cells, *bdFaces, *bdFaceIds;
PetscInt dim, numCorners, cStart, cEnd, numCells, numCoarseCells, c, vStart, vEnd, numVertices, numCoarseVertices, v, numBdFaces, f, maxConeSize, size, bdSize, coff;
#endif
PetscErrorCode ierr;
PetscFunctionBegin;
#ifdef PETSC_HAVE_PRAGMATIC
if (!mesh->coarseMesh) {
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm, &coordDM);CHKERRQ(ierr);
ierr = DMGetDefaultSection(coordDM, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(udm, &maxConeSize, NULL);CHKERRQ(ierr);
numCells = cEnd - cStart;
numVertices = vEnd - vStart;
ierr = PetscCalloc5(numVertices, &x, numVertices, &y, numVertices, &z, numVertices*PetscSqr(dim), &metric, numCells*maxConeSize, &cells);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
PetscInt off;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
x[v-vStart] = coords[off+0];
y[v-vStart] = coords[off+1];
if (dim > 2) z[v-vStart] = coords[off+2];
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (c = 0, coff = 0; c < numCells; ++c) {
const PetscInt *cone;
PetscInt coneSize, cl;
ierr = DMPlexGetConeSize(udm, c, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(udm, c, &cone);CHKERRQ(ierr);
for (cl = 0; cl < coneSize; ++cl) cells[coff++] = cone[cl] - vStart;
}
switch (dim) {
case 2:
pragmatic_2d_init(&numVertices, &numCells, cells, x, y);
break;
case 3:
pragmatic_3d_init(&numVertices, &numCells, cells, x, y, z);
break;
default:
SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No Pragmatic coarsening defined for dimension %d", dim);
}
/* Create boundary mesh */
ierr = DMLabelCreate("boundary", &bd);CHKERRQ(ierr);
ierr = DMPlexMarkBoundaryFaces(dm, bd);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(bd, 1, &bdIS);CHKERRQ(ierr);
ierr = DMLabelGetStratumSize(bd, 1, &numBdFaces);CHKERRQ(ierr);
ierr = ISGetIndices(bdIS, &faces);CHKERRQ(ierr);
for (f = 0, bdSize = 0; f < numBdFaces; ++f) {
PetscInt *closure = NULL;
PetscInt closureSize, cl;
ierr = DMPlexGetTransitiveClosure(dm, faces[f], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) ++bdSize;
}
ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
ierr = PetscMalloc2(bdSize, &bdFaces, numBdFaces, &bdFaceIds);CHKERRQ(ierr);
for (f = 0, bdSize = 0; f < numBdFaces; ++f) {
PetscInt *closure = NULL;
PetscInt closureSize, cl;
ierr = DMPlexGetTransitiveClosure(dm, faces[f], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) bdFaces[bdSize++] = closure[cl] - vStart;
}
/* TODO Fix */
bdFaceIds[f] = 1;
ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
ierr = ISDestroy(&bdIS);CHKERRQ(ierr);
ierr = DMLabelDestroy(&bd);CHKERRQ(ierr);
pragmatic_set_boundary(&numBdFaces, bdFaces, bdFaceIds);
/* Create metric */
size = (dim*(dim+1))/2;
ierr = PetscMalloc1(PetscSqr(size), &eqns);CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF, size, size, eqns, &A);CHKERRQ(ierr);
ierr = MatCreateVecs(A, &mx, &mb);CHKERRQ(ierr);
ierr = VecSet(mb, 1.0);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
const PetscScalar *sol;
PetscScalar *cellCoords = NULL;
PetscReal e[3], vol;
const PetscInt *cone;
PetscInt coneSize, cl, i, j, d, r;
ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, NULL, &cellCoords);CHKERRQ(ierr);
/* Only works for simplices */
for (i = 0, r = 0; i < dim+1; ++i) {
for (j = 0; j < i; ++j, ++r) {
for (d = 0; d < dim; ++d) e[d] = cellCoords[i*dim+d] - cellCoords[j*dim+d];
/* FORTRAN ORDERING */
if (dim == 2) {
eqns[0*size+r] = PetscSqr(e[0]);
eqns[1*size+r] = 2.0*e[0]*e[1];
eqns[2*size+r] = PetscSqr(e[1]);
} else {
eqns[0*size+r] = PetscSqr(e[0]);
eqns[1*size+r] = 2.0*e[0]*e[1];
eqns[2*size+r] = 2.0*e[0]*e[2];
eqns[3*size+r] = PetscSqr(e[1]);
eqns[4*size+r] = 2.0*e[1]*e[2];
eqns[5*size+r] = PetscSqr(e[2]);
}
}
}
ierr = MatSetUnfactored(A);CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, NULL, &cellCoords);CHKERRQ(ierr);
ierr = MatLUFactor(A, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = MatSolve(A, mb, mx);CHKERRQ(ierr);
ierr = VecGetArrayRead(mx, &sol);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
ierr = DMPlexGetCone(udm, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(udm, c, &coneSize);CHKERRQ(ierr);
for (cl = 0; cl < coneSize; ++cl) {
const PetscInt v = cone[cl] - vStart;
if (dim == 2) {
metric[v*4+0] += vol*coarseRatio*sol[0];
metric[v*4+1] += vol*coarseRatio*sol[1];
metric[v*4+2] += vol*coarseRatio*sol[1];
metric[v*4+3] += vol*coarseRatio*sol[2];
} else {
metric[v*9+0] += vol*coarseRatio*sol[0];
metric[v*9+1] += vol*coarseRatio*sol[1];
metric[v*9+3] += vol*coarseRatio*sol[1];
metric[v*9+2] += vol*coarseRatio*sol[2];
metric[v*9+6] += vol*coarseRatio*sol[2];
metric[v*9+4] += vol*coarseRatio*sol[3];
metric[v*9+5] += vol*coarseRatio*sol[4];
metric[v*9+7] += vol*coarseRatio*sol[4];
metric[v*9+8] += vol*coarseRatio*sol[5];
}
}
ierr = VecRestoreArrayRead(mx, &sol);CHKERRQ(ierr);
}
for (v = 0; v < numVertices; ++v) {
const PetscInt *support;
PetscInt supportSize, s;
PetscReal vol, totVol = 0.0;
ierr = DMPlexGetSupport(udm, v+vStart, &support);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(udm, v+vStart, &supportSize);CHKERRQ(ierr);
for (s = 0; s < supportSize; ++s) {ierr = DMPlexComputeCellGeometryFVM(dm, support[s], &vol, NULL, NULL);CHKERRQ(ierr); totVol += vol;}
for (s = 0; s < PetscSqr(dim); ++s) metric[v*PetscSqr(dim)+s] /= totVol;
}
ierr = VecDestroy(&mx);CHKERRQ(ierr);
ierr = VecDestroy(&mb);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&udm);CHKERRQ(ierr);
ierr = PetscFree(eqns);CHKERRQ(ierr);
pragmatic_set_metric(metric);
pragmatic_adapt();
/* Read out mesh */
pragmatic_get_info(&numCoarseVertices, &numCoarseCells);
ierr = PetscMalloc1(numCoarseVertices*dim, &coarseCoords);CHKERRQ(ierr);
switch (dim) {
case 2:
pragmatic_get_coords_2d(x, y);
numCorners = 3;
for (v = 0; v < numCoarseVertices; ++v) {coarseCoords[v*2+0] = x[v]; coarseCoords[v*2+1] = y[v];}
break;
case 3:
pragmatic_get_coords_3d(x, y, z);
numCorners = 4;
for (v = 0; v < numCoarseVertices; ++v) {coarseCoords[v*3+0] = x[v]; coarseCoords[v*3+1] = y[v]; coarseCoords[v*3+2] = z[v];}
break;
default:
SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No Pragmatic coarsening defined for dimension %d", dim);
}
pragmatic_get_elements(cells);
/* TODO Read out markers for boundary */
ierr = DMPlexCreateFromCellList(PetscObjectComm((PetscObject) dm), dim, numCoarseCells, numCoarseVertices, numCorners, PETSC_TRUE, cells, dim, coarseCoords, &mesh->coarseMesh);CHKERRQ(ierr);
pragmatic_finalize();
ierr = PetscFree5(x, y, z, metric, cells);CHKERRQ(ierr);
ierr = PetscFree2(bdFaces, bdFaceIds);CHKERRQ(ierr);
ierr = PetscFree(coarseCoords);CHKERRQ(ierr);
}
#endif
ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
*dmCoarsened = mesh->coarseMesh;
PetscFunctionReturn(0);
}
/*
- Checks face match
- Flips non-matching
- Inserts faces of support cells in FIFO
*/
static PetscErrorCode DMPlexCheckFace_Internal(DM dm, PetscInt *faceFIFO, PetscInt *fTop, PetscInt *fBottom, PetscInt cStart, PetscInt fStart, PetscInt fEnd, PetscBT seenCells, PetscBT flippedCells, PetscBT seenFaces)
{
const PetscInt *support, *coneA, *coneB, *coneOA, *coneOB;
PetscInt supportSize, coneSizeA, coneSizeB, posA = -1, posB = -1;
PetscInt face, dim, seenA, flippedA, seenB, flippedB, mismatch, c;
PetscErrorCode ierr;
PetscFunctionBegin;
face = faceFIFO[(*fTop)++];
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
if (supportSize < 2) PetscFunctionReturn(0);
if (supportSize != 2) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Faces should separate only two cells, not %d", supportSize);
seenA = PetscBTLookup(seenCells, support[0]-cStart);
flippedA = PetscBTLookup(flippedCells, support[0]-cStart) ? 1 : 0;
seenB = PetscBTLookup(seenCells, support[1]-cStart);
flippedB = PetscBTLookup(flippedCells, support[1]-cStart) ? 1 : 0;
ierr = DMPlexGetConeSize(dm, support[0], &coneSizeA);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, support[1], &coneSizeB);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[0], &coneA);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[1], &coneB);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[0], &coneOA);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[1], &coneOB);CHKERRQ(ierr);
for (c = 0; c < coneSizeA; ++c) {
if (!PetscBTLookup(seenFaces, coneA[c]-fStart)) {
faceFIFO[(*fBottom)++] = coneA[c];
ierr = PetscBTSet(seenFaces, coneA[c]-fStart);CHKERRQ(ierr);
}
if (coneA[c] == face) posA = c;
if (*fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], *fBottom, fEnd-fStart);
}
if (posA < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[0]);
for (c = 0; c < coneSizeB; ++c) {
if (!PetscBTLookup(seenFaces, coneB[c]-fStart)) {
faceFIFO[(*fBottom)++] = coneB[c];
ierr = PetscBTSet(seenFaces, coneB[c]-fStart);CHKERRQ(ierr);
}
if (coneB[c] == face) posB = c;
if (*fBottom > fEnd-fStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Face %d was pushed exceeding capacity %d > %d", coneA[c], *fBottom, fEnd-fStart);
}
if (posB < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d could not be located in cell %d", face, support[1]);
if (dim == 1) {
mismatch = posA == posB;
} else {
mismatch = coneOA[posA] == coneOB[posB];
}
if (mismatch ^ (flippedA ^ flippedB)) {
if (seenA && seenB) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen cells %d and %d do not match: Fault mesh is non-orientable", support[0], support[1]);
if (!seenA && !flippedA) {
ierr = PetscBTSet(flippedCells, support[0]-cStart);CHKERRQ(ierr);
} else if (!seenB && !flippedB) {
ierr = PetscBTSet(flippedCells, support[1]-cStart);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
} else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
ierr = PetscBTSet(seenCells, support[0]-cStart);CHKERRQ(ierr);
ierr = PetscBTSet(seenCells, support[1]-cStart);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexOrient - Give a consistent orientation to the input mesh
Input Parameters:
. dm - The DM
Note: The orientation data for the DM are change in-place.
$ This routine will fail for non-orientable surfaces, such as the Moebius strip.
Level: advanced
.seealso: DMCreate(), DMPLEX
@*/
PetscErrorCode DMPlexOrient(DM dm)
{
MPI_Comm comm;
PetscSF sf;
const PetscInt *lpoints;
const PetscSFNode *rpoints;
PetscSFNode *rorntComp = NULL, *lorntComp = NULL;
PetscInt *numNeighbors, **neighbors;
PetscSFNode *nrankComp;
PetscBool *match, *flipped;
PetscBT seenCells, flippedCells, seenFaces;
PetscInt *faceFIFO, fTop, fBottom, *cellComp, *faceComp;
PetscInt numLeaves, numRoots, dim, h, cStart, cEnd, c, cell, fStart, fEnd, face, off, totNeighbors = 0;
PetscMPIInt rank, size, numComponents, comp = 0;
PetscBool flg, flg2;
PetscViewer viewer = NULL, selfviewer = NULL;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view", &flg);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view_synchronized", &flg2);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
/* Truth Table
mismatch flips do action mismatch flipA ^ flipB action
F 0 flips no F F F
F 1 flip yes F T T
F 2 flips no T F T
T 0 flips yes T T F
T 1 flip no
T 2 flips yes
*/
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscCalloc3(fEnd - fStart, &faceFIFO, cEnd-cStart, &cellComp, fEnd-fStart, &faceComp);CHKERRQ(ierr);
/*
OLD STYLE
- Add an integer array over cells and faces (component) for connected component number
Foreach component
- Mark the initial cell as seen
- Process component as usual
- Set component for all seenCells
- Wipe seenCells and seenFaces (flippedCells can stay)
- Generate parallel adjacency for component using SF and seenFaces
- Collect numComponents adj data from each proc to 0
- Build same serial graph
- Use same solver
- Use Scatterv to to send back flipped flags for each component
- Negate flippedCells by component
NEW STYLE
- Create the adj on each process
- Bootstrap to complete graph on proc 0
*/
/* Loop over components */
for (cell = cStart; cell < cEnd; ++cell) cellComp[cell-cStart] = -1;
do {
/* Look for first unmarked cell */
for (cell = cStart; cell < cEnd; ++cell) if (cellComp[cell-cStart] < 0) break;
if (cell >= cEnd) break;
/* Initialize FIFO with first cell in component */
{
const PetscInt *cone;
PetscInt coneSize;
fTop = fBottom = 0;
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
faceFIFO[fBottom++] = cone[c];
ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
}
ierr = PetscBTSet(seenCells, cell-cStart);CHKERRQ(ierr);
}
/* Consider each face in FIFO */
while (fTop < fBottom) {
ierr = DMPlexCheckFace_Internal(dm, faceFIFO, &fTop, &fBottom, cStart, fStart, fEnd, seenCells, flippedCells, seenFaces);CHKERRQ(ierr);
}
/* Set component for cells and faces */
for (cell = 0; cell < cEnd-cStart; ++cell) {
if (PetscBTLookup(seenCells, cell)) cellComp[cell] = comp;
}
for (face = 0; face < fEnd-fStart; ++face) {
if (PetscBTLookup(seenFaces, face)) faceComp[face] = comp;
}
/* Wipe seenCells and seenFaces for next component */
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
++comp;
} while (1);
numComponents = comp;
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for serial flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Now all subdomains are oriented, but we need a consistent parallel orientation */
if (numLeaves >= 0) {
/* Store orientations of boundary faces*/
ierr = PetscCalloc2(numRoots,&rorntComp,numRoots,&lorntComp);CHKERRQ(ierr);
for (face = fStart; face < fEnd; ++face) {
const PetscInt *cone, *support, *ornt;
PetscInt coneSize, supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) continue;
ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
if (dim == 1) {
/* Use cone position instead, shifted to -1 or 1 */
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = 1-c*2;
else rorntComp[face].rank = c*2-1;
} else {
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = ornt[c] < 0 ? -1 : 1;
else rorntComp[face].rank = ornt[c] < 0 ? 1 : -1;
}
rorntComp[face].index = faceComp[face-fStart];
}
/* Communicate boundary edge orientations */
ierr = PetscSFBcastBegin(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
}
/* Get process adjacency */
ierr = PetscMalloc2(numComponents, &numNeighbors, numComponents, &neighbors);CHKERRQ(ierr);
viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)dm));
if (flg2) {ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
for (comp = 0; comp < numComponents; ++comp) {
PetscInt l, n;
numNeighbors[comp] = 0;
ierr = PetscMalloc1(PetscMax(numLeaves, 0), &neighbors[comp]);CHKERRQ(ierr);
/* I know this is p^2 time in general, but for bounded degree its alright */
for (l = 0; l < numLeaves; ++l) {
const PetscInt face = lpoints[l];
/* Find a representative face (edge) separating pairs of procs */
if ((face >= fStart) && (face < fEnd) && (faceComp[face-fStart] == comp)) {
const PetscInt rrank = rpoints[l].rank;
const PetscInt rcomp = lorntComp[face].index;
for (n = 0; n < numNeighbors[comp]; ++n) if ((rrank == rpoints[neighbors[comp][n]].rank) && (rcomp == lorntComp[lpoints[neighbors[comp][n]]].index)) break;
if (n >= numNeighbors[comp]) {
PetscInt supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
if (flg) {ierr = PetscViewerASCIIPrintf(selfviewer, "[%d]: component %d, Found representative leaf %d (face %d) connecting to face %d on (%d, %d) with orientation %d\n", rank, comp, l, face, rpoints[l].index, rrank, rcomp, lorntComp[face].rank);CHKERRQ(ierr);}
neighbors[comp][numNeighbors[comp]++] = l;
}
}
}
totNeighbors += numNeighbors[comp];
}
ierr = PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (flg2) {ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscMalloc2(totNeighbors, &nrankComp, totNeighbors, &match);CHKERRQ(ierr);
for (comp = 0, off = 0; comp < numComponents; ++comp) {
PetscInt n;
for (n = 0; n < numNeighbors[comp]; ++n, ++off) {
const PetscInt face = lpoints[neighbors[comp][n]];
const PetscInt o = rorntComp[face].rank*lorntComp[face].rank;
if (o < 0) match[off] = PETSC_TRUE;
else if (o > 0) match[off] = PETSC_FALSE;
else SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid face %d (%d, %d) neighbor: %d comp: %d", face, rorntComp[face], lorntComp[face], neighbors[comp][n], comp);
nrankComp[off].rank = rpoints[neighbors[comp][n]].rank;
nrankComp[off].index = lorntComp[lpoints[neighbors[comp][n]]].index;
}
ierr = PetscFree(neighbors[comp]);CHKERRQ(ierr);
}
/* Collect the graph on 0 */
if (numLeaves >= 0) {
Mat G;
PetscBT seenProcs, flippedProcs;
PetscInt *procFIFO, pTop, pBottom;
PetscInt *N = NULL, *Noff;
PetscSFNode *adj = NULL;
PetscBool *val = NULL;
PetscMPIInt *recvcounts = NULL, *displs = NULL, *Nc, p, o;
PetscMPIInt size = 0;
ierr = PetscCalloc1(numComponents, &flipped);CHKERRQ(ierr);
if (!rank) {ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);}
ierr = PetscCalloc4(size, &recvcounts, size+1, &displs, size, &Nc, size+1, &Noff);CHKERRQ(ierr);
ierr = MPI_Gather(&numComponents, 1, MPI_INT, Nc, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
if (!rank) {ierr = PetscMalloc1(displs[size],&N);CHKERRQ(ierr);}
ierr = MPI_Gatherv(numNeighbors, numComponents, MPIU_INT, N, Nc, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0, o = 0; p < size; ++p) {
recvcounts[p] = 0;
for (c = 0; c < Nc[p]; ++c, ++o) recvcounts[p] += N[o];
displs[p+1] = displs[p] + recvcounts[p];
}
if (!rank) {ierr = PetscMalloc2(displs[size], &adj, displs[size], &val);CHKERRQ(ierr);}
ierr = MPI_Gatherv(nrankComp, totNeighbors, MPIU_2INT, adj, recvcounts, displs, MPIU_2INT, 0, comm);CHKERRQ(ierr);
ierr = MPI_Gatherv(match, totNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
if (!rank) {
for (p = 1; p <= size; ++p) {Noff[p] = Noff[p-1] + Nc[p-1];}
if (flg) {
PetscInt n;
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
ierr = PetscPrintf(PETSC_COMM_SELF, "Proc %d Comp %d:\n", p, c);CHKERRQ(ierr);
for (n = 0; n < N[Noff[p]+c]; ++n, ++off) {
ierr = PetscPrintf(PETSC_COMM_SELF, " edge (%d, %d) (%d):\n", adj[off].rank, adj[off].index, val[off]);CHKERRQ(ierr);
}
}
}
}
/* Symmetrize the graph */
ierr = MatCreate(PETSC_COMM_SELF, &G);CHKERRQ(ierr);
ierr = MatSetSizes(G, Noff[size], Noff[size], Noff[size], Noff[size]);CHKERRQ(ierr);
ierr = MatSetUp(G);CHKERRQ(ierr);
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
const PetscInt r = Noff[p]+c;
PetscInt n;
for (n = 0; n < N[r]; ++n, ++off) {
const PetscInt q = Noff[adj[off].rank] + adj[off].index;
const PetscScalar o = val[off] ? 1.0 : 0.0;
ierr = MatSetValues(G, 1, &r, 1, &q, &o, INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(G, 1, &q, 1, &r, &o, INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &seenProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], seenProcs);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &flippedProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], flippedProcs);CHKERRQ(ierr);
ierr = PetscMalloc1(Noff[size], &procFIFO);CHKERRQ(ierr);
pTop = pBottom = 0;
for (p = 0; p < Noff[size]; ++p) {
if (PetscBTLookup(seenProcs, p)) continue;
/* Initialize FIFO with next proc */
procFIFO[pBottom++] = p;
ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
/* Consider each proc in FIFO */
while (pTop < pBottom) {
const PetscScalar *ornt;
const PetscInt *neighbors;
PetscInt proc, nproc, seen, flippedA, flippedB, mismatch, numNeighbors, n;
proc = procFIFO[pTop++];
flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
ierr = MatGetRow(G, proc, &numNeighbors, &neighbors, &ornt);CHKERRQ(ierr);
/* Loop over neighboring procs */
for (n = 0; n < numNeighbors; ++n) {
nproc = neighbors[n];
mismatch = PetscRealPart(ornt[n]) > 0.5 ? 0 : 1;
seen = PetscBTLookup(seenProcs, nproc);
flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
if (mismatch ^ (flippedA ^ flippedB)) {
if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
if (!flippedB) {
ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
} else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
if (!seen) {
procFIFO[pBottom++] = nproc;
ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
}
}
}
}
ierr = PetscFree(procFIFO);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
ierr = PetscFree2(adj, val);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
}
/* Scatter flip flags */
{
PetscBool *flips = NULL;
if (!rank) {
ierr = PetscMalloc1(Noff[size], &flips);CHKERRQ(ierr);
for (p = 0; p < Noff[size]; ++p) {
flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
if (flg && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc+Comp %d:\n", p);CHKERRQ(ierr);}
}
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
}
ierr = MPI_Scatterv(flips, Nc, displs, MPIU_BOOL, flipped, numComponents, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree(flips);CHKERRQ(ierr);
}
if (!rank) {ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);}
ierr = PetscFree(N);CHKERRQ(ierr);
ierr = PetscFree4(recvcounts, displs, Nc, Noff);CHKERRQ(ierr);
ierr = PetscFree2(nrankComp, match);CHKERRQ(ierr);
/* Decide whether to flip cells in each component */
for (c = 0; c < cEnd-cStart; ++c) {if (flipped[cellComp[c]]) {ierr = PetscBTNegate(flippedCells, c);CHKERRQ(ierr);}}
ierr = PetscFree(flipped);CHKERRQ(ierr);
}
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for parallel flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Reverse flipped cells in the mesh */
for (c = cStart; c < cEnd; ++c) {
if (PetscBTLookup(flippedCells, c-cStart)) {
ierr = DMPlexReverseCell(dm, c);CHKERRQ(ierr);
}
}
ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
ierr = PetscFree2(rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscFree3(faceFIFO, cellComp, faceComp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
