static PetscErrorCode DMPlexGetVTKConnectivity(DM dm,PieceInfo *piece,PetscVTKInt **oconn,PetscVTKInt **ooffsets,PetscVTKType **otypes)
{
PetscErrorCode ierr;
PetscVTKInt *conn,*offsets;
PetscVTKType *types;
PetscInt dim,vStart,vEnd,cStart,cEnd,pStart,pEnd,cellHeight,cMax,numLabelCells,hasLabel,c,v,countcell,countconn;
PetscFunctionBegin;
ierr = PetscMalloc3(piece->nconn,PetscVTKInt,&conn,piece->ncells,PetscVTKInt,&offsets,piece->ncells,PetscVTKType,&types);CHKERRQ(ierr);
ierr = DMPlexGetDimension(dm,&dim);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm,&pStart,&pEnd);CHKERRQ(ierr);
ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
ierr = DMPlexGetStratumSize(dm, "vtk", 1, &numLabelCells);CHKERRQ(ierr);
hasLabel = numLabelCells > 0 ? PETSC_TRUE : PETSC_FALSE;
countcell = 0;
countconn = 0;
for (c = cStart; c < cEnd; ++c) {
PetscInt *closure = NULL;
PetscInt closureSize,nverts,celltype,startoffset,nC=0;
if (hasLabel) {
PetscInt value;
ierr = DMPlexGetLabelValue(dm, "vtk", c, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
startoffset = countconn;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (v = 0; v < closureSize*2; v += 2) {
if ((closure[v] >= vStart) && (closure[v] < vEnd)) {
conn[countconn++] = closure[v] - vStart;
++nC;
}
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
ierr = DMPlexInvertCell(dim, nC, &conn[countconn-nC]);CHKERRQ(ierr);
offsets[countcell] = countconn;
nverts = countconn - startoffset;
ierr = DMPlexVTKGetCellType(dm,dim,nverts,&celltype);CHKERRQ(ierr);
types[countcell] = celltype;
countcell++;
}
if (countcell != piece->ncells) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Inconsistent cell count");
if (countconn != piece->nconn) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Inconsistent connectivity count");
*oconn = conn;
*ooffsets = offsets;
*otypes = types;
PetscFunctionReturn(0);
}
static PetscErrorCode AddIdentityLabel(DM dm)
{
PetscInt pStart,pEnd,p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMCreateLabel(dm, "identity");CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; p++) {ierr = DMSetLabelValue(dm, "identity", p, p);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexCreateOrderingClosure_Static(DM dm, PetscInt numPoints, const PetscInt pperm[], PetscInt **clperm, PetscInt **invclperm)
{
PetscInt *perm, *iperm;
PetscInt depth, d, pStart, pEnd, fStart, fMax, fEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscMalloc1(pEnd-pStart,&perm);CHKERRQ(ierr);
ierr = PetscMalloc1(pEnd-pStart,&iperm);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) iperm[p] = -1;
for (d = depth; d > 0; --d) {
ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, d-1, &fStart, &fEnd);CHKERRQ(ierr);
fMax = fStart;
for (p = pStart; p < pEnd; ++p) {
const PetscInt *cone;
PetscInt point, coneSize, c;
if (d == depth) {
perm[p] = pperm[p];
iperm[pperm[p]] = p;
}
point = perm[p];
ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
const PetscInt oldc = cone[c];
const PetscInt newc = iperm[oldc];
if (newc < 0) {
perm[fMax] = oldc;
iperm[oldc] = fMax++;
}
}
}
if (fMax != fEnd) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of depth %d faces %d does not match permuted nubmer %d", d, fEnd-fStart, fMax-fStart);
}
*clperm = perm;
*invclperm = iperm;
PetscFunctionReturn(0);
}
/*@
DMNetworkLayoutSetUp - Sets up the bare layout (graph) for the network
Collective on DM
Input Parameters
. DM - the dmnetwork object
Notes:
This routine should be called after the network sizes and edgelists have been provided. It creates
the bare layout of the network and sets up the network to begin insertion of components.
All the components should be registered before calling this routine.
Level: intermediate
.seealso: DMNetworkSetSizes, DMNetworkSetEdgeList
@*/
PetscErrorCode DMNetworkLayoutSetUp(DM dm)
{
PetscErrorCode ierr;
DM_Network *network = (DM_Network*) dm->data;
PetscInt dim = 1; /* One dimensional network */
PetscInt numCorners=2;
PetscInt spacedim=2;
double *vertexcoords=NULL;
PetscInt i;
PetscInt ndata;
PetscFunctionBegin;
if (network->nNodes) {
ierr = PetscMalloc1(numCorners*network->nNodes,&vertexcoords);CHKERRQ(ierr);
}
ierr = DMPlexCreateFromCellList(PetscObjectComm((PetscObject)dm),dim,network->nEdges,network->nNodes,numCorners,PETSC_FALSE,network->edges,spacedim,vertexcoords,&network->plex);CHKERRQ(ierr);
if (network->nNodes) {
ierr = PetscFree(vertexcoords);CHKERRQ(ierr);
}
ierr = DMPlexGetChart(network->plex,&network->pStart,&network->pEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(network->plex,0,&network->eStart,&network->eEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(network->plex,1,&network->vStart,&network->vEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm),&network->DataSection);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm),&network->DofSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(network->DataSection,network->pStart,network->pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(network->DofSection,network->pStart,network->pEnd);CHKERRQ(ierr);
network->dataheadersize = sizeof(struct _p_DMNetworkComponentHeader)/sizeof(DMNetworkComponentGenericDataType);
ierr = PetscMalloc1(network->pEnd-network->pStart,&network->header);CHKERRQ(ierr);
for (i = network->pStart; i < network->pEnd; i++) {
network->header[i].ndata = 0;
ndata = network->header[i].ndata;
ierr = PetscSectionAddDof(network->DataSection,i,network->dataheadersize);CHKERRQ(ierr);
network->header[i].offset[ndata] = 0;
}
ierr = PetscMalloc1(network->pEnd-network->pStart,&network->cvalue);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
// Create a section where each vertex has the given degrees of freedom
PetscSection NewSection(MPI_Comm comm, DM mesh, uint32_t dof) {
PetscSection section;
PetscSectionCreate(comm, §ion);
// Set the section size from the mesh
int chartStart, chartEnd;
DMPlexGetChart(mesh, &chartStart, &chartEnd);
PetscSectionSetChart(section, chartStart, chartEnd);
// Set the dof for each vertex
// Can also do faces/edges depending on the integer argument to
// GetDepthStratum
int vertexStart, vertexEnd;
DMPlexGetDepthStratum(mesh, 0, &vertexStart, &vertexEnd);
for (int vertex = vertexStart; vertex<vertexEnd; ++vertex) {
PetscSectionSetDof(section, vertex, dof);
}
// Finalise setup
PetscSectionSetUp(section);
return section;
}
/*@
DMPlexGetOrdering - Calculate a reordering of the mesh
Collective on DM
Input Parameter:
+ dm - The DMPlex object
. otype - type of reordering, one of the following:
$ MATORDERINGNATURAL - Natural
$ MATORDERINGND - Nested Dissection
$ MATORDERING1WD - One-way Dissection
$ MATORDERINGRCM - Reverse Cuthill-McKee
$ MATORDERINGQMD - Quotient Minimum Degree
- label - [Optional] Label used to segregate ordering into sets, or NULL
Output Parameter:
. perm - The point permutation as an IS, perm[old point number] = new point number
Note: The label is used to group sets of points together by label value. This makes it easy to reorder a mesh which
has different types of cells, and then loop over each set of reordered cells for assembly.
Level: intermediate
.keywords: mesh
.seealso: MatGetOrdering()
@*/
PetscErrorCode DMPlexGetOrdering(DM dm, MatOrderingType otype, DMLabel label, IS *perm)
{
PetscInt numCells = 0;
PetscInt *start = NULL, *adjacency = NULL, *cperm, *clperm, *invclperm, *mask, *xls, pStart, pEnd, c, i;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidPointer(perm, 3);
ierr = DMPlexCreateNeighborCSR(dm, 0, &numCells, &start, &adjacency);CHKERRQ(ierr);
ierr = PetscMalloc3(numCells,&cperm,numCells,&mask,numCells*2,&xls);CHKERRQ(ierr);
if (numCells) {
/* Shift for Fortran numbering */
for (i = 0; i < start[numCells]; ++i) ++adjacency[i];
for (i = 0; i <= numCells; ++i) ++start[i];
ierr = SPARSEPACKgenrcm(&numCells, start, adjacency, cperm, mask, xls);CHKERRQ(ierr);
}
ierr = PetscFree(start);CHKERRQ(ierr);
ierr = PetscFree(adjacency);CHKERRQ(ierr);
/* Shift for Fortran numbering */
for (c = 0; c < numCells; ++c) --cperm[c];
/* Segregate */
if (label) {
IS valueIS;
const PetscInt *values;
PetscInt numValues, numPoints = 0;
PetscInt *sperm, *vsize, *voff, v;
ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
ierr = ISSort(valueIS);CHKERRQ(ierr);
ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
ierr = PetscCalloc3(numCells,&sperm,numValues,&vsize,numValues+1,&voff);CHKERRQ(ierr);
for (v = 0; v < numValues; ++v) {
ierr = DMLabelGetStratumSize(label, values[v], &vsize[v]);CHKERRQ(ierr);
if (v < numValues-1) voff[v+2] += vsize[v] + voff[v+1];
numPoints += vsize[v];
}
if (numPoints != numCells) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Label only covers %D cells < %D total", numPoints, numCells);
for (c = 0; c < numCells; ++c) {
const PetscInt oldc = cperm[c];
PetscInt val, vloc;
ierr = DMLabelGetValue(label, oldc, &val);CHKERRQ(ierr);
if (val == -1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D not present in label", oldc);
ierr = PetscFindInt(val, numValues, values, &vloc);CHKERRQ(ierr);
if (vloc < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Value %D not present label", val);
sperm[voff[vloc+1]++] = oldc;
}
for (v = 0; v < numValues; ++v) {
if (voff[v+1] - voff[v] != vsize[v]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of %D values found is %D != %D", values[v], voff[v+1] - voff[v], vsize[v]);
}
ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
ierr = PetscMemcpy(cperm, sperm, numCells * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscFree3(sperm, vsize, voff);CHKERRQ(ierr);
}
/* Construct closure */
ierr = DMPlexCreateOrderingClosure_Static(dm, numCells, cperm, &clperm, &invclperm);CHKERRQ(ierr);
ierr = PetscFree3(cperm,mask,xls);CHKERRQ(ierr);
ierr = PetscFree(clperm);CHKERRQ(ierr);
/* Invert permutation */
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd-pStart, invclperm, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data;
FILE *fp;
PetscViewerVTKObjectLink link;
PetscSection coordSection, globalCoordSection;
PetscLayout vLayout;
Vec coordinates;
PetscReal lengthScale;
PetscInt vMax, totVertices, totCells;
PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr);
/* Vertices */
ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
if (vMax >= 0) {
PetscInt pStart, pEnd, p, localSize = 0;
ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr);
pEnd = PetscMin(pEnd, vMax);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr);
if (dof > 0) ++localSize;
}
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
} else {
ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr);
}
ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr);
ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr);
/* Cells */
ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr);
/* Vertex fields */
for (link = vtk->link; link; link = link->next) {
if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE;
if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE;
}
if (hasPoint) {
ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection, newSection = NULL;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
DMLabel subpointMap, subpointMapX;
PetscInt dim, dimX, pStart, pEnd, qStart, qEnd;
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
/* Here is where we check whether dmX is a submesh of dm */
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr);
if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) {
const PetscInt *ind = NULL;
IS subpointIS;
PetscInt n = 0, q;
ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr);
if (subpointIS) {
ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr);
ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr);
}
ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr);
for (q = qStart; q < qEnd; ++q) {
PetscInt dof, off, p;
ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr);
if (dof) {
ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr);
if (p >= pStart) {
ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr);
ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr);
}
}
}
if (subpointIS) {
ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr);
ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
}
/* No need to setup section */
section = newSection;
}
} else {
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr);
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);}
}
}
/* Cell Fields */
ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr);
if (hasCell || writePartition) {
ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
} else {
PetscContainer c;
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr);
if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
}
if (writePartition) {
ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr);
}
}
/* Cleanup */
ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
ierr = PetscFClose(comm, fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int
main (int argc, char **argv)
{
sc_MPI_Comm mpicomm;
int mpiret;
int mpisize, mpirank;
p4est_t *p4est;
p4est_connectivity_t *conn;
sc_array_t *points_per_dim, *cone_sizes, *cones,
*cone_orientations, *coords,
*children, *parents, *childids, *leaves, *remotes;
p4est_locidx_t first_local_quad = -1;
/* initialize MPI */
mpiret = sc_MPI_Init (&argc, &argv);
SC_CHECK_MPI (mpiret);
mpicomm = sc_MPI_COMM_WORLD;
mpiret = sc_MPI_Comm_size (mpicomm, &mpisize);
SC_CHECK_MPI (mpiret);
mpiret = sc_MPI_Comm_rank (mpicomm, &mpirank);
SC_CHECK_MPI (mpiret);
sc_init (mpicomm, 1, 1, NULL, SC_LP_DEFAULT);
p4est_init (NULL, SC_LP_DEFAULT);
#ifndef P4_TO_P8
conn = p4est_connectivity_new_moebius ();
#else
conn = p8est_connectivity_new_rotcubes ();
#endif
p4est = p4est_new_ext (mpicomm, conn, 0, 1, 1, 0, NULL, NULL);
p4est_refine (p4est, 1, refine_fn, NULL);
p4est_balance (p4est, P4EST_CONNECT_FULL, NULL);
p4est_partition (p4est, 0, NULL);
points_per_dim = sc_array_new (sizeof (p4est_locidx_t));
cone_sizes = sc_array_new (sizeof (p4est_locidx_t));
cones = sc_array_new (sizeof (p4est_locidx_t));
cone_orientations = sc_array_new (sizeof (p4est_locidx_t));
coords = sc_array_new (3 * sizeof (double));
children = sc_array_new (sizeof (p4est_locidx_t));
parents = sc_array_new (sizeof (p4est_locidx_t));
childids = sc_array_new (sizeof (p4est_locidx_t));
leaves = sc_array_new (sizeof (p4est_locidx_t));
remotes = sc_array_new (2 * sizeof (p4est_locidx_t));
p4est_get_plex_data (p4est, P4EST_CONNECT_FULL, (mpisize > 1) ? 2 : 0,
&first_local_quad, points_per_dim, cone_sizes, cones,
cone_orientations, coords, children, parents, childids,
leaves, remotes);
#ifdef P4EST_WITH_PETSC
{
PetscErrorCode ierr;
DM plex, refTree;
PetscInt pStart, pEnd;
PetscSection parentSection;
PetscSF pointSF;
size_t zz, count;
locidx_to_PetscInt (points_per_dim);
locidx_to_PetscInt (cone_sizes);
locidx_to_PetscInt (cones);
locidx_to_PetscInt (cone_orientations);
coords_double_to_PetscScalar (coords);
locidx_to_PetscInt (children);
locidx_to_PetscInt (parents);
locidx_to_PetscInt (childids);
locidx_to_PetscInt (leaves);
locidx_pair_to_PetscSFNode (remotes);
P4EST_GLOBAL_PRODUCTION ("Begin PETSc routines\n");
ierr = PetscInitialize (&argc, &argv, 0, help);
CHKERRQ (ierr);
ierr = DMPlexCreate (mpicomm, &plex);
CHKERRQ (ierr);
ierr = DMSetDimension (plex, P4EST_DIM);
CHKERRQ (ierr);
ierr = DMSetCoordinateDim (plex, 3);
CHKERRQ (ierr);
ierr = DMPlexCreateFromDAG (plex, P4EST_DIM,
(PetscInt *) points_per_dim->array,
(PetscInt *) cone_sizes->array,
(PetscInt *) cones->array,
(PetscInt *) cone_orientations->array,
(PetscScalar *) coords->array);
CHKERRQ (ierr);
ierr = PetscSFCreate (mpicomm, &pointSF);
CHKERRQ (ierr);
ierr =
DMPlexCreateDefaultReferenceTree (mpicomm, P4EST_DIM, PETSC_FALSE,
&refTree);
CHKERRQ (ierr);
ierr = DMPlexSetReferenceTree (plex, refTree);
CHKERRQ (ierr);
ierr = DMDestroy (&refTree);
CHKERRQ (ierr);
ierr = PetscSectionCreate (mpicomm, &parentSection);
CHKERRQ (ierr);
ierr = DMPlexGetChart (plex, &pStart, &pEnd);
CHKERRQ (ierr);
ierr = PetscSectionSetChart (parentSection, pStart, pEnd);
CHKERRQ (ierr);
count = children->elem_count;
for (zz = 0; zz < count; zz++) {
PetscInt child =
*((PetscInt *) sc_array_index (children, zz));
ierr = PetscSectionSetDof (parentSection, child, 1);
CHKERRQ (ierr);
}
ierr = PetscSectionSetUp (parentSection);
CHKERRQ (ierr);
ierr =
DMPlexSetTree (plex, parentSection, (PetscInt *) parents->array,
(PetscInt *) childids->array);
CHKERRQ (ierr);
ierr = PetscSectionDestroy (&parentSection);
CHKERRQ (ierr);
ierr =
PetscSFSetGraph (pointSF, pEnd - pStart, (PetscInt) leaves->elem_count,
(PetscInt *) leaves->array, PETSC_COPY_VALUES,
(PetscSFNode *) remotes->array, PETSC_COPY_VALUES);
CHKERRQ (ierr);
ierr = DMViewFromOptions (plex, NULL, "-dm_view");
CHKERRQ (ierr);
/* TODO: test with rigid body modes as in plex ex3 */
ierr = DMDestroy (&plex);
CHKERRQ (ierr);
ierr = PetscFinalize ();
P4EST_GLOBAL_PRODUCTION ("End PETSc routines\n");
}
#endif
sc_array_destroy (points_per_dim);
sc_array_destroy (cone_sizes);
sc_array_destroy (cones);
sc_array_destroy (cone_orientations);
sc_array_destroy (coords);
sc_array_destroy (children);
sc_array_destroy (parents);
sc_array_destroy (childids);
sc_array_destroy (leaves);
sc_array_destroy (remotes);
p4est_destroy (p4est);
p4est_connectivity_destroy (conn);
sc_finalize ();
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
PetscErrorCode DMPlexGetAdjacency_Internal(DM dm, PetscInt p, PetscBool useCone, PetscBool useTransitiveClosure, PetscBool useAnchors, PetscInt *adjSize, PetscInt *adj[])
{
static PetscInt asiz = 0;
PetscInt maxAnchors = 1;
PetscInt aStart = -1, aEnd = -1;
PetscInt maxAdjSize;
PetscSection aSec = NULL;
IS aIS = NULL;
const PetscInt *anchors;
PetscErrorCode ierr;
PetscFunctionBeginHot;
if (useAnchors) {
ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
if (aSec) {
ierr = PetscSectionGetMaxDof(aSec,&maxAnchors);CHKERRQ(ierr);
maxAnchors = PetscMax(1,maxAnchors);
ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
}
}
if (!*adj) {
PetscInt depth, coneSeries, supportSeries, maxC, maxS, pStart, pEnd;
ierr = DMPlexGetChart(dm, &pStart,&pEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(dm, &maxC, &maxS);CHKERRQ(ierr);
coneSeries = (maxC > 1) ? ((PetscPowInt(maxC,depth+1)-1)/(maxC-1)) : depth+1;
supportSeries = (maxS > 1) ? ((PetscPowInt(maxS,depth+1)-1)/(maxS-1)) : depth+1;
asiz = PetscMax(PetscPowInt(maxS,depth)*coneSeries,PetscPowInt(maxC,depth)*supportSeries);
asiz *= maxAnchors;
asiz = PetscMin(asiz,pEnd-pStart);
ierr = PetscMalloc1(asiz,adj);CHKERRQ(ierr);
}
if (*adjSize < 0) *adjSize = asiz;
maxAdjSize = *adjSize;
if (useTransitiveClosure) {
ierr = DMPlexGetAdjacency_Transitive_Internal(dm, p, useCone, adjSize, *adj);CHKERRQ(ierr);
} else if (useCone) {
ierr = DMPlexGetAdjacency_Cone_Internal(dm, p, adjSize, *adj);CHKERRQ(ierr);
} else {
ierr = DMPlexGetAdjacency_Support_Internal(dm, p, adjSize, *adj);CHKERRQ(ierr);
}
if (useAnchors && aSec) {
PetscInt origSize = *adjSize;
PetscInt numAdj = origSize;
PetscInt i = 0, j;
PetscInt *orig = *adj;
while (i < origSize) {
PetscInt p = orig[i];
PetscInt aDof = 0;
if (p >= aStart && p < aEnd) {
ierr = PetscSectionGetDof(aSec,p,&aDof);CHKERRQ(ierr);
}
if (aDof) {
PetscInt aOff;
PetscInt s, q;
for (j = i + 1; j < numAdj; j++) {
orig[j - 1] = orig[j];
}
origSize--;
numAdj--;
ierr = PetscSectionGetOffset(aSec,p,&aOff);CHKERRQ(ierr);
for (s = 0; s < aDof; ++s) {
for (q = 0; q < numAdj || (orig[numAdj++] = anchors[aOff+s],0); ++q) {
if (anchors[aOff+s] == orig[q]) break;
}
if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
}
}
else {
i++;
}
}
*adjSize = numAdj;
ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
void PETSC_STDCALL dmplexgetchart_(DM dm,PetscInt *pStart,PetscInt *pEnd, int *__ierr ){
*__ierr = DMPlexGetChart(
(DM)PetscToPointer((dm) ),pStart,pEnd);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
DM dm;
PetscBool interpolate = PETSC_TRUE;
PetscInt dim;
PetscInt pStart, pEnd;
PetscInt vStart, vEnd;
PetscInt eStart, eEnd;
PetscInt cStart, cEnd;
PetscSection s;
ierr = PetscInitialize(&argc, &argv, (char*)0, help); CHKERRQ(ierr);
ierr = DMPlexCreateGmshFromFile(MPI_COMM_WORLD, argv[1], interpolate, &dm); CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr);
printf("dim = %d\n", dim);
ierr = DMPlexGetChart(dm, &pStart, &pEnd); CHKERRQ(ierr);
printf("chart: pstart, pend = %d %d\n", pStart, pEnd);
// vertices
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd); CHKERRQ(ierr);
printf("Depth 0: vstart, vend, len = %d %d %d\n", vStart, vEnd, vEnd-vStart);
// edges
ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd); CHKERRQ(ierr);
printf("Depth 1: estart, eend, len = %d %d %d\n", eStart, eEnd, eEnd-eStart);
// cells
ierr = DMPlexGetDepthStratum(dm, 2, &cStart, &cEnd); CHKERRQ(ierr);
printf("Depth 2: cstart, cend, len = %d %d %d\n", cStart, cEnd, cEnd-cStart);
// cells adjacent to face
{
FILE * fid = fopen("face_nbr.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
PetscInt nbr;
ierr = DMPlexGetSupportSize(dm, e, &nbr); CHKERRQ(ierr);
const PetscInt *nbcells;
ierr = DMPlexGetSupport(dm, e, &nbcells); CHKERRQ(ierr);
if(nbr == 1) // boundary face
fprintf(fid, "%d %d\n",e-eStart+1,nbcells[0]-cStart+1);
else if(nbr == 2) // interior face
fprintf(fid, "%d %d %d\n",e-eStart+1,nbcells[0]-cStart+1,nbcells[1]-cStart+1);
else
{
printf("nbr is not 1 or 2\n");
exit(0);
}
}
fclose(fid);
}
// point coordinates
{
Vec coordinates;
ierr = DMGetCoordinatesLocal(dm, &coordinates); CHKERRQ(ierr);
const PetscScalar *coords;
ierr = VecGetArrayRead(coordinates, &coords); CHKERRQ(ierr);
DM dmCoord;
ierr = DMGetCoordinateDM(dm, &dmCoord); CHKERRQ(ierr);
FILE * fid = fopen("vertices.txt","w");
for(PetscInt v=vStart; v<vEnd; ++v)
{
PetscScalar *vertex;
ierr = DMPlexPointLocalRead(dmCoord, v, coords, &vertex); CHKERRQ(ierr);
fprintf(fid, "%f %f\n", vertex[0], vertex[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(coordinates, &coords); CHKERRQ(ierr);
}
// compute cell and face geometry
{
Vec cellgeom, facegeom;
ierr = DMPlexComputeGeometryFVM(dm, &cellgeom, &facegeom); CHKERRQ(ierr);
// cell information
DM dmCell;
ierr = VecGetDM(cellgeom, &dmCell); CHKERRQ(ierr);
const PetscScalar *cgeom;
ierr = VecGetArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
FILE * fid = fopen("cells.txt","w");
for(PetscInt c=cStart; c<cEnd; ++c)
{
// cell properties like volume, centroid
PetscFVCellGeom *cg;
ierr = DMPlexPointLocalRead(dmCell, c, cgeom, &cg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f\n", c-cStart+1, cg->volume,
cg->centroid[0], cg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
// face information
DM dmFace;
ierr = VecGetDM(facegeom, &dmFace); CHKERRQ(ierr);
const PetscScalar *fgeom;
ierr = VecGetArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
fid = fopen("faces.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
// face properties like area normal, centroid
PetscFVFaceGeom *fg;
ierr = DMPlexPointLocalRead(dmFace, e, fgeom, &fg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f %f\n", e-eStart+1, fg->normal[0], fg->normal[1],
fg->centroid[0], fg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
}
// create section with one variable in each cell
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &s); CHKERRQ(ierr);
ierr = PetscSectionSetChart(s, pStart, pEnd); CHKERRQ(ierr);
for(PetscInt c=cStart; c<cEnd; ++c)
{
ierr = PetscSectionSetDof(s, c, 1); CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(s); CHKERRQ(ierr);
// create vector to store solution
Vec lv, gv;
ierr = DMSetDefaultSection(dm, s); CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &lv); CHKERRQ(ierr);
ierr = DMGetGlobalVector(dm, &gv); CHKERRQ(ierr);
}
/* This interpolates the PointSF in parallel following local interpolation */
static PetscErrorCode DMPlexInterpolatePointSF(DM dm, PetscSF pointSF, PetscInt depth)
{
PetscMPIInt numProcs, rank;
PetscInt p, c, d, dof, offset;
PetscInt numLeaves, numRoots, candidatesSize, candidatesRemoteSize;
const PetscInt *localPoints;
const PetscSFNode *remotePoints;
PetscSFNode *candidates, *candidatesRemote, *claims;
PetscSection candidateSection, candidateSectionRemote, claimSection;
PetscHashI leafhash;
PetscHashIJ roothash;
PetscHashIJKey key;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
if (numProcs < 2 || numRoots < 0) PetscFunctionReturn(0);
/* Build hashes of points in the SF for efficient lookup */
PetscHashICreate(leafhash);
PetscHashIJCreate(&roothash);
ierr = PetscHashIJSetMultivalued(roothash, PETSC_FALSE);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
PetscHashIAdd(leafhash, localPoints[p], p);
key.i = remotePoints[p].index; key.j = remotePoints[p].rank;
PetscHashIJAdd(roothash, key, p);
}
/* Build a section / SFNode array of candidate points in the single-level adjacency of leaves,
where each candidate is defined by the root entry for the other vertex that defines the edge. */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &candidateSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(candidateSection, 0, numRoots);CHKERRQ(ierr);
{
PetscInt leaf, root, idx, a, *adj = NULL;
for (p = 0; p < numLeaves; ++p) {
PetscInt adjSize = PETSC_DETERMINE;
ierr = DMPlexGetAdjacency_Internal(dm, localPoints[p], PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
for (a = 0; a < adjSize; ++a) {
PetscHashIMap(leafhash, adj[a], leaf);
if (leaf >= 0) {ierr = PetscSectionAddDof(candidateSection, localPoints[p], 1);CHKERRQ(ierr);}
}
}
ierr = PetscSectionSetUp(candidateSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(candidateSection, &candidatesSize);CHKERRQ(ierr);
ierr = PetscMalloc1(candidatesSize, &candidates);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
PetscInt adjSize = PETSC_DETERMINE;
ierr = PetscSectionGetOffset(candidateSection, localPoints[p], &offset);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency_Internal(dm, localPoints[p], PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
for (idx = 0, a = 0; a < adjSize; ++a) {
PetscHashIMap(leafhash, adj[a], root);
if (root >= 0) candidates[offset+idx++] = remotePoints[root];
}
}
ierr = PetscFree(adj);CHKERRQ(ierr);
}
/* Gather candidate section / array pair into the root partition via inverse(multi(pointSF)). */
{
PetscSF sfMulti, sfInverse, sfCandidates;
PetscInt *remoteOffsets;
ierr = PetscSFGetMultiSF(pointSF, &sfMulti);CHKERRQ(ierr);
ierr = PetscSFCreateInverseSF(sfMulti, &sfInverse);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sfInverse, candidateSection, &remoteOffsets, candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfInverse, candidateSection, remoteOffsets, candidateSectionRemote, &sfCandidates);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(candidateSectionRemote, &candidatesRemoteSize);CHKERRQ(ierr);
ierr = PetscMalloc1(candidatesRemoteSize, &candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfCandidates, MPIU_2INT, candidates, candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfCandidates, MPIU_2INT, candidates, candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfInverse);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfCandidates);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
}
/* Walk local roots and check for each remote candidate whether we know all required points,
either from owning it or having a root entry in the point SF. If we do we place a claim
by replacing the vertex number with our edge ID. */
{
PetscInt idx, root, joinSize, vertices[2];
const PetscInt *rootdegree, *join = NULL;
ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
/* Loop remote edge connections and put in a claim if both vertices are known */
for (idx = 0, p = 0; p < numRoots; ++p) {
for (d = 0; d < rootdegree[p]; ++d) {
ierr = PetscSectionGetDof(candidateSectionRemote, idx, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(candidateSectionRemote, idx, &offset);CHKERRQ(ierr);
for (c = 0; c < dof; ++c) {
/* We own both vertices, so we claim the edge by replacing vertex with edge */
if (candidatesRemote[offset+c].rank == rank) {
vertices[0] = p; vertices[1] = candidatesRemote[offset+c].index;
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) candidatesRemote[offset+c].index = join[0];
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
continue;
}
/* If we own one vertex and share a root with the other, we claim it */
key.i = candidatesRemote[offset+c].index; key.j = candidatesRemote[offset+c].rank;
PetscHashIJGet(roothash, key, &root);
if (root >= 0) {
vertices[0] = p; vertices[1] = localPoints[root];
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) {
candidatesRemote[offset+c].index = join[0];
candidatesRemote[offset+c].rank = rank;
}
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
}
}
idx++;
}
}
}
/* Push claims back to receiver via the MultiSF and derive new pointSF mapping on receiver */
{
PetscSF sfMulti, sfClaims, sfPointNew;
PetscHashI claimshash;
PetscInt size, pStart, pEnd, root, joinSize, numLocalNew;
PetscInt *remoteOffsets, *localPointsNew, vertices[2];
const PetscInt *join = NULL;
PetscSFNode *remotePointsNew;
ierr = PetscSFGetMultiSF(pointSF, &sfMulti);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &claimSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sfMulti, candidateSectionRemote, &remoteOffsets, claimSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfMulti, candidateSectionRemote, remoteOffsets, claimSection, &sfClaims);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(claimSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &claims);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfClaims, MPIU_2INT, candidatesRemote, claims);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfClaims, MPIU_2INT, candidatesRemote, claims);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfClaims);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
/* Walk the original section of local supports and add an SF entry for each updated item */
PetscHashICreate(claimshash);
for (p = 0; p < numRoots; ++p) {
ierr = PetscSectionGetDof(candidateSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(candidateSection, p, &offset);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
if (candidates[offset+d].index != claims[offset+d].index) {
key.i = candidates[offset+d].index; key.j = candidates[offset+d].rank;
PetscHashIJGet(roothash, key, &root);
if (root >= 0) {
vertices[0] = p; vertices[1] = localPoints[root];
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) PetscHashIAdd(claimshash, join[0], offset+d);
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
}
}
}
}
/* Create new pointSF from hashed claims */
PetscHashISize(claimshash, numLocalNew);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeaves + numLocalNew, &localPointsNew);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeaves + numLocalNew, &remotePointsNew);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
localPointsNew[p] = localPoints[p];
remotePointsNew[p].index = remotePoints[p].index;
remotePointsNew[p].rank = remotePoints[p].rank;
}
p = numLeaves; ierr = PetscHashIGetKeys(claimshash, &p, localPointsNew);CHKERRQ(ierr);
for (p = numLeaves; p < numLeaves + numLocalNew; ++p) {
PetscHashIMap(claimshash, localPointsNew[p], offset);
remotePointsNew[p] = claims[offset];
}
ierr = PetscSFCreate(PetscObjectComm((PetscObject) dm), &sfPointNew);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sfPointNew, pEnd-pStart, numLeaves+numLocalNew, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = DMSetPointSF(dm, sfPointNew);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfPointNew);CHKERRQ(ierr);
PetscHashIDestroy(claimshash);
}
PetscHashIDestroy(leafhash);
ierr = PetscHashIJDestroy(&roothash);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&candidateSection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&claimSection);CHKERRQ(ierr);
ierr = PetscFree(candidates);CHKERRQ(ierr);
ierr = PetscFree(candidatesRemote);CHKERRQ(ierr);
ierr = PetscFree(claims);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
