PetscErrorCode DMLabelDistribute_Internal(DMLabel label, PetscSF sf, PetscSection *leafSection, PetscInt **leafStrata)
{
MPI_Comm comm;
PetscInt s, l, nroots, nleaves, dof, offset, size;
PetscInt *remoteOffsets, *rootStrata, *rootIdx;
PetscSection rootSection;
PetscSF labelSF;
PetscErrorCode ierr;
PetscFunctionBegin;
if (label) {ierr = DMLabelMakeAllValid_Private(label);CHKERRQ(ierr);}
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
/* Build a section of stratum values per point, generate the according SF
and distribute point-wise stratum values to leaves. */
ierr = PetscSFGetGraph(sf, &nroots, &nleaves, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSection, 0, nroots);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
for (l = 0; l < label->stratumSizes[s]; l++) {
ierr = PetscSectionGetDof(rootSection, label->points[s][l], &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSection, label->points[s][l], dof+1);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSection);CHKERRQ(ierr);
/* Create a point-wise array of stratum values */
ierr = PetscSectionGetStorageSize(rootSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &rootStrata);CHKERRQ(ierr);
ierr = PetscCalloc1(nroots, &rootIdx);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
for (l = 0; l < label->stratumSizes[s]; l++) {
const PetscInt p = label->points[s][l];
ierr = PetscSectionGetOffset(rootSection, p, &offset);CHKERRQ(ierr);
rootStrata[offset+rootIdx[p]++] = label->stratumValues[s];
}
}
}
/* Build SF that maps label points to remote processes */
ierr = PetscSectionCreate(comm, leafSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sf, rootSection, &remoteOffsets, *leafSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, rootSection, remoteOffsets, *leafSection, &labelSF);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
/* Send the strata for each point over the derived SF */
ierr = PetscSectionGetStorageSize(*leafSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(labelSF, MPIU_INT, rootStrata, *leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(labelSF, MPIU_INT, rootStrata, *leafStrata);CHKERRQ(ierr);
/* Clean up */
ierr = PetscFree(rootStrata);CHKERRQ(ierr);
ierr = PetscFree(rootIdx);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSection);CHKERRQ(ierr);
ierr = PetscSFDestroy(&labelSF);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexTSSetupGradient(DM dm, PetscFV fvm, DMTS_Plex *dmplexts)
{
DM dmFace, dmCell;
PetscScalar *fgeom, *cgeom;
PetscSection sectionGrad;
PetscInt dim, pdim, cStart, cEnd, cEndInterior, c;
PetscErrorCode ierr;
PetscFunctionBegin;
if (dmplexts->setupGrad) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = PetscFVGetNumComponents(fvm, &pdim);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
/* Construct the interpolant corresponding to each face from the leat-square solution over the cell neighborhood */
ierr = VecGetDM(dmplexts->facegeom, &dmFace);CHKERRQ(ierr);
ierr = VecGetDM(dmplexts->cellgeom, &dmCell);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
ierr = BuildGradientReconstruction(dm, fvm, dmFace, fgeom, dmCell, cgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
/* Create storage for gradients */
ierr = DMClone(dm, &dmplexts->dmGrad);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionGrad);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionGrad, cStart, cEnd);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {ierr = PetscSectionSetDof(sectionGrad, c, pdim*dim);CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionGrad);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmplexts->dmGrad, sectionGrad);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionGrad);CHKERRQ(ierr);
dmplexts->setupGrad = PETSC_TRUE;
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
{
DM_Plex *mesh;
PetscInt unit, d;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = PetscNewLog(dm, DM_Plex, &mesh);CHKERRQ(ierr);
dm->data = mesh;
mesh->refct = 1;
mesh->dim = 0;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection);CHKERRQ(ierr);
mesh->maxConeSize = 0;
mesh->cones = NULL;
mesh->coneOrientations = NULL;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection);CHKERRQ(ierr);
mesh->maxSupportSize = 0;
mesh->supports = NULL;
mesh->refinementUniform = PETSC_TRUE;
mesh->refinementLimit = -1.0;
mesh->facesTmp = NULL;
mesh->subpointMap = NULL;
for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
mesh->labels = NULL;
mesh->globalVertexNumbers = NULL;
mesh->globalCellNumbers = NULL;
for (d = 0; d < 8; ++d) mesh->hybridPointMax[d] = PETSC_DETERMINE;
mesh->vtkCellHeight = 0;
mesh->preallocCenterDim = -1;
mesh->integrateResidualFEM = NULL;
mesh->integrateJacobianActionFEM = NULL;
mesh->integrateJacobianFEM = NULL;
mesh->printSetValues = PETSC_FALSE;
mesh->printFEM = 0;
ierr = DMInitialize_Plex(dm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMNetworkLayoutSetUp - Sets up the bare layout (graph) for the network
Collective on DM
Input Parameters
. DM - the dmnetwork object
Notes:
This routine should be called after the network sizes and edgelists have been provided. It creates
the bare layout of the network and sets up the network to begin insertion of components.
All the components should be registered before calling this routine.
Level: intermediate
.seealso: DMNetworkSetSizes, DMNetworkSetEdgeList
@*/
PetscErrorCode DMNetworkLayoutSetUp(DM dm)
{
PetscErrorCode ierr;
DM_Network *network = (DM_Network*) dm->data;
PetscInt dim = 1; /* One dimensional network */
PetscInt numCorners=2;
PetscInt spacedim=2;
double *vertexcoords=NULL;
PetscInt i;
PetscInt ndata;
PetscFunctionBegin;
if (network->nNodes) {
ierr = PetscMalloc1(numCorners*network->nNodes,&vertexcoords);CHKERRQ(ierr);
}
ierr = DMPlexCreateFromCellList(PetscObjectComm((PetscObject)dm),dim,network->nEdges,network->nNodes,numCorners,PETSC_FALSE,network->edges,spacedim,vertexcoords,&network->plex);CHKERRQ(ierr);
if (network->nNodes) {
ierr = PetscFree(vertexcoords);CHKERRQ(ierr);
}
ierr = DMPlexGetChart(network->plex,&network->pStart,&network->pEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(network->plex,0,&network->eStart,&network->eEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(network->plex,1,&network->vStart,&network->vEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm),&network->DataSection);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm),&network->DofSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(network->DataSection,network->pStart,network->pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(network->DofSection,network->pStart,network->pEnd);CHKERRQ(ierr);
network->dataheadersize = sizeof(struct _p_DMNetworkComponentHeader)/sizeof(DMNetworkComponentGenericDataType);
ierr = PetscMalloc1(network->pEnd-network->pStart,&network->header);CHKERRQ(ierr);
for (i = network->pStart; i < network->pEnd; i++) {
network->header[i].ndata = 0;
ndata = network->header[i].ndata;
ierr = PetscSectionAddDof(network->DataSection,i,network->dataheadersize);CHKERRQ(ierr);
network->header[i].offset[ndata] = 0;
}
ierr = PetscMalloc1(network->pEnd-network->pStart,&network->cvalue);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexCopyCoordinates - Copy coordinates from one mesh to another with the same vertices
Collective on DM
Input Parameter:
. dmA - The DMPlex object with initial coordinates
Output Parameter:
. dmB - The DMPlex object with copied coordinates
Level: intermediate
Note: This is typically used when adding pieces other than vertices to a mesh
.keywords: mesh
.seealso: DMCopyLabels(), DMGetCoordinates(), DMGetCoordinatesLocal(), DMGetCoordinateDM(), DMGetCoordinateSection()
@*/
PetscErrorCode DMPlexCopyCoordinates(DM dmA, DM dmB)
{
Vec coordinatesA, coordinatesB;
PetscSection coordSectionA, coordSectionB;
PetscScalar *coordsA, *coordsB;
PetscInt spaceDim, vStartA, vStartB, vEndA, vEndB, coordSizeB, v, d;
PetscErrorCode ierr;
PetscFunctionBegin;
if (dmA == dmB) PetscFunctionReturn(0);
ierr = DMPlexGetDepthStratum(dmA, 0, &vStartA, &vEndA);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dmB, 0, &vStartB, &vEndB);CHKERRQ(ierr);
if ((vEndA-vStartA) != (vEndB-vStartB)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of vertices in first DM %d != %d in the second DM", vEndA-vStartA, vEndB-vStartB);
ierr = DMGetCoordinateSection(dmA, &coordSectionA);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dmB, &coordSectionB);CHKERRQ(ierr);
if (!coordSectionB) {
PetscInt dim;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) coordSectionA), &coordSectionB);CHKERRQ(ierr);
ierr = DMGetCoordinateDim(dmA, &dim);CHKERRQ(ierr);
ierr = DMSetCoordinateSection(dmB, dim, coordSectionB);CHKERRQ(ierr);
ierr = PetscObjectDereference((PetscObject) coordSectionB);CHKERRQ(ierr);
}
ierr = PetscSectionSetNumFields(coordSectionB, 1);CHKERRQ(ierr);
ierr = PetscSectionGetFieldComponents(coordSectionA, 0, &spaceDim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSectionB, 0, spaceDim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSectionB, vStartB, vEndB);CHKERRQ(ierr);
for (v = vStartB; v < vEndB; ++v) {
ierr = PetscSectionSetDof(coordSectionB, v, spaceDim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSectionB, v, 0, spaceDim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSectionB);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSectionB, &coordSizeB);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dmA, &coordinatesA);CHKERRQ(ierr);
ierr = VecCreate(PetscObjectComm((PetscObject) dmB), &coordinatesB);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinatesB, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinatesB, coordSizeB, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinatesB,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinatesA, &coordsA);CHKERRQ(ierr);
ierr = VecGetArray(coordinatesB, &coordsB);CHKERRQ(ierr);
for (v = 0; v < vEndB-vStartB; ++v) {
for (d = 0; d < spaceDim; ++d) {
coordsB[v*spaceDim+d] = coordsA[v*spaceDim+d];
}
}
ierr = VecRestoreArray(coordinatesA, &coordsA);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinatesB, &coordsB);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dmB, coordinatesB);CHKERRQ(ierr);
ierr = VecDestroy(&coordinatesB);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMNetworkDistribute - Distributes the network and moves associated component data.
Collective
Input Parameter:
+ oldDM - the original DMNetwork object
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
. distDM - the distributed DMNetwork object
Notes:
This routine should be called only when using multiple processors.
Distributes the network with <overlap>-overlapping partitioning of the edges.
Level: intermediate
.seealso: DMNetworkCreate
@*/
PetscErrorCode DMNetworkDistribute(DM oldDM, PetscInt overlap,DM *distDM)
{
PetscErrorCode ierr;
DM_Network *oldDMnetwork = (DM_Network*)oldDM->data;
PetscSF pointsf;
DM newDM;
DM_Network *newDMnetwork;
PetscFunctionBegin;
ierr = DMNetworkCreate(PetscObjectComm((PetscObject)oldDM),&newDM);CHKERRQ(ierr);
newDMnetwork = (DM_Network*)newDM->data;
newDMnetwork->dataheadersize = sizeof(struct _p_DMNetworkComponentHeader)/sizeof(DMNetworkComponentGenericDataType);
/* Distribute plex dm and dof section */
ierr = DMPlexDistribute(oldDMnetwork->plex,overlap,&pointsf,&newDMnetwork->plex);CHKERRQ(ierr);
/* Distribute dof section */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)oldDM),&newDMnetwork->DofSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointsf,oldDMnetwork->DofSection,NULL,newDMnetwork->DofSection);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)oldDM),&newDMnetwork->DataSection);CHKERRQ(ierr);
/* Distribute data and associated section */
ierr = DMPlexDistributeData(newDMnetwork->plex,pointsf,oldDMnetwork->DataSection,MPI_INT,(void*)oldDMnetwork->componentdataarray,newDMnetwork->DataSection,(void**)&newDMnetwork->componentdataarray);CHKERRQ(ierr);
/* Destroy point SF */
ierr = PetscSFDestroy(&pointsf);CHKERRQ(ierr);
ierr = PetscSectionGetChart(newDMnetwork->DataSection,&newDMnetwork->pStart,&newDMnetwork->pEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(newDMnetwork->plex,0, &newDMnetwork->eStart,&newDMnetwork->eEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(newDMnetwork->plex,1,&newDMnetwork->vStart,&newDMnetwork->vEnd);CHKERRQ(ierr);
newDMnetwork->nEdges = newDMnetwork->eEnd - newDMnetwork->eStart;
newDMnetwork->nNodes = newDMnetwork->vEnd - newDMnetwork->vStart;
newDMnetwork->NNodes = oldDMnetwork->NNodes;
newDMnetwork->NEdges = oldDMnetwork->NEdges;
/* Set Dof section as the default section for dm */
ierr = DMSetDefaultSection(newDMnetwork->plex,newDMnetwork->DofSection);CHKERRQ(ierr);
ierr = DMGetDefaultGlobalSection(newDMnetwork->plex,&newDMnetwork->GlobalDofSection);CHKERRQ(ierr);
*distDM = newDM;
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelConvertToSection(DMLabel label, PetscSection *section, IS *is)
{
IS vIS;
const PetscInt *values;
PetscInt *points;
PetscInt nV, vS = 0, vE = 0, v, N;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMLabelGetNumValues(label, &nV);CHKERRQ(ierr);
ierr = DMLabelGetValueIS(label, &vIS);CHKERRQ(ierr);
ierr = ISGetIndices(vIS, &values);CHKERRQ(ierr);
if (nV) {vS = values[0]; vE = values[0]+1;}
for (v = 1; v < nV; ++v) {
vS = PetscMin(vS, values[v]);
vE = PetscMax(vE, values[v]+1);
}
ierr = PetscSectionCreate(PETSC_COMM_SELF, section);CHKERRQ(ierr);
ierr = PetscSectionSetChart(*section, vS, vE);CHKERRQ(ierr);
for (v = 0; v < nV; ++v) {
PetscInt n;
ierr = DMLabelGetStratumSize(label, values[v], &n);CHKERRQ(ierr);
ierr = PetscSectionSetDof(*section, values[v], n);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(*section, &N);CHKERRQ(ierr);
ierr = PetscMalloc1(N, &points);CHKERRQ(ierr);
for (v = 0; v < nV; ++v) {
IS is;
const PetscInt *spoints;
PetscInt dof, off, p;
ierr = PetscSectionGetDof(*section, values[v], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(*section, values[v], &off);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(label, values[v], &is);CHKERRQ(ierr);
ierr = ISGetIndices(is, &spoints);CHKERRQ(ierr);
for (p = 0; p < dof; ++p) points[off+p] = spoints[p];
ierr = ISRestoreIndices(is, &spoints);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(vIS, &values);CHKERRQ(ierr);
ierr = ISDestroy(&vIS);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF, N, points, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode SetupSection(DM dm)
{
PetscSection s;
PetscInt vStart, vEnd, v;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &s);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(s, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(s, 0, 1);CHKERRQ(ierr);
ierr = PetscSectionSetChart(s, vStart, vEnd);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
ierr = PetscSectionSetDof(s, v, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(s, v, 0, 1);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(s);CHKERRQ(ierr);
ierr = DMSetSection(dm, s);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&s);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
// Create a section where each vertex has the given degrees of freedom
PetscSection NewSection(MPI_Comm comm, DM mesh, uint32_t dof) {
PetscSection section;
PetscSectionCreate(comm, §ion);
// Set the section size from the mesh
int chartStart, chartEnd;
DMPlexGetChart(mesh, &chartStart, &chartEnd);
PetscSectionSetChart(section, chartStart, chartEnd);
// Set the dof for each vertex
// Can also do faces/edges depending on the integer argument to
// GetDepthStratum
int vertexStart, vertexEnd;
DMPlexGetDepthStratum(mesh, 0, &vertexStart, &vertexEnd);
for (int vertex = vertexStart; vertex<vertexEnd; ++vertex) {
PetscSectionSetDof(section, vertex, dof);
}
// Finalise setup
PetscSectionSetUp(section);
return section;
}
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "DMCreate_Complex"
PetscErrorCode DMCreate_Complex(DM dm)
{
DM_Complex *mesh;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = PetscNewLog(dm, DM_Complex, &mesh);CHKERRQ(ierr);
dm->data = mesh;
mesh->dim = 0;
ierr = PetscSectionCreate(((PetscObject) dm)->comm, &mesh->coneSection);CHKERRQ(ierr);
mesh->maxConeSize = 0;
mesh->cones = PETSC_NULL;
mesh->coneOrientations = PETSC_NULL;
ierr = PetscSectionCreate(((PetscObject) dm)->comm, &mesh->supportSection);CHKERRQ(ierr);
mesh->maxSupportSize = 0;
mesh->supports = PETSC_NULL;
ierr = PetscSectionCreate(((PetscObject) dm)->comm, &mesh->coordSection);CHKERRQ(ierr);
ierr = VecCreate(((PetscObject) dm)->comm, &mesh->coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) mesh->coordinates, "coordinates");CHKERRQ(ierr);
mesh->refinementLimit = -1.0;
mesh->meetTmpA = PETSC_NULL;
mesh->meetTmpB = PETSC_NULL;
mesh->joinTmpA = PETSC_NULL;
mesh->joinTmpB = PETSC_NULL;
mesh->closureTmpA = PETSC_NULL;
mesh->closureTmpB = PETSC_NULL;
mesh->facesTmp = PETSC_NULL;
mesh->labels = PETSC_NULL;
mesh->printSetValues = PETSC_FALSE;
ierr = PetscStrallocpy(VECSTANDARD, &dm->vectype);CHKERRQ(ierr);
dm->ops->view = DMView_Complex;
dm->ops->setfromoptions = DMSetFromOptions_Complex;
dm->ops->setup = DMSetUp_Complex;
dm->ops->createglobalvector = PETSC_NULL;
dm->ops->createlocalvector = PETSC_NULL;
dm->ops->createlocaltoglobalmapping = PETSC_NULL;
dm->ops->createlocaltoglobalmappingblock = PETSC_NULL;
dm->ops->createfieldis = PETSC_NULL;
dm->ops->getcoloring = 0;
dm->ops->creatematrix = DMCreateMatrix_Complex;
dm->ops->createinterpolation= 0;
dm->ops->getaggregates = 0;
dm->ops->getinjection = 0;
dm->ops->refine = DMRefine_Complex;
dm->ops->coarsen = 0;
dm->ops->refinehierarchy = 0;
dm->ops->coarsenhierarchy = 0;
dm->ops->globaltolocalbegin = PETSC_NULL;
dm->ops->globaltolocalend = PETSC_NULL;
dm->ops->localtoglobalbegin = PETSC_NULL;
dm->ops->localtoglobalend = PETSC_NULL;
dm->ops->destroy = DMDestroy_Complex;
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data;
FILE *fp;
PetscViewerVTKObjectLink link;
PetscSection coordSection, globalCoordSection;
PetscLayout vLayout;
Vec coordinates;
PetscReal lengthScale;
PetscInt vMax, totVertices, totCells;
PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr);
/* Vertices */
ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
if (vMax >= 0) {
PetscInt pStart, pEnd, p, localSize = 0;
ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr);
pEnd = PetscMin(pEnd, vMax);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr);
if (dof > 0) ++localSize;
}
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
} else {
ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr);
}
ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr);
ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr);
/* Cells */
ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr);
/* Vertex fields */
for (link = vtk->link; link; link = link->next) {
if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE;
if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE;
}
if (hasPoint) {
ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection, newSection = NULL;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
DMLabel subpointMap, subpointMapX;
PetscInt dim, dimX, pStart, pEnd, qStart, qEnd;
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
/* Here is where we check whether dmX is a submesh of dm */
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr);
if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) {
const PetscInt *ind = NULL;
IS subpointIS;
PetscInt n = 0, q;
ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr);
if (subpointIS) {
ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr);
ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr);
}
ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr);
for (q = qStart; q < qEnd; ++q) {
PetscInt dof, off, p;
ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr);
if (dof) {
ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr);
if (p >= pStart) {
ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr);
ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr);
}
}
}
if (subpointIS) {
ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr);
ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
}
/* No need to setup section */
section = newSection;
}
} else {
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr);
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);}
}
}
/* Cell Fields */
ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr);
if (hasCell || writePartition) {
ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
} else {
PetscContainer c;
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr);
if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
}
if (writePartition) {
ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr);
}
}
/* Cleanup */
ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
ierr = PetscFClose(comm, fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "DMConvert_DA_Mesh"
PetscErrorCode DMConvert_DA_Mesh(DM dm, const DMType newtype, DM *dmNew)
{
PetscSection section;
DM cda;
DMDALocalInfo info;
Vec coordinates;
PetscInt *cone, *coneO;
PetscInt dim, M, N, P, numCells, numGlobalCells, numCorners, numVertices, c = 0, v = 0;
PetscInt ye, ze;
PetscInt debug = 0;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscOptionsGetInt(PETSC_NULL, "-dm_mesh_debug", &debug, PETSC_NULL);CHKERRQ(ierr);
ierr = DMDAGetInfo(dm, &dim, &M, &N, &P, 0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetLocalInfo(dm, &info);CHKERRQ(ierr);
if (info.sw > 1) SETERRQ(((PetscObject) dm)->comm, PETSC_ERR_SUP, "Currently, only DMDAs with unti stencil width can be converted to DMMeshes.");
/* In order to get a partition of cells, rather than vertices, we give each process the cells between vertices it owns
and also higher numbered ghost vertices (vertices to the right and up) */
numCorners = 1 << dim;
numCells = ((info.gxm+info.gxs - info.xs) - 1);
if (dim > 1) {numCells *= ((info.gym+info.gys - info.ys) - 1);}
if (dim > 2) {numCells *= ((info.gzm+info.gzs - info.zs) - 1);}
numVertices = (info.gxm+info.gxs - info.xs);
if (dim > 1) {numVertices *= (info.gym+info.gys - info.ys);}
if (dim > 2) {numVertices *= (info.gzm+info.gzs - info.zs);}
numGlobalCells = M-1;
if (dim > 1) {numGlobalCells *= N-1;}
if (dim > 2) {numGlobalCells *= P-1;}
ALE::Obj<PETSC_MESH_TYPE> mesh = new PETSC_MESH_TYPE(((PetscObject) dm)->comm, info.dim, debug);
ALE::Obj<PETSC_MESH_TYPE::sieve_type> sieve = new PETSC_MESH_TYPE::sieve_type(((PetscObject) dm)->comm, 0, numCells+numVertices, debug);
PETSC_MESH_TYPE::renumbering_type renumbering;
mesh->setSieve(sieve);
/* Number each cell for the vertex in the lower left corner */
if (dim < 3) {ze = 1; P = 1;} else {ze = info.gzs+info.gzm-1;}
if (dim < 2) {ye = 1; N = 1;} else {ye = info.gys+info.gym-1;}
for(PetscInt k = info.zs; k < ze; ++k) {
for(PetscInt j = info.ys; j < ye; ++j) {
for(PetscInt i = info.xs; i < info.gxs+info.gxm-1; ++i, ++c) {
PetscInt globalC = (k*(N-1) + j)*(M-1) + i;
renumbering[globalC] = c;
sieve->setConeSize(c, numCorners);
}
}
}
if (c != numCells) {SETERRQ2(((PetscObject) dm)->comm, PETSC_ERR_PLIB, "Error in generated cell numbering, %d should be %d", c, numCells);}
/* Get vertex renumbering */
for(PetscInt k = info.zs; k < info.gzs+info.gzm; ++k) {
for(PetscInt j = info.ys; j < info.gys+info.gym; ++j) {
for(PetscInt i = info.xs; i < info.gxs+info.gxm; ++i, ++v) {
PetscInt globalV = (k*N + j)*M + i + numGlobalCells;
renumbering[globalV] = v+numCells;
}
}
}
if (v != numVertices) {SETERRQ2(((PetscObject) dm)->comm, PETSC_ERR_PLIB, "Error in generated vertex numbering, %d should be %d", v, numVertices);}
/* Calculate support sizes */
for(PetscInt k = info.zs; k < ze; ++k, ++c) {
for(PetscInt j = info.ys; j < ye; ++j) {
for(PetscInt i = info.xs; i < info.gxs+info.gxm-1; ++i) {
for(PetscInt kp = k; kp <= k+(dim>2); ++kp) {
for(PetscInt jp = j; jp <= j+(dim>1); ++jp) {
for(PetscInt ip = i; ip <= i+1; ++ip) {
PetscInt globalV = (kp*N + jp)*M + ip + numGlobalCells;
sieve->addSupportSize(renumbering[globalV], 1);
}
}
}
}
}
}
sieve->allocate();
ierr = PetscMalloc2(numCorners,PetscInt,&cone,numCorners,PetscInt,&coneO);CHKERRQ(ierr);
for(PetscInt v = 0; v < numCorners; ++v) {
coneO[v] = 1;
}
for(PetscInt k = info.zs; k < ze; ++k) {
for(PetscInt j = info.ys; j < ye; ++j) {
for(PetscInt i = info.xs; i < info.gxs+info.gxm-1; ++i) {
PetscInt globalC = (k*(N-1) + j)*(M-1) + i;
PetscInt v = 0;
cone[v++] = renumbering[(k*N + j)*M + i+0 + numGlobalCells];
cone[v++] = renumbering[(k*N + j)*M + i+1 + numGlobalCells];
if (dim > 1) {
cone[v++] = renumbering[(k*N + j+1)*M + i+0 + numGlobalCells];
cone[v++] = renumbering[(k*N + j+1)*M + i+1 + numGlobalCells];
}
if (dim > 2) {
cone[v++] = renumbering[((k+1)*N + j+0)*M + i+0 + numGlobalCells];
cone[v++] = renumbering[((k+1)*N + j+0)*M + i+1 + numGlobalCells];
cone[v++] = renumbering[((k+1)*N + j+1)*M + i+0 + numGlobalCells];
cone[v++] = renumbering[((k+1)*N + j+1)*M + i+1 + numGlobalCells];
}
sieve->setCone(cone, renumbering[globalC]);
sieve->setConeOrientation(coneO, renumbering[globalC]);
}
}
}
ierr = PetscFree2(cone,coneO);CHKERRQ(ierr);
sieve->symmetrize();
mesh->stratify();
/* Create boundary marker */
{
const Obj<PETSC_MESH_TYPE::label_type>& boundary = mesh->createLabel("marker");
for(PetscInt k = info.zs; k < info.gzs+info.gzm; ++k) {
for(PetscInt j = info.ys; j < info.gys+info.gym; ++j) {
if (info.xs == 0) {
PetscInt globalV = (k*N + j)*M + info.xs + numGlobalCells;
mesh->setValue(boundary, renumbering[globalV], 1);
}
if (info.gxs+info.gxm-1 == M-1) {
PetscInt globalV = (k*N + j)*M + info.gxs+info.gxm-1 + numGlobalCells;
mesh->setValue(boundary, renumbering[globalV], 1);
}
}
}
if (dim > 1) {
for(PetscInt k = info.zs; k < info.gzs+info.gzm; ++k) {
for(PetscInt i = info.xs; i < info.gxs+info.gxm; ++i) {
if (info.ys == 0) {
PetscInt globalV = (k*N + info.ys)*M + i + numGlobalCells;
mesh->setValue(boundary, renumbering[globalV], 1);
}
if (info.gys+info.gym-1 == N-1) {
PetscInt globalV = (k*N + info.gys+info.gym-1)*M + i + numGlobalCells;
mesh->setValue(boundary, renumbering[globalV], 1);
}
}
}
}
if (dim > 2) {
for(PetscInt j = info.ys; j < info.gys+info.gym; ++j) {
for(PetscInt i = info.xs; i < info.gxs+info.gxm; ++i) {
if (info.zs == 0) {
PetscInt globalV = (info.zs*N + j)*M + i + numGlobalCells;
mesh->setValue(boundary, renumbering[globalV], 1);
}
if (info.gzs+info.gzm-1 == P-1) {
PetscInt globalV = ((info.gzs+info.gzm-1)*N + j)*M + i + numGlobalCells;
mesh->setValue(boundary, renumbering[globalV], 1);
}
}
}
}
}
/* Create new DM */
ierr = DMMeshCreate(((PetscObject) dm)->comm, dmNew);CHKERRQ(ierr);
ierr = DMMeshSetMesh(*dmNew, mesh);CHKERRQ(ierr);
/* Set coordinates */
ierr = PetscSectionCreate(((PetscObject) dm)->comm, §ion);CHKERRQ(ierr);
ierr = PetscSectionSetChart(section, numCells, numCells+numVertices);CHKERRQ(ierr);
for(PetscInt v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(section, v, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(section);CHKERRQ(ierr);
ierr = DMMeshSetCoordinateSection(*dmNew, section);CHKERRQ(ierr);
ierr = DMDAGetCoordinateDA(dm, &cda);CHKERRQ(ierr);
ierr = DMDAGetGhostedCoordinates(dm, &coordinates);CHKERRQ(ierr);
{
Obj<PETSC_MESH_TYPE::real_section_type> coordSection = mesh->getRealSection("coordinates");
switch(dim) {
case 1:
{
PetscScalar **coords;
ierr = DMDAVecGetArrayDOF(cda, coordinates, &coords);CHKERRQ(ierr);
for(PetscInt i = info.xs; i < info.gxs+info.gxm; ++i) {
PetscInt globalV = i + numGlobalCells;
coordSection->updatePoint(renumbering[globalV], coords[i]);
}
ierr = DMDAVecRestoreArrayDOF(cda, coordinates, &coords);CHKERRQ(ierr);
break;
}
case 2:
{
PetscScalar ***coords;
ierr = DMDAVecGetArrayDOF(cda, coordinates, &coords);CHKERRQ(ierr);
for(PetscInt j = info.ys; j < info.gys+info.gym; ++j) {
for(PetscInt i = info.xs; i < info.gxs+info.gxm; ++i) {
PetscInt globalV = j*M + i + numGlobalCells;
coordSection->updatePoint(renumbering[globalV], coords[j][i]);
}
}
ierr = DMDAVecRestoreArrayDOF(cda, coordinates, &coords);CHKERRQ(ierr);
break;
}
case 3:
{
PetscScalar ****coords;
ierr = DMDAVecGetArrayDOF(cda, coordinates, &coords);CHKERRQ(ierr);
for(PetscInt k = info.zs; k < info.gzs+info.gzm; ++k, ++v) {
for(PetscInt j = info.ys; j < info.gys+info.gym; ++j) {
for(PetscInt i = info.xs; i < info.gxs+info.gxm; ++i) {
PetscInt globalV = (k*N + j)*M + i + numGlobalCells;
coordSection->updatePoint(renumbering[globalV], coords[k][j][i]);
}
}
}
ierr = DMDAVecRestoreArrayDOF(cda, coordinates, &coords);CHKERRQ(ierr);
break;
}
default:
SETERRQ1(((PetscObject) dm)->comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid DMDA dimension %d", dim);
}
}
/* Get overlap for interdomain communication */
{
typedef PETSC_MESH_TYPE::point_type point_type;
PETSc::Log::Event("CreateOverlap").begin();
ALE::Obj<PETSC_MESH_TYPE::send_overlap_type> sendParallelMeshOverlap = mesh->getSendOverlap();
ALE::Obj<PETSC_MESH_TYPE::recv_overlap_type> recvParallelMeshOverlap = mesh->getRecvOverlap();
// Can I figure this out in a nicer way?
ALE::SetFromMap<std::map<point_type,point_type> > globalPoints(renumbering);
ALE::OverlapBuilder<>::constructOverlap(globalPoints, renumbering, sendParallelMeshOverlap, recvParallelMeshOverlap);
if (debug) {
sendParallelMeshOverlap->view("Send Overlap");
recvParallelMeshOverlap->view("Recieve Overlap");
}
mesh->setCalculatedOverlap(true);
PETSc::Log::Event("CreateOverlap").end();
}
PetscFunctionReturn(0);
}
/*@C
PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
the local section and an SF describing the section point overlap.
Input Parameters:
+ s - The PetscSection for the local field layout
. sf - The SF describing parallel layout of the section points
. includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
. label - The label specifying the points
- labelValue - The label stratum specifying the points
Output Parameter:
. gsection - The PetscSection for the global field layout
Note: This gives negative sizes and offsets to points not owned by this process
Level: developer
.seealso: PetscSectionCreate()
@*/
PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
{
PetscInt *neg = NULL, *tmpOff = NULL;
PetscInt pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) s), gsection);CHKERRQ(ierr);
ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
if (nroots >= 0) {
if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
} else {
tmpOff = &(*gsection)->atlasDof[-pStart];
}
}
/* Mark ghost points with negative dof */
for (p = pStart; p < pEnd; ++p) {
PetscInt value;
ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
if (value != labelValue) continue;
ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
if (neg) neg[p] = -(dof+1);
}
ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
if (nroots >= 0) {
ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
}
}
/* Calculate new sizes, get proccess offset, and calculate point offsets */
for (p = 0, off = 0; p < pEnd-pStart; ++p) {
cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
(*gsection)->atlasOff[p] = off;
off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
}
ierr = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) s));CHKERRQ(ierr);
globalOff -= off;
for (p = 0, off = 0; p < pEnd-pStart; ++p) {
(*gsection)->atlasOff[p] += globalOff;
if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
}
/* Put in negative offsets for ghost points */
if (nroots >= 0) {
ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
}
}
if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
ierr = PetscFree(neg);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexPreallocateOperator(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix)
{
MPI_Comm comm;
MatType mtype;
PetscSF sf, sfDof, sfAdj;
PetscSection leafSectionAdj, rootSectionAdj, sectionAdj, anchorSectionAdj;
PetscInt nroots, nleaves, l, p;
const PetscInt *leaves;
const PetscSFNode *remotes;
PetscInt dim, pStart, pEnd, numDof, globalOffStart, globalOffEnd, numCols;
PetscInt *tmpAdj = NULL, *adj, *rootAdj, *anchorAdj = NULL, *cols, *remoteOffsets;
PetscInt adjSize;
PetscLayout rLayout;
PetscInt locRows, rStart, rEnd, r;
PetscMPIInt size;
PetscBool doCommLocal, doComm, debug = PETSC_FALSE, isSymBlock, isSymSeqBlock, isSymMPIBlock;
PetscBool useAnchors;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 3);
PetscValidHeaderSpecific(sectionGlobal, PETSC_SECTION_CLASSID, 4);
PetscValidHeaderSpecific(A, MAT_CLASSID, 9);
if (dnz) PetscValidPointer(dnz,5);
if (onz) PetscValidPointer(onz,6);
if (dnzu) PetscValidPointer(dnzu,7);
if (onzu) PetscValidPointer(onzu,8);
ierr = PetscLogEventBegin(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL, "-dm_view_preallocation", &debug, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
doCommLocal = (size > 1) && (nroots >= 0) ? PETSC_TRUE : PETSC_FALSE;
ierr = MPI_Allreduce(&doCommLocal, &doComm, 1, MPIU_BOOL, MPI_LAND, comm);CHKERRQ(ierr);
/* Create dof SF based on point SF */
if (debug) {
ierr = PetscPrintf(comm, "Input Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input Global Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionGlobal, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sf, NULL);CHKERRQ(ierr);
}
ierr = PetscSFCreateRemoteOffsets(sf, section, section, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, section, remoteOffsets, section, &sfDof);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Dof SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfDof, NULL);CHKERRQ(ierr);
}
/* Create section for dof adjacency (dof ==> # adj dof) */
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(section, &numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(leafSectionAdj, 0, numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSectionAdj, 0, numDof);CHKERRQ(ierr);
/* Fill in the ghost dofs on the interface */
ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, &remotes);CHKERRQ(ierr);
/* use constraints in finding adjacency in this routine */
ierr = DMPlexGetAdjacencyUseAnchors(dm,&useAnchors);CHKERRQ(ierr);
ierr = DMPlexSetAdjacencyUseAnchors(dm,PETSC_TRUE);CHKERRQ(ierr);
/*
section - maps points to (# dofs, local dofs)
sectionGlobal - maps points to (# dofs, global dofs)
leafSectionAdj - maps unowned local dofs to # adj dofs
rootSectionAdj - maps owned local dofs to # adj dofs
adj - adj global dofs indexed by leafSectionAdj
rootAdj - adj global dofs indexed by rootSectionAdj
sf - describes shared points across procs
sfDof - describes shared dofs across procs
sfAdj - describes shared adjacent dofs across procs
** The bootstrapping process involves six rounds with similar structure of visiting neighbors of each point.
(0). If there are point-to-point constraints, add the adjacencies of constrained points to anchors in anchorAdj
(This is done in DMPlexComputeAnchorAdjacencies())
1. Visit unowned points on interface, count adjacencies placing in leafSectionAdj
Reduce those counts to rootSectionAdj (now redundantly counting some interface points)
2. Visit owned points on interface, count adjacencies placing in rootSectionAdj
Create sfAdj connecting rootSectionAdj and leafSectionAdj
3. Visit unowned points on interface, write adjacencies to adj
Gather adj to rootAdj (note that there is redundancy in rootAdj when multiple procs find the same adjacencies)
4. Visit owned points on interface, write adjacencies to rootAdj
Remove redundancy in rootAdj
** The last two traversals use transitive closure
5. Visit all owned points in the subdomain, count dofs for each point (sectionAdj)
Allocate memory addressed by sectionAdj (cols)
6. Visit all owned points in the subdomain, insert dof adjacencies into cols
** Knowing all the column adjacencies, check ownership and sum into dnz and onz
*/
ierr = DMPlexComputeAnchorAdjacencies(dm,section,sectionGlobal,&anchorSectionAdj,&anchorAdj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation on Leaves:\n");CHKERRQ(ierr);
ierr = PetscSectionView(leafSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get maximum remote adjacency sizes for owned dofs on interface (roots) */
if (doComm) {
ierr = PetscSFReduceBegin(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
}
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Add in local adjacency sizes for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after local additions:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create adj SF based on dof SF */
ierr = PetscSFCreateRemoteOffsets(sfDof, rootSectionAdj, leafSectionAdj, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfDof, rootSectionAdj, remoteOffsets, leafSectionAdj, &sfAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfAdj, NULL);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfDof);CHKERRQ(ierr);
/* Create leaf adjacency */
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(leafSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscCalloc1(adjSize, &adj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof, anOff;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt aoff, i = 0;
ierr = PetscSectionGetOffset(leafSectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
adj[aoff+i] = (ngoff < 0 ? -(ngoff+1) : ngoff) + nd;
++i;
}
}
for (q = 0; q < anDof; q++) {
adj[aoff+i] = anchorAdj[anOff+q];
++i;
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Leaf adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, adj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Gather adjacenct indices to root */
ierr = PetscSectionGetStorageSize(rootSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize, &rootAdj);CHKERRQ(ierr);
for (r = 0; r < adjSize; ++r) rootAdj[r] = -1;
if (doComm) {
ierr = PetscSFGatherBegin(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
ierr = PetscSFGatherEnd(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfAdj);CHKERRQ(ierr);
ierr = PetscFree(adj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices after gather\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Add in local adjacency indices for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof, anOff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt adof, aoff, i;
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
i = adof-1;
for (q = 0; q < anDof; q++) {
rootAdj[aoff+i] = anchorAdj[anOff+q];
--i;
}
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
rootAdj[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
--i;
}
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Compress indices */
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, cdof, off, d;
PetscInt adof, aoff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
for (d = off; d < off+dof-cdof; ++d) {
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
ierr = PetscSortRemoveDupsInt(&adof, &rootAdj[aoff]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSectionAdj, d, adof);CHKERRQ(ierr);
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after compression:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Root adjacency indices after compression\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Build adjacency section: Maps global indices to sets of adjacent global indices */
ierr = PetscSectionGetOffsetRange(sectionGlobal, &globalOffStart, &globalOffEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, §ionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionAdj, globalOffStart, globalOffEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
ierr = PetscSectionSetDof(sectionAdj, goff+d, rdof);CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, noff;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, padj, &noff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(sectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get adjacent indices */
ierr = PetscSectionGetStorageSize(sectionAdj, &numCols);CHKERRQ(ierr);
ierr = PetscMalloc1(numCols, &cols);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof, anOff;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
PetscInt aoff, roff;
ierr = PetscSectionGetOffset(sectionAdj, goff+d, &aoff);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, off+d, &roff);CHKERRQ(ierr);
ierr = PetscMemcpy(&cols[aoff], &rootAdj[roff], rdof * sizeof(PetscInt));CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
PetscInt adof, aoff, i = 0;
ierr = PetscSectionGetDof(sectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
const PetscInt *ncind;
/* Adjacent points may not be in the section chart */
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintIndices(section, padj, &ncind);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd, ++i) {
cols[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
}
}
for (q = 0; q < anDof; q++, i++) {
cols[aoff+i] = anchorAdj[anOff + q];
}
if (i != adof) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of entries %D != %D for dof %D (point %D)", i, adof, d, p);
}
}
ierr = PetscSectionDestroy(&anchorSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSectionAdj);CHKERRQ(ierr);
ierr = PetscFree(anchorAdj);CHKERRQ(ierr);
ierr = PetscFree(rootAdj);CHKERRQ(ierr);
ierr = PetscFree(tmpAdj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Column indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, numCols, cols, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Create allocation vectors from adjacency graph */
ierr = MatGetLocalSize(A, &locRows, NULL);CHKERRQ(ierr);
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)A), &rLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr);
/* Only loop over blocks of rows */
if (rStart%bs || rEnd%bs) SETERRQ3(PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Invalid layout [%d, %d) for matrix, must be divisible by block size %d", rStart, rEnd, bs);
for (r = rStart/bs; r < rEnd/bs; ++r) {
const PetscInt row = r*bs;
PetscInt numCols, cStart, c;
ierr = PetscSectionGetDof(sectionAdj, row, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, row, &cStart);CHKERRQ(ierr);
for (c = cStart; c < cStart+numCols; ++c) {
if ((cols[c] >= rStart*bs) && (cols[c] < rEnd*bs)) {
++dnz[r-rStart];
if (cols[c] >= row) ++dnzu[r-rStart];
} else {
++onz[r-rStart];
if (cols[c] >= row) ++onzu[r-rStart];
}
}
}
if (bs > 1) {
for (r = 0; r < locRows/bs; ++r) {
dnz[r] /= bs;
onz[r] /= bs;
dnzu[r] /= bs;
onzu[r] /= bs;
}
}
/* Set matrix pattern */
ierr = MatXAIJSetPreallocation(A, bs, dnz, onz, dnzu, onzu);CHKERRQ(ierr);
ierr = MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
/* Check for symmetric storage */
ierr = MatGetType(A, &mtype);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
if (isSymBlock || isSymSeqBlock || isSymMPIBlock) {ierr = MatSetOption(A, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);CHKERRQ(ierr);}
/* Fill matrix with zeros */
if (fillMatrix) {
PetscScalar *values;
PetscInt maxRowLen = 0;
for (r = rStart; r < rEnd; ++r) {
PetscInt len;
ierr = PetscSectionGetDof(sectionAdj, r, &len);CHKERRQ(ierr);
maxRowLen = PetscMax(maxRowLen, len);
}
ierr = PetscCalloc1(maxRowLen, &values);CHKERRQ(ierr);
for (r = rStart; r < rEnd; ++r) {
PetscInt numCols, cStart;
ierr = PetscSectionGetDof(sectionAdj, r, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, r, &cStart);CHKERRQ(ierr);
ierr = MatSetValues(A, 1, &r, numCols, &cols[cStart], values, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree(values);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* restore original useAnchors */
ierr = DMPlexSetAdjacencyUseAnchors(dm,useAnchors);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionAdj);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* get adjacencies due to point-to-point constraints that can't be found with DMPlexGetAdjacency() */
static PetscErrorCode DMPlexComputeAnchorAdjacencies(DM dm, PetscSection section, PetscSection sectionGlobal, PetscSection *anchorSectionAdj, PetscInt *anchorAdj[])
{
PetscInt pStart, pEnd;
PetscSection adjSec, aSec;
IS aIS;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)section),&adjSec);CHKERRQ(ierr);
ierr = PetscSectionGetChart(section,&pStart,&pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(adjSec,pStart,pEnd);CHKERRQ(ierr);
ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
if (aSec) {
const PetscInt *anchors;
PetscInt p, q, a, aSize, *offsets, aStart, aEnd, *inverse, iSize, *adj, adjSize;
PetscInt *tmpAdjP = NULL, *tmpAdjQ = NULL;
PetscSection inverseSec;
/* invert the constraint-to-anchor map */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)aSec),&inverseSec);CHKERRQ(ierr);
ierr = PetscSectionSetChart(inverseSec,pStart,pEnd);CHKERRQ(ierr);
ierr = ISGetLocalSize(aIS, &aSize);CHKERRQ(ierr);
ierr = ISGetIndices(aIS, &anchors);CHKERRQ(ierr);
for (p = 0; p < aSize; p++) {
PetscInt a = anchors[p];
ierr = PetscSectionAddDof(inverseSec,a,1);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(inverseSec);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(inverseSec,&iSize);CHKERRQ(ierr);
ierr = PetscMalloc1(iSize,&inverse);CHKERRQ(ierr);
ierr = PetscCalloc1(pEnd-pStart,&offsets);CHKERRQ(ierr);
ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
for (p = aStart; p < aEnd; p++) {
PetscInt dof, off;
ierr = PetscSectionGetDof(aSec, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(aSec, p, &off);CHKERRQ(ierr);
for (q = 0; q < dof; q++) {
PetscInt iOff;
a = anchors[off + q];
ierr = PetscSectionGetOffset(inverseSec, a, &iOff);CHKERRQ(ierr);
inverse[iOff + offsets[a-pStart]++] = p;
}
}
ierr = ISRestoreIndices(aIS, &anchors);CHKERRQ(ierr);
ierr = PetscFree(offsets);CHKERRQ(ierr);
/* construct anchorAdj and adjSec
*
* loop over anchors:
* construct anchor adjacency
* loop over constrained:
* construct constrained adjacency
* if not in anchor adjacency, add to dofs
* setup adjSec, allocate anchorAdj
* loop over anchors:
* construct anchor adjacency
* loop over constrained:
* construct constrained adjacency
* if not in anchor adjacency
* if not already in list, put in list
* sort, unique, reduce dof count
* optional: compactify
*/
for (p = pStart; p < pEnd; p++) {
PetscInt iDof, iOff, i, r, s, numAdjP = PETSC_DETERMINE;
ierr = PetscSectionGetDof(inverseSec,p,&iDof);CHKERRQ(ierr);
if (!iDof) continue;
ierr = PetscSectionGetOffset(inverseSec,p,&iOff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm,p,&numAdjP,&tmpAdjP);CHKERRQ(ierr);
for (i = 0; i < iDof; i++) {
PetscInt iNew = 0, qAdj, qAdjDof, qAdjCDof, numAdjQ = PETSC_DETERMINE;
q = inverse[iOff + i];
ierr = DMPlexGetAdjacency(dm,q,&numAdjQ,&tmpAdjQ);CHKERRQ(ierr);
for (r = 0; r < numAdjQ; r++) {
qAdj = tmpAdjQ[r];
if ((qAdj < pStart) || (qAdj >= pEnd)) continue;
for (s = 0; s < numAdjP; s++) {
if (qAdj == tmpAdjP[s]) break;
}
if (s < numAdjP) continue;
ierr = PetscSectionGetDof(section,qAdj,&qAdjDof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section,qAdj,&qAdjCDof);CHKERRQ(ierr);
iNew += qAdjDof - qAdjCDof;
}
ierr = PetscSectionAddDof(adjSec,p,iNew);CHKERRQ(ierr);
}
}
ierr = PetscSectionSetUp(adjSec);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(adjSec,&adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize,&adj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; p++) {
PetscInt iDof, iOff, i, r, s, aOff, aOffOrig, aDof, numAdjP = PETSC_DETERMINE;
ierr = PetscSectionGetDof(inverseSec,p,&iDof);CHKERRQ(ierr);
if (!iDof) continue;
ierr = PetscSectionGetOffset(inverseSec,p,&iOff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm,p,&numAdjP,&tmpAdjP);CHKERRQ(ierr);
ierr = PetscSectionGetDof(adjSec,p,&aDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(adjSec,p,&aOff);CHKERRQ(ierr);
aOffOrig = aOff;
for (i = 0; i < iDof; i++) {
PetscInt qAdj, qAdjDof, qAdjCDof, qAdjOff, nd, numAdjQ = PETSC_DETERMINE;
q = inverse[iOff + i];
ierr = DMPlexGetAdjacency(dm,q,&numAdjQ,&tmpAdjQ);CHKERRQ(ierr);
for (r = 0; r < numAdjQ; r++) {
qAdj = tmpAdjQ[r];
if ((qAdj < pStart) || (qAdj >= pEnd)) continue;
for (s = 0; s < numAdjP; s++) {
if (qAdj == tmpAdjP[s]) break;
}
if (s < numAdjP) continue;
ierr = PetscSectionGetDof(section,qAdj,&qAdjDof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section,qAdj,&qAdjCDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal,qAdj,&qAdjOff);CHKERRQ(ierr);
for (nd = 0; nd < qAdjDof-qAdjCDof; ++nd) {
adj[aOff++] = (qAdjOff < 0 ? -(qAdjOff+1) : qAdjOff) + nd;
}
}
}
ierr = PetscSortRemoveDupsInt(&aDof,&adj[aOffOrig]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(adjSec,p,aDof);CHKERRQ(ierr);
}
*anchorAdj = adj;
/* clean up */
ierr = PetscSectionDestroy(&inverseSec);CHKERRQ(ierr);
ierr = PetscFree(inverse);CHKERRQ(ierr);
ierr = PetscFree(tmpAdjP);CHKERRQ(ierr);
ierr = PetscFree(tmpAdjQ);CHKERRQ(ierr);
}
else {
*anchorAdj = NULL;
ierr = PetscSectionSetUp(adjSec);CHKERRQ(ierr);
}
*anchorSectionAdj = adjSec;
PetscFunctionReturn(0);
}
PetscErrorCode DMCreate_Mesh(DM dm)
{
DM_Mesh *mesh;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = PetscNewLog(dm, DM_Mesh, &mesh);CHKERRQ(ierr);
dm->data = mesh;
new(&mesh->m) ALE::Obj<PETSC_MESH_TYPE>(PETSC_NULL);
mesh->globalScatter = PETSC_NULL;
mesh->defaultSection = PETSC_NULL;
mesh->lf = PETSC_NULL;
mesh->lj = PETSC_NULL;
mesh->useNewImpl = PETSC_FALSE;
mesh->dim = 0;
mesh->sf = PETSC_NULL;
ierr = PetscSectionCreate(((PetscObject) dm)->comm, &mesh->coneSection);CHKERRQ(ierr);
mesh->maxConeSize = 0;
mesh->cones = PETSC_NULL;
ierr = PetscSectionCreate(((PetscObject) dm)->comm, &mesh->supportSection);CHKERRQ(ierr);
mesh->maxSupportSize = 0;
mesh->supports = PETSC_NULL;
ierr = PetscSectionCreate(((PetscObject) dm)->comm, &mesh->coordSection);CHKERRQ(ierr);
ierr = VecCreate(((PetscObject) dm)->comm, &mesh->coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) mesh->coordinates, "coordinates");CHKERRQ(ierr);
mesh->meetTmpA = PETSC_NULL;
mesh->meetTmpB = PETSC_NULL;
mesh->joinTmpA = PETSC_NULL;
mesh->joinTmpB = PETSC_NULL;
mesh->closureTmpA = PETSC_NULL;
mesh->closureTmpB = PETSC_NULL;
ierr = PetscStrallocpy(VECSTANDARD, &dm->vectype);CHKERRQ(ierr);
dm->ops->view = DMView_Mesh;
dm->ops->setfromoptions = DMSetFromOptions_Mesh;
dm->ops->setup = 0;
dm->ops->createglobalvector = DMCreateGlobalVector_Mesh;
dm->ops->createlocalvector = DMCreateLocalVector_Mesh;
dm->ops->createlocaltoglobalmapping = DMCreateLocalToGlobalMapping_Mesh;
dm->ops->createlocaltoglobalmappingblock = 0;
dm->ops->getcoloring = 0;
dm->ops->creatematrix = DMCreateMatrix_Mesh;
dm->ops->createinterpolation = DMCreateInterpolation_Mesh;
dm->ops->getaggregates = 0;
dm->ops->getinjection = 0;
dm->ops->refine = DMRefine_Mesh;
dm->ops->coarsen = 0;
dm->ops->refinehierarchy = 0;
dm->ops->coarsenhierarchy = DMCoarsenHierarchy_Mesh;
dm->ops->globaltolocalbegin = DMGlobalToLocalBegin_Mesh;
dm->ops->globaltolocalend = DMGlobalToLocalEnd_Mesh;
dm->ops->localtoglobalbegin = DMLocalToGlobalBegin_Mesh;
dm->ops->localtoglobalend = DMLocalToGlobalEnd_Mesh;
dm->ops->destroy = DMDestroy_Mesh;
ierr = PetscObjectComposeFunction((PetscObject) dm, "DMConvert_da_mesh_C", "DMConvert_DA_Mesh", (void (*)(void)) DMConvert_DA_Mesh);CHKERRQ(ierr);
/* NEW_MESH_IMPL */
ierr = PetscOptionsBool("-dm_mesh_new_impl", "Use the new C unstructured mesh implementation", "DMCreate", PETSC_FALSE, &mesh->useNewImpl, PETSC_NULL);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int
main (int argc, char **argv)
{
sc_MPI_Comm mpicomm;
int mpiret;
int mpisize, mpirank;
p4est_t *p4est;
p4est_connectivity_t *conn;
sc_array_t *points_per_dim, *cone_sizes, *cones,
*cone_orientations, *coords,
*children, *parents, *childids, *leaves, *remotes;
p4est_locidx_t first_local_quad = -1;
/* initialize MPI */
mpiret = sc_MPI_Init (&argc, &argv);
SC_CHECK_MPI (mpiret);
mpicomm = sc_MPI_COMM_WORLD;
mpiret = sc_MPI_Comm_size (mpicomm, &mpisize);
SC_CHECK_MPI (mpiret);
mpiret = sc_MPI_Comm_rank (mpicomm, &mpirank);
SC_CHECK_MPI (mpiret);
sc_init (mpicomm, 1, 1, NULL, SC_LP_DEFAULT);
p4est_init (NULL, SC_LP_DEFAULT);
#ifndef P4_TO_P8
conn = p4est_connectivity_new_moebius ();
#else
conn = p8est_connectivity_new_rotcubes ();
#endif
p4est = p4est_new_ext (mpicomm, conn, 0, 1, 1, 0, NULL, NULL);
p4est_refine (p4est, 1, refine_fn, NULL);
p4est_balance (p4est, P4EST_CONNECT_FULL, NULL);
p4est_partition (p4est, 0, NULL);
points_per_dim = sc_array_new (sizeof (p4est_locidx_t));
cone_sizes = sc_array_new (sizeof (p4est_locidx_t));
cones = sc_array_new (sizeof (p4est_locidx_t));
cone_orientations = sc_array_new (sizeof (p4est_locidx_t));
coords = sc_array_new (3 * sizeof (double));
children = sc_array_new (sizeof (p4est_locidx_t));
parents = sc_array_new (sizeof (p4est_locidx_t));
childids = sc_array_new (sizeof (p4est_locidx_t));
leaves = sc_array_new (sizeof (p4est_locidx_t));
remotes = sc_array_new (2 * sizeof (p4est_locidx_t));
p4est_get_plex_data (p4est, P4EST_CONNECT_FULL, (mpisize > 1) ? 2 : 0,
&first_local_quad, points_per_dim, cone_sizes, cones,
cone_orientations, coords, children, parents, childids,
leaves, remotes);
#ifdef P4EST_WITH_PETSC
{
PetscErrorCode ierr;
DM plex, refTree;
PetscInt pStart, pEnd;
PetscSection parentSection;
PetscSF pointSF;
size_t zz, count;
locidx_to_PetscInt (points_per_dim);
locidx_to_PetscInt (cone_sizes);
locidx_to_PetscInt (cones);
locidx_to_PetscInt (cone_orientations);
coords_double_to_PetscScalar (coords);
locidx_to_PetscInt (children);
locidx_to_PetscInt (parents);
locidx_to_PetscInt (childids);
locidx_to_PetscInt (leaves);
locidx_pair_to_PetscSFNode (remotes);
P4EST_GLOBAL_PRODUCTION ("Begin PETSc routines\n");
ierr = PetscInitialize (&argc, &argv, 0, help);
CHKERRQ (ierr);
ierr = DMPlexCreate (mpicomm, &plex);
CHKERRQ (ierr);
ierr = DMSetDimension (plex, P4EST_DIM);
CHKERRQ (ierr);
ierr = DMSetCoordinateDim (plex, 3);
CHKERRQ (ierr);
ierr = DMPlexCreateFromDAG (plex, P4EST_DIM,
(PetscInt *) points_per_dim->array,
(PetscInt *) cone_sizes->array,
(PetscInt *) cones->array,
(PetscInt *) cone_orientations->array,
(PetscScalar *) coords->array);
CHKERRQ (ierr);
ierr = PetscSFCreate (mpicomm, &pointSF);
CHKERRQ (ierr);
ierr =
DMPlexCreateDefaultReferenceTree (mpicomm, P4EST_DIM, PETSC_FALSE,
&refTree);
CHKERRQ (ierr);
ierr = DMPlexSetReferenceTree (plex, refTree);
CHKERRQ (ierr);
ierr = DMDestroy (&refTree);
CHKERRQ (ierr);
ierr = PetscSectionCreate (mpicomm, &parentSection);
CHKERRQ (ierr);
ierr = DMPlexGetChart (plex, &pStart, &pEnd);
CHKERRQ (ierr);
ierr = PetscSectionSetChart (parentSection, pStart, pEnd);
CHKERRQ (ierr);
count = children->elem_count;
for (zz = 0; zz < count; zz++) {
PetscInt child =
*((PetscInt *) sc_array_index (children, zz));
ierr = PetscSectionSetDof (parentSection, child, 1);
CHKERRQ (ierr);
}
ierr = PetscSectionSetUp (parentSection);
CHKERRQ (ierr);
ierr =
DMPlexSetTree (plex, parentSection, (PetscInt *) parents->array,
(PetscInt *) childids->array);
CHKERRQ (ierr);
ierr = PetscSectionDestroy (&parentSection);
CHKERRQ (ierr);
ierr =
PetscSFSetGraph (pointSF, pEnd - pStart, (PetscInt) leaves->elem_count,
(PetscInt *) leaves->array, PETSC_COPY_VALUES,
(PetscSFNode *) remotes->array, PETSC_COPY_VALUES);
CHKERRQ (ierr);
ierr = DMViewFromOptions (plex, NULL, "-dm_view");
CHKERRQ (ierr);
/* TODO: test with rigid body modes as in plex ex3 */
ierr = DMDestroy (&plex);
CHKERRQ (ierr);
ierr = PetscFinalize ();
P4EST_GLOBAL_PRODUCTION ("End PETSc routines\n");
}
#endif
sc_array_destroy (points_per_dim);
sc_array_destroy (cone_sizes);
sc_array_destroy (cones);
sc_array_destroy (cone_orientations);
sc_array_destroy (coords);
sc_array_destroy (children);
sc_array_destroy (parents);
sc_array_destroy (childids);
sc_array_destroy (leaves);
sc_array_destroy (remotes);
p4est_destroy (p4est);
p4est_connectivity_destroy (conn);
sc_finalize ();
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
DM dm;
PetscBool interpolate = PETSC_TRUE;
PetscInt dim;
PetscInt pStart, pEnd;
PetscInt vStart, vEnd;
PetscInt eStart, eEnd;
PetscInt cStart, cEnd;
PetscSection s;
ierr = PetscInitialize(&argc, &argv, (char*)0, help); CHKERRQ(ierr);
ierr = DMPlexCreateGmshFromFile(MPI_COMM_WORLD, argv[1], interpolate, &dm); CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr);
printf("dim = %d\n", dim);
ierr = DMPlexGetChart(dm, &pStart, &pEnd); CHKERRQ(ierr);
printf("chart: pstart, pend = %d %d\n", pStart, pEnd);
// vertices
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd); CHKERRQ(ierr);
printf("Depth 0: vstart, vend, len = %d %d %d\n", vStart, vEnd, vEnd-vStart);
// edges
ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd); CHKERRQ(ierr);
printf("Depth 1: estart, eend, len = %d %d %d\n", eStart, eEnd, eEnd-eStart);
// cells
ierr = DMPlexGetDepthStratum(dm, 2, &cStart, &cEnd); CHKERRQ(ierr);
printf("Depth 2: cstart, cend, len = %d %d %d\n", cStart, cEnd, cEnd-cStart);
// cells adjacent to face
{
FILE * fid = fopen("face_nbr.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
PetscInt nbr;
ierr = DMPlexGetSupportSize(dm, e, &nbr); CHKERRQ(ierr);
const PetscInt *nbcells;
ierr = DMPlexGetSupport(dm, e, &nbcells); CHKERRQ(ierr);
if(nbr == 1) // boundary face
fprintf(fid, "%d %d\n",e-eStart+1,nbcells[0]-cStart+1);
else if(nbr == 2) // interior face
fprintf(fid, "%d %d %d\n",e-eStart+1,nbcells[0]-cStart+1,nbcells[1]-cStart+1);
else
{
printf("nbr is not 1 or 2\n");
exit(0);
}
}
fclose(fid);
}
// point coordinates
{
Vec coordinates;
ierr = DMGetCoordinatesLocal(dm, &coordinates); CHKERRQ(ierr);
const PetscScalar *coords;
ierr = VecGetArrayRead(coordinates, &coords); CHKERRQ(ierr);
DM dmCoord;
ierr = DMGetCoordinateDM(dm, &dmCoord); CHKERRQ(ierr);
FILE * fid = fopen("vertices.txt","w");
for(PetscInt v=vStart; v<vEnd; ++v)
{
PetscScalar *vertex;
ierr = DMPlexPointLocalRead(dmCoord, v, coords, &vertex); CHKERRQ(ierr);
fprintf(fid, "%f %f\n", vertex[0], vertex[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(coordinates, &coords); CHKERRQ(ierr);
}
// compute cell and face geometry
{
Vec cellgeom, facegeom;
ierr = DMPlexComputeGeometryFVM(dm, &cellgeom, &facegeom); CHKERRQ(ierr);
// cell information
DM dmCell;
ierr = VecGetDM(cellgeom, &dmCell); CHKERRQ(ierr);
const PetscScalar *cgeom;
ierr = VecGetArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
FILE * fid = fopen("cells.txt","w");
for(PetscInt c=cStart; c<cEnd; ++c)
{
// cell properties like volume, centroid
PetscFVCellGeom *cg;
ierr = DMPlexPointLocalRead(dmCell, c, cgeom, &cg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f\n", c-cStart+1, cg->volume,
cg->centroid[0], cg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
// face information
DM dmFace;
ierr = VecGetDM(facegeom, &dmFace); CHKERRQ(ierr);
const PetscScalar *fgeom;
ierr = VecGetArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
fid = fopen("faces.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
// face properties like area normal, centroid
PetscFVFaceGeom *fg;
ierr = DMPlexPointLocalRead(dmFace, e, fgeom, &fg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f %f\n", e-eStart+1, fg->normal[0], fg->normal[1],
fg->centroid[0], fg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
}
// create section with one variable in each cell
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &s); CHKERRQ(ierr);
ierr = PetscSectionSetChart(s, pStart, pEnd); CHKERRQ(ierr);
for(PetscInt c=cStart; c<cEnd; ++c)
{
ierr = PetscSectionSetDof(s, c, 1); CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(s); CHKERRQ(ierr);
// create vector to store solution
Vec lv, gv;
ierr = DMSetDefaultSection(dm, s); CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &lv); CHKERRQ(ierr);
ierr = DMGetGlobalVector(dm, &gv); CHKERRQ(ierr);
}
static PetscErrorCode DMPlexTSSetupGeometry(DM dm, PetscFV fvm, DMTS_Plex *dmplexts)
{
DM dmFace, dmCell;
DMLabel ghostLabel;
PetscSection sectionFace, sectionCell;
PetscSection coordSection;
Vec coordinates;
PetscReal minradius;
PetscScalar *fgeom, *cgeom;
PetscInt dim, cStart, cEnd, cEndInterior, c, fStart, fEnd, f;
PetscErrorCode ierr;
PetscFunctionBegin;
if (dmplexts->setupGeom) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
/* Make cell centroids and volumes */
ierr = DMClone(dm, &dmCell);CHKERRQ(ierr);
ierr = DMSetCoordinateSection(dmCell, coordSection);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dmCell, coordinates);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionCell);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionCell, cStart, cEnd);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {ierr = PetscSectionSetDof(sectionCell, c, sizeof(CellGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionCell);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmCell, sectionCell);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionCell);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmCell, &dmplexts->cellgeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
for (c = cStart; c < cEndInterior; ++c) {
CellGeom *cg;
ierr = DMPlexPointLocalRef(dmCell, c, cgeom, &cg);CHKERRQ(ierr);
ierr = PetscMemzero(cg, sizeof(*cg));CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dmCell, c, &cg->volume, cg->centroid, NULL);CHKERRQ(ierr);
}
/* Compute face normals and minimum cell radius */
ierr = DMClone(dm, &dmFace);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionFace);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionFace, fStart, fEnd);CHKERRQ(ierr);
for (f = fStart; f < fEnd; ++f) {ierr = PetscSectionSetDof(sectionFace, f, sizeof(FaceGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionFace);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmFace, sectionFace);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionFace);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmFace, &dmplexts->facegeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = DMPlexGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
minradius = PETSC_MAX_REAL;
for (f = fStart; f < fEnd; ++f) {
FaceGeom *fg;
PetscReal area;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, f, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexPointLocalRef(dmFace, f, fgeom, &fg);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, f, &area, fg->centroid, fg->normal);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) fg->normal[d] *= area;
/* Flip face orientation if necessary to match ordering in support, and Update minimum radius */
{
CellGeom *cL, *cR;
const PetscInt *cells;
PetscReal *lcentroid, *rcentroid;
PetscReal v[3];
ierr = DMPlexGetSupport(dm, f, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[0], cgeom, &cL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[1], cgeom, &cR);CHKERRQ(ierr);
lcentroid = cells[0] >= cEndInterior ? fg->centroid : cL->centroid;
rcentroid = cells[1] >= cEndInterior ? fg->centroid : cR->centroid;
WaxpyD(dim, -1, lcentroid, rcentroid, v);
if (DotRealD(dim, fg->normal, v) < 0) {
for (d = 0; d < dim; ++d) fg->normal[d] = -fg->normal[d];
}
if (DotRealD(dim, fg->normal, v) <= 0) {
if (dim == 2) SETERRQ5(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g) v (%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) v[0], (double) v[1]);
if (dim == 3) SETERRQ7(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g,%g) v (%g,%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) fg->normal[2], (double) v[0], (double) v[1], (double) v[2]);
SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed", f);
}
if (cells[0] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cL->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
if (cells[1] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cR->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
}
}
ierr = MPI_Allreduce(&minradius, &dmplexts->minradius, 1, MPIU_REAL, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
/* Compute centroids of ghost cells */
for (c = cEndInterior; c < cEnd; ++c) {
FaceGeom *fg;
const PetscInt *cone, *support;
PetscInt coneSize, supportSize, s;
ierr = DMPlexGetConeSize(dmCell, c, &coneSize);CHKERRQ(ierr);
if (coneSize != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ghost cell %d has cone size %d != 1", c, coneSize);
ierr = DMPlexGetCone(dmCell, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(dmCell, cone[0], &supportSize);CHKERRQ(ierr);
if (supportSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d has support size %d != 1", cone[0], supportSize);
ierr = DMPlexGetSupport(dmCell, cone[0], &support);CHKERRQ(ierr);
ierr = DMPlexPointLocalRef(dmFace, cone[0], fgeom, &fg);CHKERRQ(ierr);
for (s = 0; s < 2; ++s) {
/* Reflect ghost centroid across plane of face */
if (support[s] == c) {
const CellGeom *ci;
CellGeom *cg;
PetscReal c2f[3], a;
ierr = DMPlexPointLocalRead(dmCell, support[(s+1)%2], cgeom, &ci);CHKERRQ(ierr);
WaxpyD(dim, -1, ci->centroid, fg->centroid, c2f); /* cell to face centroid */
a = DotRealD(dim, c2f, fg->normal)/DotRealD(dim, fg->normal, fg->normal);
ierr = DMPlexPointLocalRef(dmCell, support[s], cgeom, &cg);CHKERRQ(ierr);
WaxpyD(dim, 2*a, fg->normal, ci->centroid, cg->centroid);
cg->volume = ci->volume;
}
}
}
ierr = VecRestoreArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
ierr = DMDestroy(&dmCell);CHKERRQ(ierr);
ierr = DMDestroy(&dmFace);CHKERRQ(ierr);
dmplexts->setupGeom = PETSC_TRUE;
PetscFunctionReturn(0);
}
/* This interpolates the PointSF in parallel following local interpolation */
static PetscErrorCode DMPlexInterpolatePointSF(DM dm, PetscSF pointSF, PetscInt depth)
{
PetscMPIInt numProcs, rank;
PetscInt p, c, d, dof, offset;
PetscInt numLeaves, numRoots, candidatesSize, candidatesRemoteSize;
const PetscInt *localPoints;
const PetscSFNode *remotePoints;
PetscSFNode *candidates, *candidatesRemote, *claims;
PetscSection candidateSection, candidateSectionRemote, claimSection;
PetscHashI leafhash;
PetscHashIJ roothash;
PetscHashIJKey key;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
if (numProcs < 2 || numRoots < 0) PetscFunctionReturn(0);
/* Build hashes of points in the SF for efficient lookup */
PetscHashICreate(leafhash);
PetscHashIJCreate(&roothash);
ierr = PetscHashIJSetMultivalued(roothash, PETSC_FALSE);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
PetscHashIAdd(leafhash, localPoints[p], p);
key.i = remotePoints[p].index; key.j = remotePoints[p].rank;
PetscHashIJAdd(roothash, key, p);
}
/* Build a section / SFNode array of candidate points in the single-level adjacency of leaves,
where each candidate is defined by the root entry for the other vertex that defines the edge. */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &candidateSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(candidateSection, 0, numRoots);CHKERRQ(ierr);
{
PetscInt leaf, root, idx, a, *adj = NULL;
for (p = 0; p < numLeaves; ++p) {
PetscInt adjSize = PETSC_DETERMINE;
ierr = DMPlexGetAdjacency_Internal(dm, localPoints[p], PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
for (a = 0; a < adjSize; ++a) {
PetscHashIMap(leafhash, adj[a], leaf);
if (leaf >= 0) {ierr = PetscSectionAddDof(candidateSection, localPoints[p], 1);CHKERRQ(ierr);}
}
}
ierr = PetscSectionSetUp(candidateSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(candidateSection, &candidatesSize);CHKERRQ(ierr);
ierr = PetscMalloc1(candidatesSize, &candidates);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
PetscInt adjSize = PETSC_DETERMINE;
ierr = PetscSectionGetOffset(candidateSection, localPoints[p], &offset);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency_Internal(dm, localPoints[p], PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
for (idx = 0, a = 0; a < adjSize; ++a) {
PetscHashIMap(leafhash, adj[a], root);
if (root >= 0) candidates[offset+idx++] = remotePoints[root];
}
}
ierr = PetscFree(adj);CHKERRQ(ierr);
}
/* Gather candidate section / array pair into the root partition via inverse(multi(pointSF)). */
{
PetscSF sfMulti, sfInverse, sfCandidates;
PetscInt *remoteOffsets;
ierr = PetscSFGetMultiSF(pointSF, &sfMulti);CHKERRQ(ierr);
ierr = PetscSFCreateInverseSF(sfMulti, &sfInverse);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sfInverse, candidateSection, &remoteOffsets, candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfInverse, candidateSection, remoteOffsets, candidateSectionRemote, &sfCandidates);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(candidateSectionRemote, &candidatesRemoteSize);CHKERRQ(ierr);
ierr = PetscMalloc1(candidatesRemoteSize, &candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfCandidates, MPIU_2INT, candidates, candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfCandidates, MPIU_2INT, candidates, candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfInverse);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfCandidates);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
}
/* Walk local roots and check for each remote candidate whether we know all required points,
either from owning it or having a root entry in the point SF. If we do we place a claim
by replacing the vertex number with our edge ID. */
{
PetscInt idx, root, joinSize, vertices[2];
const PetscInt *rootdegree, *join = NULL;
ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
/* Loop remote edge connections and put in a claim if both vertices are known */
for (idx = 0, p = 0; p < numRoots; ++p) {
for (d = 0; d < rootdegree[p]; ++d) {
ierr = PetscSectionGetDof(candidateSectionRemote, idx, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(candidateSectionRemote, idx, &offset);CHKERRQ(ierr);
for (c = 0; c < dof; ++c) {
/* We own both vertices, so we claim the edge by replacing vertex with edge */
if (candidatesRemote[offset+c].rank == rank) {
vertices[0] = p; vertices[1] = candidatesRemote[offset+c].index;
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) candidatesRemote[offset+c].index = join[0];
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
continue;
}
/* If we own one vertex and share a root with the other, we claim it */
key.i = candidatesRemote[offset+c].index; key.j = candidatesRemote[offset+c].rank;
PetscHashIJGet(roothash, key, &root);
if (root >= 0) {
vertices[0] = p; vertices[1] = localPoints[root];
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) {
candidatesRemote[offset+c].index = join[0];
candidatesRemote[offset+c].rank = rank;
}
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
}
}
idx++;
}
}
}
/* Push claims back to receiver via the MultiSF and derive new pointSF mapping on receiver */
{
PetscSF sfMulti, sfClaims, sfPointNew;
PetscHashI claimshash;
PetscInt size, pStart, pEnd, root, joinSize, numLocalNew;
PetscInt *remoteOffsets, *localPointsNew, vertices[2];
const PetscInt *join = NULL;
PetscSFNode *remotePointsNew;
ierr = PetscSFGetMultiSF(pointSF, &sfMulti);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &claimSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sfMulti, candidateSectionRemote, &remoteOffsets, claimSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfMulti, candidateSectionRemote, remoteOffsets, claimSection, &sfClaims);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(claimSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &claims);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfClaims, MPIU_2INT, candidatesRemote, claims);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfClaims, MPIU_2INT, candidatesRemote, claims);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfClaims);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
/* Walk the original section of local supports and add an SF entry for each updated item */
PetscHashICreate(claimshash);
for (p = 0; p < numRoots; ++p) {
ierr = PetscSectionGetDof(candidateSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(candidateSection, p, &offset);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
if (candidates[offset+d].index != claims[offset+d].index) {
key.i = candidates[offset+d].index; key.j = candidates[offset+d].rank;
PetscHashIJGet(roothash, key, &root);
if (root >= 0) {
vertices[0] = p; vertices[1] = localPoints[root];
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) PetscHashIAdd(claimshash, join[0], offset+d);
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
}
}
}
}
/* Create new pointSF from hashed claims */
PetscHashISize(claimshash, numLocalNew);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeaves + numLocalNew, &localPointsNew);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeaves + numLocalNew, &remotePointsNew);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
localPointsNew[p] = localPoints[p];
remotePointsNew[p].index = remotePoints[p].index;
remotePointsNew[p].rank = remotePoints[p].rank;
}
p = numLeaves; ierr = PetscHashIGetKeys(claimshash, &p, localPointsNew);CHKERRQ(ierr);
for (p = numLeaves; p < numLeaves + numLocalNew; ++p) {
PetscHashIMap(claimshash, localPointsNew[p], offset);
remotePointsNew[p] = claims[offset];
}
ierr = PetscSFCreate(PetscObjectComm((PetscObject) dm), &sfPointNew);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sfPointNew, pEnd-pStart, numLeaves+numLocalNew, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = DMSetPointSF(dm, sfPointNew);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfPointNew);CHKERRQ(ierr);
PetscHashIDestroy(claimshash);
}
PetscHashIDestroy(leafhash);
ierr = PetscHashIJDestroy(&roothash);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&candidateSection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&claimSection);CHKERRQ(ierr);
ierr = PetscFree(candidates);CHKERRQ(ierr);
ierr = PetscFree(candidatesRemote);CHKERRQ(ierr);
ierr = PetscFree(claims);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
pmesh->useAnchors = mesh->useAnchors;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
const PetscReal *maxCell, *L;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
ierr = DMGetPeriodicity(dm, &maxCell, &L);CHKERRQ(ierr);
if (L) {ierr = DMSetPeriodicity(*dmParallel, maxCell, L);CHKERRQ(ierr);}
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Set up tree */
{
DM refTree;
PetscSection origParentSection, newParentSection;
PetscInt *origParents, *origChildIDs;
ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
ierr = DMPlexSetReferenceTree(*dmParallel,refTree);CHKERRQ(ierr);
ierr = DMPlexGetTree(dm,&origParentSection,&origParents,&origChildIDs,NULL,NULL);CHKERRQ(ierr);
if (origParentSection) {
PetscInt pStart, pEnd;
PetscInt *newParents, *newChildIDs;
PetscInt *remoteOffsetsParents, newParentSize;
PetscSF parentSF;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)*dmParallel),&newParentSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newParentSection,pStart,pEnd);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, origParentSection, &remoteOffsetsParents, newParentSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(pointSF, origParentSection, remoteOffsetsParents, newParentSection, &parentSF);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newParentSection,&newParentSize);CHKERRQ(ierr);
ierr = PetscMalloc2(newParentSize,&newParents,newParentSize,&newChildIDs);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(parentSF, MPIU_INT, origParents, newParents);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(parentSF, MPIU_INT, origParents, newParents);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(parentSF, MPIU_INT, origChildIDs, newChildIDs);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(parentSF, MPIU_INT, origChildIDs, newChildIDs);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering,IS_GTOLM_MASK, newParentSize, newParents, NULL, newParents);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-parents_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Parent Section: \n");CHKERRQ(ierr);
ierr = PetscSectionView(origParentSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Parent Section: \n");CHKERRQ(ierr);
ierr = PetscSectionView(newParentSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(parentSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexSetTree(*dmParallel,newParentSection,newParents,newChildIDs);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&newParentSection);CHKERRQ(ierr);
ierr = PetscFree2(newParents,newChildIDs);CHKERRQ(ierr);
ierr = PetscSFDestroy(&parentSF);CHKERRQ(ierr);
}
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelDistribute(DMLabel label, PetscSF sf, DMLabel *labelNew)
{
MPI_Comm comm;
PetscSection rootSection, leafSection;
PetscSF labelSF;
PetscInt p, pStart, pEnd, l, lStart, lEnd, s, nroots, nleaves, size, dof, offset, stratum;
PetscInt *remoteOffsets, *rootStrata, *rootIdx, *leafStrata, *strataIdx;
char *name;
PetscInt nameSize;
size_t len = 0;
PetscMPIInt rank, numProcs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (label) {ierr = DMLabelMakeAllValid_Private(label);CHKERRQ(ierr);}
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
/* Bcast name */
if (!rank) {ierr = PetscStrlen(label->name, &len);CHKERRQ(ierr);}
nameSize = len;
ierr = MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1(nameSize+1, &name);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy(name, label->name, nameSize+1);CHKERRQ(ierr);}
ierr = MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = DMLabelCreate(name, labelNew);CHKERRQ(ierr);
ierr = PetscFree(name);CHKERRQ(ierr);
/* Bcast numStrata */
if (!rank) (*labelNew)->numStrata = label->numStrata;
ierr = MPI_Bcast(&(*labelNew)->numStrata, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
/* Bcast stratumValues */
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->stratumValues);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy((*labelNew)->stratumValues, label->stratumValues, label->numStrata * sizeof(PetscInt));CHKERRQ(ierr);}
ierr = MPI_Bcast((*labelNew)->stratumValues, (*labelNew)->numStrata, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->arrayValid);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) (*labelNew)->arrayValid[s] = PETSC_TRUE;
/* Build a section detailing strata-per-point, distribute and build SF
from that and then send our points. */
ierr = PetscSFGetGraph(sf, &nroots, &nleaves, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSection, 0, nroots);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
lStart = 0;
lEnd = label->stratumSizes[s];
for (l=lStart; l<lEnd; l++) {
ierr = PetscSectionGetDof(rootSection, label->points[s][l], &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSection, label->points[s][l], dof+1);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSection);CHKERRQ(ierr);
/* Create a point-wise array of point strata */
ierr = PetscSectionGetStorageSize(rootSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &rootStrata);CHKERRQ(ierr);
ierr = PetscCalloc1(nroots, &rootIdx);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
lStart = 0;
lEnd = label->stratumSizes[s];
for (l=lStart; l<lEnd; l++) {
p = label->points[s][l];
ierr = PetscSectionGetOffset(rootSection, p, &offset);CHKERRQ(ierr);
rootStrata[offset+rootIdx[p]++] = s;
}
}
}
/* Build SF that maps label points to remote processes */
ierr = PetscSectionCreate(comm, &leafSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sf, rootSection, &remoteOffsets, leafSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, rootSection, remoteOffsets, leafSection, &labelSF);CHKERRQ(ierr);
/* Send the strata for each point over the derived SF */
ierr = PetscSectionGetStorageSize(leafSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(labelSF, MPIU_INT, rootStrata, leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(labelSF, MPIU_INT, rootStrata, leafStrata);CHKERRQ(ierr);
/* Rebuild the point strata on the receiver */
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->stratumSizes);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
(*labelNew)->stratumSizes[leafStrata[offset+s]]++;
}
}
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->ht);CHKERRQ(ierr);
ierr = PetscMalloc1((*labelNew)->numStrata,&(*labelNew)->points);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) {
PetscHashICreate((*labelNew)->ht[s]);
ierr = PetscMalloc1((*labelNew)->stratumSizes[s], &(*labelNew)->points[s]);CHKERRQ(ierr);
}
/* Insert points into new strata */
ierr = PetscCalloc1((*labelNew)->numStrata, &strataIdx);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
stratum = leafStrata[offset+s];
(*labelNew)->points[stratum][strataIdx[stratum]++] = p;
}
}
ierr = PetscFree(rootStrata);CHKERRQ(ierr);
ierr = PetscFree(leafStrata);CHKERRQ(ierr);
ierr = PetscFree(rootIdx);CHKERRQ(ierr);
ierr = PetscFree(strataIdx);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSection);CHKERRQ(ierr);
ierr = PetscSFDestroy(&labelSF);CHKERRQ(ierr);
PetscFunctionReturn(0);
}