PETSC_STATIC_INLINE PetscErrorCode DMPlexGetGlobalFieldOffset_Private(DM dm,PetscInt point,PetscInt field,PetscInt *offset)
{
PetscFunctionBegin;
#if defined(PETSC_USE_DEBUG)
{
PetscErrorCode ierr;
PetscInt loff,lfoff,dof,cdof;
ierr = PetscSectionGetOffset(dm->defaultGlobalSection,point,offset);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(dm->defaultSection,point,&loff);CHKERRQ(ierr);
ierr = PetscSectionGetFieldOffset(dm->defaultSection,point,field,&lfoff);CHKERRQ(ierr);
ierr = PetscSectionGetFieldDof(dm->defaultSection,point,field,&dof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(dm->defaultSection,point,field,&cdof);CHKERRQ(ierr);
*offset += lfoff - loff;
if (dof-cdof <= 0) *offset = -1; /* Indicates no data */
}
#else
{
PetscSection s = dm->defaultSection;
PetscSection fs = dm->defaultSection->field[field];
PetscSection gs = dm->defaultGlobalSection;
PetscInt dof,cdof,loff,lfoff;
loff = s->atlasOff[point - s->pStart];
lfoff = fs->atlasOff[point - s->pStart];
dof = s->atlasDof[point - s->pStart];
cdof = s->bc ? s->bc->atlasDof[point - s->bc->pStart] : 0;
*offset = gs->atlasOff[point - s->pStart] + lfoff - loff;
if (dof-cdof <= 0) *offset = -1;
}
#endif
PetscFunctionReturn(0);
}
PetscErrorCode DMNetworkComponentSetUp(DM dm)
{
PetscErrorCode ierr;
DM_Network *network = (DM_Network*)dm->data;
PetscInt arr_size;
PetscInt p,offset,offsetp;
DMNetworkComponentHeader header;
DMNetworkComponentValue cvalue;
DMNetworkComponentGenericDataType *componentdataarray;
PetscInt ncomp, i;
PetscFunctionBegin;
ierr = PetscSectionSetUp(network->DataSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(network->DataSection,&arr_size);CHKERRQ(ierr);
ierr = PetscMalloc1(arr_size,&network->componentdataarray);CHKERRQ(ierr);
componentdataarray = network->componentdataarray;
for (p = network->pStart; p < network->pEnd; p++) {
ierr = PetscSectionGetOffset(network->DataSection,p,&offsetp);CHKERRQ(ierr);
/* Copy header */
header = &network->header[p];
ierr = PetscMemcpy(componentdataarray+offsetp,header,network->dataheadersize*sizeof(DMNetworkComponentGenericDataType));CHKERRQ(ierr);
/* Copy data */
cvalue = &network->cvalue[p];
ncomp = header->ndata;
for (i = 0; i < ncomp; i++) {
offset = offsetp + network->dataheadersize + header->offset[i];
ierr = PetscMemcpy(componentdataarray+offset,cvalue->data[i],header->size[i]*sizeof(DMNetworkComponentGenericDataType));CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
PETSC_STATIC_INLINE PetscErrorCode FillClosureArray_Static(DM dm, PetscSection section, PetscInt nP, const PetscInt points[], PetscScalar *vArray, PetscInt *csize, PetscScalar **array)
{
PetscScalar *a;
PetscInt pStart, pEnd, size = 0, dof, off, d, k, i;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
for (i = 0; i < nP; ++i) {
const PetscInt p = points[i];
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
size += dof;
}
if (csize) *csize = size;
if (array) {
ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &a);CHKERRQ(ierr);
for (i = 0, k = 0; i < nP; ++i) {
const PetscInt p = points[i];
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
for (d = 0; d < dof; ++d, ++k) a[k] = vArray[off+d];
}
*array = a;
}
PetscFunctionReturn(0);
}
static PetscErrorCode CreateSpectralPlanes(DM dm, PetscInt numPlanes, const PetscInt planeDir[], const PetscReal planeCoord[], AppCtx *user)
{
PetscSection coordSection;
Vec coordinates;
const PetscScalar *coords;
PetscInt dim, p, vStart, vEnd, v;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (p = 0; p < numPlanes; ++p) {
DMLabel label;
char name[PETSC_MAX_PATH_LEN];
ierr = PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "spectral_plane_%D", p);CHKERRQ(ierr);
ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
ierr = DMLabelAddStratum(label, 1);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
PetscInt off;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
if (PetscAbsReal(planeCoord[p] - PetscRealPart(coords[off+planeDir[p]])) < PETSC_SMALL) {
ierr = DMLabelSetValue(label, v, 1);CHKERRQ(ierr);
}
}
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMNetworkGetVariableGlobalOffset - Get the global offset for the variable associated with the given vertex/edge from the global vector.
Not Collective
Input Parameters:
+ dm - The DMNetwork object
- p - the edge/vertex point
Output Parameters:
. offsetg - the offset
Level: intermediate
.seealso: DMNetworkGetVariableOffset, DMGetLocalVector
@*/
PetscErrorCode DMNetworkGetVariableGlobalOffset(DM dm,PetscInt p,PetscInt *offsetg)
{
PetscErrorCode ierr;
DM_Network *network = (DM_Network*)dm->data;
PetscFunctionBegin;
ierr = PetscSectionGetOffset(network->GlobalDofSection,p,offsetg);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelDistribute_Internal(DMLabel label, PetscSF sf, PetscSection *leafSection, PetscInt **leafStrata)
{
MPI_Comm comm;
PetscInt s, l, nroots, nleaves, dof, offset, size;
PetscInt *remoteOffsets, *rootStrata, *rootIdx;
PetscSection rootSection;
PetscSF labelSF;
PetscErrorCode ierr;
PetscFunctionBegin;
if (label) {ierr = DMLabelMakeAllValid_Private(label);CHKERRQ(ierr);}
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
/* Build a section of stratum values per point, generate the according SF
and distribute point-wise stratum values to leaves. */
ierr = PetscSFGetGraph(sf, &nroots, &nleaves, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSection, 0, nroots);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
for (l = 0; l < label->stratumSizes[s]; l++) {
ierr = PetscSectionGetDof(rootSection, label->points[s][l], &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSection, label->points[s][l], dof+1);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSection);CHKERRQ(ierr);
/* Create a point-wise array of stratum values */
ierr = PetscSectionGetStorageSize(rootSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &rootStrata);CHKERRQ(ierr);
ierr = PetscCalloc1(nroots, &rootIdx);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
for (l = 0; l < label->stratumSizes[s]; l++) {
const PetscInt p = label->points[s][l];
ierr = PetscSectionGetOffset(rootSection, p, &offset);CHKERRQ(ierr);
rootStrata[offset+rootIdx[p]++] = label->stratumValues[s];
}
}
}
/* Build SF that maps label points to remote processes */
ierr = PetscSectionCreate(comm, leafSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sf, rootSection, &remoteOffsets, *leafSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, rootSection, remoteOffsets, *leafSection, &labelSF);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
/* Send the strata for each point over the derived SF */
ierr = PetscSectionGetStorageSize(*leafSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(labelSF, MPIU_INT, rootStrata, *leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(labelSF, MPIU_INT, rootStrata, *leafStrata);CHKERRQ(ierr);
/* Clean up */
ierr = PetscFree(rootStrata);CHKERRQ(ierr);
ierr = PetscFree(rootIdx);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSection);CHKERRQ(ierr);
ierr = PetscSFDestroy(&labelSF);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMNetworkGetNumComponents - Get the number of components at a vertex/edge
Not Collective
Input Parameters:
+ dm - The DMNetwork object
. p - vertex/edge point
Output Parameters:
. numcomponents - Number of components at the vertex/edge
Level: intermediate
.seealso: DMNetworkRegisterComponent, DMNetworkAddComponent
@*/
PetscErrorCode DMNetworkGetNumComponents(DM dm,PetscInt p,PetscInt *numcomponents)
{
PetscErrorCode ierr;
PetscInt offset;
DM_Network *network = (DM_Network*)dm->data;
PetscFunctionBegin;
ierr = PetscSectionGetOffset(network->DataSection,p,&offset);CHKERRQ(ierr);
*numcomponents = ((DMNetworkComponentHeader)(network->componentdataarray+offset))->ndata;
PetscFunctionReturn(0);
}
/*@
DMPlexGetPointGlobal - get location of point data in global Vec
Not Collective
Input Arguments:
+ dm - DM defining the topological space
- point - topological point
Output Arguments:
+ start - start of point data; returns -(global_start+1) if point is not owned
- end - end of point data; returns -(global_end+1) if point is not owned
Level: intermediate
.seealso: DMGetDefaultSection(), PetscSectionGetOffset(), PetscSectionGetDof(), DMPlexPointGlobalRead(), DMPlexGetPointLocal(), DMPlexPointGlobalRead(), DMPlexPointGlobalRef()
@*/
PetscErrorCode DMPlexGetPointGlobal(DM dm,PetscInt point,PetscInt *start,PetscInt *end)
{
PetscErrorCode ierr;
PetscInt offset,dof;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm,DM_CLASSID,1);
ierr = PetscSectionGetOffset(dm->defaultGlobalSection,point,&offset);CHKERRQ(ierr);
ierr = PetscSectionGetDof(dm->defaultGlobalSection,point,&dof);CHKERRQ(ierr);
if (start) *start = offset;
if (end) *end = offset + dof;
PetscFunctionReturn(0);
}
/*
This takes as input the raw Hasse Diagram data
*/
PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
{
Vec coordinates;
PetscSection coordSection;
PetscScalar *coords;
PetscInt coordSize, firstVertex = numPoints[depth], pStart = 0, pEnd = 0, p, v, dim, d, off;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
ierr = DMPlexSetChart(dm, pStart, pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
ierr = DMPlexSetConeSize(dm, p, coneSize[p-pStart]);CHKERRQ(ierr);
}
ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
ierr = DMPlexSetCone(dm, p, &cones[off]);CHKERRQ(ierr);
ierr = DMPlexSetConeOrientation(dm, p, &coneOrientations[off]);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
ierr = DMPlexStratify(dm);CHKERRQ(ierr);
/* Build coordinates */
ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]);CHKERRQ(ierr);
for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PetscObjectComm((PetscObject)dm), &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetFromOptions(coordinates);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
for (v = 0; v < numPoints[0]; ++v) {
PetscInt off;
ierr = PetscSectionGetOffset(coordSection, v+firstVertex, &off);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) {
coords[off+d] = vertexCoords[v*dim+d];
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_STATIC_INLINE PetscErrorCode FillClosureVec_Private(DM dm, PetscSection section, PetscInt nP, const PetscInt points[], PetscScalar *vArray, const PetscScalar *array, InsertMode mode)
{
PetscInt dof, off, d, k, i;
PetscErrorCode ierr;
PetscFunctionBegin;
if ((mode == INSERT_VALUES) || (mode == INSERT_ALL_VALUES)) {
for (i = 0, k = 0; i < nP; ++i) {
ierr = PetscSectionGetDof(section, points[i], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, points[i], &off);CHKERRQ(ierr);
for (d = 0; d < dof; ++d, ++k) vArray[off+d] = array[k];
}
} else {
for (i = 0, k = 0; i < nP; ++i) {
ierr = PetscSectionGetDof(section, points[i], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, points[i], &off);CHKERRQ(ierr);
for (d = 0; d < dof; ++d, ++k) vArray[off+d] += array[k];
}
}
PetscFunctionReturn(0);
}
/*@
DMNetworkIsGhostVertex - Returns TRUE if the vertex is a ghost vertex
Not Collective
Input Parameters:
+ dm - The DMNetwork object
. p - the vertex point
Output Parameter:
. isghost - TRUE if the vertex is a ghost point
Level: intermediate
.seealso: DMNetworkCreate, DMNetworkGetConnectedNodes, DMNetworkGetVertexRange
@*/
PetscErrorCode DMNetworkIsGhostVertex(DM dm,PetscInt p,PetscBool *isghost)
{
PetscErrorCode ierr;
DM_Network *network = (DM_Network*)dm->data;
PetscInt offsetg;
PetscSection sectiong;
PetscFunctionBegin;
*isghost = PETSC_FALSE;
ierr = DMGetDefaultGlobalSection(network->plex,§iong);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectiong,p,&offsetg);CHKERRQ(ierr);
if (offsetg < 0) *isghost = PETSC_TRUE;
PetscFunctionReturn(0);
}
/*@
DMNetworkGetComponentTypeOffset - Gets the type along with the offset for indexing the
component value from the component data array
Not Collective
Input Parameters:
+ dm - The DMNetwork object
. p - vertex/edge point
- compnum - component number
Output Parameters:
+ compkey - the key obtained when registering the component
- offset - offset into the component data array associated with the vertex/edge point
Notes:
Typical usage:
DMNetworkGetComponentDataArray(dm, &arr);
DMNetworkGetVertex/EdgeRange(dm,&Start,&End);
Loop over vertices or edges
DMNetworkGetNumComponents(dm,v,&numcomps);
Loop over numcomps
DMNetworkGetComponentTypeOffset(dm,v,compnum,&key,&offset);
compdata = (UserCompDataType)(arr+offset);
Level: intermediate
.seealso: DMNetworkGetNumComponents, DMNetworkGetComponentDataArray,
@*/
PetscErrorCode DMNetworkGetComponentTypeOffset(DM dm,PetscInt p, PetscInt compnum, PetscInt *compkey, PetscInt *offset)
{
PetscErrorCode ierr;
PetscInt offsetp;
DMNetworkComponentHeader header;
DM_Network *network = (DM_Network*)dm->data;
PetscFunctionBegin;
ierr = PetscSectionGetOffset(network->DataSection,p,&offsetp);CHKERRQ(ierr);
header = (DMNetworkComponentHeader)(network->componentdataarray+offsetp);
*compkey = header->key[compnum];
*offset = offsetp+network->dataheadersize+header->offset[compnum];
PetscFunctionReturn(0);
}
PETSC_STATIC_INLINE PetscErrorCode DMPlexGetLocalOffset_Private(DM dm,PetscInt point,PetscInt *offset)
{
PetscFunctionBegin;
#if defined(PETSC_USE_DEBUG)
{
PetscErrorCode ierr;
ierr = PetscSectionGetOffset(dm->defaultSection,point,offset);CHKERRQ(ierr);
}
#else
{
PetscSection s = dm->defaultSection;
*offset = s->atlasOff[point - s->pStart];
}
#endif
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelConvertToSection(DMLabel label, PetscSection *section, IS *is)
{
IS vIS;
const PetscInt *values;
PetscInt *points;
PetscInt nV, vS = 0, vE = 0, v, N;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMLabelGetNumValues(label, &nV);CHKERRQ(ierr);
ierr = DMLabelGetValueIS(label, &vIS);CHKERRQ(ierr);
ierr = ISGetIndices(vIS, &values);CHKERRQ(ierr);
if (nV) {vS = values[0]; vE = values[0]+1;}
for (v = 1; v < nV; ++v) {
vS = PetscMin(vS, values[v]);
vE = PetscMax(vE, values[v]+1);
}
ierr = PetscSectionCreate(PETSC_COMM_SELF, section);CHKERRQ(ierr);
ierr = PetscSectionSetChart(*section, vS, vE);CHKERRQ(ierr);
for (v = 0; v < nV; ++v) {
PetscInt n;
ierr = DMLabelGetStratumSize(label, values[v], &n);CHKERRQ(ierr);
ierr = PetscSectionSetDof(*section, values[v], n);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(*section);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(*section, &N);CHKERRQ(ierr);
ierr = PetscMalloc1(N, &points);CHKERRQ(ierr);
for (v = 0; v < nV; ++v) {
IS is;
const PetscInt *spoints;
PetscInt dof, off, p;
ierr = PetscSectionGetDof(*section, values[v], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(*section, values[v], &off);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(label, values[v], &is);CHKERRQ(ierr);
ierr = ISGetIndices(is, &spoints);CHKERRQ(ierr);
for (p = 0; p < dof; ++p) points[off+p] = spoints[p];
ierr = ISRestoreIndices(is, &spoints);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(vIS, &values);CHKERRQ(ierr);
ierr = ISDestroy(&vIS);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF, N, points, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ChangeCoordinates(DM dm, PetscInt spaceDim, PetscScalar vertexCoords[])
{
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt vStart, vEnd, v, d, coordSize;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, spaceDim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, vStart, vEnd);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
ierr = PetscSectionSetDof(coordSection, v, spaceDim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, spaceDim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PetscObjectComm((PetscObject) dm), &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetFromOptions(coordinates);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
PetscInt off;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
for (d = 0; d < spaceDim; ++d) {
coords[off+d] = vertexCoords[(v-vStart)*spaceDim+d];
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
ierr = DMSetFromOptions(dm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PetscSectionGetField_Internal(PetscSection section, PetscSection sectionGlobal, Vec v, PetscInt field, PetscInt pStart, PetscInt pEnd, IS *is, Vec *subv)
{
PetscInt *subIndices;
PetscInt Nc, subSize = 0, subOff = 0, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionGetFieldComponents(section, field, &Nc);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof, fdof = 0;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);}
subSize += fdof;
}
ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof, goff;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
PetscInt fdof, fc, f2, poff = 0;
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
/* Can get rid of this loop by storing field information in the global section */
for (f2 = 0; f2 < field; ++f2) {
ierr = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
poff += fdof;
}
ierr = PetscSectionGetFieldDof(section, p, field, &fdof);CHKERRQ(ierr);
for (fc = 0; fc < fdof; ++fc, ++subOff) subIndices[subOff] = goff+poff+fc;
}
}
ierr = ISCreateGeneral(PetscObjectComm((PetscObject) v), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
ierr = VecGetSubVector(v, *is, subv);CHKERRQ(ierr);
ierr = VecSetBlockSize(*subv, Nc);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_STATIC_INLINE PetscErrorCode DMPlexGetGlobalOffset_Private(DM dm,PetscInt point,PetscInt *offset)
{
PetscFunctionBegin;
#if defined(PETSC_USE_DEBUG)
{
PetscErrorCode ierr;
PetscInt dof,cdof;
ierr = PetscSectionGetOffset(dm->defaultGlobalSection,point,offset);CHKERRQ(ierr);
ierr = PetscSectionGetDof(dm->defaultGlobalSection,point,&dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(dm->defaultGlobalSection,point,&cdof);CHKERRQ(ierr);
if (dof-cdof <= 0) *offset = -1; /* Indicates no data */
}
#else
{
PetscSection s = dm->defaultGlobalSection;
PetscInt dof,cdof;
*offset = s->atlasOff[point - s->pStart];
dof = s->atlasDof[point - s->pStart];
cdof = s->bc ? s->bc->atlasDof[point - s->bc->pStart] : 0;
if (dof-cdof <= 0) *offset = -1;
}
#endif
PetscFunctionReturn(0);
}
/*@
DMPlexPermute - Reorder the mesh according to the input permutation
Collective on DM
Input Parameter:
+ dm - The DMPlex object
- perm - The point permutation, perm[old point number] = new point number
Output Parameter:
. pdm - The permuted DM
Level: intermediate
.keywords: mesh
.seealso: MatPermute()
@*/
PetscErrorCode DMPlexPermute(DM dm, IS perm, DM *pdm)
{
DM_Plex *plex = (DM_Plex *) dm->data, *plexNew;
PetscSection section, sectionNew;
PetscInt dim;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(perm, IS_CLASSID, 2);
PetscValidPointer(pdm, 3);
ierr = DMCreate(PetscObjectComm((PetscObject) dm), pdm);CHKERRQ(ierr);
ierr = DMSetType(*pdm, DMPLEX);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMSetDimension(*pdm, dim);CHKERRQ(ierr);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
if (section) {
ierr = PetscSectionPermute(section, perm, §ionNew);CHKERRQ(ierr);
ierr = DMSetDefaultSection(*pdm, sectionNew);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionNew);CHKERRQ(ierr);
}
plexNew = (DM_Plex *) (*pdm)->data;
/* Ignore ltogmap, ltogmapb */
/* Ignore sf, defaultSF */
/* Ignore globalVertexNumbers, globalCellNumbers */
/* Remap coordinates */
{
DM cdm, cdmNew;
PetscSection csection, csectionNew;
Vec coordinates, coordinatesNew;
PetscScalar *coords, *coordsNew;
const PetscInt *pperm;
PetscInt pStart, pEnd, p;
const char *name;
ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
ierr = DMGetDefaultSection(cdm, &csection);CHKERRQ(ierr);
ierr = PetscSectionPermute(csection, perm, &csectionNew);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = VecDuplicate(coordinates, &coordinatesNew);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)coordinatesNew,name);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
ierr = VecGetArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
ierr = PetscSectionGetChart(csectionNew, &pStart, &pEnd);CHKERRQ(ierr);
ierr = ISGetIndices(perm, &pperm);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(csectionNew, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csectionNew, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) coordsNew[offNew+d] = coords[off+d];
}
ierr = ISRestoreIndices(perm, &pperm);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(*pdm, &cdmNew);CHKERRQ(ierr);
ierr = DMSetDefaultSection(cdmNew, csectionNew);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*pdm, coordinatesNew);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&csectionNew);CHKERRQ(ierr);
ierr = VecDestroy(&coordinatesNew);CHKERRQ(ierr);
}
/* Reorder labels */
{
PetscInt numLabels, l;
DMLabel label, labelNew;
ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
for (l = numLabels-1; l >= 0; --l) {
ierr = DMGetLabelByNum(dm, l, &label);CHKERRQ(ierr);
ierr = DMLabelPermute(label, perm, &labelNew);CHKERRQ(ierr);
ierr = DMAddLabel(*pdm, labelNew);CHKERRQ(ierr);
}
if (plex->subpointMap) {ierr = DMLabelPermute(plex->subpointMap, perm, &plexNew->subpointMap);CHKERRQ(ierr);}
}
/* Reorder topology */
{
const PetscInt *pperm;
PetscInt maxConeSize, maxSupportSize, n, pStart, pEnd, p;
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
plexNew->maxConeSize = maxConeSize;
plexNew->maxSupportSize = maxSupportSize;
ierr = PetscSectionDestroy(&plexNew->coneSection);CHKERRQ(ierr);
ierr = PetscSectionPermute(plex->coneSection, perm, &plexNew->coneSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(plexNew->coneSection, &n);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->cones);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->coneOrientations);CHKERRQ(ierr);
ierr = ISGetIndices(perm, &pperm);CHKERRQ(ierr);
ierr = PetscSectionGetChart(plex->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(plexNew->coneSection, pperm[p], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plex->coneSection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plexNew->coneSection, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
plexNew->cones[offNew+d] = pperm[plex->cones[off+d]];
plexNew->coneOrientations[offNew+d] = plex->coneOrientations[off+d];
}
}
ierr = PetscSectionDestroy(&plexNew->supportSection);CHKERRQ(ierr);
ierr = PetscSectionPermute(plex->supportSection, perm, &plexNew->supportSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(plexNew->supportSection, &n);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->supports);CHKERRQ(ierr);
ierr = PetscSectionGetChart(plex->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(plexNew->supportSection, pperm[p], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plex->supportSection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plexNew->supportSection, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
plexNew->supports[offNew+d] = pperm[plex->supports[off+d]];
}
}
ierr = ISRestoreIndices(perm, &pperm);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@
DMDASetUniformCoordinates - Sets a DMDA coordinates to be a uniform grid
Collective on DMDA
Input Parameters:
+ da - the distributed array object
. xmin,xmax - extremes in the x direction
. ymin,ymax - extremes in the y direction (use NULL for 1 dimensional problems)
- zmin,zmax - extremes in the z direction (use NULL for 1 or 2 dimensional problems)
Level: beginner
.seealso: DMSetCoordinates(), DMGetCoordinates(), DMDACreate1d(), DMDACreate2d(), DMDACreate3d()
@*/
PetscErrorCode DMDASetUniformCoordinates(DM da,PetscReal xmin,PetscReal xmax,PetscReal ymin,PetscReal ymax,PetscReal zmin,PetscReal zmax)
{
MPI_Comm comm;
PetscSection section;
DM cda;
DMBoundaryType bx,by,bz;
Vec xcoor;
PetscScalar *coors;
PetscReal hx,hy,hz_;
PetscInt i,j,k,M,N,P,istart,isize,jstart,jsize,kstart,ksize,dim,cnt;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
ierr = DMDAGetInfo(da,&dim,&M,&N,&P,0,0,0,0,0,&bx,&by,&bz,0);CHKERRQ(ierr);
if (xmax <= xmin) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_INCOMP,"xmax must be larger than xmin %g %g",(double)xmin,(double)xmax);
if ((ymax <= ymin) && (dim > 1)) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_INCOMP,"ymax must be larger than ymin %g %g",(double)ymin,(double)ymax);
if ((zmax <= zmin) && (dim > 2)) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_INCOMP,"zmax must be larger than zmin %g %g",(double)zmin,(double)zmax);
ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr);
ierr = DMGetDefaultSection(da,§ion);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&istart,&jstart,&kstart,&isize,&jsize,&ksize);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da, &cda);CHKERRQ(ierr);
if (section) {
/* This would be better as a vector, but this is compatible */
PetscInt numComp[3] = {1, 1, 1};
PetscInt numVertexDof[3] = {1, 1, 1};
ierr = DMDASetFieldName(cda, 0, "x");CHKERRQ(ierr);
if (dim > 1) {ierr = DMDASetFieldName(cda, 1, "y");CHKERRQ(ierr);}
if (dim > 2) {ierr = DMDASetFieldName(cda, 2, "z");CHKERRQ(ierr);}
ierr = DMDACreateSection(cda, numComp, numVertexDof, NULL, NULL);CHKERRQ(ierr);
}
ierr = DMCreateGlobalVector(cda, &xcoor);CHKERRQ(ierr);
if (section) {
PetscSection csection;
PetscInt vStart, vEnd;
ierr = DMGetDefaultGlobalSection(cda,&csection);CHKERRQ(ierr);
ierr = VecGetArray(xcoor,&coors);CHKERRQ(ierr);
ierr = DMDAGetHeightStratum(da, dim, &vStart, &vEnd);CHKERRQ(ierr);
if (bx == DM_BOUNDARY_PERIODIC) hx = (xmax-xmin)/(M+1);
else hx = (xmax-xmin)/(M ? M : 1);
if (by == DM_BOUNDARY_PERIODIC) hy = (ymax-ymin)/(N+1);
else hy = (ymax-ymin)/(N ? N : 1);
if (bz == DM_BOUNDARY_PERIODIC) hz_ = (zmax-zmin)/(P+1);
else hz_ = (zmax-zmin)/(P ? P : 1);
switch (dim) {
case 1:
for (i = 0; i < isize+1; ++i) {
PetscInt v = i+vStart, dof, off;
ierr = PetscSectionGetDof(csection, v, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csection, v, &off);CHKERRQ(ierr);
if (off >= 0) {
coors[off] = xmin + hx*(i+istart);
}
}
break;
case 2:
for (j = 0; j < jsize+1; ++j) {
for (i = 0; i < isize+1; ++i) {
PetscInt v = j*(isize+1)+i+vStart, dof, off;
ierr = PetscSectionGetDof(csection, v, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csection, v, &off);CHKERRQ(ierr);
if (off >= 0) {
coors[off+0] = xmin + hx*(i+istart);
coors[off+1] = ymin + hy*(j+jstart);
}
}
}
break;
case 3:
for (k = 0; k < ksize+1; ++k) {
for (j = 0; j < jsize+1; ++j) {
for (i = 0; i < isize+1; ++i) {
PetscInt v = (k*(jsize+1)+j)*(isize+1)+i+vStart, dof, off;
ierr = PetscSectionGetDof(csection, v, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csection, v, &off);CHKERRQ(ierr);
if (off >= 0) {
coors[off+0] = xmin + hx*(i+istart);
coors[off+1] = ymin + hy*(j+jstart);
coors[off+2] = zmin + hz_*(k+kstart);
}
}
}
}
break;
default:
SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Cannot create uniform coordinates for this dimension %D\n",dim);
}
ierr = VecRestoreArray(xcoor,&coors);CHKERRQ(ierr);
ierr = DMSetCoordinates(da,xcoor);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)da,(PetscObject)xcoor);CHKERRQ(ierr);
ierr = VecDestroy(&xcoor);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
if (dim == 1) {
if (bx == DM_BOUNDARY_PERIODIC) hx = (xmax-xmin)/M;
else hx = (xmax-xmin)/(M-1);
ierr = VecGetArray(xcoor,&coors);CHKERRQ(ierr);
for (i=0; i<isize; i++) {
coors[i] = xmin + hx*(i+istart);
}
ierr = VecRestoreArray(xcoor,&coors);CHKERRQ(ierr);
} else if (dim == 2) {
if (bx == DM_BOUNDARY_PERIODIC) hx = (xmax-xmin)/(M);
else hx = (xmax-xmin)/(M-1);
if (by == DM_BOUNDARY_PERIODIC) hy = (ymax-ymin)/(N);
else hy = (ymax-ymin)/(N-1);
ierr = VecGetArray(xcoor,&coors);CHKERRQ(ierr);
cnt = 0;
for (j=0; j<jsize; j++) {
for (i=0; i<isize; i++) {
coors[cnt++] = xmin + hx*(i+istart);
coors[cnt++] = ymin + hy*(j+jstart);
}
}
ierr = VecRestoreArray(xcoor,&coors);CHKERRQ(ierr);
} else if (dim == 3) {
if (bx == DM_BOUNDARY_PERIODIC) hx = (xmax-xmin)/(M);
else hx = (xmax-xmin)/(M-1);
if (by == DM_BOUNDARY_PERIODIC) hy = (ymax-ymin)/(N);
else hy = (ymax-ymin)/(N-1);
if (bz == DM_BOUNDARY_PERIODIC) hz_ = (zmax-zmin)/(P);
else hz_ = (zmax-zmin)/(P-1);
ierr = VecGetArray(xcoor,&coors);CHKERRQ(ierr);
cnt = 0;
for (k=0; k<ksize; k++) {
for (j=0; j<jsize; j++) {
for (i=0; i<isize; i++) {
coors[cnt++] = xmin + hx*(i+istart);
coors[cnt++] = ymin + hy*(j+jstart);
coors[cnt++] = zmin + hz_*(k+kstart);
}
}
}
ierr = VecRestoreArray(xcoor,&coors);CHKERRQ(ierr);
} else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Cannot create uniform coordinates for this dimension %D\n",dim);
ierr = DMSetCoordinates(da,xcoor);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)da,(PetscObject)xcoor);CHKERRQ(ierr);
ierr = VecDestroy(&xcoor);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data;
FILE *fp;
PetscViewerVTKObjectLink link;
PetscSection coordSection, globalCoordSection;
PetscLayout vLayout;
Vec coordinates;
PetscReal lengthScale;
PetscInt vMax, totVertices, totCells;
PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr);
/* Vertices */
ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
if (vMax >= 0) {
PetscInt pStart, pEnd, p, localSize = 0;
ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr);
pEnd = PetscMin(pEnd, vMax);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr);
if (dof > 0) ++localSize;
}
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
} else {
ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr);
}
ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr);
ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr);
/* Cells */
ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr);
/* Vertex fields */
for (link = vtk->link; link; link = link->next) {
if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE;
if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE;
}
if (hasPoint) {
ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection, newSection = NULL;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
DMLabel subpointMap, subpointMapX;
PetscInt dim, dimX, pStart, pEnd, qStart, qEnd;
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
/* Here is where we check whether dmX is a submesh of dm */
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr);
if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) {
const PetscInt *ind = NULL;
IS subpointIS;
PetscInt n = 0, q;
ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr);
if (subpointIS) {
ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr);
ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr);
}
ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr);
for (q = qStart; q < qEnd; ++q) {
PetscInt dof, off, p;
ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr);
if (dof) {
ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr);
if (p >= pStart) {
ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr);
ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr);
}
}
}
if (subpointIS) {
ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr);
ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
}
/* No need to setup section */
section = newSection;
}
} else {
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr);
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);}
}
}
/* Cell Fields */
ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr);
if (hasCell || writePartition) {
ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
} else {
PetscContainer c;
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr);
if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
}
if (writePartition) {
ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr);
}
}
/* Cleanup */
ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
ierr = PetscFClose(comm, fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexVTKWriteSection_ASCII(DM dm, PetscSection section, PetscSection globalSection, Vec v, FILE *fp, PetscInt enforceDof, PetscInt precision, PetscReal scale)
{
MPI_Comm comm;
const MPI_Datatype mpiType = MPIU_SCALAR;
PetscScalar *array;
PetscInt numDof = 0, maxDof;
PetscInt numLabelCells, cellHeight, cMax, cStart, cEnd, numLabelVertices, vMax, vStart, vEnd, pStart, pEnd, p;
PetscMPIInt numProcs, rank, proc, tag;
PetscBool hasLabel;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
PetscValidHeaderSpecific(dm,DM_CLASSID,1);
PetscValidHeaderSpecific(v,VEC_CLASSID,4);
if (precision < 0) precision = 6;
ierr = PetscCommGetNewTag(comm, &tag);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
/* VTK only wants the values at cells or vertices */
ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, &vMax);CHKERRQ(ierr);
if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
pStart = PetscMax(PetscMin(cStart, vStart), pStart);
pEnd = PetscMin(PetscMax(cEnd, vEnd), pEnd);
ierr = DMPlexGetStratumSize(dm, "vtk", 1, &numLabelCells);CHKERRQ(ierr);
ierr = DMPlexGetStratumSize(dm, "vtk", 2, &numLabelVertices);CHKERRQ(ierr);
hasLabel = numLabelCells > 0 || numLabelVertices > 0 ? PETSC_TRUE : PETSC_FALSE;
for (p = pStart; p < pEnd; ++p) {
/* Reject points not either cells or vertices */
if (((p < cStart) || (p >= cEnd)) && ((p < vStart) || (p >= vEnd))) continue;
if (hasLabel) {
PetscInt value;
if (((p >= cStart) && (p < cEnd) && numLabelCells) ||
((p >= vStart) && (p < vEnd) && numLabelVertices)) {
ierr = DMPlexGetLabelValue(dm, "vtk", p, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
}
ierr = PetscSectionGetDof(section, p, &numDof);CHKERRQ(ierr);
if (numDof) break;
}
ierr = MPI_Allreduce(&numDof, &maxDof, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
enforceDof = PetscMax(enforceDof, maxDof);
ierr = VecGetArray(v, &array);CHKERRQ(ierr);
if (!rank) {
char formatString[8];
ierr = PetscSNPrintf(formatString, 8, "%%.%de", precision);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
/* Here we lose a way to filter points by keeping them out of the Numbering */
PetscInt dof, off, goff, d;
/* Reject points not either cells or vertices */
if (((p < cStart) || (p >= cEnd)) && ((p < vStart) || (p >= vEnd))) continue;
if (hasLabel) {
PetscInt value;
if (((p >= cStart) && (p < cEnd) && numLabelCells) ||
((p >= vStart) && (p < vEnd) && numLabelVertices)) {
ierr = DMPlexGetLabelValue(dm, "vtk", p, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
}
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(globalSection, p, &goff);CHKERRQ(ierr);
if (dof && goff >= 0) {
for (d = 0; d < dof; d++) {
if (d > 0) {
ierr = PetscFPrintf(comm, fp, " ");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fp, formatString, PetscRealPart(array[off+d])*scale);CHKERRQ(ierr);
}
for (d = dof; d < enforceDof; d++) {
ierr = PetscFPrintf(comm, fp, " 0.0");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fp, "\n");CHKERRQ(ierr);
}
}
for (proc = 1; proc < numProcs; ++proc) {
PetscScalar *remoteValues;
PetscInt size = 0, d;
MPI_Status status;
ierr = MPI_Recv(&size, 1, MPIU_INT, proc, tag, comm, &status);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &remoteValues);CHKERRQ(ierr);
ierr = MPI_Recv(remoteValues, size, mpiType, proc, tag, comm, &status);CHKERRQ(ierr);
for (p = 0; p < size/maxDof; ++p) {
for (d = 0; d < maxDof; ++d) {
if (d > 0) {
ierr = PetscFPrintf(comm, fp, " ");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fp, formatString, PetscRealPart(remoteValues[p*maxDof+d])*scale);CHKERRQ(ierr);
}
for (d = maxDof; d < enforceDof; ++d) {
ierr = PetscFPrintf(comm, fp, " 0.0");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fp, "\n");CHKERRQ(ierr);
}
ierr = PetscFree(remoteValues);CHKERRQ(ierr);
}
} else {
PetscScalar *localValues;
PetscInt size, k = 0;
ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &localValues);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, goff, d;
/* Reject points not either cells or vertices */
if (((p < cStart) || (p >= cEnd)) && ((p < vStart) || (p >= vEnd))) continue;
if (hasLabel) {
PetscInt value;
if (((p >= cStart) && (p < cEnd) && numLabelCells) ||
((p >= vStart) && (p < vEnd) && numLabelVertices)) {
ierr = DMPlexGetLabelValue(dm, "vtk", p, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
}
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(globalSection, p, &goff);CHKERRQ(ierr);
if (goff >= 0) {
for (d = 0; d < dof; ++d) {
localValues[k++] = array[off+d];
}
}
}
ierr = MPI_Send(&k, 1, MPIU_INT, 0, tag, comm);CHKERRQ(ierr);
ierr = MPI_Send(localValues, k, mpiType, 0, tag, comm);CHKERRQ(ierr);
ierr = PetscFree(localValues);CHKERRQ(ierr);
}
ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelDistribute(DMLabel label, PetscSF sf, DMLabel *labelNew)
{
MPI_Comm comm;
PetscSection leafSection;
PetscInt p, pStart, pEnd, s, size, dof, offset, stratum;
PetscInt *leafStrata, *strataIdx;
char *name;
PetscInt nameSize;
PetscHashI stratumHash;
PETSC_UNUSED PetscHashIIter ret, iter;
size_t len = 0;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
if (label) {ierr = DMLabelMakeAllValid_Private(label);CHKERRQ(ierr);}
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
/* Bcast name */
if (!rank) {ierr = PetscStrlen(label->name, &len);CHKERRQ(ierr);}
nameSize = len;
ierr = MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1(nameSize+1, &name);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy(name, label->name, nameSize+1);CHKERRQ(ierr);}
ierr = MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = DMLabelCreate(name, labelNew);CHKERRQ(ierr);
ierr = PetscFree(name);CHKERRQ(ierr);
/* Bcast defaultValue */
if (!rank) (*labelNew)->defaultValue = label->defaultValue;
ierr = MPI_Bcast(&(*labelNew)->defaultValue, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
/* Distribute stratum values over the SF and get the point mapping on the receiver */
ierr = DMLabelDistribute_Internal(label, sf, &leafSection, &leafStrata);CHKERRQ(ierr);
/* Determine received stratum values and initialise new label*/
PetscHashICreate(stratumHash);
ierr = PetscSectionGetStorageSize(leafSection, &size);CHKERRQ(ierr);
for (p = 0; p < size; ++p) PetscHashIPut(stratumHash, leafStrata[p], ret, iter);
PetscHashISize(stratumHash, (*labelNew)->numStrata);
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->arrayValid);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) (*labelNew)->arrayValid[s] = PETSC_TRUE;
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->stratumValues);CHKERRQ(ierr);
/* Turn leafStrata into indices rather than stratum values */
offset = 0;
ierr = PetscHashIGetKeys(stratumHash, &offset, (*labelNew)->stratumValues);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) {
for (p = 0; p < size; ++p) {
if (leafStrata[p] == (*labelNew)->stratumValues[s]) leafStrata[p] = s;
}
}
/* Rebuild the point strata on the receiver */
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->stratumSizes);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
(*labelNew)->stratumSizes[leafStrata[offset+s]]++;
}
}
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->ht);CHKERRQ(ierr);
ierr = PetscMalloc1((*labelNew)->numStrata,&(*labelNew)->points);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) {
PetscHashICreate((*labelNew)->ht[s]);
ierr = PetscMalloc1((*labelNew)->stratumSizes[s], &(*labelNew)->points[s]);CHKERRQ(ierr);
}
/* Insert points into new strata */
ierr = PetscCalloc1((*labelNew)->numStrata, &strataIdx);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
stratum = leafStrata[offset+s];
(*labelNew)->points[stratum][strataIdx[stratum]++] = p;
}
}
PetscHashIDestroy(stratumHash);
ierr = PetscFree(leafStrata);CHKERRQ(ierr);
ierr = PetscFree(strataIdx);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSection);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
{
MPI_Comm comm;
const PetscInt dim = 3;
const char *labelName = "marker";
PLC *in, *out;
DMLabel label;
PetscInt verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = MPIU_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMGetLabel(dm, labelName, &label);CHKERRQ(ierr);
ierr = PLCCreate(&in);CHKERRQ(ierr);
ierr = PLCCreate(&out);CHKERRQ(ierr);
in->numberofpoints = vEnd - vStart;
if (in->numberofpoints > 0) {
PetscSection coordSection;
Vec coordinates;
PetscScalar *array;
ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
ierr = PetscMalloc1(in->numberofpoints, &in->pointmarkerlist);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
const PetscInt idx = v - vStart;
PetscInt off, d, m = -1;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) {
in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
}
if (label) {ierr = DMLabelGetValue(label, v, &m);CHKERRQ(ierr);}
in->pointmarkerlist[idx] = (int) m;
}
ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
}
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
in->numberofcorners = 4;
in->numberoftetrahedra = cEnd - cStart;
in->tetrahedronvolumelist = maxVolumes;
if (in->numberoftetrahedra > 0) {
ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {
const PetscInt idx = c - cStart;
PetscInt *closure = NULL;
PetscInt closureSize;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
for (v = 0; v < 4; ++v) {
in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
}
if (!rank) {
TetGenOpts t;
ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
t.in = dm; /* Should go away */
t.refine = 1;
t.varvolume = 1;
t.quality = 1;
t.edgesout = 1;
t.zeroindex = 1;
t.quiet = 1;
t.verbose = verbose; /* Change this */
ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
}
in->tetrahedronvolumelist = NULL;
{
DMLabel rlabel = NULL;
const PetscInt numCorners = 4;
const PetscInt numCells = out->numberoftetrahedra;
const PetscInt numVertices = out->numberofpoints;
double *meshCoords;
int *cells = out->tetrahedronlist;
PetscBool interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
if (sizeof (PetscReal) == sizeof (double)) {
meshCoords = (double *) out->pointlist;
}
else {
PetscInt i;
ierr = PetscMalloc1(3 * numVertices,&meshCoords);CHKERRQ(ierr);
for (i = 0; i < 3 * numVertices; i++) {
meshCoords[i] = (PetscReal) out->pointlist[i];
}
}
ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
if (sizeof (PetscReal) != sizeof (double)) {
ierr = PetscFree(meshCoords);CHKERRQ(ierr);
}
if (label) {ierr = DMCreateLabel(*dmRefined, labelName); ierr = DMGetLabel(*dmRefined, labelName, &rlabel);}
/* Set labels */
for (v = 0; v < numVertices; ++v) {
if (out->pointmarkerlist[v]) {
if (rlabel) {ierr = DMLabelSetValue(rlabel, v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);}
}
}
if (interpolate) {
PetscInt e, f;
for (e = 0; e < out->numberofedges; e++) {
if (out->edgemarkerlist[e]) {
const PetscInt vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
const PetscInt *edges;
PetscInt numEdges;
ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
if (rlabel) {ierr = DMLabelSetValue(rlabel, edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
}
}
for (f = 0; f < out->numberoftrifaces; f++) {
if (out->trifacemarkerlist[f]) {
const PetscInt vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
const PetscInt *faces;
PetscInt numFaces;
ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
if (rlabel) {ierr = DMLabelSetValue(rlabel, faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
}
}
}
ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
}
ierr = PLCDestroy(&in);CHKERRQ(ierr);
ierr = PLCDestroy(&out);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* This assumes that the DM has been cloned prior to the call */
PetscErrorCode DMCreateSubDM_Section_Private(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
{
PetscSection section, sectionGlobal;
PetscInt *subIndices;
PetscInt subSize = 0, subOff = 0, nF, f, pStart, pEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!numFields) PetscFunctionReturn(0);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = DMGetDefaultGlobalSection(dm, §ionGlobal);CHKERRQ(ierr);
if (!section) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting fields");
if (!sectionGlobal) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default global section for DM before splitting fields");
ierr = PetscSectionGetNumFields(section, &nF);CHKERRQ(ierr);
if (numFields > nF) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Number of requested fields %d greater than number of DM fields %d", numFields, nF);
if (is) {
ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof;
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
subSize += fdof-fcdof;
}
}
}
ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof, goff;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof, fc, f2, poff = 0;
/* Can get rid of this loop by storing field information in the global section */
for (f2 = 0; f2 < fields[f]; ++f2) {
ierr = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
poff += fdof-fcdof;
}
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
subIndices[subOff] = goff+poff+fc;
}
}
}
}
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
}
if (subdm) {
PetscSection subsection;
PetscBool haveNull = PETSC_FALSE;
PetscInt f, nf = 0;
ierr = PetscSectionCreateSubsection(section, numFields, fields, &subsection);CHKERRQ(ierr);
ierr = DMSetDefaultSection(*subdm, subsection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
(*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[fields[f]];
if ((*subdm)->nullspaceConstructors[f]) {
haveNull = PETSC_TRUE;
nf = f;
}
}
if (haveNull && is) {
MatNullSpace nullSpace;
ierr = (*(*subdm)->nullspaceConstructors[nf])(*subdm, nf, &nullSpace);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
}
if (dm->prob) {
PetscInt Nf;
ierr = PetscDSGetNumFields(dm->prob, &Nf);CHKERRQ(ierr);
if (nF != Nf) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "The number of DM fields %d does not match the number of Section fields %d", Nf, nF);
ierr = DMSetNumFields(*subdm, numFields);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
PetscObject disc;
ierr = DMGetField(dm, fields[f], &disc);CHKERRQ(ierr);
ierr = DMSetField(*subdm, f, disc);CHKERRQ(ierr);
}
if (numFields == 1 && is) {
PetscObject disc, space, pmat;
ierr = DMGetField(*subdm, 0, &disc);CHKERRQ(ierr);
ierr = PetscObjectQuery(disc, "nullspace", &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nullspace", space);CHKERRQ(ierr);}
ierr = PetscObjectQuery(disc, "nearnullspace", &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nearnullspace", space);CHKERRQ(ierr);}
ierr = PetscObjectQuery(disc, "pmat", &pmat);CHKERRQ(ierr);
if (pmat) {ierr = PetscObjectCompose((PetscObject) *is, "pmat", pmat);CHKERRQ(ierr);}
}
}
}
#if 0
/* We need a way to filter the original SF for given fields:
- Keeping the original section around it too much I think
- We could keep the distributed section, and subset it
*/
if (dm->sfNatural) {
PetscSF sfNatural;
ierr = PetscSectionCreateSubsection(dm->originalSection, numFields, fields, &(*subdm)->originalSection);CHKERRQ(ierr);
ierr = DMPlexCreateGlobalToNaturalPetscSF(*subdm, &sfNatural);CHKERRQ(ierr);
ierr = DMPlexSetGlobalToNaturalPetscSF(*subdm, sfNatural);CHKERRQ(ierr);
}
#endif
PetscFunctionReturn(0);
}
/*@
DMLabelGather - Gather all label values from leafs into roots
Input Parameters:
+ label - the DMLabel
. point - the Star Forest point communication map
Input Parameters:
+ label - the new DMLabel with localised leaf values
Level: developer
Note: This is the inverse operation to DMLabelDistribute.
.seealso: DMLabelDistribute()
@*/
PetscErrorCode DMLabelGather(DMLabel label, PetscSF sf, DMLabel *labelNew)
{
MPI_Comm comm;
PetscSection rootSection;
PetscSF sfLabel;
PetscSFNode *rootPoints, *leafPoints;
PetscInt p, s, d, nroots, nleaves, nmultiroots, idx, dof, offset;
const PetscInt *rootDegree, *ilocal;
PetscInt *rootStrata;
char *name;
PetscInt nameSize;
size_t len = 0;
PetscMPIInt rank, numProcs;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
/* Bcast name */
if (!rank) {ierr = PetscStrlen(label->name, &len);CHKERRQ(ierr);}
nameSize = len;
ierr = MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1(nameSize+1, &name);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy(name, label->name, nameSize+1);CHKERRQ(ierr);}
ierr = MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = DMLabelCreate(name, labelNew);CHKERRQ(ierr);
ierr = PetscFree(name);CHKERRQ(ierr);
/* Gather rank/index pairs of leaves into local roots to build
an inverse, multi-rooted SF. Note that this ignores local leaf
indexing due to the use of the multiSF in PetscSFGather. */
ierr = PetscSFGetGraph(sf, &nroots, &nleaves, &ilocal, NULL);CHKERRQ(ierr);
ierr = PetscMalloc1(nleaves, &leafPoints);CHKERRQ(ierr);
for (p = 0; p < nleaves; p++) {
leafPoints[p].index = ilocal[p];
leafPoints[p].rank = rank;
}
ierr = PetscSFComputeDegreeBegin(sf, &rootDegree);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(sf, &rootDegree);CHKERRQ(ierr);
for (p = 0, nmultiroots = 0; p < nroots; ++p) nmultiroots += rootDegree[p];
ierr = PetscMalloc1(nmultiroots, &rootPoints);CHKERRQ(ierr);
ierr = PetscSFGatherBegin(sf, MPIU_2INT, leafPoints, rootPoints);CHKERRQ(ierr);
ierr = PetscSFGatherEnd(sf, MPIU_2INT, leafPoints, rootPoints);CHKERRQ(ierr);
ierr = PetscSFCreate(comm,& sfLabel);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sfLabel, nroots, nmultiroots, NULL, PETSC_OWN_POINTER, rootPoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
/* Migrate label over inverted SF to pull stratum values at leaves into roots. */
ierr = DMLabelDistribute_Internal(label, sfLabel, &rootSection, &rootStrata);CHKERRQ(ierr);
/* Rebuild the point strata on the receiver */
for (p = 0, idx = 0; p < nroots; p++) {
for (d = 0; d < rootDegree[p]; d++) {
ierr = PetscSectionGetDof(rootSection, idx+d, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSection, idx+d, &offset);CHKERRQ(ierr);
for (s = 0; s < dof; s++) {ierr = DMLabelSetValue(*labelNew, p, rootStrata[offset+s]);CHKERRQ(ierr);}
}
idx += rootDegree[p];
}
ierr = PetscFree(leafPoints);CHKERRQ(ierr);
ierr = PetscFree(rootStrata);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSection);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfLabel);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexGetAdjacency_Internal(DM dm, PetscInt p, PetscBool useCone, PetscBool useTransitiveClosure, PetscBool useAnchors, PetscInt *adjSize, PetscInt *adj[])
{
static PetscInt asiz = 0;
PetscInt maxAnchors = 1;
PetscInt aStart = -1, aEnd = -1;
PetscInt maxAdjSize;
PetscSection aSec = NULL;
IS aIS = NULL;
const PetscInt *anchors;
PetscErrorCode ierr;
PetscFunctionBeginHot;
if (useAnchors) {
ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
if (aSec) {
ierr = PetscSectionGetMaxDof(aSec,&maxAnchors);CHKERRQ(ierr);
maxAnchors = PetscMax(1,maxAnchors);
ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
}
}
if (!*adj) {
PetscInt depth, coneSeries, supportSeries, maxC, maxS, pStart, pEnd;
ierr = DMPlexGetChart(dm, &pStart,&pEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(dm, &maxC, &maxS);CHKERRQ(ierr);
coneSeries = (maxC > 1) ? ((PetscPowInt(maxC,depth+1)-1)/(maxC-1)) : depth+1;
supportSeries = (maxS > 1) ? ((PetscPowInt(maxS,depth+1)-1)/(maxS-1)) : depth+1;
asiz = PetscMax(PetscPowInt(maxS,depth)*coneSeries,PetscPowInt(maxC,depth)*supportSeries);
asiz *= maxAnchors;
asiz = PetscMin(asiz,pEnd-pStart);
ierr = PetscMalloc1(asiz,adj);CHKERRQ(ierr);
}
if (*adjSize < 0) *adjSize = asiz;
maxAdjSize = *adjSize;
if (useTransitiveClosure) {
ierr = DMPlexGetAdjacency_Transitive_Internal(dm, p, useCone, adjSize, *adj);CHKERRQ(ierr);
} else if (useCone) {
ierr = DMPlexGetAdjacency_Cone_Internal(dm, p, adjSize, *adj);CHKERRQ(ierr);
} else {
ierr = DMPlexGetAdjacency_Support_Internal(dm, p, adjSize, *adj);CHKERRQ(ierr);
}
if (useAnchors && aSec) {
PetscInt origSize = *adjSize;
PetscInt numAdj = origSize;
PetscInt i = 0, j;
PetscInt *orig = *adj;
while (i < origSize) {
PetscInt p = orig[i];
PetscInt aDof = 0;
if (p >= aStart && p < aEnd) {
ierr = PetscSectionGetDof(aSec,p,&aDof);CHKERRQ(ierr);
}
if (aDof) {
PetscInt aOff;
PetscInt s, q;
for (j = i + 1; j < numAdj; j++) {
orig[j - 1] = orig[j];
}
origSize--;
numAdj--;
ierr = PetscSectionGetOffset(aSec,p,&aOff);CHKERRQ(ierr);
for (s = 0; s < aDof; ++s) {
for (q = 0; q < numAdj || (orig[numAdj++] = anchors[aOff+s],0); ++q) {
if (anchors[aOff+s] == orig[q]) break;
}
if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
}
}
else {
i++;
}
}
*adjSize = numAdj;
ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/* get adjacencies due to point-to-point constraints that can't be found with DMPlexGetAdjacency() */
static PetscErrorCode DMPlexComputeAnchorAdjacencies(DM dm, PetscSection section, PetscSection sectionGlobal, PetscSection *anchorSectionAdj, PetscInt *anchorAdj[])
{
PetscInt pStart, pEnd;
PetscSection adjSec, aSec;
IS aIS;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)section),&adjSec);CHKERRQ(ierr);
ierr = PetscSectionGetChart(section,&pStart,&pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(adjSec,pStart,pEnd);CHKERRQ(ierr);
ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
if (aSec) {
const PetscInt *anchors;
PetscInt p, q, a, aSize, *offsets, aStart, aEnd, *inverse, iSize, *adj, adjSize;
PetscInt *tmpAdjP = NULL, *tmpAdjQ = NULL;
PetscSection inverseSec;
/* invert the constraint-to-anchor map */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)aSec),&inverseSec);CHKERRQ(ierr);
ierr = PetscSectionSetChart(inverseSec,pStart,pEnd);CHKERRQ(ierr);
ierr = ISGetLocalSize(aIS, &aSize);CHKERRQ(ierr);
ierr = ISGetIndices(aIS, &anchors);CHKERRQ(ierr);
for (p = 0; p < aSize; p++) {
PetscInt a = anchors[p];
ierr = PetscSectionAddDof(inverseSec,a,1);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(inverseSec);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(inverseSec,&iSize);CHKERRQ(ierr);
ierr = PetscMalloc1(iSize,&inverse);CHKERRQ(ierr);
ierr = PetscCalloc1(pEnd-pStart,&offsets);CHKERRQ(ierr);
ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
for (p = aStart; p < aEnd; p++) {
PetscInt dof, off;
ierr = PetscSectionGetDof(aSec, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(aSec, p, &off);CHKERRQ(ierr);
for (q = 0; q < dof; q++) {
PetscInt iOff;
a = anchors[off + q];
ierr = PetscSectionGetOffset(inverseSec, a, &iOff);CHKERRQ(ierr);
inverse[iOff + offsets[a-pStart]++] = p;
}
}
ierr = ISRestoreIndices(aIS, &anchors);CHKERRQ(ierr);
ierr = PetscFree(offsets);CHKERRQ(ierr);
/* construct anchorAdj and adjSec
*
* loop over anchors:
* construct anchor adjacency
* loop over constrained:
* construct constrained adjacency
* if not in anchor adjacency, add to dofs
* setup adjSec, allocate anchorAdj
* loop over anchors:
* construct anchor adjacency
* loop over constrained:
* construct constrained adjacency
* if not in anchor adjacency
* if not already in list, put in list
* sort, unique, reduce dof count
* optional: compactify
*/
for (p = pStart; p < pEnd; p++) {
PetscInt iDof, iOff, i, r, s, numAdjP = PETSC_DETERMINE;
ierr = PetscSectionGetDof(inverseSec,p,&iDof);CHKERRQ(ierr);
if (!iDof) continue;
ierr = PetscSectionGetOffset(inverseSec,p,&iOff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm,p,&numAdjP,&tmpAdjP);CHKERRQ(ierr);
for (i = 0; i < iDof; i++) {
PetscInt iNew = 0, qAdj, qAdjDof, qAdjCDof, numAdjQ = PETSC_DETERMINE;
q = inverse[iOff + i];
ierr = DMPlexGetAdjacency(dm,q,&numAdjQ,&tmpAdjQ);CHKERRQ(ierr);
for (r = 0; r < numAdjQ; r++) {
qAdj = tmpAdjQ[r];
if ((qAdj < pStart) || (qAdj >= pEnd)) continue;
for (s = 0; s < numAdjP; s++) {
if (qAdj == tmpAdjP[s]) break;
}
if (s < numAdjP) continue;
ierr = PetscSectionGetDof(section,qAdj,&qAdjDof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section,qAdj,&qAdjCDof);CHKERRQ(ierr);
iNew += qAdjDof - qAdjCDof;
}
ierr = PetscSectionAddDof(adjSec,p,iNew);CHKERRQ(ierr);
}
}
ierr = PetscSectionSetUp(adjSec);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(adjSec,&adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize,&adj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; p++) {
PetscInt iDof, iOff, i, r, s, aOff, aOffOrig, aDof, numAdjP = PETSC_DETERMINE;
ierr = PetscSectionGetDof(inverseSec,p,&iDof);CHKERRQ(ierr);
if (!iDof) continue;
ierr = PetscSectionGetOffset(inverseSec,p,&iOff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm,p,&numAdjP,&tmpAdjP);CHKERRQ(ierr);
ierr = PetscSectionGetDof(adjSec,p,&aDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(adjSec,p,&aOff);CHKERRQ(ierr);
aOffOrig = aOff;
for (i = 0; i < iDof; i++) {
PetscInt qAdj, qAdjDof, qAdjCDof, qAdjOff, nd, numAdjQ = PETSC_DETERMINE;
q = inverse[iOff + i];
ierr = DMPlexGetAdjacency(dm,q,&numAdjQ,&tmpAdjQ);CHKERRQ(ierr);
for (r = 0; r < numAdjQ; r++) {
qAdj = tmpAdjQ[r];
if ((qAdj < pStart) || (qAdj >= pEnd)) continue;
for (s = 0; s < numAdjP; s++) {
if (qAdj == tmpAdjP[s]) break;
}
if (s < numAdjP) continue;
ierr = PetscSectionGetDof(section,qAdj,&qAdjDof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section,qAdj,&qAdjCDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal,qAdj,&qAdjOff);CHKERRQ(ierr);
for (nd = 0; nd < qAdjDof-qAdjCDof; ++nd) {
adj[aOff++] = (qAdjOff < 0 ? -(qAdjOff+1) : qAdjOff) + nd;
}
}
}
ierr = PetscSortRemoveDupsInt(&aDof,&adj[aOffOrig]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(adjSec,p,aDof);CHKERRQ(ierr);
}
*anchorAdj = adj;
/* clean up */
ierr = PetscSectionDestroy(&inverseSec);CHKERRQ(ierr);
ierr = PetscFree(inverse);CHKERRQ(ierr);
ierr = PetscFree(tmpAdjP);CHKERRQ(ierr);
ierr = PetscFree(tmpAdjQ);CHKERRQ(ierr);
}
else {
*anchorAdj = NULL;
ierr = PetscSectionSetUp(adjSec);CHKERRQ(ierr);
}
*anchorSectionAdj = adjSec;
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexPreallocateOperator(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix)
{
MPI_Comm comm;
MatType mtype;
PetscSF sf, sfDof, sfAdj;
PetscSection leafSectionAdj, rootSectionAdj, sectionAdj, anchorSectionAdj;
PetscInt nroots, nleaves, l, p;
const PetscInt *leaves;
const PetscSFNode *remotes;
PetscInt dim, pStart, pEnd, numDof, globalOffStart, globalOffEnd, numCols;
PetscInt *tmpAdj = NULL, *adj, *rootAdj, *anchorAdj = NULL, *cols, *remoteOffsets;
PetscInt adjSize;
PetscLayout rLayout;
PetscInt locRows, rStart, rEnd, r;
PetscMPIInt size;
PetscBool doCommLocal, doComm, debug = PETSC_FALSE, isSymBlock, isSymSeqBlock, isSymMPIBlock;
PetscBool useAnchors;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 3);
PetscValidHeaderSpecific(sectionGlobal, PETSC_SECTION_CLASSID, 4);
PetscValidHeaderSpecific(A, MAT_CLASSID, 9);
if (dnz) PetscValidPointer(dnz,5);
if (onz) PetscValidPointer(onz,6);
if (dnzu) PetscValidPointer(dnzu,7);
if (onzu) PetscValidPointer(onzu,8);
ierr = PetscLogEventBegin(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL, "-dm_view_preallocation", &debug, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
doCommLocal = (size > 1) && (nroots >= 0) ? PETSC_TRUE : PETSC_FALSE;
ierr = MPI_Allreduce(&doCommLocal, &doComm, 1, MPIU_BOOL, MPI_LAND, comm);CHKERRQ(ierr);
/* Create dof SF based on point SF */
if (debug) {
ierr = PetscPrintf(comm, "Input Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input Global Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionGlobal, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sf, NULL);CHKERRQ(ierr);
}
ierr = PetscSFCreateRemoteOffsets(sf, section, section, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, section, remoteOffsets, section, &sfDof);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Dof SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfDof, NULL);CHKERRQ(ierr);
}
/* Create section for dof adjacency (dof ==> # adj dof) */
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(section, &numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(leafSectionAdj, 0, numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSectionAdj, 0, numDof);CHKERRQ(ierr);
/* Fill in the ghost dofs on the interface */
ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, &remotes);CHKERRQ(ierr);
/* use constraints in finding adjacency in this routine */
ierr = DMPlexGetAdjacencyUseAnchors(dm,&useAnchors);CHKERRQ(ierr);
ierr = DMPlexSetAdjacencyUseAnchors(dm,PETSC_TRUE);CHKERRQ(ierr);
/*
section - maps points to (# dofs, local dofs)
sectionGlobal - maps points to (# dofs, global dofs)
leafSectionAdj - maps unowned local dofs to # adj dofs
rootSectionAdj - maps owned local dofs to # adj dofs
adj - adj global dofs indexed by leafSectionAdj
rootAdj - adj global dofs indexed by rootSectionAdj
sf - describes shared points across procs
sfDof - describes shared dofs across procs
sfAdj - describes shared adjacent dofs across procs
** The bootstrapping process involves six rounds with similar structure of visiting neighbors of each point.
(0). If there are point-to-point constraints, add the adjacencies of constrained points to anchors in anchorAdj
(This is done in DMPlexComputeAnchorAdjacencies())
1. Visit unowned points on interface, count adjacencies placing in leafSectionAdj
Reduce those counts to rootSectionAdj (now redundantly counting some interface points)
2. Visit owned points on interface, count adjacencies placing in rootSectionAdj
Create sfAdj connecting rootSectionAdj and leafSectionAdj
3. Visit unowned points on interface, write adjacencies to adj
Gather adj to rootAdj (note that there is redundancy in rootAdj when multiple procs find the same adjacencies)
4. Visit owned points on interface, write adjacencies to rootAdj
Remove redundancy in rootAdj
** The last two traversals use transitive closure
5. Visit all owned points in the subdomain, count dofs for each point (sectionAdj)
Allocate memory addressed by sectionAdj (cols)
6. Visit all owned points in the subdomain, insert dof adjacencies into cols
** Knowing all the column adjacencies, check ownership and sum into dnz and onz
*/
ierr = DMPlexComputeAnchorAdjacencies(dm,section,sectionGlobal,&anchorSectionAdj,&anchorAdj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation on Leaves:\n");CHKERRQ(ierr);
ierr = PetscSectionView(leafSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get maximum remote adjacency sizes for owned dofs on interface (roots) */
if (doComm) {
ierr = PetscSFReduceBegin(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
}
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Add in local adjacency sizes for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after local additions:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create adj SF based on dof SF */
ierr = PetscSFCreateRemoteOffsets(sfDof, rootSectionAdj, leafSectionAdj, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfDof, rootSectionAdj, remoteOffsets, leafSectionAdj, &sfAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfAdj, NULL);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfDof);CHKERRQ(ierr);
/* Create leaf adjacency */
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(leafSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscCalloc1(adjSize, &adj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof, anOff;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt aoff, i = 0;
ierr = PetscSectionGetOffset(leafSectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
adj[aoff+i] = (ngoff < 0 ? -(ngoff+1) : ngoff) + nd;
++i;
}
}
for (q = 0; q < anDof; q++) {
adj[aoff+i] = anchorAdj[anOff+q];
++i;
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Leaf adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, adj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Gather adjacenct indices to root */
ierr = PetscSectionGetStorageSize(rootSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize, &rootAdj);CHKERRQ(ierr);
for (r = 0; r < adjSize; ++r) rootAdj[r] = -1;
if (doComm) {
ierr = PetscSFGatherBegin(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
ierr = PetscSFGatherEnd(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfAdj);CHKERRQ(ierr);
ierr = PetscFree(adj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices after gather\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Add in local adjacency indices for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof, anOff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt adof, aoff, i;
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
i = adof-1;
for (q = 0; q < anDof; q++) {
rootAdj[aoff+i] = anchorAdj[anOff+q];
--i;
}
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
rootAdj[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
--i;
}
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Compress indices */
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, cdof, off, d;
PetscInt adof, aoff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
for (d = off; d < off+dof-cdof; ++d) {
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
ierr = PetscSortRemoveDupsInt(&adof, &rootAdj[aoff]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSectionAdj, d, adof);CHKERRQ(ierr);
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after compression:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Root adjacency indices after compression\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Build adjacency section: Maps global indices to sets of adjacent global indices */
ierr = PetscSectionGetOffsetRange(sectionGlobal, &globalOffStart, &globalOffEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, §ionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionAdj, globalOffStart, globalOffEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
ierr = PetscSectionSetDof(sectionAdj, goff+d, rdof);CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, noff;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, padj, &noff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(sectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get adjacent indices */
ierr = PetscSectionGetStorageSize(sectionAdj, &numCols);CHKERRQ(ierr);
ierr = PetscMalloc1(numCols, &cols);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof, anOff;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
PetscInt aoff, roff;
ierr = PetscSectionGetOffset(sectionAdj, goff+d, &aoff);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, off+d, &roff);CHKERRQ(ierr);
ierr = PetscMemcpy(&cols[aoff], &rootAdj[roff], rdof * sizeof(PetscInt));CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
PetscInt adof, aoff, i = 0;
ierr = PetscSectionGetDof(sectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
const PetscInt *ncind;
/* Adjacent points may not be in the section chart */
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintIndices(section, padj, &ncind);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd, ++i) {
cols[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
}
}
for (q = 0; q < anDof; q++, i++) {
cols[aoff+i] = anchorAdj[anOff + q];
}
if (i != adof) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of entries %D != %D for dof %D (point %D)", i, adof, d, p);
}
}
ierr = PetscSectionDestroy(&anchorSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSectionAdj);CHKERRQ(ierr);
ierr = PetscFree(anchorAdj);CHKERRQ(ierr);
ierr = PetscFree(rootAdj);CHKERRQ(ierr);
ierr = PetscFree(tmpAdj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Column indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, numCols, cols, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Create allocation vectors from adjacency graph */
ierr = MatGetLocalSize(A, &locRows, NULL);CHKERRQ(ierr);
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)A), &rLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr);
/* Only loop over blocks of rows */
if (rStart%bs || rEnd%bs) SETERRQ3(PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Invalid layout [%d, %d) for matrix, must be divisible by block size %d", rStart, rEnd, bs);
for (r = rStart/bs; r < rEnd/bs; ++r) {
const PetscInt row = r*bs;
PetscInt numCols, cStart, c;
ierr = PetscSectionGetDof(sectionAdj, row, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, row, &cStart);CHKERRQ(ierr);
for (c = cStart; c < cStart+numCols; ++c) {
if ((cols[c] >= rStart*bs) && (cols[c] < rEnd*bs)) {
++dnz[r-rStart];
if (cols[c] >= row) ++dnzu[r-rStart];
} else {
++onz[r-rStart];
if (cols[c] >= row) ++onzu[r-rStart];
}
}
}
if (bs > 1) {
for (r = 0; r < locRows/bs; ++r) {
dnz[r] /= bs;
onz[r] /= bs;
dnzu[r] /= bs;
onzu[r] /= bs;
}
}
/* Set matrix pattern */
ierr = MatXAIJSetPreallocation(A, bs, dnz, onz, dnzu, onzu);CHKERRQ(ierr);
ierr = MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
/* Check for symmetric storage */
ierr = MatGetType(A, &mtype);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
if (isSymBlock || isSymSeqBlock || isSymMPIBlock) {ierr = MatSetOption(A, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);CHKERRQ(ierr);}
/* Fill matrix with zeros */
if (fillMatrix) {
PetscScalar *values;
PetscInt maxRowLen = 0;
for (r = rStart; r < rEnd; ++r) {
PetscInt len;
ierr = PetscSectionGetDof(sectionAdj, r, &len);CHKERRQ(ierr);
maxRowLen = PetscMax(maxRowLen, len);
}
ierr = PetscCalloc1(maxRowLen, &values);CHKERRQ(ierr);
for (r = rStart; r < rEnd; ++r) {
PetscInt numCols, cStart;
ierr = PetscSectionGetDof(sectionAdj, r, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, r, &cStart);CHKERRQ(ierr);
ierr = MatSetValues(A, 1, &r, numCols, &cols[cStart], values, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree(values);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* restore original useAnchors */
ierr = DMPlexSetAdjacencyUseAnchors(dm,useAnchors);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionAdj);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexGetIndex - Get the index of 'point'. It returns the global indices.
Not collective
Input Parameters:
+ dm - The DM
. section - The section describing the layout in v, or NULL to use the default section
. globalSection - The section describing the layout in v, or NULL to use the default global section
. point - The sieve point in the DM
. NumOfIndices - The number of the indices for this point
. indices - The indices of this point
Level: intermediate
.seealso DMPlexVecSetClosure()
@*/
PetscErrorCode DMPlexGetIndex(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *NumOfIndices, PetscInt index[])
{
PetscSection clSection;
IS clPoints;
PetscInt *points = NULL;
const PetscInt *clp;
PetscInt *indices;
PetscInt offsets[32];
PetscInt numFields, numIndices, numPoints, dof, off, globalOff, pStart, pEnd, p, q, f;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (!section) {ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);}
PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
if (!clPoints) {
ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
/* Compress out points not in the section */
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
for (p = 0, q = 0; p < numPoints*2; p += 2) {
if ((points[p] >= pStart) && (points[p] < pEnd)) {
points[q*2] = points[p];
points[q*2+1] = points[p+1];
++q;
}
}
numPoints = q;
} else {
PetscInt dof, off;
ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
numPoints = dof/2;
ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
points = (PetscInt *) &clp[off];
}
for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
PetscInt fdof;
ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
offsets[f+1] += fdof;
}
numIndices += dof;
}
for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
if (numFields) {
for (p = 0; p < numPoints*2; p += 2) {
PetscInt o = points[p+1];
ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
}
} else {
for (p = 0, off = 0; p < numPoints*2; p += 2) {
PetscInt o = points[p+1];
ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
}
}
if (!clPoints) {
ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
} else {
ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
}
{
/*Out put the number of indices and the value of indices on this point*/
PetscInt j;
*NumOfIndices = numIndices;
for(j = 0; j < numIndices; j++){
index[j] = indices[j];
}
}
ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
