PetscErrorCode DMPlexVTKWriteField_ASCII(DM dm, PetscSection section, PetscSection globalSection, Vec field, const char name[], FILE *fp, PetscInt enforceDof, PetscInt precision, PetscReal scale)
{
MPI_Comm comm;
PetscInt numDof = 0, maxDof;
PetscInt pStart, pEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
ierr = PetscSectionGetDof(section, p, &numDof);CHKERRQ(ierr);
if (numDof) break;
}
numDof = PetscMax(numDof, enforceDof);
ierr = MPI_Allreduce(&numDof, &maxDof, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
if (!name) name = "Unknown";
if (maxDof == 3) {
ierr = PetscFPrintf(comm, fp, "VECTORS %s double\n", name);CHKERRQ(ierr);
} else {
ierr = PetscFPrintf(comm, fp, "SCALARS %s double %d\n", name, maxDof);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
}
ierr = DMPlexVTKWriteSection_ASCII(dm, section, globalSection, field, fp, enforceDof, precision, scale);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PetscSectionVecNorm - Computes the vector norm, separated into field components.
Input Parameters:
+ s - the local Section
. gs - the global section
. x - the vector
- type - one of NORM_1, NORM_2, NORM_INFINITY.
Output Parameter:
. val - the array of norms
Level: intermediate
.seealso: VecNorm(), PetscSectionCreate()
@*/
PetscErrorCode PetscSectionVecNorm(PetscSection s, PetscSection gs, Vec x, NormType type, PetscReal val[])
{
PetscInt Nf, f, pStart, pEnd;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(s, PETSC_SECTION_CLASSID, 1);
PetscValidHeaderSpecific(gs, PETSC_SECTION_CLASSID, 2);
PetscValidHeaderSpecific(x, VEC_CLASSID, 3);
PetscValidPointer(val, 5);
ierr = PetscSectionGetNumFields(s, &Nf);CHKERRQ(ierr);
if (Nf < 2) {ierr = VecNorm(x, type, val);CHKERRQ(ierr);}
else {
ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
for (f = 0; f < Nf; ++f) {
Vec subv;
IS is;
ierr = PetscSectionGetField_Internal(s, gs, x, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
ierr = VecNorm(subv, type, &val[f]);CHKERRQ(ierr);
ierr = PetscSectionRestoreField_Internal(s, gs, x, f, pStart, pEnd, &is, &subv);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
PETSC_STATIC_INLINE PetscErrorCode FillClosureArray_Static(DM dm, PetscSection section, PetscInt nP, const PetscInt points[], PetscScalar *vArray, PetscInt *csize, PetscScalar **array)
{
PetscScalar *a;
PetscInt pStart, pEnd, size = 0, dof, off, d, k, i;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
for (i = 0; i < nP; ++i) {
const PetscInt p = points[i];
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
size += dof;
}
if (csize) *csize = size;
if (array) {
ierr = DMGetWorkArray(dm, size, PETSC_SCALAR, &a);CHKERRQ(ierr);
for (i = 0, k = 0; i < nP; ++i) {
const PetscInt p = points[i];
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
for (d = 0; d < dof; ++d, ++k) a[k] = vArray[off+d];
}
*array = a;
}
PetscFunctionReturn(0);
}
PetscErrorCode DMCreateGlobalVector_Section_Private(DM dm,Vec *vec)
{
PetscSection gSection;
PetscInt localSize, bs, blockSize = -1, pStart, pEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMGetDefaultGlobalSection(dm, &gSection);CHKERRQ(ierr);
ierr = PetscSectionGetChart(gSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, cdof;
ierr = PetscSectionGetDof(gSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(gSection, p, &cdof);CHKERRQ(ierr);
if ((blockSize < 0) && (dof > 0) && (dof-cdof > 0)) blockSize = dof-cdof;
if ((dof > 0) && (dof-cdof != blockSize)) {
blockSize = 1;
break;
}
}
if (blockSize < 0) blockSize = PETSC_MAX_INT;
ierr = MPIU_Allreduce(&blockSize, &bs, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
if (blockSize == PETSC_MAX_INT) blockSize = 1; /* Everyone was empty */
ierr = PetscSectionGetConstrainedStorageSize(gSection, &localSize);CHKERRQ(ierr);
if (localSize%blockSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Mismatch between blocksize %d and local storage size %d", blockSize, localSize);
ierr = VecCreate(PetscObjectComm((PetscObject)dm), vec);CHKERRQ(ierr);
ierr = VecSetSizes(*vec, localSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(*vec, bs);CHKERRQ(ierr);
ierr = VecSetType(*vec,dm->vectype);CHKERRQ(ierr);
ierr = VecSetDM(*vec, dm);CHKERRQ(ierr);
/* ierr = VecSetLocalToGlobalMapping(*vec, dm->ltogmap);CHKERRQ(ierr); */
PetscFunctionReturn(0);
}
PetscErrorCode DMCreateLocalVector_Section_Private(DM dm,Vec *vec)
{
PetscSection section;
PetscInt localSize, blockSize = -1, pStart, pEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
if ((blockSize < 0) && (dof > 0)) blockSize = dof;
if ((dof > 0) && (dof != blockSize)) {
blockSize = 1;
break;
}
}
ierr = PetscSectionGetStorageSize(section, &localSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, vec);CHKERRQ(ierr);
ierr = VecSetSizes(*vec, localSize, localSize);CHKERRQ(ierr);
ierr = VecSetBlockSize(*vec, blockSize);CHKERRQ(ierr);
ierr = VecSetType(*vec,dm->vectype);CHKERRQ(ierr);
ierr = VecSetDM(*vec, dm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode CreatePressureNullSpace(DM dm, AppCtx *user, MatNullSpace *nullSpace)
{
Vec vec, localVec;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DMGetGlobalVector(dm, &vec);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &localVec);CHKERRQ(ierr);
ierr = VecSet(vec, 0.0);CHKERRQ(ierr);
/* Put a constant in for all pressures
Could change this to project the constant function onto the pressure space (when that is finished) */
{
PetscSection section;
PetscInt pStart, pEnd, p;
PetscScalar *a;
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
ierr = VecGetArray(localVec, &a);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt fDim, off, d;
ierr = PetscSectionGetFieldDof(section, p, 1, &fDim);CHKERRQ(ierr);
ierr = PetscSectionGetFieldOffset(section, p, 1, &off);CHKERRQ(ierr);
for (d = 0; d < fDim; ++d) a[off+d] = 1.0;
}
ierr = VecRestoreArray(localVec, &a);CHKERRQ(ierr);
}
ierr = DMLocalToGlobalBegin(dm, localVec, INSERT_VALUES, vec);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(dm, localVec, INSERT_VALUES, vec);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &localVec);CHKERRQ(ierr);
ierr = VecNormalize(vec, NULL);CHKERRQ(ierr);
if (user->debug) {
ierr = PetscPrintf(PetscObjectComm((PetscObject)dm), "Pressure Null Space\n");CHKERRQ(ierr);
ierr = VecView(vec, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)dm), PETSC_FALSE, 1, &vec, nullSpace);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(dm, &vec);CHKERRQ(ierr);
/* New style for field null spaces */
{
PetscObject pressure;
MatNullSpace nullSpacePres;
ierr = DMGetField(dm, 1, &pressure);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(PetscObjectComm(pressure), PETSC_TRUE, 0, NULL, &nullSpacePres);CHKERRQ(ierr);
ierr = PetscObjectCompose(pressure, "nullspace", (PetscObject) nullSpacePres);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullSpacePres);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexPreallocateOperator_2(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix)
{
PetscInt *tmpClosure,*tmpAdj,*visits;
PetscInt c,cStart,cEnd,pStart,pEnd;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
maxClosureSize = 2*PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1));
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
npoints = pEnd - pStart;
ierr = PetscMalloc3(maxClosureSize,&tmpClosure,npoints,&lvisits,npoints,&visits);CHKERRQ(ierr);
ierr = PetscMemzero(lvisits,(pEnd-pStart)*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemzero(visits,(pEnd-pStart)*sizeof(PetscInt));CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
for (c=cStart; c<cEnd; c++) {
PetscInt *support = tmpClosure;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_FALSE, &supportSize, (PetscInt**)&support);CHKERRQ(ierr);
for (p=0; p<supportSize; p++) lvisits[support[p]]++;
}
ierr = PetscSFReduceBegin(sf,MPIU_INT,lvisits,visits,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd (sf,MPIU_INT,lvisits,visits,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,visits,lvisits);CHKERRQ(ierr);
ierr = PetscSFBcastEnd (sf,MPIU_INT,visits,lvisits);CHKERRQ(ierr);
ierr = PetscSFGetRanks();CHKERRQ(ierr);
ierr = PetscMalloc2(maxClosureSize*maxClosureSize,&cellmat,npoints,&owner);CHKERRQ(ierr);
for (c=cStart; c<cEnd; c++) {
ierr = PetscMemzero(cellmat,maxClosureSize*maxClosureSize*sizeof(PetscInt));CHKERRQ(ierr);
/*
Depth-first walk of transitive closure.
At each leaf frame f of transitive closure that we see, add 1/visits[f] to each pair (p,q) not marked as done in cellmat.
This contribution is added to dnz if owning ranks of p and q match, to onz otherwise.
*/
}
ierr = PetscSFReduceBegin(sf,MPIU_INT,ldnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd (sf,MPIU_INT,lonz,onz,MPI_SUM);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMNetworkDistribute - Distributes the network and moves associated component data.
Collective
Input Parameter:
+ oldDM - the original DMNetwork object
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
. distDM - the distributed DMNetwork object
Notes:
This routine should be called only when using multiple processors.
Distributes the network with <overlap>-overlapping partitioning of the edges.
Level: intermediate
.seealso: DMNetworkCreate
@*/
PetscErrorCode DMNetworkDistribute(DM oldDM, PetscInt overlap,DM *distDM)
{
PetscErrorCode ierr;
DM_Network *oldDMnetwork = (DM_Network*)oldDM->data;
PetscSF pointsf;
DM newDM;
DM_Network *newDMnetwork;
PetscFunctionBegin;
ierr = DMNetworkCreate(PetscObjectComm((PetscObject)oldDM),&newDM);CHKERRQ(ierr);
newDMnetwork = (DM_Network*)newDM->data;
newDMnetwork->dataheadersize = sizeof(struct _p_DMNetworkComponentHeader)/sizeof(DMNetworkComponentGenericDataType);
/* Distribute plex dm and dof section */
ierr = DMPlexDistribute(oldDMnetwork->plex,overlap,&pointsf,&newDMnetwork->plex);CHKERRQ(ierr);
/* Distribute dof section */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)oldDM),&newDMnetwork->DofSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointsf,oldDMnetwork->DofSection,NULL,newDMnetwork->DofSection);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)oldDM),&newDMnetwork->DataSection);CHKERRQ(ierr);
/* Distribute data and associated section */
ierr = DMPlexDistributeData(newDMnetwork->plex,pointsf,oldDMnetwork->DataSection,MPI_INT,(void*)oldDMnetwork->componentdataarray,newDMnetwork->DataSection,(void**)&newDMnetwork->componentdataarray);CHKERRQ(ierr);
/* Destroy point SF */
ierr = PetscSFDestroy(&pointsf);CHKERRQ(ierr);
ierr = PetscSectionGetChart(newDMnetwork->DataSection,&newDMnetwork->pStart,&newDMnetwork->pEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(newDMnetwork->plex,0, &newDMnetwork->eStart,&newDMnetwork->eEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(newDMnetwork->plex,1,&newDMnetwork->vStart,&newDMnetwork->vEnd);CHKERRQ(ierr);
newDMnetwork->nEdges = newDMnetwork->eEnd - newDMnetwork->eStart;
newDMnetwork->nNodes = newDMnetwork->vEnd - newDMnetwork->vStart;
newDMnetwork->NNodes = oldDMnetwork->NNodes;
newDMnetwork->NEdges = oldDMnetwork->NEdges;
/* Set Dof section as the default section for dm */
ierr = DMSetDefaultSection(newDMnetwork->plex,newDMnetwork->DofSection);CHKERRQ(ierr);
ierr = DMGetDefaultGlobalSection(newDMnetwork->plex,&newDMnetwork->GlobalDofSection);CHKERRQ(ierr);
*distDM = newDM;
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelDistribute(DMLabel label, PetscSF sf, DMLabel *labelNew)
{
MPI_Comm comm;
PetscSection leafSection;
PetscInt p, pStart, pEnd, s, size, dof, offset, stratum;
PetscInt *leafStrata, *strataIdx;
char *name;
PetscInt nameSize;
PetscHashI stratumHash;
PETSC_UNUSED PetscHashIIter ret, iter;
size_t len = 0;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
if (label) {ierr = DMLabelMakeAllValid_Private(label);CHKERRQ(ierr);}
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
/* Bcast name */
if (!rank) {ierr = PetscStrlen(label->name, &len);CHKERRQ(ierr);}
nameSize = len;
ierr = MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1(nameSize+1, &name);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy(name, label->name, nameSize+1);CHKERRQ(ierr);}
ierr = MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = DMLabelCreate(name, labelNew);CHKERRQ(ierr);
ierr = PetscFree(name);CHKERRQ(ierr);
/* Bcast defaultValue */
if (!rank) (*labelNew)->defaultValue = label->defaultValue;
ierr = MPI_Bcast(&(*labelNew)->defaultValue, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
/* Distribute stratum values over the SF and get the point mapping on the receiver */
ierr = DMLabelDistribute_Internal(label, sf, &leafSection, &leafStrata);CHKERRQ(ierr);
/* Determine received stratum values and initialise new label*/
PetscHashICreate(stratumHash);
ierr = PetscSectionGetStorageSize(leafSection, &size);CHKERRQ(ierr);
for (p = 0; p < size; ++p) PetscHashIPut(stratumHash, leafStrata[p], ret, iter);
PetscHashISize(stratumHash, (*labelNew)->numStrata);
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->arrayValid);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) (*labelNew)->arrayValid[s] = PETSC_TRUE;
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->stratumValues);CHKERRQ(ierr);
/* Turn leafStrata into indices rather than stratum values */
offset = 0;
ierr = PetscHashIGetKeys(stratumHash, &offset, (*labelNew)->stratumValues);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) {
for (p = 0; p < size; ++p) {
if (leafStrata[p] == (*labelNew)->stratumValues[s]) leafStrata[p] = s;
}
}
/* Rebuild the point strata on the receiver */
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->stratumSizes);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
(*labelNew)->stratumSizes[leafStrata[offset+s]]++;
}
}
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->ht);CHKERRQ(ierr);
ierr = PetscMalloc1((*labelNew)->numStrata,&(*labelNew)->points);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) {
PetscHashICreate((*labelNew)->ht[s]);
ierr = PetscMalloc1((*labelNew)->stratumSizes[s], &(*labelNew)->points[s]);CHKERRQ(ierr);
}
/* Insert points into new strata */
ierr = PetscCalloc1((*labelNew)->numStrata, &strataIdx);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
stratum = leafStrata[offset+s];
(*labelNew)->points[stratum][strataIdx[stratum]++] = p;
}
}
PetscHashIDestroy(stratumHash);
ierr = PetscFree(leafStrata);CHKERRQ(ierr);
ierr = PetscFree(strataIdx);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSection);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
the local section and an SF describing the section point overlap.
Input Parameters:
+ s - The PetscSection for the local field layout
. sf - The SF describing parallel layout of the section points
. includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
. label - The label specifying the points
- labelValue - The label stratum specifying the points
Output Parameter:
. gsection - The PetscSection for the global field layout
Note: This gives negative sizes and offsets to points not owned by this process
Level: developer
.seealso: PetscSectionCreate()
@*/
PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
{
PetscInt *neg = NULL, *tmpOff = NULL;
PetscInt pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) s), gsection);CHKERRQ(ierr);
ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
if (nroots >= 0) {
if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
} else {
tmpOff = &(*gsection)->atlasDof[-pStart];
}
}
/* Mark ghost points with negative dof */
for (p = pStart; p < pEnd; ++p) {
PetscInt value;
ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
if (value != labelValue) continue;
ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
if (neg) neg[p] = -(dof+1);
}
ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
if (nroots >= 0) {
ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
}
}
/* Calculate new sizes, get proccess offset, and calculate point offsets */
for (p = 0, off = 0; p < pEnd-pStart; ++p) {
cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
(*gsection)->atlasOff[p] = off;
off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
}
ierr = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) s));CHKERRQ(ierr);
globalOff -= off;
for (p = 0, off = 0; p < pEnd-pStart; ++p) {
(*gsection)->atlasOff[p] += globalOff;
if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
}
/* Put in negative offsets for ghost points */
if (nroots >= 0) {
ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
}
}
if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
ierr = PetscFree(neg);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* get adjacencies due to point-to-point constraints that can't be found with DMPlexGetAdjacency() */
static PetscErrorCode DMPlexComputeAnchorAdjacencies(DM dm, PetscSection section, PetscSection sectionGlobal, PetscSection *anchorSectionAdj, PetscInt *anchorAdj[])
{
PetscInt pStart, pEnd;
PetscSection adjSec, aSec;
IS aIS;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)section),&adjSec);CHKERRQ(ierr);
ierr = PetscSectionGetChart(section,&pStart,&pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(adjSec,pStart,pEnd);CHKERRQ(ierr);
ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
if (aSec) {
const PetscInt *anchors;
PetscInt p, q, a, aSize, *offsets, aStart, aEnd, *inverse, iSize, *adj, adjSize;
PetscInt *tmpAdjP = NULL, *tmpAdjQ = NULL;
PetscSection inverseSec;
/* invert the constraint-to-anchor map */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)aSec),&inverseSec);CHKERRQ(ierr);
ierr = PetscSectionSetChart(inverseSec,pStart,pEnd);CHKERRQ(ierr);
ierr = ISGetLocalSize(aIS, &aSize);CHKERRQ(ierr);
ierr = ISGetIndices(aIS, &anchors);CHKERRQ(ierr);
for (p = 0; p < aSize; p++) {
PetscInt a = anchors[p];
ierr = PetscSectionAddDof(inverseSec,a,1);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(inverseSec);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(inverseSec,&iSize);CHKERRQ(ierr);
ierr = PetscMalloc1(iSize,&inverse);CHKERRQ(ierr);
ierr = PetscCalloc1(pEnd-pStart,&offsets);CHKERRQ(ierr);
ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
for (p = aStart; p < aEnd; p++) {
PetscInt dof, off;
ierr = PetscSectionGetDof(aSec, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(aSec, p, &off);CHKERRQ(ierr);
for (q = 0; q < dof; q++) {
PetscInt iOff;
a = anchors[off + q];
ierr = PetscSectionGetOffset(inverseSec, a, &iOff);CHKERRQ(ierr);
inverse[iOff + offsets[a-pStart]++] = p;
}
}
ierr = ISRestoreIndices(aIS, &anchors);CHKERRQ(ierr);
ierr = PetscFree(offsets);CHKERRQ(ierr);
/* construct anchorAdj and adjSec
*
* loop over anchors:
* construct anchor adjacency
* loop over constrained:
* construct constrained adjacency
* if not in anchor adjacency, add to dofs
* setup adjSec, allocate anchorAdj
* loop over anchors:
* construct anchor adjacency
* loop over constrained:
* construct constrained adjacency
* if not in anchor adjacency
* if not already in list, put in list
* sort, unique, reduce dof count
* optional: compactify
*/
for (p = pStart; p < pEnd; p++) {
PetscInt iDof, iOff, i, r, s, numAdjP = PETSC_DETERMINE;
ierr = PetscSectionGetDof(inverseSec,p,&iDof);CHKERRQ(ierr);
if (!iDof) continue;
ierr = PetscSectionGetOffset(inverseSec,p,&iOff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm,p,&numAdjP,&tmpAdjP);CHKERRQ(ierr);
for (i = 0; i < iDof; i++) {
PetscInt iNew = 0, qAdj, qAdjDof, qAdjCDof, numAdjQ = PETSC_DETERMINE;
q = inverse[iOff + i];
ierr = DMPlexGetAdjacency(dm,q,&numAdjQ,&tmpAdjQ);CHKERRQ(ierr);
for (r = 0; r < numAdjQ; r++) {
qAdj = tmpAdjQ[r];
if ((qAdj < pStart) || (qAdj >= pEnd)) continue;
for (s = 0; s < numAdjP; s++) {
if (qAdj == tmpAdjP[s]) break;
}
if (s < numAdjP) continue;
ierr = PetscSectionGetDof(section,qAdj,&qAdjDof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section,qAdj,&qAdjCDof);CHKERRQ(ierr);
iNew += qAdjDof - qAdjCDof;
}
ierr = PetscSectionAddDof(adjSec,p,iNew);CHKERRQ(ierr);
}
}
ierr = PetscSectionSetUp(adjSec);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(adjSec,&adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize,&adj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; p++) {
PetscInt iDof, iOff, i, r, s, aOff, aOffOrig, aDof, numAdjP = PETSC_DETERMINE;
ierr = PetscSectionGetDof(inverseSec,p,&iDof);CHKERRQ(ierr);
if (!iDof) continue;
ierr = PetscSectionGetOffset(inverseSec,p,&iOff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm,p,&numAdjP,&tmpAdjP);CHKERRQ(ierr);
ierr = PetscSectionGetDof(adjSec,p,&aDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(adjSec,p,&aOff);CHKERRQ(ierr);
aOffOrig = aOff;
for (i = 0; i < iDof; i++) {
PetscInt qAdj, qAdjDof, qAdjCDof, qAdjOff, nd, numAdjQ = PETSC_DETERMINE;
q = inverse[iOff + i];
ierr = DMPlexGetAdjacency(dm,q,&numAdjQ,&tmpAdjQ);CHKERRQ(ierr);
for (r = 0; r < numAdjQ; r++) {
qAdj = tmpAdjQ[r];
if ((qAdj < pStart) || (qAdj >= pEnd)) continue;
for (s = 0; s < numAdjP; s++) {
if (qAdj == tmpAdjP[s]) break;
}
if (s < numAdjP) continue;
ierr = PetscSectionGetDof(section,qAdj,&qAdjDof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section,qAdj,&qAdjCDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal,qAdj,&qAdjOff);CHKERRQ(ierr);
for (nd = 0; nd < qAdjDof-qAdjCDof; ++nd) {
adj[aOff++] = (qAdjOff < 0 ? -(qAdjOff+1) : qAdjOff) + nd;
}
}
}
ierr = PetscSortRemoveDupsInt(&aDof,&adj[aOffOrig]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(adjSec,p,aDof);CHKERRQ(ierr);
}
*anchorAdj = adj;
/* clean up */
ierr = PetscSectionDestroy(&inverseSec);CHKERRQ(ierr);
ierr = PetscFree(inverse);CHKERRQ(ierr);
ierr = PetscFree(tmpAdjP);CHKERRQ(ierr);
ierr = PetscFree(tmpAdjQ);CHKERRQ(ierr);
}
else {
*anchorAdj = NULL;
ierr = PetscSectionSetUp(adjSec);CHKERRQ(ierr);
}
*anchorSectionAdj = adjSec;
PetscFunctionReturn(0);
}
/* This assumes that the DM has been cloned prior to the call */
PetscErrorCode DMCreateSubDM_Section_Private(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
{
PetscSection section, sectionGlobal;
PetscInt *subIndices;
PetscInt subSize = 0, subOff = 0, nF, f, pStart, pEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!numFields) PetscFunctionReturn(0);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = DMGetDefaultGlobalSection(dm, §ionGlobal);CHKERRQ(ierr);
if (!section) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting fields");
if (!sectionGlobal) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default global section for DM before splitting fields");
ierr = PetscSectionGetNumFields(section, &nF);CHKERRQ(ierr);
if (numFields > nF) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Number of requested fields %d greater than number of DM fields %d", numFields, nF);
if (is) {
ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof;
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
subSize += fdof-fcdof;
}
}
}
ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof, goff;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof, fc, f2, poff = 0;
/* Can get rid of this loop by storing field information in the global section */
for (f2 = 0; f2 < fields[f]; ++f2) {
ierr = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
poff += fdof-fcdof;
}
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
subIndices[subOff] = goff+poff+fc;
}
}
}
}
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
}
if (subdm) {
PetscSection subsection;
PetscBool haveNull = PETSC_FALSE;
PetscInt f, nf = 0;
ierr = PetscSectionCreateSubsection(section, numFields, fields, &subsection);CHKERRQ(ierr);
ierr = DMSetDefaultSection(*subdm, subsection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
(*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[fields[f]];
if ((*subdm)->nullspaceConstructors[f]) {
haveNull = PETSC_TRUE;
nf = f;
}
}
if (haveNull) {
MatNullSpace nullSpace;
ierr = (*(*subdm)->nullspaceConstructors[nf])(*subdm, nf, &nullSpace);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
}
if (dm->fields) {
if (nF != dm->numFields) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "The number of DM fields %d does not match the number of Section fields %d", dm->numFields, nF);
ierr = DMSetNumFields(*subdm, numFields);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
ierr = PetscObjectListDuplicate(dm->fields[fields[f]]->olist, &(*subdm)->fields[f]->olist);CHKERRQ(ierr);
}
if (numFields == 1) {
MatNullSpace space;
Mat pmat;
ierr = PetscObjectQuery((*subdm)->fields[0], "nullspace", (PetscObject*) &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) space);CHKERRQ(ierr);}
ierr = PetscObjectQuery((*subdm)->fields[0], "nearnullspace", (PetscObject*) &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nearnullspace", (PetscObject) space);CHKERRQ(ierr);}
ierr = PetscObjectQuery((*subdm)->fields[0], "pmat", (PetscObject*) &pmat);CHKERRQ(ierr);
if (pmat) {ierr = PetscObjectCompose((PetscObject) *is, "pmat", (PetscObject) pmat);CHKERRQ(ierr);}
}
}
}
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data;
FILE *fp;
PetscViewerVTKObjectLink link;
PetscSection coordSection, globalCoordSection;
PetscLayout vLayout;
Vec coordinates;
PetscReal lengthScale;
PetscInt vMax, totVertices, totCells;
PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr);
/* Vertices */
ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
if (vMax >= 0) {
PetscInt pStart, pEnd, p, localSize = 0;
ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr);
pEnd = PetscMin(pEnd, vMax);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr);
if (dof > 0) ++localSize;
}
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
} else {
ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr);
}
ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr);
ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr);
/* Cells */
ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr);
/* Vertex fields */
for (link = vtk->link; link; link = link->next) {
if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE;
if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE;
}
if (hasPoint) {
ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection, newSection = NULL;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
DMLabel subpointMap, subpointMapX;
PetscInt dim, dimX, pStart, pEnd, qStart, qEnd;
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
/* Here is where we check whether dmX is a submesh of dm */
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr);
if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) {
const PetscInt *ind = NULL;
IS subpointIS;
PetscInt n = 0, q;
ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr);
if (subpointIS) {
ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr);
ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr);
}
ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr);
for (q = qStart; q < qEnd; ++q) {
PetscInt dof, off, p;
ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr);
if (dof) {
ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr);
if (p >= pStart) {
ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr);
ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr);
}
}
}
if (subpointIS) {
ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr);
ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
}
/* No need to setup section */
section = newSection;
}
} else {
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr);
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);}
}
}
/* Cell Fields */
ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr);
if (hasCell || writePartition) {
ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
} else {
PetscContainer c;
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr);
if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
}
if (writePartition) {
ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr);
}
}
/* Cleanup */
ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
ierr = PetscFClose(comm, fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexGetIndex - Get the index of 'point'. It returns the global indices.
Not collective
Input Parameters:
+ dm - The DM
. section - The section describing the layout in v, or NULL to use the default section
. globalSection - The section describing the layout in v, or NULL to use the default global section
. point - The sieve point in the DM
. NumOfIndices - The number of the indices for this point
. indices - The indices of this point
Level: intermediate
.seealso DMPlexVecSetClosure()
@*/
PetscErrorCode DMPlexGetIndex(DM dm, PetscSection section, PetscSection globalSection, PetscInt point, PetscInt *NumOfIndices, PetscInt index[])
{
PetscSection clSection;
IS clPoints;
PetscInt *points = NULL;
const PetscInt *clp;
PetscInt *indices;
PetscInt offsets[32];
PetscInt numFields, numIndices, numPoints, dof, off, globalOff, pStart, pEnd, p, q, f;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (!section) {ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);}
PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
if (!globalSection) {ierr = DMGetDefaultGlobalSection(dm, &globalSection);CHKERRQ(ierr);}
PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
ierr = PetscSectionGetNumFields(section, &numFields);CHKERRQ(ierr);
if (numFields > 31) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %D limited to 31", numFields);
ierr = PetscMemzero(offsets, 32 * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscSectionGetClosureIndex(section, (PetscObject) dm, &clSection, &clPoints);CHKERRQ(ierr);
if (!clPoints) {
ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
/* Compress out points not in the section */
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
for (p = 0, q = 0; p < numPoints*2; p += 2) {
if ((points[p] >= pStart) && (points[p] < pEnd)) {
points[q*2] = points[p];
points[q*2+1] = points[p+1];
++q;
}
}
numPoints = q;
} else {
PetscInt dof, off;
ierr = PetscSectionGetDof(clSection, point, &dof);CHKERRQ(ierr);
numPoints = dof/2;
ierr = PetscSectionGetOffset(clSection, point, &off);CHKERRQ(ierr);
ierr = ISGetIndices(clPoints, &clp);CHKERRQ(ierr);
points = (PetscInt *) &clp[off];
}
for (p = 0, numIndices = 0; p < numPoints*2; p += 2) {
PetscInt fdof;
ierr = PetscSectionGetDof(section, points[p], &dof);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
ierr = PetscSectionGetFieldDof(section, points[p], f, &fdof);CHKERRQ(ierr);
offsets[f+1] += fdof;
}
numIndices += dof;
}
for (f = 1; f < numFields; ++f) offsets[f+1] += offsets[f];
if (numFields && offsets[numFields] != numIndices) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %d should be %d", offsets[numFields], numIndices);
ierr = DMGetWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
if (numFields) {
for (p = 0; p < numPoints*2; p += 2) {
PetscInt o = points[p+1];
ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
indicesPointFields_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, offsets, PETSC_FALSE, o, indices);
}
} else {
for (p = 0, off = 0; p < numPoints*2; p += 2) {
PetscInt o = points[p+1];
ierr = PetscSectionGetOffset(globalSection, points[p], &globalOff);CHKERRQ(ierr);
indicesPoint_private(section, points[p], globalOff < 0 ? -(globalOff+1) : globalOff, &off, PETSC_FALSE, o, indices);
}
}
if (!clPoints) {
ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
} else {
ierr = ISRestoreIndices(clPoints, &clp);CHKERRQ(ierr);
}
{
/*Out put the number of indices and the value of indices on this point*/
PetscInt j;
*NumOfIndices = numIndices;
for(j = 0; j < numIndices; j++){
index[j] = indices[j];
}
}
ierr = DMRestoreWorkArray(dm, numIndices, PETSC_INT, &indices);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexGetAdjacency_Internal(DM dm, PetscInt p, PetscBool useCone, PetscBool useTransitiveClosure, PetscBool useAnchors, PetscInt *adjSize, PetscInt *adj[])
{
static PetscInt asiz = 0;
PetscInt maxAnchors = 1;
PetscInt aStart = -1, aEnd = -1;
PetscInt maxAdjSize;
PetscSection aSec = NULL;
IS aIS = NULL;
const PetscInt *anchors;
PetscErrorCode ierr;
PetscFunctionBeginHot;
if (useAnchors) {
ierr = DMPlexGetAnchors(dm,&aSec,&aIS);CHKERRQ(ierr);
if (aSec) {
ierr = PetscSectionGetMaxDof(aSec,&maxAnchors);CHKERRQ(ierr);
maxAnchors = PetscMax(1,maxAnchors);
ierr = PetscSectionGetChart(aSec,&aStart,&aEnd);CHKERRQ(ierr);
ierr = ISGetIndices(aIS,&anchors);CHKERRQ(ierr);
}
}
if (!*adj) {
PetscInt depth, coneSeries, supportSeries, maxC, maxS, pStart, pEnd;
ierr = DMPlexGetChart(dm, &pStart,&pEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(dm, &maxC, &maxS);CHKERRQ(ierr);
coneSeries = (maxC > 1) ? ((PetscPowInt(maxC,depth+1)-1)/(maxC-1)) : depth+1;
supportSeries = (maxS > 1) ? ((PetscPowInt(maxS,depth+1)-1)/(maxS-1)) : depth+1;
asiz = PetscMax(PetscPowInt(maxS,depth)*coneSeries,PetscPowInt(maxC,depth)*supportSeries);
asiz *= maxAnchors;
asiz = PetscMin(asiz,pEnd-pStart);
ierr = PetscMalloc1(asiz,adj);CHKERRQ(ierr);
}
if (*adjSize < 0) *adjSize = asiz;
maxAdjSize = *adjSize;
if (useTransitiveClosure) {
ierr = DMPlexGetAdjacency_Transitive_Internal(dm, p, useCone, adjSize, *adj);CHKERRQ(ierr);
} else if (useCone) {
ierr = DMPlexGetAdjacency_Cone_Internal(dm, p, adjSize, *adj);CHKERRQ(ierr);
} else {
ierr = DMPlexGetAdjacency_Support_Internal(dm, p, adjSize, *adj);CHKERRQ(ierr);
}
if (useAnchors && aSec) {
PetscInt origSize = *adjSize;
PetscInt numAdj = origSize;
PetscInt i = 0, j;
PetscInt *orig = *adj;
while (i < origSize) {
PetscInt p = orig[i];
PetscInt aDof = 0;
if (p >= aStart && p < aEnd) {
ierr = PetscSectionGetDof(aSec,p,&aDof);CHKERRQ(ierr);
}
if (aDof) {
PetscInt aOff;
PetscInt s, q;
for (j = i + 1; j < numAdj; j++) {
orig[j - 1] = orig[j];
}
origSize--;
numAdj--;
ierr = PetscSectionGetOffset(aSec,p,&aOff);CHKERRQ(ierr);
for (s = 0; s < aDof; ++s) {
for (q = 0; q < numAdj || (orig[numAdj++] = anchors[aOff+s],0); ++q) {
if (anchors[aOff+s] == orig[q]) break;
}
if (numAdj > maxAdjSize) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid mesh exceeded adjacency allocation (%D)", maxAdjSize);
}
}
else {
i++;
}
}
*adjSize = numAdj;
ierr = ISRestoreIndices(aIS,&anchors);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/* This assumes that the DM has been cloned prior to the call */
PetscErrorCode DMCreateSubDM_Section_Private(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
{
PetscSection section, sectionGlobal;
PetscInt *subIndices;
PetscInt subSize = 0, subOff = 0, nF, f, pStart, pEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!numFields) PetscFunctionReturn(0);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = DMGetDefaultGlobalSection(dm, §ionGlobal);CHKERRQ(ierr);
if (!section) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting fields");
if (!sectionGlobal) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default global section for DM before splitting fields");
ierr = PetscSectionGetNumFields(section, &nF);CHKERRQ(ierr);
if (numFields > nF) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Number of requested fields %d greater than number of DM fields %d", numFields, nF);
if (is) {
ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof;
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
subSize += fdof-fcdof;
}
}
}
ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof, goff;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof, fc, f2, poff = 0;
/* Can get rid of this loop by storing field information in the global section */
for (f2 = 0; f2 < fields[f]; ++f2) {
ierr = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
poff += fdof-fcdof;
}
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
subIndices[subOff] = goff+poff+fc;
}
}
}
}
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
}
if (subdm) {
PetscSection subsection;
PetscBool haveNull = PETSC_FALSE;
PetscInt f, nf = 0;
ierr = PetscSectionCreateSubsection(section, numFields, fields, &subsection);CHKERRQ(ierr);
ierr = DMSetDefaultSection(*subdm, subsection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
(*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[fields[f]];
if ((*subdm)->nullspaceConstructors[f]) {
haveNull = PETSC_TRUE;
nf = f;
}
}
if (haveNull && is) {
MatNullSpace nullSpace;
ierr = (*(*subdm)->nullspaceConstructors[nf])(*subdm, nf, &nullSpace);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
}
if (dm->prob) {
PetscInt Nf;
ierr = PetscDSGetNumFields(dm->prob, &Nf);CHKERRQ(ierr);
if (nF != Nf) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "The number of DM fields %d does not match the number of Section fields %d", Nf, nF);
ierr = DMSetNumFields(*subdm, numFields);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
PetscObject disc;
ierr = DMGetField(dm, fields[f], &disc);CHKERRQ(ierr);
ierr = DMSetField(*subdm, f, disc);CHKERRQ(ierr);
}
if (numFields == 1 && is) {
PetscObject disc, space, pmat;
ierr = DMGetField(*subdm, 0, &disc);CHKERRQ(ierr);
ierr = PetscObjectQuery(disc, "nullspace", &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nullspace", space);CHKERRQ(ierr);}
ierr = PetscObjectQuery(disc, "nearnullspace", &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nearnullspace", space);CHKERRQ(ierr);}
ierr = PetscObjectQuery(disc, "pmat", &pmat);CHKERRQ(ierr);
if (pmat) {ierr = PetscObjectCompose((PetscObject) *is, "pmat", pmat);CHKERRQ(ierr);}
}
}
}
#if 0
/* We need a way to filter the original SF for given fields:
- Keeping the original section around it too much I think
- We could keep the distributed section, and subset it
*/
if (dm->sfNatural) {
PetscSF sfNatural;
ierr = PetscSectionCreateSubsection(dm->originalSection, numFields, fields, &(*subdm)->originalSection);CHKERRQ(ierr);
ierr = DMPlexCreateGlobalToNaturalPetscSF(*subdm, &sfNatural);CHKERRQ(ierr);
ierr = DMPlexSetGlobalToNaturalPetscSF(*subdm, sfNatural);CHKERRQ(ierr);
}
#endif
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexVTKWriteSection_ASCII(DM dm, PetscSection section, PetscSection globalSection, Vec v, FILE *fp, PetscInt enforceDof, PetscInt precision, PetscReal scale)
{
MPI_Comm comm;
const MPI_Datatype mpiType = MPIU_SCALAR;
PetscScalar *array;
PetscInt numDof = 0, maxDof;
PetscInt numLabelCells, cellHeight, cMax, cStart, cEnd, numLabelVertices, vMax, vStart, vEnd, pStart, pEnd, p;
PetscMPIInt numProcs, rank, proc, tag;
PetscBool hasLabel;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
PetscValidHeaderSpecific(dm,DM_CLASSID,1);
PetscValidHeaderSpecific(v,VEC_CLASSID,4);
if (precision < 0) precision = 6;
ierr = PetscCommGetNewTag(comm, &tag);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
/* VTK only wants the values at cells or vertices */
ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, &vMax);CHKERRQ(ierr);
if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
if (vMax >= 0) vEnd = PetscMin(vEnd, vMax);
pStart = PetscMax(PetscMin(cStart, vStart), pStart);
pEnd = PetscMin(PetscMax(cEnd, vEnd), pEnd);
ierr = DMPlexGetStratumSize(dm, "vtk", 1, &numLabelCells);CHKERRQ(ierr);
ierr = DMPlexGetStratumSize(dm, "vtk", 2, &numLabelVertices);CHKERRQ(ierr);
hasLabel = numLabelCells > 0 || numLabelVertices > 0 ? PETSC_TRUE : PETSC_FALSE;
for (p = pStart; p < pEnd; ++p) {
/* Reject points not either cells or vertices */
if (((p < cStart) || (p >= cEnd)) && ((p < vStart) || (p >= vEnd))) continue;
if (hasLabel) {
PetscInt value;
if (((p >= cStart) && (p < cEnd) && numLabelCells) ||
((p >= vStart) && (p < vEnd) && numLabelVertices)) {
ierr = DMPlexGetLabelValue(dm, "vtk", p, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
}
ierr = PetscSectionGetDof(section, p, &numDof);CHKERRQ(ierr);
if (numDof) break;
}
ierr = MPI_Allreduce(&numDof, &maxDof, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
enforceDof = PetscMax(enforceDof, maxDof);
ierr = VecGetArray(v, &array);CHKERRQ(ierr);
if (!rank) {
char formatString[8];
ierr = PetscSNPrintf(formatString, 8, "%%.%de", precision);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
/* Here we lose a way to filter points by keeping them out of the Numbering */
PetscInt dof, off, goff, d;
/* Reject points not either cells or vertices */
if (((p < cStart) || (p >= cEnd)) && ((p < vStart) || (p >= vEnd))) continue;
if (hasLabel) {
PetscInt value;
if (((p >= cStart) && (p < cEnd) && numLabelCells) ||
((p >= vStart) && (p < vEnd) && numLabelVertices)) {
ierr = DMPlexGetLabelValue(dm, "vtk", p, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
}
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(globalSection, p, &goff);CHKERRQ(ierr);
if (dof && goff >= 0) {
for (d = 0; d < dof; d++) {
if (d > 0) {
ierr = PetscFPrintf(comm, fp, " ");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fp, formatString, PetscRealPart(array[off+d])*scale);CHKERRQ(ierr);
}
for (d = dof; d < enforceDof; d++) {
ierr = PetscFPrintf(comm, fp, " 0.0");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fp, "\n");CHKERRQ(ierr);
}
}
for (proc = 1; proc < numProcs; ++proc) {
PetscScalar *remoteValues;
PetscInt size = 0, d;
MPI_Status status;
ierr = MPI_Recv(&size, 1, MPIU_INT, proc, tag, comm, &status);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &remoteValues);CHKERRQ(ierr);
ierr = MPI_Recv(remoteValues, size, mpiType, proc, tag, comm, &status);CHKERRQ(ierr);
for (p = 0; p < size/maxDof; ++p) {
for (d = 0; d < maxDof; ++d) {
if (d > 0) {
ierr = PetscFPrintf(comm, fp, " ");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fp, formatString, PetscRealPart(remoteValues[p*maxDof+d])*scale);CHKERRQ(ierr);
}
for (d = maxDof; d < enforceDof; ++d) {
ierr = PetscFPrintf(comm, fp, " 0.0");CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fp, "\n");CHKERRQ(ierr);
}
ierr = PetscFree(remoteValues);CHKERRQ(ierr);
}
} else {
PetscScalar *localValues;
PetscInt size, k = 0;
ierr = PetscSectionGetStorageSize(section, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &localValues);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, goff, d;
/* Reject points not either cells or vertices */
if (((p < cStart) || (p >= cEnd)) && ((p < vStart) || (p >= vEnd))) continue;
if (hasLabel) {
PetscInt value;
if (((p >= cStart) && (p < cEnd) && numLabelCells) ||
((p >= vStart) && (p < vEnd) && numLabelVertices)) {
ierr = DMPlexGetLabelValue(dm, "vtk", p, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
}
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(globalSection, p, &goff);CHKERRQ(ierr);
if (goff >= 0) {
for (d = 0; d < dof; ++d) {
localValues[k++] = array[off+d];
}
}
}
ierr = MPI_Send(&k, 1, MPIU_INT, 0, tag, comm);CHKERRQ(ierr);
ierr = MPI_Send(localValues, k, mpiType, 0, tag, comm);CHKERRQ(ierr);
ierr = PetscFree(localValues);CHKERRQ(ierr);
}
ierr = VecRestoreArray(v, &array);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void test(DM mesh) {
const int dim = 2;
const int field_count = 1;
int components[field_count];
int dof[field_count*(dim+1)];
// Field 0 - temperature
components[0] = 1;
dof[0*(dim+1) + 0] = 1; // vertices
dof[0*(dim+1) + 1] = 0; // faces
// Field 1 - pressure
components[1] = 1;
dof[1*(dim+1) + 0] = 64; // vertices
dof[1*(dim+1) + 1] = 0; // faces
PetscSection section;
DMPlexCreateSection(mesh,
dim, // Spatial dimension
field_count, // Number of fields
components, // Component count for each field
dof, // DoF for each field component
0, // Number of boundary conditions
NULL, // BC Field
NULL, // BC Points
§ion);
DMSetDefaultSection(mesh, section);
// Temperature
// Create a global vector
Vec temperature = NULL;
DMCreateGlobalVector(mesh, &temperature);
// Get a temporary local vector & a pointer to its data
Vec localTemp = NULL;
DMGetLocalVector(mesh, &localTemp);
PetscScalar * data = NULL;
VecGetArray(localTemp, &data);
// Loop over points in the section
int start, end;
DMGetDefaultSection(mesh, §ion);
PetscSectionGetChart(section, &start, &end);
for (int point=start; point<end; ++point) {
// Get the size & offset of field 0
int dof, offset;
PetscSectionGetFieldDof(section, point, 0, &dof);
PetscSectionGetFieldOffset(section, point, 0, &offset);
// Set values
for (int d=0; d<dof; ++d){
data[offset+d] = point * 100 + d;
}
}
VecRestoreArray(localTemp, &data);
// Broadcast values
DMLocalToGlobalBegin(mesh, localTemp, INSERT_VALUES, temperature);
DMLocalToGlobalEnd(mesh, localTemp, INSERT_VALUES, temperature);
// Now each node should have all temperature values
// Free the temp vector
DMRestoreLocalVector(mesh,&localTemp);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelDistribute(DMLabel label, PetscSF sf, DMLabel *labelNew)
{
MPI_Comm comm;
PetscSection rootSection, leafSection;
PetscSF labelSF;
PetscInt p, pStart, pEnd, l, lStart, lEnd, s, nroots, nleaves, size, dof, offset, stratum;
PetscInt *remoteOffsets, *rootStrata, *rootIdx, *leafStrata, *strataIdx;
char *name;
PetscInt nameSize;
size_t len = 0;
PetscMPIInt rank, numProcs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (label) {ierr = DMLabelMakeAllValid_Private(label);CHKERRQ(ierr);}
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
/* Bcast name */
if (!rank) {ierr = PetscStrlen(label->name, &len);CHKERRQ(ierr);}
nameSize = len;
ierr = MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1(nameSize+1, &name);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy(name, label->name, nameSize+1);CHKERRQ(ierr);}
ierr = MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = DMLabelCreate(name, labelNew);CHKERRQ(ierr);
ierr = PetscFree(name);CHKERRQ(ierr);
/* Bcast numStrata */
if (!rank) (*labelNew)->numStrata = label->numStrata;
ierr = MPI_Bcast(&(*labelNew)->numStrata, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
/* Bcast stratumValues */
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->stratumValues);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy((*labelNew)->stratumValues, label->stratumValues, label->numStrata * sizeof(PetscInt));CHKERRQ(ierr);}
ierr = MPI_Bcast((*labelNew)->stratumValues, (*labelNew)->numStrata, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->arrayValid);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) (*labelNew)->arrayValid[s] = PETSC_TRUE;
/* Build a section detailing strata-per-point, distribute and build SF
from that and then send our points. */
ierr = PetscSFGetGraph(sf, &nroots, &nleaves, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSection, 0, nroots);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
lStart = 0;
lEnd = label->stratumSizes[s];
for (l=lStart; l<lEnd; l++) {
ierr = PetscSectionGetDof(rootSection, label->points[s][l], &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSection, label->points[s][l], dof+1);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSection);CHKERRQ(ierr);
/* Create a point-wise array of point strata */
ierr = PetscSectionGetStorageSize(rootSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &rootStrata);CHKERRQ(ierr);
ierr = PetscCalloc1(nroots, &rootIdx);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
lStart = 0;
lEnd = label->stratumSizes[s];
for (l=lStart; l<lEnd; l++) {
p = label->points[s][l];
ierr = PetscSectionGetOffset(rootSection, p, &offset);CHKERRQ(ierr);
rootStrata[offset+rootIdx[p]++] = s;
}
}
}
/* Build SF that maps label points to remote processes */
ierr = PetscSectionCreate(comm, &leafSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sf, rootSection, &remoteOffsets, leafSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, rootSection, remoteOffsets, leafSection, &labelSF);CHKERRQ(ierr);
/* Send the strata for each point over the derived SF */
ierr = PetscSectionGetStorageSize(leafSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(labelSF, MPIU_INT, rootStrata, leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(labelSF, MPIU_INT, rootStrata, leafStrata);CHKERRQ(ierr);
/* Rebuild the point strata on the receiver */
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->stratumSizes);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
(*labelNew)->stratumSizes[leafStrata[offset+s]]++;
}
}
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->ht);CHKERRQ(ierr);
ierr = PetscMalloc1((*labelNew)->numStrata,&(*labelNew)->points);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) {
PetscHashICreate((*labelNew)->ht[s]);
ierr = PetscMalloc1((*labelNew)->stratumSizes[s], &(*labelNew)->points[s]);CHKERRQ(ierr);
}
/* Insert points into new strata */
ierr = PetscCalloc1((*labelNew)->numStrata, &strataIdx);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
stratum = leafStrata[offset+s];
(*labelNew)->points[stratum][strataIdx[stratum]++] = p;
}
}
ierr = PetscFree(rootStrata);CHKERRQ(ierr);
ierr = PetscFree(leafStrata);CHKERRQ(ierr);
ierr = PetscFree(rootIdx);CHKERRQ(ierr);
ierr = PetscFree(strataIdx);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSection);CHKERRQ(ierr);
ierr = PetscSFDestroy(&labelSF);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexPermute - Reorder the mesh according to the input permutation
Collective on DM
Input Parameter:
+ dm - The DMPlex object
- perm - The point permutation, perm[old point number] = new point number
Output Parameter:
. pdm - The permuted DM
Level: intermediate
.keywords: mesh
.seealso: MatPermute()
@*/
PetscErrorCode DMPlexPermute(DM dm, IS perm, DM *pdm)
{
DM_Plex *plex = (DM_Plex *) dm->data, *plexNew;
PetscSection section, sectionNew;
PetscInt dim;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(perm, IS_CLASSID, 2);
PetscValidPointer(pdm, 3);
ierr = DMCreate(PetscObjectComm((PetscObject) dm), pdm);CHKERRQ(ierr);
ierr = DMSetType(*pdm, DMPLEX);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMSetDimension(*pdm, dim);CHKERRQ(ierr);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
if (section) {
ierr = PetscSectionPermute(section, perm, §ionNew);CHKERRQ(ierr);
ierr = DMSetDefaultSection(*pdm, sectionNew);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionNew);CHKERRQ(ierr);
}
plexNew = (DM_Plex *) (*pdm)->data;
/* Ignore ltogmap, ltogmapb */
/* Ignore sf, defaultSF */
/* Ignore globalVertexNumbers, globalCellNumbers */
/* Remap coordinates */
{
DM cdm, cdmNew;
PetscSection csection, csectionNew;
Vec coordinates, coordinatesNew;
PetscScalar *coords, *coordsNew;
const PetscInt *pperm;
PetscInt pStart, pEnd, p;
const char *name;
ierr = DMGetCoordinateDM(dm, &cdm);CHKERRQ(ierr);
ierr = DMGetDefaultSection(cdm, &csection);CHKERRQ(ierr);
ierr = PetscSectionPermute(csection, perm, &csectionNew);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = VecDuplicate(coordinates, &coordinatesNew);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)coordinatesNew,name);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
ierr = VecGetArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
ierr = PetscSectionGetChart(csectionNew, &pStart, &pEnd);CHKERRQ(ierr);
ierr = ISGetIndices(perm, &pperm);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(csectionNew, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(csectionNew, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) coordsNew[offNew+d] = coords[off+d];
}
ierr = ISRestoreIndices(perm, &pperm);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinatesNew, &coordsNew);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(*pdm, &cdmNew);CHKERRQ(ierr);
ierr = DMSetDefaultSection(cdmNew, csectionNew);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*pdm, coordinatesNew);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&csectionNew);CHKERRQ(ierr);
ierr = VecDestroy(&coordinatesNew);CHKERRQ(ierr);
}
/* Reorder labels */
{
PetscInt numLabels, l;
DMLabel label, labelNew;
ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
for (l = numLabels-1; l >= 0; --l) {
ierr = DMGetLabelByNum(dm, l, &label);CHKERRQ(ierr);
ierr = DMLabelPermute(label, perm, &labelNew);CHKERRQ(ierr);
ierr = DMAddLabel(*pdm, labelNew);CHKERRQ(ierr);
}
if (plex->subpointMap) {ierr = DMLabelPermute(plex->subpointMap, perm, &plexNew->subpointMap);CHKERRQ(ierr);}
}
/* Reorder topology */
{
const PetscInt *pperm;
PetscInt maxConeSize, maxSupportSize, n, pStart, pEnd, p;
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
plexNew->maxConeSize = maxConeSize;
plexNew->maxSupportSize = maxSupportSize;
ierr = PetscSectionDestroy(&plexNew->coneSection);CHKERRQ(ierr);
ierr = PetscSectionPermute(plex->coneSection, perm, &plexNew->coneSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(plexNew->coneSection, &n);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->cones);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->coneOrientations);CHKERRQ(ierr);
ierr = ISGetIndices(perm, &pperm);CHKERRQ(ierr);
ierr = PetscSectionGetChart(plex->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(plexNew->coneSection, pperm[p], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plex->coneSection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plexNew->coneSection, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
plexNew->cones[offNew+d] = pperm[plex->cones[off+d]];
plexNew->coneOrientations[offNew+d] = plex->coneOrientations[off+d];
}
}
ierr = PetscSectionDestroy(&plexNew->supportSection);CHKERRQ(ierr);
ierr = PetscSectionPermute(plex->supportSection, perm, &plexNew->supportSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(plexNew->supportSection, &n);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &plexNew->supports);CHKERRQ(ierr);
ierr = PetscSectionGetChart(plex->supportSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, off, offNew, d;
ierr = PetscSectionGetDof(plexNew->supportSection, pperm[p], &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plex->supportSection, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(plexNew->supportSection, pperm[p], &offNew);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
plexNew->supports[offNew+d] = pperm[plex->supports[off+d]];
}
}
ierr = ISRestoreIndices(perm, &pperm);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexPreallocateOperator(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix)
{
MPI_Comm comm;
MatType mtype;
PetscSF sf, sfDof, sfAdj;
PetscSection leafSectionAdj, rootSectionAdj, sectionAdj, anchorSectionAdj;
PetscInt nroots, nleaves, l, p;
const PetscInt *leaves;
const PetscSFNode *remotes;
PetscInt dim, pStart, pEnd, numDof, globalOffStart, globalOffEnd, numCols;
PetscInt *tmpAdj = NULL, *adj, *rootAdj, *anchorAdj = NULL, *cols, *remoteOffsets;
PetscInt adjSize;
PetscLayout rLayout;
PetscInt locRows, rStart, rEnd, r;
PetscMPIInt size;
PetscBool doCommLocal, doComm, debug = PETSC_FALSE, isSymBlock, isSymSeqBlock, isSymMPIBlock;
PetscBool useAnchors;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 3);
PetscValidHeaderSpecific(sectionGlobal, PETSC_SECTION_CLASSID, 4);
PetscValidHeaderSpecific(A, MAT_CLASSID, 9);
if (dnz) PetscValidPointer(dnz,5);
if (onz) PetscValidPointer(onz,6);
if (dnzu) PetscValidPointer(dnzu,7);
if (onzu) PetscValidPointer(onzu,8);
ierr = PetscLogEventBegin(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL, "-dm_view_preallocation", &debug, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
doCommLocal = (size > 1) && (nroots >= 0) ? PETSC_TRUE : PETSC_FALSE;
ierr = MPI_Allreduce(&doCommLocal, &doComm, 1, MPIU_BOOL, MPI_LAND, comm);CHKERRQ(ierr);
/* Create dof SF based on point SF */
if (debug) {
ierr = PetscPrintf(comm, "Input Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input Global Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionGlobal, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sf, NULL);CHKERRQ(ierr);
}
ierr = PetscSFCreateRemoteOffsets(sf, section, section, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, section, remoteOffsets, section, &sfDof);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Dof SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfDof, NULL);CHKERRQ(ierr);
}
/* Create section for dof adjacency (dof ==> # adj dof) */
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(section, &numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(leafSectionAdj, 0, numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSectionAdj, 0, numDof);CHKERRQ(ierr);
/* Fill in the ghost dofs on the interface */
ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, &remotes);CHKERRQ(ierr);
/* use constraints in finding adjacency in this routine */
ierr = DMPlexGetAdjacencyUseAnchors(dm,&useAnchors);CHKERRQ(ierr);
ierr = DMPlexSetAdjacencyUseAnchors(dm,PETSC_TRUE);CHKERRQ(ierr);
/*
section - maps points to (# dofs, local dofs)
sectionGlobal - maps points to (# dofs, global dofs)
leafSectionAdj - maps unowned local dofs to # adj dofs
rootSectionAdj - maps owned local dofs to # adj dofs
adj - adj global dofs indexed by leafSectionAdj
rootAdj - adj global dofs indexed by rootSectionAdj
sf - describes shared points across procs
sfDof - describes shared dofs across procs
sfAdj - describes shared adjacent dofs across procs
** The bootstrapping process involves six rounds with similar structure of visiting neighbors of each point.
(0). If there are point-to-point constraints, add the adjacencies of constrained points to anchors in anchorAdj
(This is done in DMPlexComputeAnchorAdjacencies())
1. Visit unowned points on interface, count adjacencies placing in leafSectionAdj
Reduce those counts to rootSectionAdj (now redundantly counting some interface points)
2. Visit owned points on interface, count adjacencies placing in rootSectionAdj
Create sfAdj connecting rootSectionAdj and leafSectionAdj
3. Visit unowned points on interface, write adjacencies to adj
Gather adj to rootAdj (note that there is redundancy in rootAdj when multiple procs find the same adjacencies)
4. Visit owned points on interface, write adjacencies to rootAdj
Remove redundancy in rootAdj
** The last two traversals use transitive closure
5. Visit all owned points in the subdomain, count dofs for each point (sectionAdj)
Allocate memory addressed by sectionAdj (cols)
6. Visit all owned points in the subdomain, insert dof adjacencies into cols
** Knowing all the column adjacencies, check ownership and sum into dnz and onz
*/
ierr = DMPlexComputeAnchorAdjacencies(dm,section,sectionGlobal,&anchorSectionAdj,&anchorAdj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation on Leaves:\n");CHKERRQ(ierr);
ierr = PetscSectionView(leafSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get maximum remote adjacency sizes for owned dofs on interface (roots) */
if (doComm) {
ierr = PetscSFReduceBegin(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
}
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Add in local adjacency sizes for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after local additions:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create adj SF based on dof SF */
ierr = PetscSFCreateRemoteOffsets(sfDof, rootSectionAdj, leafSectionAdj, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfDof, rootSectionAdj, remoteOffsets, leafSectionAdj, &sfAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfAdj, NULL);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfDof);CHKERRQ(ierr);
/* Create leaf adjacency */
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(leafSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscCalloc1(adjSize, &adj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof, anOff;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt aoff, i = 0;
ierr = PetscSectionGetOffset(leafSectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
adj[aoff+i] = (ngoff < 0 ? -(ngoff+1) : ngoff) + nd;
++i;
}
}
for (q = 0; q < anDof; q++) {
adj[aoff+i] = anchorAdj[anOff+q];
++i;
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Leaf adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, adj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Gather adjacenct indices to root */
ierr = PetscSectionGetStorageSize(rootSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize, &rootAdj);CHKERRQ(ierr);
for (r = 0; r < adjSize; ++r) rootAdj[r] = -1;
if (doComm) {
ierr = PetscSFGatherBegin(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
ierr = PetscSFGatherEnd(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfAdj);CHKERRQ(ierr);
ierr = PetscFree(adj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices after gather\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Add in local adjacency indices for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof, anOff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt adof, aoff, i;
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
i = adof-1;
for (q = 0; q < anDof; q++) {
rootAdj[aoff+i] = anchorAdj[anOff+q];
--i;
}
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
rootAdj[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
--i;
}
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Compress indices */
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, cdof, off, d;
PetscInt adof, aoff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
for (d = off; d < off+dof-cdof; ++d) {
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
ierr = PetscSortRemoveDupsInt(&adof, &rootAdj[aoff]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSectionAdj, d, adof);CHKERRQ(ierr);
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after compression:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Root adjacency indices after compression\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Build adjacency section: Maps global indices to sets of adjacent global indices */
ierr = PetscSectionGetOffsetRange(sectionGlobal, &globalOffStart, &globalOffEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, §ionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionAdj, globalOffStart, globalOffEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
ierr = PetscSectionSetDof(sectionAdj, goff+d, rdof);CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, noff;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, padj, &noff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(sectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get adjacent indices */
ierr = PetscSectionGetStorageSize(sectionAdj, &numCols);CHKERRQ(ierr);
ierr = PetscMalloc1(numCols, &cols);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof, anOff;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
PetscInt aoff, roff;
ierr = PetscSectionGetOffset(sectionAdj, goff+d, &aoff);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, off+d, &roff);CHKERRQ(ierr);
ierr = PetscMemcpy(&cols[aoff], &rootAdj[roff], rdof * sizeof(PetscInt));CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
PetscInt adof, aoff, i = 0;
ierr = PetscSectionGetDof(sectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
const PetscInt *ncind;
/* Adjacent points may not be in the section chart */
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintIndices(section, padj, &ncind);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd, ++i) {
cols[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
}
}
for (q = 0; q < anDof; q++, i++) {
cols[aoff+i] = anchorAdj[anOff + q];
}
if (i != adof) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of entries %D != %D for dof %D (point %D)", i, adof, d, p);
}
}
ierr = PetscSectionDestroy(&anchorSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSectionAdj);CHKERRQ(ierr);
ierr = PetscFree(anchorAdj);CHKERRQ(ierr);
ierr = PetscFree(rootAdj);CHKERRQ(ierr);
ierr = PetscFree(tmpAdj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Column indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, numCols, cols, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Create allocation vectors from adjacency graph */
ierr = MatGetLocalSize(A, &locRows, NULL);CHKERRQ(ierr);
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)A), &rLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr);
/* Only loop over blocks of rows */
if (rStart%bs || rEnd%bs) SETERRQ3(PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Invalid layout [%d, %d) for matrix, must be divisible by block size %d", rStart, rEnd, bs);
for (r = rStart/bs; r < rEnd/bs; ++r) {
const PetscInt row = r*bs;
PetscInt numCols, cStart, c;
ierr = PetscSectionGetDof(sectionAdj, row, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, row, &cStart);CHKERRQ(ierr);
for (c = cStart; c < cStart+numCols; ++c) {
if ((cols[c] >= rStart*bs) && (cols[c] < rEnd*bs)) {
++dnz[r-rStart];
if (cols[c] >= row) ++dnzu[r-rStart];
} else {
++onz[r-rStart];
if (cols[c] >= row) ++onzu[r-rStart];
}
}
}
if (bs > 1) {
for (r = 0; r < locRows/bs; ++r) {
dnz[r] /= bs;
onz[r] /= bs;
dnzu[r] /= bs;
onzu[r] /= bs;
}
}
/* Set matrix pattern */
ierr = MatXAIJSetPreallocation(A, bs, dnz, onz, dnzu, onzu);CHKERRQ(ierr);
ierr = MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
/* Check for symmetric storage */
ierr = MatGetType(A, &mtype);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
if (isSymBlock || isSymSeqBlock || isSymMPIBlock) {ierr = MatSetOption(A, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);CHKERRQ(ierr);}
/* Fill matrix with zeros */
if (fillMatrix) {
PetscScalar *values;
PetscInt maxRowLen = 0;
for (r = rStart; r < rEnd; ++r) {
PetscInt len;
ierr = PetscSectionGetDof(sectionAdj, r, &len);CHKERRQ(ierr);
maxRowLen = PetscMax(maxRowLen, len);
}
ierr = PetscCalloc1(maxRowLen, &values);CHKERRQ(ierr);
for (r = rStart; r < rEnd; ++r) {
PetscInt numCols, cStart;
ierr = PetscSectionGetDof(sectionAdj, r, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, r, &cStart);CHKERRQ(ierr);
ierr = MatSetValues(A, 1, &r, numCols, &cols[cStart], values, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree(values);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* restore original useAnchors */
ierr = DMPlexSetAdjacencyUseAnchors(dm,useAnchors);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionAdj);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
