/* This assumes that the DM has been cloned prior to the call */
PetscErrorCode DMCreateSubDM_Section_Private(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
{
PetscSection section, sectionGlobal;
PetscInt *subIndices;
PetscInt subSize = 0, subOff = 0, nF, f, pStart, pEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!numFields) PetscFunctionReturn(0);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = DMGetDefaultGlobalSection(dm, §ionGlobal);CHKERRQ(ierr);
if (!section) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting fields");
if (!sectionGlobal) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default global section for DM before splitting fields");
ierr = PetscSectionGetNumFields(section, &nF);CHKERRQ(ierr);
if (numFields > nF) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Number of requested fields %d greater than number of DM fields %d", numFields, nF);
if (is) {
ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof;
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
subSize += fdof-fcdof;
}
}
}
ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof, goff;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof, fc, f2, poff = 0;
/* Can get rid of this loop by storing field information in the global section */
for (f2 = 0; f2 < fields[f]; ++f2) {
ierr = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
poff += fdof-fcdof;
}
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
subIndices[subOff] = goff+poff+fc;
}
}
}
}
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
}
if (subdm) {
PetscSection subsection;
PetscBool haveNull = PETSC_FALSE;
PetscInt f, nf = 0;
ierr = PetscSectionCreateSubsection(section, numFields, fields, &subsection);CHKERRQ(ierr);
ierr = DMSetDefaultSection(*subdm, subsection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
(*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[fields[f]];
if ((*subdm)->nullspaceConstructors[f]) {
haveNull = PETSC_TRUE;
nf = f;
}
}
if (haveNull && is) {
MatNullSpace nullSpace;
ierr = (*(*subdm)->nullspaceConstructors[nf])(*subdm, nf, &nullSpace);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
}
if (dm->prob) {
PetscInt Nf;
ierr = PetscDSGetNumFields(dm->prob, &Nf);CHKERRQ(ierr);
if (nF != Nf) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "The number of DM fields %d does not match the number of Section fields %d", Nf, nF);
ierr = DMSetNumFields(*subdm, numFields);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
PetscObject disc;
ierr = DMGetField(dm, fields[f], &disc);CHKERRQ(ierr);
ierr = DMSetField(*subdm, f, disc);CHKERRQ(ierr);
}
if (numFields == 1 && is) {
PetscObject disc, space, pmat;
ierr = DMGetField(*subdm, 0, &disc);CHKERRQ(ierr);
ierr = PetscObjectQuery(disc, "nullspace", &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nullspace", space);CHKERRQ(ierr);}
ierr = PetscObjectQuery(disc, "nearnullspace", &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nearnullspace", space);CHKERRQ(ierr);}
ierr = PetscObjectQuery(disc, "pmat", &pmat);CHKERRQ(ierr);
if (pmat) {ierr = PetscObjectCompose((PetscObject) *is, "pmat", pmat);CHKERRQ(ierr);}
}
}
}
#if 0
/* We need a way to filter the original SF for given fields:
- Keeping the original section around it too much I think
- We could keep the distributed section, and subset it
*/
if (dm->sfNatural) {
PetscSF sfNatural;
ierr = PetscSectionCreateSubsection(dm->originalSection, numFields, fields, &(*subdm)->originalSection);CHKERRQ(ierr);
ierr = DMPlexCreateGlobalToNaturalPetscSF(*subdm, &sfNatural);CHKERRQ(ierr);
ierr = DMPlexSetGlobalToNaturalPetscSF(*subdm, sfNatural);CHKERRQ(ierr);
}
#endif
PetscFunctionReturn(0);
}
//! Help PETSc identify the coarser DM dmc given the fine DM dmf
PetscErrorCode libmesh_petsc_DMCoarsen(DM dmf, MPI_Comm /*comm*/, DM * dmc)
{
libmesh_assert(dmc);
libmesh_assert(dmf);
PetscErrorCode ierr;
// Extract our context from the incoming dmf
void * ctx_f = nullptr;
ierr = DMShellGetContext(dmf, &ctx_f);LIBMESH_CHKERR(ierr);
libmesh_assert(ctx_f);
PetscDMContext * p_ctx = static_cast<PetscDMContext*>(ctx_f);
// First, ensure that there exists a coarse DM that we want to
// set. There ought to be as we created it while walking the
// hierarchy.
libmesh_assert(p_ctx->coarser_dm);
libmesh_assert(*(p_ctx->coarser_dm));
// In situations using fieldsplit we need to (potentially)
// provide a coarser DM which only has the relevant subfields in
// it. Since we create global DMs for each mesh level, we need
// to extract the section from the DM, and check the number of
// fields. When less than all the fields are used, we need to
// create the proper subsections.
// Get the number of fields and their names from the incomming
// fine DM and the global reference DM
PetscInt nfieldsf, nfieldsg;
char ** fieldnamesf;
char ** fieldnamesg;
libmesh_assert(p_ctx->global_dm);
DM * globaldm = p_ctx->global_dm;
ierr = DMCreateFieldIS(dmf, &nfieldsf, &fieldnamesf, nullptr);
LIBMESH_CHKERR(ierr);
ierr = DMCreateFieldIS(*globaldm, &nfieldsg, &fieldnamesg, nullptr);
LIBMESH_CHKERR(ierr);
// If the probed number of fields is less than the number of
// global fields, this amounts to PETSc 'indicating' to us we
// are doing FS. So, we must create subsections for the coarser
// DMs.
if ( nfieldsf < nfieldsg )
{
PetscSection section;
PetscSection subsection;
std::vector<PetscInt> subfields(nfieldsf); // extracted fields
// First, get the section from the coarse DM
#if PETSC_VERSION_LESS_THAN(3,10,0)
ierr = DMGetDefaultSection(*(p_ctx->coarser_dm), §ion);
#else
ierr = DMGetSection(*(p_ctx->coarser_dm), §ion);
#endif
LIBMESH_CHKERR(ierr);
// Now, match fine grid DM field names to their global DM
// counterparts. Since PETSc can internally reassign field
// numbering under a fieldsplit, we must extract
// subsections via the field names. This is admittedly
// gross, but c'est la vie.
for (int i = 0; i < nfieldsf ; i++)
{
for (int j = 0; j < nfieldsg ;j++)
if ( strcmp( fieldnamesg[j], fieldnamesf[i] ) == 0 )
subfields[i] = j;
}
// Next, for the found fields we now make a subsection and set it for the coarser DM
ierr = PetscSectionCreateSubsection(section, nfieldsf, subfields.data(), &subsection);
LIBMESH_CHKERR(ierr);
#if PETSC_VERSION_LESS_THAN(3,10,0)
ierr = DMSetDefaultSection(*(p_ctx->coarser_dm) , subsection);
#else
ierr = DMSetSection(*(p_ctx->coarser_dm) , subsection);
#endif
LIBMESH_CHKERR(ierr);
ierr = PetscSectionDestroy(&subsection);
LIBMESH_CHKERR(ierr);
}
// Finally, set the coarser DM
*(dmc) = *(p_ctx->coarser_dm);
return 0;
}
/* This assumes that the DM has been cloned prior to the call */
PetscErrorCode DMCreateSubDM_Section_Private(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm)
{
PetscSection section, sectionGlobal;
PetscInt *subIndices;
PetscInt subSize = 0, subOff = 0, nF, f, pStart, pEnd, p;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!numFields) PetscFunctionReturn(0);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = DMGetDefaultGlobalSection(dm, §ionGlobal);CHKERRQ(ierr);
if (!section) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting fields");
if (!sectionGlobal) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default global section for DM before splitting fields");
ierr = PetscSectionGetNumFields(section, &nF);CHKERRQ(ierr);
if (numFields > nF) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Number of requested fields %d greater than number of DM fields %d", numFields, nF);
if (is) {
ierr = PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof;
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
subSize += fdof-fcdof;
}
}
}
ierr = PetscMalloc1(subSize, &subIndices);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt gdof, goff;
ierr = PetscSectionGetDof(sectionGlobal, p, &gdof);CHKERRQ(ierr);
if (gdof > 0) {
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
PetscInt fdof, fcdof, fc, f2, poff = 0;
/* Can get rid of this loop by storing field information in the global section */
for (f2 = 0; f2 < fields[f]; ++f2) {
ierr = PetscSectionGetFieldDof(section, p, f2, &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof);CHKERRQ(ierr);
poff += fdof-fcdof;
}
ierr = PetscSectionGetFieldDof(section, p, fields[f], &fdof);CHKERRQ(ierr);
ierr = PetscSectionGetFieldConstraintDof(section, p, fields[f], &fcdof);CHKERRQ(ierr);
for (fc = 0; fc < fdof-fcdof; ++fc, ++subOff) {
subIndices[subOff] = goff+poff+fc;
}
}
}
}
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is);CHKERRQ(ierr);
}
if (subdm) {
PetscSection subsection;
PetscBool haveNull = PETSC_FALSE;
PetscInt f, nf = 0;
ierr = PetscSectionCreateSubsection(section, numFields, fields, &subsection);CHKERRQ(ierr);
ierr = DMSetDefaultSection(*subdm, subsection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&subsection);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
(*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[fields[f]];
if ((*subdm)->nullspaceConstructors[f]) {
haveNull = PETSC_TRUE;
nf = f;
}
}
if (haveNull) {
MatNullSpace nullSpace;
ierr = (*(*subdm)->nullspaceConstructors[nf])(*subdm, nf, &nullSpace);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) nullSpace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullSpace);CHKERRQ(ierr);
}
if (dm->fields) {
if (nF != dm->numFields) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "The number of DM fields %d does not match the number of Section fields %d", dm->numFields, nF);
ierr = DMSetNumFields(*subdm, numFields);CHKERRQ(ierr);
for (f = 0; f < numFields; ++f) {
ierr = PetscObjectListDuplicate(dm->fields[fields[f]]->olist, &(*subdm)->fields[f]->olist);CHKERRQ(ierr);
}
if (numFields == 1) {
MatNullSpace space;
Mat pmat;
ierr = PetscObjectQuery((*subdm)->fields[0], "nullspace", (PetscObject*) &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nullspace", (PetscObject) space);CHKERRQ(ierr);}
ierr = PetscObjectQuery((*subdm)->fields[0], "nearnullspace", (PetscObject*) &space);CHKERRQ(ierr);
if (space) {ierr = PetscObjectCompose((PetscObject) *is, "nearnullspace", (PetscObject) space);CHKERRQ(ierr);}
ierr = PetscObjectQuery((*subdm)->fields[0], "pmat", (PetscObject*) &pmat);CHKERRQ(ierr);
if (pmat) {ierr = PetscObjectCompose((PetscObject) *is, "pmat", (PetscObject) pmat);CHKERRQ(ierr);}
}
}
}
PetscFunctionReturn(0);
}
