static PetscErrorCode CreateInterpolation(DM dm1,DM dm2,Mat *mat,Vec *vec)
{
DM da1,da2;
PetscErrorCode ierr;
ierr = DMShellGetContext(dm1,(void**)&da1);CHKERRQ(ierr);
ierr = DMShellGetContext(dm2,(void**)&da2);CHKERRQ(ierr);
ierr = DMCreateInterpolation(da1,da2,mat,vec);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode CreateRestriction(DM dm1,DM dm2,Mat *mat)
{
DM da1,da2;
PetscErrorCode ierr;
Mat tmat;
ierr = DMShellGetContext(dm1,(void**)&da1);CHKERRQ(ierr);
ierr = DMShellGetContext(dm2,(void**)&da2);CHKERRQ(ierr);
ierr = DMCreateInterpolation(da1,da2,&tmat,NULL);CHKERRQ(ierr);
ierr = MatTranspose(tmat,MAT_INITIAL_MATRIX,mat);CHKERRQ(ierr);
ierr = MatDestroy(&tmat);CHKERRQ(ierr);
return 0;
}
//! Function to give PETSc that sets the Restriction Matrix between two DMs
PetscErrorCode
libmesh_petsc_DMCreateRestriction (DM dmc /*coarse*/, DM dmf/*fine*/, Mat * mat)
{
libmesh_assert(dmc);
libmesh_assert(dmf);
libmesh_assert(mat);
PetscErrorCode ierr;
// get a communicator from incomming DM
MPI_Comm comm;
PetscObjectGetComm((PetscObject)dmc, &comm);
// extract our fine context from the incoming DM
void * ctx_f = nullptr;
ierr = DMShellGetContext(dmf, &ctx_f);LIBMESH_CHKERR(ierr);
libmesh_assert(ctx_f);
PetscDMContext * p_ctx_f = static_cast<PetscDMContext*>(ctx_f);
// check / give PETSc its matrix
libmesh_assert(p_ctx_f->K_restrict_ptr);
*(mat) = p_ctx_f->K_restrict_ptr->mat();
return 0;
}
static PetscErrorCode ComputeMatrix(KSP ksp,Mat J,Mat jac,void *ctx)
{
PetscErrorCode ierr;
PetscInt i,mx,xm,xs;
PetscScalar v[3],h;
MatStencil row,col[3];
DM da,shell;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&shell);CHKERRQ(ierr);
ierr = DMShellGetContext(shell,(void**)&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xs,0,0,&xm,0,0);CHKERRQ(ierr);
h = 1.0/(mx-1);
for (i=xs; i<xs+xm; i++) {
row.i = i;
if (i==0 || i==mx-1) {
v[0] = 2.0/h;
ierr = MatSetValuesStencil(jac,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
} else {
v[0] = (-1.0)/h;col[0].i = i-1;
v[1] = (2.0)/h;col[1].i = row.i;
v[2] = (-1.0)/h;col[2].i = i+1;
ierr = MatSetValuesStencil(jac,1,&row,3,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode CreateMatrix(DM shell,Mat *A)
{
PetscErrorCode ierr;
DM da;
ierr = DMShellGetContext(shell,(void**)&da);CHKERRQ(ierr);
ierr = DMCreateMatrix(da,A);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode Refine(DM shell,MPI_Comm comm,DM *dmnew)
{
PetscErrorCode ierr;
DM da,dafine;
ierr = DMShellGetContext(shell,(void**)&da);CHKERRQ(ierr);
ierr = DMRefine(da,comm,&dafine);CHKERRQ(ierr);
ierr = MyDMShellCreate(PetscObjectComm((PetscObject)shell),dafine,dmnew);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode CreateLocalVector(DM shell,Vec *x)
{
PetscErrorCode ierr;
DM da;
ierr = DMShellGetContext(shell,(void**)&da);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,x);CHKERRQ(ierr);
ierr = VecSetDM(*x,shell);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode Coarsen(DM shell,MPI_Comm comm,DM *dmnew)
{
PetscErrorCode ierr;
DM da,dacoarse;
ierr = DMShellGetContext(shell,(void**)&da);CHKERRQ(ierr);
ierr = DMCoarsen(da,comm,&dacoarse);CHKERRQ(ierr);
ierr = MyDMShellCreate(PetscObjectComm((PetscObject)shell),dacoarse,dmnew);CHKERRQ(ierr);
/* discard an "extra" reference count to dacoarse */
ierr = DMDestroy(&dacoarse);CHKERRQ(ierr);
return 0;
}
//! Help PETSc identify the finer DM given a dmc
PetscErrorCode libmesh_petsc_DMRefine(DM dmc, MPI_Comm /*comm*/, DM * dmf)
{
libmesh_assert(dmc);
libmesh_assert(dmf);
PetscErrorCode ierr;
// extract our context from the incoming dmc
void * ctx_c = nullptr;
ierr = DMShellGetContext(dmc, & ctx_c);LIBMESH_CHKERR(ierr);
libmesh_assert(ctx_c);
PetscDMContext * p_ctx = static_cast<PetscDMContext * >(ctx_c);
// check / set the finer DM
libmesh_assert(p_ctx->finer_dm);
libmesh_assert(*(p_ctx->finer_dm));
*(dmf) = *(p_ctx->finer_dm);
return 0;
}
static PetscErrorCode ComputeRHS(KSP ksp,Vec b,void *ctx)
{
PetscErrorCode ierr;
PetscInt mx,idx[2];
PetscScalar h,v[2];
DM da,shell;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&shell);CHKERRQ(ierr);
ierr = DMShellGetContext(shell,(void**)&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
h = 1.0/((mx-1));
ierr = VecSet(b,h);CHKERRQ(ierr);
idx[0] = 0; idx[1] = mx -1;
v[0] = v[1] = 0.0;
ierr = VecSetValues(b,2,idx,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
//! Function to give PETSc that sets the Interpolation Matrix between two DMs
PetscErrorCode
libmesh_petsc_DMCreateInterpolation (DM dmc /*coarse*/, DM dmf /*fine*/,
Mat * mat ,Vec * vec)
{
libmesh_assert(dmc);
libmesh_assert(dmf);
libmesh_assert(mat);
libmesh_assert(vec); // Optional scaling (not needed for mg)
// Get a communicator from incoming DM
PetscErrorCode ierr;
MPI_Comm comm;
PetscObjectGetComm((PetscObject)dmc, &comm);
// Extract our coarse context from the incoming DM
void * ctx_c = nullptr;
ierr = DMShellGetContext(dmc, &ctx_c);
LIBMESH_CHKERR(ierr);
libmesh_assert(ctx_c);
PetscDMContext * p_ctx_c = static_cast<PetscDMContext*>(ctx_c);
// Extract our fine context from the incoming DM
void * ctx_f = nullptr;
ierr = DMShellGetContext(dmf, &ctx_f);LIBMESH_CHKERR(ierr);
libmesh_assert(ctx_f);
PetscDMContext * p_ctx_f = static_cast<PetscDMContext*>(ctx_f);
// Check for existing projection matrix
libmesh_assert(p_ctx_c->K_interp_ptr);
libmesh_assert(p_ctx_c->K_sub_interp_ptr);
// If were doing fieldsplit we need to construct sub projection
// matrices. We compare the passed in number of DMs fields to a
// global DM in order to determine if a subprojection is needed.
PetscInt nfieldsc,nfieldsf, nfieldsg;
libmesh_assert(p_ctx_c->global_dm);
DM * globaldm = p_ctx_c->global_dm;
ierr = DMCreateFieldIS(dmc, &nfieldsc, nullptr, nullptr);
LIBMESH_CHKERR(ierr);
ierr = DMCreateFieldIS(dmf, &nfieldsf, nullptr, nullptr);
LIBMESH_CHKERR(ierr);
ierr = DMCreateFieldIS(*globaldm, &nfieldsg, nullptr, nullptr);
LIBMESH_CHKERR(ierr);
// If subfields are identified, were doing FS so we need to create the subProjectionMatrix
if (nfieldsc < nfieldsg)
{
// Loop over the fields and merge their index sets.
std::vector<std::vector<numeric_index_type>> allrows,allcols;
std::vector<numeric_index_type> rows,cols;
allrows = p_ctx_f->dof_vec;
allcols = p_ctx_c->dof_vec;
// For internal libmesh submat extraction need to merge all
// field dofs and then sort the vectors so that they match
// the Projection Matrix ordering
const int n_subfields = nfieldsc;
if ( n_subfields > 1 )
{
for (int i = 0 ; i < n_subfields ; i++)
{
rows.insert(rows.end(), allrows[i].begin(), allrows[i].end());
cols.insert(cols.end(), allcols[i].begin(), allcols[i].end());
}
std::sort(rows.begin(),rows.end());
std::sort(cols.begin(),cols.end());
}
// Now that we have merged the fine and coarse index sets
// were ready to make the submatrix and pass it off to PETSc
p_ctx_c->K_interp_ptr->create_submatrix (*p_ctx_c->K_sub_interp_ptr, rows, cols);
*(mat) = p_ctx_c->K_sub_interp_ptr->mat();
}
else // We are not doing fieldsplit, so return entire projection
*(mat) = p_ctx_c->K_interp_ptr->mat();
// Vec scaling isnt needed so were done.
*(vec) = PETSC_NULL;
return 0;
} // end libmesh_petsc_DMCreateInterpolation
//! Help PETSc identify the coarser DM dmc given the fine DM dmf
PetscErrorCode libmesh_petsc_DMCoarsen(DM dmf, MPI_Comm /*comm*/, DM * dmc)
{
libmesh_assert(dmc);
libmesh_assert(dmf);
PetscErrorCode ierr;
// Extract our context from the incoming dmf
void * ctx_f = nullptr;
ierr = DMShellGetContext(dmf, &ctx_f);LIBMESH_CHKERR(ierr);
libmesh_assert(ctx_f);
PetscDMContext * p_ctx = static_cast<PetscDMContext*>(ctx_f);
// First, ensure that there exists a coarse DM that we want to
// set. There ought to be as we created it while walking the
// hierarchy.
libmesh_assert(p_ctx->coarser_dm);
libmesh_assert(*(p_ctx->coarser_dm));
// In situations using fieldsplit we need to (potentially)
// provide a coarser DM which only has the relevant subfields in
// it. Since we create global DMs for each mesh level, we need
// to extract the section from the DM, and check the number of
// fields. When less than all the fields are used, we need to
// create the proper subsections.
// Get the number of fields and their names from the incomming
// fine DM and the global reference DM
PetscInt nfieldsf, nfieldsg;
char ** fieldnamesf;
char ** fieldnamesg;
libmesh_assert(p_ctx->global_dm);
DM * globaldm = p_ctx->global_dm;
ierr = DMCreateFieldIS(dmf, &nfieldsf, &fieldnamesf, nullptr);
LIBMESH_CHKERR(ierr);
ierr = DMCreateFieldIS(*globaldm, &nfieldsg, &fieldnamesg, nullptr);
LIBMESH_CHKERR(ierr);
// If the probed number of fields is less than the number of
// global fields, this amounts to PETSc 'indicating' to us we
// are doing FS. So, we must create subsections for the coarser
// DMs.
if ( nfieldsf < nfieldsg )
{
PetscSection section;
PetscSection subsection;
std::vector<PetscInt> subfields(nfieldsf); // extracted fields
// First, get the section from the coarse DM
#if PETSC_VERSION_LESS_THAN(3,10,0)
ierr = DMGetDefaultSection(*(p_ctx->coarser_dm), §ion);
#else
ierr = DMGetSection(*(p_ctx->coarser_dm), §ion);
#endif
LIBMESH_CHKERR(ierr);
// Now, match fine grid DM field names to their global DM
// counterparts. Since PETSc can internally reassign field
// numbering under a fieldsplit, we must extract
// subsections via the field names. This is admittedly
// gross, but c'est la vie.
for (int i = 0; i < nfieldsf ; i++)
{
for (int j = 0; j < nfieldsg ;j++)
if ( strcmp( fieldnamesg[j], fieldnamesf[i] ) == 0 )
subfields[i] = j;
}
// Next, for the found fields we now make a subsection and set it for the coarser DM
ierr = PetscSectionCreateSubsection(section, nfieldsf, subfields.data(), &subsection);
LIBMESH_CHKERR(ierr);
#if PETSC_VERSION_LESS_THAN(3,10,0)
ierr = DMSetDefaultSection(*(p_ctx->coarser_dm) , subsection);
#else
ierr = DMSetSection(*(p_ctx->coarser_dm) , subsection);
#endif
LIBMESH_CHKERR(ierr);
ierr = PetscSectionDestroy(&subsection);
LIBMESH_CHKERR(ierr);
}
// Finally, set the coarser DM
*(dmc) = *(p_ctx->coarser_dm);
return 0;
}
