static PetscErrorCode KSPSetFromOptions_SpecEst(KSP ksp)
{
PetscErrorCode ierr;
KSP_SpecEst *spec = (KSP_SpecEst*)ksp->data;
char prefix[256];
PetscFunctionBegin;
ierr = PetscOptionsHead("KSP SpecEst Options");CHKERRQ(ierr);
ierr = PetscOptionsReal("-ksp_specest_minfactor","Multiplier on the minimum eigen/singular value","None",spec->minfactor,&spec->minfactor,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-ksp_specest_maxfactor","Multiplier on the maximum eigen/singular value","None",spec->maxfactor,&spec->maxfactor,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-ksp_specest_richfactor","Multiplier on the richimum eigen/singular value","None",spec->richfactor,&spec->richfactor,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsTail();CHKERRQ(ierr);
/* Mask the PC so that PCSetFromOptions does not do anything */
ierr = KSPSetPC(spec->kspest,spec->pcnone);CHKERRQ(ierr);
ierr = KSPSetPC(spec->kspcheap,spec->pcnone);CHKERRQ(ierr);
ierr = PetscSNPrintf(prefix,sizeof prefix,"%sspecest_",((PetscObject)ksp)->prefix?((PetscObject)ksp)->prefix:"");CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(spec->kspest,prefix);CHKERRQ(ierr);
ierr = PetscSNPrintf(prefix,sizeof prefix,"%sspeccheap_",((PetscObject)ksp)->prefix?((PetscObject)ksp)->prefix:"");CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(spec->kspcheap,prefix);CHKERRQ(ierr);
if (!((PetscObject)spec->kspest)->type_name) {
ierr = KSPSetType(spec->kspest,KSPGMRES);CHKERRQ(ierr);
}
if (!((PetscObject)spec->kspcheap)->type_name) {
ierr = KSPSetType(spec->kspcheap,KSPCHEBYSHEV);CHKERRQ(ierr);
}
ierr = KSPSetFromOptions(spec->kspest);CHKERRQ(ierr);
ierr = KSPSetFromOptions(spec->kspcheap);CHKERRQ(ierr);
/* Unmask the PC */
ierr = KSPSetPC(spec->kspest,ksp->pc);CHKERRQ(ierr);
ierr = KSPSetPC(spec->kspcheap,ksp->pc);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode PCKSPCreateKSP_KSP(PC pc)
{
PetscErrorCode ierr;
const char *prefix;
PC_KSP *jac = (PC_KSP*)pc->data;
PetscFunctionBegin;
ierr = KSPCreate(PetscObjectComm((PetscObject)pc),&jac->ksp);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(jac->ksp,pc->erroriffailure);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)jac->ksp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(jac->ksp,prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(jac->ksp,"ksp_");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
virtual PetscErrorCode setup_KSP() override
{
PetscErrorCode ierr;
PC pc;
PetscFunctionBegin;
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(ksp, "NtoD_");CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);CHKERRQ(ierr);
ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr);
ierr = PCSetType(pc, PCNONE);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
PCMGGetSmootherUp - Gets the KSP context to be used as smoother after
coarse grid correction (post-smoother).
Not Collective, KSP returned is parallel if PC is
Input Parameters:
+ pc - the multigrid context
- l - the level (0 is coarsest) to supply
Ouput Parameters:
. ksp - the smoother
Level: advanced
Notes: calling this will result in a different pre and post smoother so you may need to
set options on the pre smoother also
.keywords: MG, multigrid, get, smoother, up, post-smoother, level
.seealso: PCMGGetSmootherUp(), PCMGGetSmootherDown()
@*/
PetscErrorCode PCMGGetSmootherUp(PC pc,PetscInt l,KSP *ksp)
{
PC_MG *mg = (PC_MG*)pc->data;
PC_MG_Levels **mglevels = mg->levels;
PetscErrorCode ierr;
const char *prefix;
MPI_Comm comm;
PetscFunctionBegin;
PetscValidHeaderSpecific(pc,PC_CLASSID,1);
/*
This is called only if user wants a different pre-smoother from post.
Thus we check if a different one has already been allocated,
if not we allocate it.
*/
if (!l) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_OUTOFRANGE,"There is no such thing as a up smoother on the coarse grid");
if (mglevels[l]->smoothu == mglevels[l]->smoothd) {
KSPType ksptype;
PCType pctype;
PC ipc;
PetscReal rtol,abstol,dtol;
PetscInt maxits;
KSPNormType normtype;
ierr = PetscObjectGetComm((PetscObject)mglevels[l]->smoothd,&comm);CHKERRQ(ierr);
ierr = KSPGetOptionsPrefix(mglevels[l]->smoothd,&prefix);CHKERRQ(ierr);
ierr = KSPGetTolerances(mglevels[l]->smoothd,&rtol,&abstol,&dtol,&maxits);CHKERRQ(ierr);
ierr = KSPGetType(mglevels[l]->smoothd,&ksptype);CHKERRQ(ierr);
ierr = KSPGetNormType(mglevels[l]->smoothd,&normtype);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[l]->smoothd,&ipc);CHKERRQ(ierr);
ierr = PCGetType(ipc,&pctype);CHKERRQ(ierr);
ierr = KSPCreate(comm,&mglevels[l]->smoothu);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(mglevels[l]->smoothu,pc->erroriffailure);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)mglevels[l]->smoothu,(PetscObject)pc,mglevels[0]->levels-l);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(mglevels[l]->smoothu,prefix);CHKERRQ(ierr);
ierr = KSPSetTolerances(mglevels[l]->smoothu,rtol,abstol,dtol,maxits);CHKERRQ(ierr);
ierr = KSPSetType(mglevels[l]->smoothu,ksptype);CHKERRQ(ierr);
ierr = KSPSetNormType(mglevels[l]->smoothu,normtype);CHKERRQ(ierr);
ierr = KSPSetConvergenceTest(mglevels[l]->smoothu,KSPConvergedSkip,NULL,NULL);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[l]->smoothu,&ipc);CHKERRQ(ierr);
ierr = PCSetType(ipc,pctype);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)mglevels[l]->smoothu);CHKERRQ(ierr);
ierr = PetscObjectComposedDataSetInt((PetscObject) mglevels[l]->smoothu, PetscMGLevelId, mglevels[l]->level);CHKERRQ(ierr);
}
if (ksp) *ksp = mglevels[l]->smoothu;
PetscFunctionReturn(0);
}
// -------------------------------------------------------------
// PetscNonlinearSolverImplementation::p_build
// -------------------------------------------------------------
void
PetscNonlinearSolverImplementation::p_build(const std::string& option_prefix)
{
PetscErrorCode ierr(0);
try {
ierr = SNESCreate(this->communicator(), &p_snes); CHKERRXX(ierr);
p_petsc_F = PETScVector(*p_F);
if (!p_function.empty()) {
ierr = SNESSetFunction(p_snes, *p_petsc_F, FormFunction,
static_cast<void *>(this)); CHKERRXX(ierr);
}
p_petsc_J = PETScMatrix(*p_J);
if (!p_jacobian.empty()) {
ierr = SNESSetJacobian(p_snes, *p_petsc_J, *p_petsc_J, FormJacobian,
static_cast<void *>(this)); CHKERRXX(ierr);
}
// set the
ierr = SNESSetOptionsPrefix(p_snes, option_prefix.c_str()); CHKERRXX(ierr);
KSP ksp;
ierr = SNESGetKSP(p_snes, &ksp); CHKERRXX(ierr);
ierr = KSPSetOptionsPrefix(ksp, option_prefix.c_str()); CHKERRXX(ierr);
PC pc;
ierr = KSPGetPC(ksp, &pc); CHKERRXX(ierr);
ierr = PCSetOptionsPrefix(pc, option_prefix.c_str()); CHKERRXX(ierr);
ierr = SNESMonitorSet(p_snes, MonitorNorms, PETSC_NULL, PETSC_NULL); CHKERRXX(ierr);
ierr = SNESSetTolerances(p_snes,
p_functionTolerance,
PETSC_DEFAULT,
p_solutionTolerance,
p_maxIterations,
PETSC_DEFAULT);
ierr = SNESSetFromOptions(p_snes); CHKERRXX(ierr);
} catch (const PETSC_EXCEPTION_TYPE& e) {
throw PETScException(ierr, e);
}
}
/*@C
NEPAppendOptionsPrefix - Appends to the prefix used for searching for all
NEP options in the database.
Logically Collective on NEP
Input Parameters:
+ nep - the nonlinear eigensolver context
- prefix - the prefix string to prepend to all NEP option requests
Notes:
A hyphen (-) must NOT be given at the beginning of the prefix name.
The first character of all runtime options is AUTOMATICALLY the hyphen.
Level: advanced
.seealso: NEPSetOptionsPrefix(), NEPGetOptionsPrefix()
@*/
PetscErrorCode NEPAppendOptionsPrefix(NEP nep,const char *prefix)
{
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(nep,NEP_CLASSID,1);
if (!nep->V) { ierr = NEPGetBV(nep,&nep->V);CHKERRQ(ierr); }
ierr = BVSetOptionsPrefix(nep->V,prefix);CHKERRQ(ierr);
if (!nep->ds) { ierr = NEPGetDS(nep,&nep->ds);CHKERRQ(ierr); }
ierr = DSSetOptionsPrefix(nep->ds,prefix);CHKERRQ(ierr);
if (!nep->rg) { ierr = NEPGetRG(nep,&nep->rg);CHKERRQ(ierr); }
ierr = RGSetOptionsPrefix(nep->rg,prefix);CHKERRQ(ierr);
if (!nep->ksp) { ierr = NEPGetKSP(nep,&nep->ksp);CHKERRQ(ierr); }
ierr = KSPSetOptionsPrefix(nep->ksp,prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(nep->ksp,"nep_");CHKERRQ(ierr);
ierr = PetscObjectAppendOptionsPrefix((PetscObject)nep,prefix);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCSetFromOptions_Exotic(PetscOptions *PetscOptionsObject,PC pc)
{
PetscErrorCode ierr;
PetscBool flg;
PC_MG *mg = (PC_MG*)pc->data;
PCExoticType mgctype;
PC_Exotic *ctx = (PC_Exotic*) mg->innerctx;
PetscFunctionBegin;
ierr = PetscOptionsHead(PetscOptionsObject,"Exotic coarse space options");
CHKERRQ(ierr);
ierr = PetscOptionsEnum("-pc_exotic_type","face or wirebasket","PCExoticSetType",PCExoticTypes,(PetscEnum)ctx->type,(PetscEnum*)&mgctype,&flg);
CHKERRQ(ierr);
if (flg) {
ierr = PCExoticSetType(pc,mgctype);
CHKERRQ(ierr);
}
ierr = PetscOptionsBool("-pc_exotic_direct_solver","use direct solver to construct interpolation","None",ctx->directSolve,&ctx->directSolve,NULL);
CHKERRQ(ierr);
if (!ctx->directSolve) {
if (!ctx->ksp) {
const char *prefix;
ierr = KSPCreate(PETSC_COMM_SELF,&ctx->ksp);
CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(ctx->ksp,pc->erroriffailure);
CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)ctx->ksp,(PetscObject)pc,1);
CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)ctx->ksp);
CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);
CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(ctx->ksp,prefix);
CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(ctx->ksp,"exotic_");
CHKERRQ(ierr);
}
ierr = KSPSetFromOptions(ctx->ksp);
CHKERRQ(ierr);
}
ierr = PetscOptionsTail();
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode TaoCreate_SSFLS(Tao tao)
{
TAO_SSLS *ssls;
PetscErrorCode ierr;
const char *armijo_type = TAOLINESEARCHARMIJO;
PetscFunctionBegin;
ierr = PetscNewLog(tao,&ssls);CHKERRQ(ierr);
tao->data = (void*)ssls;
tao->ops->solve=TaoSolve_SSFLS;
tao->ops->setup=TaoSetUp_SSFLS;
tao->ops->view=TaoView_SSLS;
tao->ops->setfromoptions = TaoSetFromOptions_SSLS;
tao->ops->destroy = TaoDestroy_SSFLS;
ssls->delta = 1e-10;
ssls->rho = 2.1;
ierr = TaoLineSearchCreate(((PetscObject)tao)->comm,&tao->linesearch);CHKERRQ(ierr);
ierr = TaoLineSearchSetType(tao->linesearch,armijo_type);CHKERRQ(ierr);
ierr = TaoLineSearchSetOptionsPrefix(tao->linesearch,tao->hdr.prefix);CHKERRQ(ierr);
ierr = TaoLineSearchSetFromOptions(tao->linesearch);CHKERRQ(ierr);
/* Linesearch objective and objectivegradient routines are set in solve routine */
ierr = KSPCreate(((PetscObject)tao)->comm,&tao->ksp);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(tao->ksp,tao->hdr.prefix);CHKERRQ(ierr);
/* Override default settings (unless already changed) */
if (!tao->max_it_changed) tao->max_it = 2000;
if (!tao->max_funcs_changed) tao->max_funcs = 4000;
if (!tao->fatol_changed) tao->fatol = 0;
if (!tao->frtol_changed) tao->frtol = 0;
if (!tao->gttol_changed) tao->gttol = 0;
if (!tao->grtol_changed) tao->grtol = 0;
#if defined(PETSC_USE_REAL_SINGLE)
if (!tao->gatol_changed) tao->gatol = 1.0e-6;
if (!tao->fmin_changed) tao->fmin = 1.0e-4;
#else
if (!tao->gatol_changed) tao->gatol = 1.0e-16;
if (!tao->fmin_changed) tao->fmin = 1.0e-8;
#endif
PetscFunctionReturn(0);
}
static PetscErrorCode KSPChebyshevSetEstimateEigenvalues_Chebyshev(KSP ksp,PetscReal a,PetscReal b,PetscReal c,PetscReal d)
{
KSP_Chebyshev *cheb = (KSP_Chebyshev*)ksp->data;
PetscErrorCode ierr;
PetscFunctionBegin;
if (a != 0.0 || b != 0.0 || c != 0.0 || d != 0.0) {
if (!cheb->kspest) { /* should this block of code be moved to KSPSetUp_Chebyshev()? */
PetscBool nonzero;
ierr = KSPCreate(PetscObjectComm((PetscObject)ksp),&cheb->kspest);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)cheb->kspest,(PetscObject)ksp,1);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(cheb->kspest,((PetscObject)ksp)->prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(cheb->kspest,"est_");CHKERRQ(ierr);
ierr = KSPGetPC(cheb->kspest,&cheb->pcnone);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)cheb->pcnone);CHKERRQ(ierr);
ierr = PCSetType(cheb->pcnone,PCNONE);CHKERRQ(ierr);
ierr = KSPSetPC(cheb->kspest,ksp->pc);CHKERRQ(ierr);
ierr = KSPGetInitialGuessNonzero(ksp,&nonzero);CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(cheb->kspest,nonzero);CHKERRQ(ierr);
ierr = KSPSetComputeEigenvalues(cheb->kspest,PETSC_TRUE);CHKERRQ(ierr);
/* Estimate with a fixed number of iterations */
ierr = KSPSetConvergenceTest(cheb->kspest,KSPConvergedSkip,0,0);CHKERRQ(ierr);
ierr = KSPSetNormType(cheb->kspest,KSP_NORM_NONE);CHKERRQ(ierr);
ierr = KSPSetTolerances(cheb->kspest,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,cheb->eststeps);CHKERRQ(ierr);
}
if (a >= 0) cheb->tform[0] = a;
if (b >= 0) cheb->tform[1] = b;
if (c >= 0) cheb->tform[2] = c;
if (d >= 0) cheb->tform[3] = d;
cheb->estimate_current = PETSC_FALSE;
} else {
ierr = KSPDestroy(&cheb->kspest);CHKERRQ(ierr);
ierr = PCDestroy(&cheb->pcnone);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode PCRedundantGetKSP_Redundant(PC pc,KSP *innerksp)
{
PetscErrorCode ierr;
PC_Redundant *red = (PC_Redundant*)pc->data;
MPI_Comm comm,subcomm;
const char *prefix;
PetscFunctionBegin;
if (!red->psubcomm) {
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
ierr = PetscSubcommCreate(comm,&red->psubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetNumber(red->psubcomm,red->nsubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetType(red->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr);
ierr = PetscSubcommSetOptionsPrefix(red->psubcomm,prefix);CHKERRQ(ierr);
ierr = PetscSubcommSetFromOptions(red->psubcomm);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));CHKERRQ(ierr);
/* create a new PC that processors in each subcomm have copy of */
subcomm = PetscSubcommChild(red->psubcomm);
ierr = KSPCreate(subcomm,&red->ksp);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(red->ksp,pc->erroriffailure);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)red->ksp);CHKERRQ(ierr);
ierr = KSPSetType(red->ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(red->ksp,&red->pc);CHKERRQ(ierr);
ierr = PCSetType(red->pc,PCLU);CHKERRQ(ierr);
if (red->shifttypeset) {
ierr = PCFactorSetShiftType(red->pc,red->shifttype);CHKERRQ(ierr);
red->shifttypeset = PETSC_FALSE;
}
ierr = KSPSetOptionsPrefix(red->ksp,prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(red->ksp,"redundant_");CHKERRQ(ierr);
}
*innerksp = red->ksp;
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode TaoCreate_BQNLS(Tao tao)
{
TAO_BNK *bnk;
TAO_BQNK *bqnk;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = TaoCreate_BQNK(tao);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(tao->ksp, "unused");CHKERRQ(ierr);
tao->ops->solve = TaoSolve_BNLS;
tao->ops->setfromoptions = TaoSetFromOptions_BQNLS;
bnk = (TAO_BNK*)tao->data;
bnk->update_type = BNK_UPDATE_STEP;
bnk->computehessian = TaoBQNLSComputeHessian;
bnk->computestep = TaoBQNLSComputeStep;
bqnk = (TAO_BQNK*)bnk->ctx;
ierr = MatSetOptionsPrefix(bqnk->B, "tao_bqnls_");CHKERRQ(ierr);
ierr = MatSetType(bqnk->B, MATLMVMBFGS);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode PCLSCAllocate_Private(PC pc)
{
PC_LSC *lsc = (PC_LSC*)pc->data;
Mat A;
PetscErrorCode ierr;
PetscFunctionBegin;
if (lsc->allocated) PetscFunctionReturn(0);
ierr = KSPCreate(PetscObjectComm((PetscObject)pc),&lsc->kspL);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(lsc->kspL,pc->erroriffailure);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)lsc->kspL,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetType(lsc->kspL,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(lsc->kspL,((PetscObject)pc)->prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(lsc->kspL,"lsc_");CHKERRQ(ierr);
ierr = MatSchurComplementGetSubMatrices(pc->mat,&A,NULL,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = MatCreateVecs(A,&lsc->x0,&lsc->y0);CHKERRQ(ierr);
ierr = MatCreateVecs(pc->pmat,&lsc->x1,NULL);CHKERRQ(ierr);
if (lsc->scalediag) {
ierr = VecDuplicate(lsc->x0,&lsc->scale);CHKERRQ(ierr);
}
lsc->allocated = PETSC_TRUE;
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode TaoCreate_TRON(Tao tao)
{
TAO_TRON *tron;
PetscErrorCode ierr;
const char *morethuente_type = TAOLINESEARCHMT;
PetscFunctionBegin;
tao->ops->setup = TaoSetup_TRON;
tao->ops->solve = TaoSolve_TRON;
tao->ops->view = TaoView_TRON;
tao->ops->setfromoptions = TaoSetFromOptions_TRON;
tao->ops->destroy = TaoDestroy_TRON;
tao->ops->computedual = TaoComputeDual_TRON;
ierr = PetscNewLog(tao,&tron);CHKERRQ(ierr);
tao->data = (void*)tron;
/* Override default settings (unless already changed) */
if (!tao->max_it_changed) tao->max_it = 50;
#if defined(PETSC_USE_REAL_SINGLE)
if (!tao->steptol_changed) tao->steptol = 1.0e-6;
#else
if (!tao->steptol_changed) tao->steptol = 1.0e-12;
#endif
if (!tao->trust0_changed) tao->trust0 = 1.0;
/* Initialize pointers and variables */
tron->n = 0;
tron->maxgpits = 3;
tron->pg_ftol = 0.001;
tron->eta1 = 1.0e-4;
tron->eta2 = 0.25;
tron->eta3 = 0.50;
tron->eta4 = 0.90;
tron->sigma1 = 0.5;
tron->sigma2 = 2.0;
tron->sigma3 = 4.0;
tron->gp_iterates = 0; /* Cumulative number */
tron->total_gp_its = 0;
tron->n_free = 0;
tron->DXFree=NULL;
tron->R=NULL;
tron->X_New=NULL;
tron->G_New=NULL;
tron->Work=NULL;
tron->Free_Local=NULL;
tron->H_sub=NULL;
tron->Hpre_sub=NULL;
tao->subset_type = TAO_SUBSET_SUBVEC;
ierr = TaoLineSearchCreate(((PetscObject)tao)->comm, &tao->linesearch);CHKERRQ(ierr);
ierr = TaoLineSearchSetType(tao->linesearch,morethuente_type);CHKERRQ(ierr);
ierr = TaoLineSearchUseTaoRoutines(tao->linesearch,tao);CHKERRQ(ierr);
ierr = TaoLineSearchSetOptionsPrefix(tao->linesearch,tao->hdr.prefix);CHKERRQ(ierr);
ierr = KSPCreate(((PetscObject)tao)->comm, &tao->ksp);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(tao->ksp, tao->hdr.prefix);CHKERRQ(ierr);
ierr = KSPSetType(tao->ksp,KSPSTCG);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode BSSCR_DRIVER_flex( KSP ksp, Mat stokes_A, Vec stokes_x, Vec stokes_b, Mat approxS, KSP ksp_K,
MatStokesBlockScaling BA, PetscTruth sym, KSP_BSSCR * bsscrp_self )
{
char name[PETSC_MAX_PATH_LEN];
char ubefore[100];
char uafter[100];
char pbefore[100];
char pafter[100];
PetscTruth flg, flg2, truth, useAcceleratingSmoothingMG, useFancySmoothingMG;
PetscTruth usePreviousGuess, useNormInfStoppingConditions, useNormInfMonitor, found, extractMats;
Mat K,G,D,C;
Vec u,p,f,h;
Mat S;
Vec h_hat,t,t2,q,v;
KSP ksp_inner;
KSP ksp_S;
KSP ksp_cleaner;
KSPType ksp_inner_type;
PetscTruth has_cnst_nullspace;
PC pc_S, pc_MG, pcInner;
PetscInt monitor_index,max_it,min_it;
Vec nsp_vec = PETSC_NULL;
PetscReal scr_rtol;
PetscReal inner_rtol;
PetscReal vSolver_rtol;
PetscScalar uNormInf, pNormInf;
PetscScalar uNorm, pNorm, rNorm, fNorm;
PetscInt uSize, pSize;
PetscInt lmin,lmax;
PetscInt iterations;
PetscReal min,max;
PetscReal p_sum;
MGContext mgCtx;
PC shellPC;
double t0, t1;
double mgSetupTime, scrSolveTime, a11SingleSolveTime, solutionAnalysisTime;
Index nx,ny,nz;
PetscInt j,start,end;
static int been_here = 0; /* Ha Ha Ha !! */
/* get sub matrix / vector objects */
MatNestGetSubMat( stokes_A, 0,0, &K );
MatNestGetSubMat( stokes_A, 0,1, &G );
MatNestGetSubMat( stokes_A, 1,0, &D );
MatNestGetSubMat( stokes_A, 1,1, &C );
VecNestGetSubVec( stokes_x, 0, &u );
VecNestGetSubVec( stokes_x, 1, &p );
VecNestGetSubVec( stokes_b, 0, &f );
VecNestGetSubVec( stokes_b, 1, &h );
/* PetscPrintf( PETSC_COMM_WORLD, "\t Adress of stokes_x is %p\n", stokes_x); */
/* VecNorm( u, NORM_2, &uNorm ); */
/* PetscPrintf( PETSC_COMM_WORLD, "\t u Norm is %.6e in %s: address is %p\n",uNorm,__func__,u); */
/* VecNorm( p, NORM_2, &pNorm ); */
/* PetscPrintf( PETSC_COMM_WORLD, "\t p Norm is %.6e in %s: addres is %p\n",pNorm,__func__,p); */
/* Create Schur complement matrix */
//MatCreateSchurFromBlock( stokes_A, 0.0, "MatSchur_A11", &S );
MatCreateSchurComplement(K,K,G,D,C, &S);
/* configure inner solver */
if (ksp_K!=PETSC_NULL) {
MatSchurComplementSetKSP( S, ksp_K );
MatSchurComplementGetKSP( S, &ksp_inner );
}
else {
abort();
MatSchurComplementGetKSP( S, &ksp_inner );
KSPSetType( ksp_inner, "cg" );
}
KSPGetPC( ksp_inner, &pcInner );
/* If we're using multigrid, replace the preconditioner here
so we get the same options prefix. */
if(bsscrp_self->mg) {
mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx );
}
/* SETFROMOPTIONS MIGHT F**K MG UP */
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest( ksp_inner );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfMonitor, PETSC_NULL, PETSC_NULL );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_to_norm_2_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/* create right hand side */
/* h_hat = G'*inv(K)*f - h */
MatGetVecs(K,PETSC_NULL,&t);
MatGetVecs( S, PETSC_NULL, &h_hat );
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
KSPSolve(ksp_inner,f,t);/* t=f/K */
//bsscr_writeVec( t, "ts", "Writing t vector");
MatMult(D,t,h_hat);/* G'*t */
VecAXPY(h_hat, -1.0, h);/* h_hat = h_hat - h */
Stg_VecDestroy(&t);
//bsscr_writeVec( h_hat, "h_hat", "Writing h_hat Vector in Solver");
//MatSchurApplyReductionToVecFromBlock( S, stokes_b, h_hat );
/* create solver for S p = h_hat */
KSPCreate( PETSC_COMM_WORLD, &ksp_S );
KSPSetOptionsPrefix( ksp_S, "scr_");
Stg_KSPSetOperators( ksp_S, S,S, SAME_NONZERO_PATTERN );
KSPSetType( ksp_S, "cg" );
/* Build preconditioner for S */
KSPGetPC( ksp_S, &pc_S );
BSSCR_BSSCR_StokesCreatePCSchur2( K,G,D,C,approxS,pc_S,sym, bsscrp_self );
KSPSetFromOptions(ksp_S);
/* Set specific monitor test */
KSPGetTolerances( ksp_S, PETSC_NULL, PETSC_NULL, PETSC_NULL, &max_it );
//BSSCR_KSPLogSetMonitor( ksp_S, max_it, &monitor_index );
/* Pressure / Velocity Solve */
scrSolveTime = MPI_Wtime();
PetscPrintf( PETSC_COMM_WORLD, "\t* Pressure / Velocity Solve \n");
usePreviousGuess = PETSC_FALSE;
if(been_here)
PetscOptionsGetTruth( PETSC_NULL, "-scr_use_previous_guess", &usePreviousGuess, &found );
if(usePreviousGuess) { /* Note this should actually look at checkpoint information */
KSPSetInitialGuessNonzero( ksp_S, PETSC_TRUE );
}
else {
KSPSetInitialGuessNonzero( ksp_S, PETSC_FALSE );
}
//KSPSetRelativeRhsConvergenceTest( ksp_S );
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-scr_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest(ksp_S);
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_S, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
PetscPrintf( PETSC_COMM_WORLD, "\t* KSPSolve( ksp_S, h_hat, p )\n");
/* if h_hat needs to be fixed up ..take out any nullspace vectors here */
/* we want to check that there is no "noise" in the null-space in the h vector */
/* this causes problems when we are trying to solve a Jacobian system when the Residual is almost converged */
if(bsscrp_self->check_pressureNS){
bsscrp_self->buildPNS(ksp);/* build and set nullspace vectors on bsscr - which is on ksp (function pointer is set in KSPSetUp_BSSCR */
}
PetscScalar norm, a, a1, a2, hnorm, pnorm, gnorm;
MatNorm(G,NORM_INFINITY,&gnorm);
VecNorm(h_hat, NORM_2, &hnorm);
hnorm=hnorm/gnorm;
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,1.0/hnorm);
}
/* test to see if v or t are in nullspace of G and orthogonalize wrt h_hat if needed */
KSPRemovePressureNullspace_BSSCR(ksp, h_hat);
/***************************************/
/* set convergence test to use min_it */
found = PETSC_FALSE;
min_it = 0;
PetscOptionsGetInt( PETSC_NULL,"-scr_ksp_set_min_it_converge", &min_it, &found);
if(found && min_it > 0){
BSSCR_KSPSetConvergenceMinIts(ksp_S, min_it, bsscrp_self);
}
KSPSolve( ksp_S, h_hat, p );
sprintf(pafter,"psafter_%d",been_here);
bsscr_writeVec( p, pafter, "Writing p Vector in Solver");
/***************************************/
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,hnorm);
VecScale(p,hnorm);
}
KSPRemovePressureNullspace_BSSCR(ksp, p);
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve( ksp_S, h_hat, p ) Solve Finished in time: %lf seconds\n\n", scrSolveTime);
/* Resolve with this pressure to obtain solution for u */
/* obtain solution for u */
VecDuplicate( u, &t );
MatMult( G, p, t);
VecAYPX( t, -1.0, f ); /* t <- -t + f */
MatSchurComplementGetKSP( S, &ksp_inner );
a11SingleSolveTime = MPI_Wtime(); /* ---------------------------------- Final V Solve */
if(usePreviousGuess)
KSPSetInitialGuessNonzero( ksp_inner, PETSC_TRUE );
KSPSetOptionsPrefix( ksp_inner, "backsolveA11_" );
KSPSetFromOptions( ksp_inner );
KSPSolve( ksp_inner, t, u ); /* Solve, then restore default tolerance and initial guess */
a11SingleSolveTime = MPI_Wtime() - a11SingleSolveTime; /* ------------------ Final V Solve */
PetscPrintf( PETSC_COMM_WORLD, "\n\nSCR Solver Summary:\n\n");
if(bsscrp_self->mg)
PetscPrintf( PETSC_COMM_WORLD, " Multigrid setup: = %.4g secs \n", mgSetupTime);
KSPGetIterationNumber( ksp_S, &iterations);
PetscPrintf( PETSC_COMM_WORLD, " Pressure Solve: = %.4g secs / %d its\n", scrSolveTime, iterations);
KSPGetIterationNumber( ksp_inner, &iterations);
PetscPrintf( PETSC_COMM_WORLD, " Final V Solve: = %.4g secs / %d its\n\n", a11SingleSolveTime, iterations);
/* Analysis of solution:
This can be somewhat time consuming as it requires allocation / de-allocation,
computing vector norms etc. So we make it optional..
This should be put into a proper KSP monitor now?
*/
flg = PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_solution_summary", &flg, &found );
if(flg) {
solutionAnalysisTime = MPI_Wtime();
VecGetSize( u, &uSize );
VecGetSize( p, &pSize );
VecDuplicate( u, &t2 );
MatMult( K, u, t2);
VecAYPX( t2, -1.0, t ); /* t2 <- -t2 + t ... should be the formal residual vector */
VecNorm( t2, NORM_2, &rNorm );
VecNorm( f, NORM_2, &fNorm );
PetscPrintf( PETSC_COMM_WORLD, " |f - K u - G p|/|f| = %.6e\n", rNorm/fNorm );
VecDuplicate( p, &q );
MatMult( D, u, q );
VecNorm( u, NORM_2, &uNorm );
VecNorm( q, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_2/|u|_2 = %.6e\n", sqrt( (double) uSize / (double) pSize ) * rNorm / uNorm);
VecNorm( q, NORM_INFINITY, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_infty/|u|_2 = %.6e\n", sqrt( (double) uSize ) * rNorm / uNorm);
VecNorm( u, NORM_INFINITY, &uNormInf );
VecNorm( u, NORM_2, &uNorm );
VecGetSize( u, &uSize );
VecNorm( p, NORM_INFINITY, &pNormInf );
VecNorm( p, NORM_2, &pNorm );
PetscPrintf( PETSC_COMM_WORLD, " |u|_{\\infty} = %.6e , u_rms = %.6e\n",
uNormInf, uNorm / sqrt( (double) uSize ) );
PetscPrintf( PETSC_COMM_WORLD, " |p|_{\\infty} = %.6e , p_rms = %.6e\n",
pNormInf, pNorm / sqrt( (double) pSize ) );
VecMax( u, &lmax, &max );
VecMin( u, &lmin, &min );
PetscPrintf( PETSC_COMM_WORLD, " min/max(u) = %.6e [%d] / %.6e [%d]\n",min,lmin,max,lmax);
VecMax( p, &lmax, &max );
VecMin( p, &lmin, &min );
PetscPrintf( PETSC_COMM_WORLD, " min/max(p) = %.6e [%d] / %.6e [%d]\n",min,lmin,max,lmax);
VecSum( p, &p_sum );
PetscPrintf( PETSC_COMM_WORLD, " \\sum_i p_i = %.6e \n", p_sum );
solutionAnalysisTime = MPI_Wtime() - solutionAnalysisTime;
PetscPrintf( PETSC_COMM_WORLD, "\n Time for this analysis = %.4g secs\n\n",solutionAnalysisTime);
Stg_VecDestroy(&t2 );
Stg_VecDestroy(&q );
}
if(bsscrp_self->mg) {
//MG_inner_solver_pcmg_shutdown( pcInner );
}
Stg_VecDestroy(&t );
// KSPLogDestroyMonitor( ksp_S );
Stg_KSPDestroy(&ksp_S );
//Stg_KSPDestroy(&ksp_inner );
Stg_VecDestroy(&h_hat );
Stg_MatDestroy(&S );
/* Destroy nullspace vector if it exists. */
if(nsp_vec)
Stg_VecDestroy(&nsp_vec);
//been_here = 1;
been_here++;
PetscFunctionReturn(0);
}
void PETScMGSolver_Setup( void* matrixSolver, void* rhs, void* solution ) {
PETScMGSolver* self = (PETScMGSolver*)matrixSolver;
Bool rebuildOps;
double wallTime;
PetscErrorCode ec;
assert( self && Stg_CheckType( self, PETScMGSolver ) );
wallTime = MPI_Wtime();
//_MatrixSolver_Setup( self, rhs, solution );
self->mgData->curRHS = rhs;
self->mgData->curSolution = solution;
self->mgData->expiredResidual = True;
PetscPrintf( PETSC_COMM_WORLD, "_MatrixSolver_Setup %g\n", MPI_Wtime() - wallTime);
if( self->opGen )
rebuildOps = MGOpGenerator_HasExpired( self->opGen );
else
rebuildOps = False;
if( !rebuildOps ) {
unsigned l_i;
for( l_i = 1; l_i < self->nLevels; l_i++ ) {
if( !self->levels[l_i].R || !self->levels[l_i].P ) {
rebuildOps = True;
break;
}
}
}
if( self->solversChanged || rebuildOps || self->opsChanged ) {
wallTime = MPI_Wtime();
PETScMGSolver_UpdateSolvers( self );
PetscPrintf( PETSC_COMM_WORLD, "PETScMGSolver_UpdateSolvers %g\n", MPI_Wtime() - wallTime);
}
if( rebuildOps ) {
wallTime = MPI_Wtime();
PETScMGSolver_UpdateOps( self );
PetscPrintf( PETSC_COMM_WORLD, "PETScMGSolver_UpdateOps %g\n", MPI_Wtime() - wallTime);
}
//if( self->matrixChanged || self->solversChanged || rebuildOps || self->opsChanged ) {
if( self->mgData->matrixChanged || self->solversChanged || rebuildOps || self->opsChanged ) {
wallTime = MPI_Wtime();
PETScMGSolver_UpdateMatrices( self );
PETScMGSolver_UpdateWorkVectors( self );
PetscPrintf( PETSC_COMM_WORLD, "PETScMGSolver_UpdateMats-WorkVecs %g\n", MPI_Wtime() - wallTime);
}
self->solversChanged = False;
//self->matrixChanged = False;
self->mgData->matrixChanged = False;
self->opsChanged = False;
//if( !self->optionsReady ) {
if( !self->mgData->optionsReady ) {
KSPSetOptionsPrefix( self->mgData->ksp, "A11_" );
ec = KSPSetFromOptions( self->mgData->ksp );
CheckPETScError( ec );
//self->optionsReady = True;
self->mgData->optionsReady = True;
}
}
/*
I should not modify setup called!!
This is handled via petsc.
*/
PetscErrorCode BSSCR_PCSetUp_GtKG( PC pc )
{
PC_GtKG ctx = (PC_GtKG)pc->data;
PetscReal fill;
Mat Ident;
Vec diag;
PetscInt M,N, m,n;
MPI_Comm comm;
PetscInt nnz_I, nnz_G;
const MatType mtype;
const char *prefix;
PetscTruth wasSetup;
if( ctx->K == PETSC_NULL ) {
Stg_SETERRQ( PETSC_ERR_SUP, "gtkg: K not set" );
}
if( ctx->G == PETSC_NULL ) {
Stg_SETERRQ( PETSC_ERR_SUP, "gtkg: G not set" );
}
PetscObjectGetComm( (PetscObject)ctx->K, &comm );
/* Check for existence of objects and trash any which exist */
if( ctx->form_GtG == PETSC_TRUE && ctx->GtG != PETSC_NULL ) {
Stg_MatDestroy(&ctx->GtG );
ctx->GtG = PETSC_NULL;
}
if( ctx->s != PETSC_NULL ) {
Stg_VecDestroy(&ctx->s );
ctx->s = PETSC_NULL;
}
if( ctx->X != PETSC_NULL ) {
Stg_VecDestroy(&ctx->X );
ctx->X = PETSC_NULL;
}
if( ctx->t != PETSC_NULL ) {
Stg_VecDestroy(&ctx->t );
ctx->t = PETSC_NULL;
}
if( ctx->inv_diag_M != PETSC_NULL ) {
Stg_VecDestroy(&ctx->inv_diag_M );
ctx->inv_diag_M = PETSC_NULL;
}
/* Create vectors */
MatGetVecs( ctx->K, &ctx->s, &ctx->X );
MatGetVecs( ctx->G, &ctx->t, PETSC_NULL );
if( ctx->M != PETSC_NULL ) {
MatGetVecs( ctx->K, &ctx->inv_diag_M, PETSC_NULL );
MatGetDiagonal( ctx->M, ctx->inv_diag_M );
VecReciprocal( ctx->inv_diag_M );
/* change the pc_apply routines */
pc->ops->apply = BSSCR_BSSCR_PCApply_GtKG_diagonal_scaling;
pc->ops->applytranspose = BSSCR_BSSCR_PCApplyTranspose_GtKG_diagonal_scaling;
}
/* Assemble GtG */
MatGetSize( ctx->G, &M, &N );
MatGetLocalSize( ctx->G, &m, &n );
MatGetVecs( ctx->G, PETSC_NULL, &diag );
VecSet( diag, 1.0 );
MatCreate( comm, &Ident );
MatSetSizes( Ident, m,m , M, M );
#if (((PETSC_VERSION_MAJOR==3) && (PETSC_VERSION_MINOR>=3)) || (PETSC_VERSION_MAJOR>3) )
MatSetUp(Ident);
#endif
MatGetType( ctx->G, &mtype );
MatSetType( Ident, mtype );
if( ctx->M == PETSC_NULL ) {
MatDiagonalSet( Ident, diag, INSERT_VALUES );
}
else {
MatDiagonalSet( Ident, ctx->inv_diag_M, INSERT_VALUES );
}
BSSCR_get_number_nonzeros_AIJ( Ident, &nnz_I );
BSSCR_get_number_nonzeros_AIJ( ctx->G, &nnz_G );
//fill = 1.0;
/*
Not sure the best way to estimate the fill factor.
GtG is a laplacian on the pressure space.
This might tell us something useful...
*/
fill = (PetscReal)(nnz_G)/(PetscReal)( nnz_I );
MatPtAP( Ident, ctx->G, MAT_INITIAL_MATRIX, fill, &ctx->GtG );
Stg_MatDestroy(&Ident);
Stg_VecDestroy(&diag );
Stg_KSPSetOperators( ctx->ksp, ctx->GtG, ctx->GtG, SAME_NONZERO_PATTERN );
if (!pc->setupcalled) {
wasSetup = PETSC_FALSE;
PCGetOptionsPrefix( pc,&prefix );
KSPSetOptionsPrefix( ctx->ksp, prefix );
KSPAppendOptionsPrefix( ctx->ksp, "pc_gtkg_" ); /* -pc_GtKG_ksp_type <type>, -ksp_GtKG_pc_type <type> */
}
else {
wasSetup = PETSC_TRUE;
}
// if (!wasSetup && pc->setfromoptionscalled) {
if (!wasSetup) {
KSPSetFromOptions(ctx->ksp);
}
PetscFunctionReturn(0);
}
static PetscErrorCode PCSetUp_PCD_Feelpp(PC pc)
{
PetscErrorCode ierr;
PC_PCD_Feelpp *pcpcd = (PC_PCD_Feelpp*)pc->data;
if ( !pcpcd->allocated )
{
ierr = KSPCreate(PetscObjectComm((PetscObject)pc),&pcpcd->kspAp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcpcd->kspAp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetType(pcpcd->kspAp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcpcd->kspAp,((PetscObject)pc)->prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(pcpcd->kspAp,"pcd_Ap");CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcpcd->kspAp);CHKERRQ(ierr);
ierr = KSPCreate(PetscObjectComm((PetscObject)pc),&pcpcd->kspMp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcpcd->kspMp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetType(pcpcd->kspMp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcpcd->kspMp,((PetscObject)pc)->prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(pcpcd->kspMp,"pcd_Mp");CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcpcd->kspMp);CHKERRQ(ierr);
#if PETSC_VERSION_LESS_THAN(3,6,0)
ierr = MatGetVecs(pcpcd->matFp,&pcpcd->x2,&pcpcd->x1);CHKERRQ(ierr);
#else
ierr = MatCreateVecs(pcpcd->matFp,&pcpcd->x2,&pcpcd->x1);CHKERRQ(ierr);
#endif
pcpcd->allocated = PETSC_TRUE;
}
PetscBool isBTBt;
PetscStrcmp(pcpcd->pcdApType,"BTBt",&isBTBt);
if ( isBTBt )
{
if (!pcpcd->MvDiag )
{
#if PETSC_VERSION_LESS_THAN(3,6,0)
ierr = MatGetVecs(pcpcd->matMv,&pcpcd->MvDiag,NULL);CHKERRQ(ierr);
#else
ierr = MatCreateVecs(pcpcd->matMv,&pcpcd->MvDiag,NULL);CHKERRQ(ierr);
#endif
}
Mat B, C;
#if PETSC_VERSION_GREATER_OR_EQUAL_THAN( 3,5,0 )
ierr = MatSchurComplementGetSubMatrices(pc->mat,NULL,NULL,&B,&C,NULL);CHKERRQ(ierr);
#else
ierr = MatSchurComplementGetSubmatrices(pc->mat,NULL,NULL,&B,&C,NULL);CHKERRQ(ierr);
#endif
ierr = MatGetDiagonal(pcpcd->matMv,pcpcd->MvDiag);CHKERRQ(ierr);
ierr = VecReciprocal(pcpcd->MvDiag);CHKERRQ(ierr);
// diag(F)^-1 * B
ierr = MatDiagonalScale( B, pcpcd->MvDiag ,NULL);CHKERRQ(ierr);
// C* diag(F)^-1 * B
if ( !pcpcd->matApBTBt )
ierr = MatMatMult(C,B,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&pcpcd->matApBTBt);
else
ierr = MatMatMult(C,B,MAT_REUSE_MATRIX,PETSC_DEFAULT,&pcpcd->matApBTBt);
CHKERRQ(ierr);
ierr = VecReciprocal(pcpcd->MvDiag);CHKERRQ(ierr);
ierr = MatDiagonalScale( B, pcpcd->MvDiag ,NULL);CHKERRQ(ierr);
pcpcd->matAp = pcpcd->matApBTBt;
}
else
pcpcd->matAp = pcpcd->matApLaplacian;
#if PETSC_VERSION_GREATER_OR_EQUAL_THAN( 3,5,0 )
ierr = KSPSetOperators(pcpcd->kspAp,pcpcd->matAp,pcpcd->matAp);CHKERRQ(ierr);
ierr = KSPSetOperators(pcpcd->kspMp,pcpcd->matMp,pcpcd->matMp);CHKERRQ(ierr);
#else
ierr = KSPSetOperators(pcpcd->kspAp,pcpcd->matAp,pcpcd->matAp,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPSetOperators(pcpcd->kspMp,pcpcd->matMp,pcpcd->matMp,SAME_PRECONDITIONER);CHKERRQ(ierr);
#endif
PetscFunctionReturn(0);
}
void
IBImplicitModHelmholtzPETScLevelSolver::initializeSolverState(
const SAMRAIVectorReal<NDIM,double>& x,
const SAMRAIVectorReal<NDIM,double>& b)
{
IBAMR_TIMER_START(t_initialize_solver_state);
// Rudimentary error checking.
#ifdef DEBUG_CHECK_ASSERTIONS
if (x.getNumberOfComponents() != b.getNumberOfComponents())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " vectors must have the same number of components" << std::endl);
}
const Pointer<PatchHierarchy<NDIM> >& patch_hierarchy = x.getPatchHierarchy();
if (patch_hierarchy != b.getPatchHierarchy())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " vectors must have the same hierarchy" << std::endl);
}
const int coarsest_ln = x.getCoarsestLevelNumber();
if (coarsest_ln < 0)
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " coarsest level number must not be negative" << std::endl);
}
if (coarsest_ln != b.getCoarsestLevelNumber())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " vectors must have same coarsest level number" << std::endl);
}
const int finest_ln = x.getFinestLevelNumber();
if (finest_ln < coarsest_ln)
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " finest level number must be >= coarsest level number" << std::endl);
}
if (finest_ln != b.getFinestLevelNumber())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " vectors must have same finest level number" << std::endl);
}
for (int ln = coarsest_ln; ln <= finest_ln; ++ln)
{
if (patch_hierarchy->getPatchLevel(ln).isNull())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " hierarchy level " << ln << " does not exist" << std::endl);
}
}
if (coarsest_ln != finest_ln)
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " coarsest_ln != finest_ln in IBImplicitModHelmholtzPETScLevelSolver" << std::endl);
}
#endif
// Deallocate the solver state if the solver is already initialized.
if (d_is_initialized) deallocateSolverState();
// Get the hierarchy information.
d_hierarchy = x.getPatchHierarchy();
d_level_num = x.getCoarsestLevelNumber();
#ifdef DEBUG_CHECK_ASSERTIONS
TBOX_ASSERT(d_level_num == x.getFinestLevelNumber());
#endif
const int x_idx = x.getComponentDescriptorIndex(0);
Pointer<SideVariable<NDIM,double> > x_var = x.getComponentVariable(0);
const int b_idx = b.getComponentDescriptorIndex(0);
Pointer<SideVariable<NDIM,double> > b_var = b.getComponentVariable(0);
// Allocate DOF index data.
VariableDatabase<NDIM>* var_db = VariableDatabase<NDIM>::getDatabase();
Pointer<SideDataFactory<NDIM,double> > x_fac =
var_db->getPatchDescriptor()->getPatchDataFactory(x_idx);
const int depth = x_fac->getDefaultDepth();
Pointer<SideDataFactory<NDIM,int> > dof_index_fac =
var_db->getPatchDescriptor()->getPatchDataFactory(d_dof_index_idx);
dof_index_fac->setDefaultDepth(depth);
Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(d_level_num);
if (!level->checkAllocated(d_dof_index_idx)) level->allocatePatchData(d_dof_index_idx);
// Setup PETSc objects.
int ierr;
PETScVecUtilities::constructPatchLevelVec(d_petsc_x, x_idx, x_var, level);
PETScVecUtilities::constructPatchLevelVec(d_petsc_b, b_idx, b_var, level);
PETScVecUtilities::constructPatchLevelDOFIndices(d_dof_index_idx, d_dof_index_var, x_idx, x_var, level);
const double C = d_poisson_spec.cIsZero() ? 0.0 : d_poisson_spec.getCConstant();
const double D = d_poisson_spec.getDConstant();
PETScMatUtilities::constructPatchLevelLaplaceOp(d_petsc_mat, C, D, x_idx, x_var, d_dof_index_idx, d_dof_index_var, level, d_dof_index_fill);
if (d_SJR_mat != PETSC_NULL)
{
ierr = PETScMatOps::MatAXPY(d_petsc_mat, 1.0, d_SJR_mat); IBTK_CHKERRQ(ierr);
}
ierr = MatSetBlockSize(d_petsc_mat, NDIM); IBTK_CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD, &d_petsc_ksp); IBTK_CHKERRQ(ierr);
ierr = KSPSetOperators(d_petsc_ksp, d_petsc_mat, d_petsc_mat, SAME_PRECONDITIONER); IBTK_CHKERRQ(ierr);
if (!d_options_prefix.empty())
{
ierr = KSPSetOptionsPrefix(d_petsc_ksp, d_options_prefix.c_str()); IBTK_CHKERRQ(ierr);
}
ierr = KSPSetFromOptions(d_petsc_ksp); IBTK_CHKERRQ(ierr);
// Indicate that the solver is initialized.
d_is_initialized = true;
IBAMR_TIMER_STOP(t_initialize_solver_state);
return;
}// initializeSolverState
void createInnerKsp(LocalData* data) {
int rank, npes;
MPI_Comm_rank(data->commAll, &rank);
MPI_Comm_size(data->commAll, &npes);
if((rank%2) == 0) {
if(rank < (npes - 1)) {
KSPCreate(data->commLow, &(data->lowSchurKsp));
PetscObjectIncrementTabLevel((PetscObject)(data->lowSchurKsp), PETSC_NULL, 1);
KSPSetType(data->lowSchurKsp, KSPFGMRES);
KSPSetTolerances(data->lowSchurKsp, 1.0e-12, 1.0e-12, PETSC_DEFAULT, 2);
KSPSetOptionsPrefix(data->lowSchurKsp, "inner_");
PC pc;
KSPGetPC(data->lowSchurKsp, &pc);
PCSetType(pc, PCNONE);
KSPSetFromOptions(data->lowSchurKsp);
KSPSetOperators(data->lowSchurKsp, data->lowSchurMat,
data->lowSchurMat, SAME_NONZERO_PATTERN);
KSPSetUp(data->lowSchurKsp);
} else {
data->lowSchurKsp = PETSC_NULL;
}
} else {
KSPCreate(data->commHigh, &(data->highSchurKsp));
PetscObjectIncrementTabLevel((PetscObject)(data->highSchurKsp), PETSC_NULL, 1);
KSPSetType(data->highSchurKsp, KSPFGMRES);
KSPSetTolerances(data->highSchurKsp, 1.0e-12, 1.0e-12, PETSC_DEFAULT, 2);
KSPSetOptionsPrefix(data->highSchurKsp, "inner_");
PC pc;
KSPGetPC(data->highSchurKsp, &pc);
PCSetType(pc, PCNONE);
KSPSetFromOptions(data->highSchurKsp);
KSPSetOperators(data->highSchurKsp, data->highSchurMat,
data->highSchurMat, SAME_NONZERO_PATTERN);
KSPSetUp(data->highSchurKsp);
}
if((rank%2) == 0) {
if(rank > 0) {
KSPCreate(data->commHigh, &(data->highSchurKsp));
PetscObjectIncrementTabLevel((PetscObject)(data->highSchurKsp), PETSC_NULL, 1);
KSPSetType(data->highSchurKsp, KSPFGMRES);
KSPSetTolerances(data->highSchurKsp, 1.0e-12, 1.0e-12, PETSC_DEFAULT, 2);
KSPSetOptionsPrefix(data->highSchurKsp, "inner_");
PC pc;
KSPGetPC(data->highSchurKsp, &pc);
PCSetType(pc, PCNONE);
KSPSetFromOptions(data->highSchurKsp);
KSPSetOperators(data->highSchurKsp, data->highSchurMat,
data->highSchurMat, SAME_NONZERO_PATTERN);
KSPSetUp(data->highSchurKsp);
} else {
data->highSchurKsp = PETSC_NULL;
}
} else {
if(rank < (npes - 1)) {
KSPCreate(data->commLow, &(data->lowSchurKsp));
PetscObjectIncrementTabLevel((PetscObject)(data->lowSchurKsp), PETSC_NULL, 1);
KSPSetType(data->lowSchurKsp, KSPFGMRES);
KSPSetTolerances(data->lowSchurKsp, 1.0e-12, 1.0e-12, PETSC_DEFAULT, 2);
KSPSetOptionsPrefix(data->lowSchurKsp, "inner_");
PC pc;
KSPGetPC(data->lowSchurKsp, &pc);
PCSetType(pc, PCNONE);
KSPSetFromOptions(data->lowSchurKsp);
KSPSetOperators(data->lowSchurKsp, data->lowSchurMat,
data->lowSchurMat, SAME_NONZERO_PATTERN);
KSPSetUp(data->lowSchurKsp);
} else {
data->lowSchurKsp = PETSC_NULL;
}
}
}
PETSC_EXTERN PetscErrorCode TaoCreate_NTR(Tao tao)
{
TAO_NTR *tr;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscNewLog(tao,&tr);CHKERRQ(ierr);
tao->ops->setup = TaoSetUp_NTR;
tao->ops->solve = TaoSolve_NTR;
tao->ops->view = TaoView_NTR;
tao->ops->setfromoptions = TaoSetFromOptions_NTR;
tao->ops->destroy = TaoDestroy_NTR;
/* Override default settings (unless already changed) */
if (!tao->max_it_changed) tao->max_it = 50;
if (!tao->trust0_changed) tao->trust0 = 100.0;
tao->data = (void*)tr;
/* Standard trust region update parameters */
tr->eta1 = 1.0e-4;
tr->eta2 = 0.25;
tr->eta3 = 0.50;
tr->eta4 = 0.90;
tr->alpha1 = 0.25;
tr->alpha2 = 0.50;
tr->alpha3 = 1.00;
tr->alpha4 = 2.00;
tr->alpha5 = 4.00;
/* Interpolation trust region update parameters */
tr->mu1 = 0.10;
tr->mu2 = 0.50;
tr->gamma1 = 0.25;
tr->gamma2 = 0.50;
tr->gamma3 = 2.00;
tr->gamma4 = 4.00;
tr->theta = 0.05;
/* Interpolation parameters for initialization */
tr->mu1_i = 0.35;
tr->mu2_i = 0.50;
tr->gamma1_i = 0.0625;
tr->gamma2_i = 0.50;
tr->gamma3_i = 2.00;
tr->gamma4_i = 5.00;
tr->theta_i = 0.25;
tr->min_radius = 1.0e-10;
tr->max_radius = 1.0e10;
tr->epsilon = 1.0e-6;
tr->pc_type = NTR_PC_BFGS;
tr->bfgs_scale_type = BFGS_SCALE_AHESS;
tr->init_type = NTR_INIT_INTERPOLATION;
tr->update_type = NTR_UPDATE_REDUCTION;
/* Set linear solver to default for trust region */
ierr = KSPCreate(((PetscObject)tao)->comm,&tao->ksp);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(tao->ksp,tao->hdr.prefix);CHKERRQ(ierr);
ierr = KSPSetType(tao->ksp,KSPCGSTCG);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMMGSetKSP - Sets the linear solver object that will use the grid hierarchy
Collective on DMMG
Input Parameter:
+ dmmg - the context
. func - function to compute linear system matrix on each grid level
- rhs - function to compute right hand side on each level (need only work on the finest grid
if you do not use grid sequencing)
Level: advanced
Notes: For linear problems my be called more than once, reevaluates the matrices if it is called more
than once. Call DMMGSolve() directly several times to solve with the same matrix but different
right hand sides.
.seealso DMMGCreate(), DMMGDestroy, DMMGSetDM(), DMMGSolve(), DMMGSetMatType()
@*/
PetscErrorCode PETSCSNES_DLLEXPORT DMMGSetKSP(DMMG *dmmg,PetscErrorCode (*rhs)(DMMG,Vec),PetscErrorCode (*func)(DMMG,Mat,Mat))
{
PetscErrorCode ierr;
PetscInt i,nlevels = dmmg[0]->nlevels,level;
PetscTruth ismg,galerkin=PETSC_FALSE;
PC pc;
KSP lksp;
PetscFunctionBegin;
if (!dmmg) SETERRQ(PETSC_ERR_ARG_NULL,"Passing null as DMMG");
if (!dmmg[0]->ksp) {
/* create solvers for each level if they don't already exist*/
for (i=0; i<nlevels; i++) {
ierr = KSPCreate(dmmg[i]->comm,&dmmg[i]->ksp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)dmmg[i]->ksp,PETSC_NULL,nlevels-i);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(dmmg[i]->ksp,dmmg[i]->prefix);CHKERRQ(ierr);
ierr = DMMGSetUpLevel(dmmg,dmmg[i]->ksp,i+1);CHKERRQ(ierr);
ierr = KSPSetFromOptions(dmmg[i]->ksp);CHKERRQ(ierr);
/* if the multigrid is being run with Galerkin then these matrices do not need to be created except on the finest level
we do not take advantage of this because it may be that Galerkin has not yet been selected for the KSP object
These are also used if grid sequencing is selected for the linear problem. We should probably turn off grid sequencing
for the linear problem */
if (!dmmg[i]->B) {
ierr = DMGetMatrix(dmmg[i]->dm,dmmg[nlevels-1]->mtype,&dmmg[i]->B);CHKERRQ(ierr);
}
if (!dmmg[i]->J) {
dmmg[i]->J = dmmg[i]->B;
ierr = PetscObjectReference((PetscObject) dmmg[i]->J);CHKERRQ(ierr);
}
dmmg[i]->solve = DMMGSolveKSP;
dmmg[i]->rhs = rhs;
}
}
/* evalute matrix on each level */
ierr = KSPGetPC(dmmg[nlevels-1]->ksp,&pc);CHKERRQ(ierr);
ierr = PetscTypeCompare((PetscObject)pc,PCMG,&ismg);CHKERRQ(ierr);
if (ismg) {
ierr = PCMGGetGalerkin(pc,&galerkin);CHKERRQ(ierr);
}
if (func) {
if (galerkin) {
ierr = (*func)(dmmg[nlevels-1],dmmg[nlevels-1]->J,dmmg[nlevels-1]->B);CHKERRQ(ierr);
} else {
for (i=0; i<nlevels; i++) {
ierr = (*func)(dmmg[i],dmmg[i]->J,dmmg[i]->B);CHKERRQ(ierr);
}
}
}
for (i=0; i<nlevels-1; i++) {
ierr = KSPSetOptionsPrefix(dmmg[i]->ksp,"dmmg_");CHKERRQ(ierr);
}
for (level=0; level<nlevels; level++) {
ierr = KSPSetOperators(dmmg[level]->ksp,dmmg[level]->J,dmmg[level]->B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPGetPC(dmmg[level]->ksp,&pc);CHKERRQ(ierr);
if (ismg) {
for (i=0; i<=level; i++) {
ierr = PCMGGetSmoother(pc,i,&lksp);CHKERRQ(ierr);
ierr = KSPSetOperators(lksp,dmmg[i]->J,dmmg[i]->B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
}
}
PetscFunctionReturn(0);
}
PetscErrorCode BSSCR_DRIVER_auglag( KSP ksp, Mat stokes_A, Vec stokes_x, Vec stokes_b, Mat approxS,
MatStokesBlockScaling BA, PetscTruth sym, KSP_BSSCR * bsscrp_self )
{
AugLagStokes_SLE * stokesSLE = (AugLagStokes_SLE*)bsscrp_self->solver->st_sle;
PetscTruth uzawastyle, KisJustK=PETSC_TRUE, restorek, change_A11rhspresolve;
PetscTruth usePreviousGuess, useNormInfStoppingConditions, useNormInfMonitor, found, forcecorrection;
PetscTruth change_backsolve, mg_active;
PetscErrorCode ierr;
//PetscInt monitor_index;
PetscInt max_it,min_it;
KSP ksp_inner, ksp_S, ksp_new_inner;
PC pc_S, pcInner;
Mat K,G,D,C, S, K2;// Korig;
Vec u,p,f,f2=0,f3=0,h, h_hat,t;
MGContext mgCtx;
double mgSetupTime, scrSolveTime, a11SingleSolveTime, penaltyNumber;// hFactor;
static int been_here = 0; /* Ha Ha Ha !! */
char name[PETSC_MAX_PATH_LEN];
char matname[PETSC_MAX_PATH_LEN];
char suffix[PETSC_MAX_PATH_LEN];
char str[PETSC_MAX_PATH_LEN];
PetscTruth flg, extractMats;
PetscLogDouble flopsA,flopsB;
/***************************************************************************************************************/
/***************************************************************************************************************/
//if( bsscrp_self->solver->st_sle->context->loadFromCheckPoint ){
// been_here=1;
//}
/* get sub matrix / vector objects */
/* note that here, the matrix D should always exist. It is set up in _StokesBlockKSPInterface_Solve in StokesBlockKSPInterface.c */
/* now extract K,G etc from a MatNest object */
MatNestGetSubMat( stokes_A, 0,0, &K );
MatNestGetSubMat( stokes_A, 0,1, &G );
MatNestGetSubMat( stokes_A, 1,0, &D );if(!D){ PetscPrintf( PETSC_COMM_WORLD, "D does not exist but should!!\n"); exit(1); }
MatNestGetSubMat( stokes_A, 1,1, &C );
VecNestGetSubVec( stokes_x, 0, &u );
VecNestGetSubVec( stokes_x, 1, &p );
VecNestGetSubVec( stokes_b, 0, &f );
VecNestGetSubVec( stokes_b, 1, &h );
PetscPrintf( PETSC_COMM_WORLD, "\n\n---------- AUGMENTED LAGRANGIAN K2 METHOD ---------\n\n" );
PetscPrintf( PETSC_COMM_WORLD, "----------- Penalty = %f\n\n", bsscrp_self->solver->penaltyNumber );
sprintf(suffix,"%s","x");
PetscOptionsGetString( PETSC_NULL, "-matsuffix", suffix, PETSC_MAX_PATH_LEN-1, &extractMats );
flg=0;
PetscOptionsGetString( PETSC_NULL, "-matdumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"K%s",suffix);
bsscr_dirwriteMat( K, matname,str, "Writing K matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"G%s",suffix);
bsscr_dirwriteMat( G, matname,str, "Writing G matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"D%s",suffix);
bsscr_dirwriteMat( D, matname,str, "Writing D matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"f%s",suffix);
bsscr_dirwriteVec( f, matname,str, "Writing f vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"h%s",suffix);
bsscr_dirwriteVec( h, matname,str, "Writing h vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"Shat%s",suffix);
bsscr_dirwriteMat( approxS, matname,str, "Writing Shat matrix in al Solver");
if(C){
sprintf(str,"%s/",name); sprintf(matname,"C%s",suffix);
bsscr_dirwriteMat( C, matname,str, "Writing C matrix in al Solver");
}
}
mg_active=PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_mg_active", &mg_active, &found );
bsscrp_self->solver->mg_active=mg_active;
penaltyNumber = bsscrp_self->solver->penaltyNumber;
//hFactor = stokesSLE->hFactor;
/***************************************************************************************************************/
/***************************************************************************************************************/
/****** GET K2 ****************************************************************************************/
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
flg=0;
PetscOptionsGetString( PETSC_NULL, "-matdumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"K2%s",suffix);
bsscr_dirwriteMat( bsscrp_self->K2, matname,str, "Writing K2 matrix in al Solver");
}
K2=bsscrp_self->K2;
scrSolveTime = MPI_Wtime();
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* K+p*K2 in time: %lf seconds\n\n", scrSolveTime);
KisJustK=PETSC_FALSE;
forcecorrection=PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL ,"-force_correction", &forcecorrection, &found );
if(forcecorrection){
if(bsscrp_self->f2 && forcecorrection){
f2=bsscrp_self->f2;
ierr=VecAXPY(f,penaltyNumber,f2);/* f <- f +a*f2 */
}else{
switch (bsscrp_self->k2type) {
case (K2_GG):
{
VecDuplicate( u, &f3 );
MatMult( G, h, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
}
break;
case (K2_GMG):
{
Mat M;
Vec Mdiag;
VecDuplicate( u, &f3 );
M = bsscrp_self->solver->mStiffMat->matrix;
MatGetVecs( M, &Mdiag, PETSC_NULL );
MatGetDiagonal( M, Mdiag );
VecReciprocal(Mdiag);
VecPointwiseMult(Mdiag,Mdiag,h);
MatMult( G, Mdiag, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
Stg_VecDestroy(&Mdiag);
}
break;
case (K2_DGMGD):
{
Mat M;
Vec Mdiag;
VecDuplicate( u, &f3 );
M = bsscrp_self->solver->mStiffMat->matrix;
MatGetVecs( M, &Mdiag, PETSC_NULL );
MatGetDiagonal( M, Mdiag );
VecReciprocal(Mdiag);
VecPointwiseMult(Mdiag,Mdiag,h);
MatMult( G, Mdiag, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
Stg_VecDestroy(&Mdiag);
}
break;
case (K2_NULL):
{
;
}
break;
case (K2_SLE):
{
;
}
break;
}
}
}
}
/* Create Schur complement matrix */
MatCreateSchurComplement(K,K,G,D,C, &S);
MatSchurComplementGetKSP( S, &ksp_inner);
KSPGetPC( ksp_inner, &pcInner );
/***************************************************************************************************************/
/***************************************************************************************************************/
/********* SET PREFIX FOR INNER/VELOCITY KSP *************************************************************/
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
Stg_KSPSetOperators(ksp_inner, K, K, DIFFERENT_NONZERO_PATTERN);
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest( ksp_inner );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor) KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfMonitor, PETSC_NULL, PETSC_NULL );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_to_norm_2_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor) KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/***************************************************************************************************************/
/***************************************************************************************************************/
/* If multigrid is enabled, set it now. */
change_A11rhspresolve = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-change_A11rhspresolve", &change_A11rhspresolve, &found );
if(bsscrp_self->solver->mg_active && !change_A11rhspresolve) { mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx ); }
/***************************************************************************************************************/
/***************************************************************************************************************/
/* create right hand side */
if(change_A11rhspresolve){
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix(ksp_new_inner, "rhsA11_");
MatSchurComplementSetKSP( S, ksp_new_inner );/* this call destroys the ksp_inner that is already set on S */
ksp_inner=ksp_new_inner;
KSPGetPC( ksp_inner, &pcInner );
KSPSetFromOptions(ksp_inner); /* make sure we are setting up our solver how we want it */
}
MatGetVecs( S, PETSC_NULL, &h_hat );
Vec f_tmp;
/* It may be the case that the current velocity solution might not be bad guess for f_tmp? <-- maybe not */
MatGetVecs( K, PETSC_NULL, &f_tmp );
scrSolveTime = MPI_Wtime();
KSPSolve(ksp_inner, f, f_tmp);
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve for RHS setup Finished in time: %lf seconds\n\n", scrSolveTime);
MatMult(D, f_tmp, h_hat);
VecAYPX(h_hat, -1.0, h); /* Computes y = x + alpha y. h_hat -> h - Gt*K^(-1)*f*/
Stg_VecDestroy(&f_tmp);
if(bsscrp_self->solver->mg_active && change_A11rhspresolve) {
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix( ksp_new_inner, "A11_" );
MatSchurComplementSetKSP( S, ksp_new_inner );
ksp_inner=ksp_new_inner;
//MatSchurSetKSP( S, ksp_inner );
KSPGetPC( ksp_inner, &pcInner );
KSPSetFromOptions( ksp_inner );
mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx );
}
/* create solver for S p = h_hat */
KSPCreate( PETSC_COMM_WORLD, &ksp_S );
KSPSetOptionsPrefix( ksp_S, "scr_");
/* By default use the UW approxS Schur preconditioner -- same as the one used by the Uzawa solver */
/* Note that if scaling is activated then the approxS matrix has been scaled already */
/* so no need to rebuild in the case of scaling as we have been doing */
if(!approxS){ PetscPrintf( PETSC_COMM_WORLD, "WARNING approxS is NULL\n"); }
Stg_KSPSetOperators( ksp_S, S, S, SAME_NONZERO_PATTERN );
KSPSetType( ksp_S, "cg" );
KSPGetPC( ksp_S, &pc_S );
BSSCR_BSSCR_StokesCreatePCSchur2( K,G,D,C,approxS, pc_S, sym, bsscrp_self );
flg=0;
PetscOptionsGetString( PETSC_NULL, "-NN", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
Mat Smat, Pmat;
//MatStructure mstruct;
Stg_PCGetOperators( pc_S, &Smat, &Pmat, NULL );
sprintf(str,"%s/",name); sprintf(matname,"Pmat%s",suffix);
bsscr_dirwriteMat( Pmat, matname,str, "Writing Pmat matrix in al Solver");
}
uzawastyle=PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-uzawa_style", &uzawastyle, &found );
if(uzawastyle){
/* now want to set up the ksp_S->pc to be of type ksp (gmres) by default to match Uzawa */
KSP pc_ksp;
KSPGetPC( ksp_S, &pc_S );
PCSetType(pc_S,PCKSP);
PCKSPGetKSP( pc_S, &pc_ksp);
KSPSetType(pc_ksp, "gmres" );
KSPSetOptionsPrefix( pc_ksp, "scrPCKSP_");
KSPSetFromOptions( pc_ksp );
}
KSPSetFromOptions( ksp_S );
/* Set specific monitor test */
KSPGetTolerances( ksp_S, PETSC_NULL, PETSC_NULL, PETSC_NULL, &max_it );
// Weirdness with petsc 3.2 here...look at it later
//BSSCR_KSPLogSetMonitor( ksp_S, max_it, &monitor_index );
/***************************************************************************************************************/
/* Pressure / Velocity Solve */
/***************************************************************************************************************/
PetscPrintf( PETSC_COMM_WORLD, "\t* Pressure / Velocity Solve \n");
/***************************************************************************************************************/
/***************************************************************************************************************/
usePreviousGuess = PETSC_FALSE;
if(been_here)
PetscOptionsGetTruth( PETSC_NULL, "-scr_use_previous_guess", &usePreviousGuess, &found );
if(usePreviousGuess) { /* Note this should actually look at checkpoint information */
KSPSetInitialGuessNonzero( ksp_S, PETSC_TRUE ); }
else {
KSPSetInitialGuessNonzero( ksp_S, PETSC_FALSE ); }
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* SET CONVERGENCE TESTS *************************************************************************/
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-scr_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest(ksp_S);
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_S, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* PRESSURE SOLVE ************************************************************************************/
PetscPrintf( PETSC_COMM_WORLD, "\t* KSPSolve( ksp_S, h_hat, p )\n");
/* if h_hat needs to be fixed up ..take out any nullspace vectors here */
/* we want to check that there is no "noise" in the null-space in the h vector */
/* this causes problems when we are trying to solve a Jacobian system when the Residual is almost converged */
if(bsscrp_self->check_pressureNS){
bsscrp_self->buildPNS(ksp);/* build and set nullspace vectors on bsscr - which is on ksp (function pointer is set in KSPSetUp_BSSCR) */
}
PetscScalar hnorm, gnorm;
MatNorm(G,NORM_INFINITY,&gnorm);
VecNorm(h_hat, NORM_2, &hnorm);
hnorm=hnorm/gnorm;
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,1.0/hnorm);
}
/* test to see if v or t are in nullspace of G and orthogonalize wrt h_hat if needed */
KSPRemovePressureNullspace_BSSCR(ksp, h_hat);
/* set convergence test to use min_it */
found = PETSC_FALSE;
min_it = 0;
PetscOptionsGetInt( PETSC_NULL,"-scr_ksp_set_min_it_converge", &min_it, &found);
if(found && min_it > 0){
BSSCR_KSPSetConvergenceMinIts(ksp_S, min_it, bsscrp_self);
}
/** Pressure Solve **/
PetscGetFlops(&flopsA);
scrSolveTime = MPI_Wtime();
KSPSolve( ksp_S, h_hat, p );
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscGetFlops(&flopsB);
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve( ksp_S, h_hat, p ) Solve Finished in time: %lf seconds\n\n", scrSolveTime);
bsscrp_self->solver->stats.pressure_time=scrSolveTime;
bsscrp_self->solver->stats.pressure_flops=(double)(flopsB-flopsA);
/***************************************/
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,hnorm);
VecScale(p,hnorm);
}
KSPRemovePressureNullspace_BSSCR(ksp, p);
/***************************************************************************************************************/
/***************************************************************************************************************/
/* restore K and f for the Velocity back solve */
found = PETSC_FALSE;
restorek = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-restore_K", &restorek, &found);
//PetscOptionsGetString( PETSC_NULL, "-restore_K", name, PETSC_MAX_PATH_LEN-1, &flg );
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
if(restorek){
penaltyNumber = -penaltyNumber;
if(f2) { ierr=VecAXPY(f,penaltyNumber,f2); }/* f <- f +a*f2 */
if(f3) { ierr=VecAXPY(f,penaltyNumber,f3); }/* f <- f +a*f3 */
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
//K=Korig;
Stg_KSPSetOperators(ksp_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KisJustK=PETSC_TRUE;
}
}
if(f3){ Stg_VecDestroy(&f3 ); } /* always destroy this local vector if was created */
/* obtain solution for u */
VecDuplicate( u, &t ); MatMult( G, p, t); VecAYPX( t, -1.0, f ); /*** t <- -t + f = f - G*p ***/
MatSchurComplementGetKSP( S, &ksp_inner );
a11SingleSolveTime = MPI_Wtime(); /* ---------------------------------- Final V Solve */
if(usePreviousGuess) KSPSetInitialGuessNonzero( ksp_inner, PETSC_TRUE );
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* VELOCITY SOLVE ************************************************************************************/
/** Easier to just create a new KSP here if we want to do backsolve diffferently. (getting petsc errors now when switching from fgmres) */
change_backsolve=PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-change_backsolve", &change_backsolve, &found );
if(change_backsolve){
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix(ksp_new_inner, "backsolveA11_");
KSPSetFromOptions(ksp_new_inner); /* make sure we are setting up our solver how we want it */
MatSchurComplementSetKSP( S, ksp_new_inner );/* need to give the Schur it's inner ksp back for when we destroy it at end */
ksp_inner=ksp_new_inner;
}
PetscGetFlops(&flopsA);
KSPSolve(ksp_inner, t, u); /* Solve, then restore default tolerance and initial guess */
PetscGetFlops(&flopsB);
bsscrp_self->solver->stats.velocity_backsolve_flops=(double)(flopsB-flopsA);
a11SingleSolveTime = MPI_Wtime() - a11SingleSolveTime; /* ------------------ Final V Solve */
bsscrp_self->solver->stats.velocity_backsolve_time=a11SingleSolveTime;
flg=0;
PetscOptionsGetString( PETSC_NULL, "-solutiondumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"p%s",suffix);
bsscr_dirwriteVec( p, matname,str, "Writing p vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"u%s",suffix);
bsscr_dirwriteVec( u, matname,str, "Writing u vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"h_hat%s",suffix);
bsscr_dirwriteVec( h_hat, matname,str, "Writing h_hat vector in al Solver");
}
found = PETSC_FALSE;
restorek = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-restore_K_after_solve", &restorek, &found);
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
if(restorek){
penaltyNumber = -penaltyNumber;
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
KisJustK=PETSC_TRUE;
}
}
/***************************************************************************************************************/
/***************************************************************************************************************/
/****** SOLUTION SUMMARY ******************************************************************************/
bsscr_summary(bsscrp_self,ksp_S,ksp_inner,K,K2,D,G,C,u,p,f,h,t,penaltyNumber,KisJustK, mgSetupTime, scrSolveTime, a11SingleSolveTime);
//bsscr_summary(bsscrp_self,ksp_S,ksp_inner,K,Korig,K2,D,G,C,u,p,f,h,t,penaltyNumber,KisJustK, mgSetupTime, scrSolveTime, a11SingleSolveTime);
/***************************************************************************************************************/
/***************************************************************************************************************/
Stg_VecDestroy(&t );
Stg_KSPDestroy(&ksp_S );
Stg_VecDestroy(&h_hat );
Stg_MatDestroy(&S );//This will destroy ksp_inner: also.. pcInner == pc_MG and is destroyed when ksp_inner is
been_here = 1;
PetscFunctionReturn(0);
}
static PetscErrorCode PCSetUp_Redundant(PC pc)
{
PC_Redundant *red = (PC_Redundant*)pc->data;
PetscErrorCode ierr;
PetscInt mstart,mend,mlocal,M;
PetscMPIInt size;
MPI_Comm comm,subcomm;
Vec x;
const char *prefix;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
/* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size == 1) red->useparallelmat = PETSC_FALSE;
if (!pc->setupcalled) {
PetscInt mloc_sub;
if (!red->psubcomm) {
ierr = PetscSubcommCreate(comm,&red->psubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetNumber(red->psubcomm,red->nsubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetType(red->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr);
/* enable runtime switch of psubcomm type, e.g., '-psubcomm_type interlaced */
ierr = PetscSubcommSetFromOptions(red->psubcomm);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));CHKERRQ(ierr);
/* create a new PC that processors in each subcomm have copy of */
subcomm = PetscSubcommChild(red->psubcomm);
ierr = KSPCreate(subcomm,&red->ksp);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(red->ksp,pc->erroriffailure);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)red->ksp);CHKERRQ(ierr);
ierr = KSPSetType(red->ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(red->ksp,&red->pc);CHKERRQ(ierr);
ierr = PCSetType(red->pc,PCLU);CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(red->ksp,prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(red->ksp,"redundant_");CHKERRQ(ierr);
} else {
subcomm = PetscSubcommChild(red->psubcomm);
}
if (red->useparallelmat) {
/* grab the parallel matrix and put it into processors of a subcomminicator */
ierr = MatCreateRedundantMatrix(pc->pmat,red->psubcomm->n,subcomm,MAT_INITIAL_MATRIX,&red->pmats);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr);
/* get working vectors xsub and ysub */
ierr = MatCreateVecs(red->pmats,&red->xsub,&red->ysub);CHKERRQ(ierr);
/* create working vectors xdup and ydup.
xdup concatenates all xsub's contigously to form a mpi vector over dupcomm (see PetscSubcommCreate_interlaced())
ydup concatenates all ysub and has empty local arrays because ysub's arrays will be place into it.
Note: we use communicator dupcomm, not PetscObjectComm((PetscObject)pc)! */
ierr = MatGetLocalSize(red->pmats,&mloc_sub,NULL);CHKERRQ(ierr);
ierr = VecCreateMPI(PetscSubcommContiguousParent(red->psubcomm),mloc_sub,PETSC_DECIDE,&red->xdup);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PetscSubcommContiguousParent(red->psubcomm),1,mloc_sub,PETSC_DECIDE,NULL,&red->ydup);CHKERRQ(ierr);
/* create vecscatters */
if (!red->scatterin) { /* efficiency of scatterin is independent from psubcomm_type! */
IS is1,is2;
PetscInt *idx1,*idx2,i,j,k;
ierr = MatCreateVecs(pc->pmat,&x,0);CHKERRQ(ierr);
ierr = VecGetSize(x,&M);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x,&mstart,&mend);CHKERRQ(ierr);
mlocal = mend - mstart;
ierr = PetscMalloc2(red->psubcomm->n*mlocal,&idx1,red->psubcomm->n*mlocal,&idx2);CHKERRQ(ierr);
j = 0;
for (k=0; k<red->psubcomm->n; k++) {
for (i=mstart; i<mend; i++) {
idx1[j] = i;
idx2[j++] = i + M*k;
}
}
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,red->xdup,is2,&red->scatterin);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
/* Impl below is good for PETSC_SUBCOMM_INTERLACED (no inter-process communication) and PETSC_SUBCOMM_CONTIGUOUS (communication within subcomm) */
ierr = ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*M,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(comm,mlocal,mstart,1,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(red->xdup,is1,x,is2,&red->scatterout);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFree2(idx1,idx2);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
}
} else { /* !red->useparallelmat */
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr);
}
} else { /* pc->setupcalled */
if (red->useparallelmat) {
MatReuse reuse;
/* grab the parallel matrix and put it into processors of a subcomminicator */
/*--------------------------------------------------------------------------*/
if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
/* destroy old matrices */
ierr = MatDestroy(&red->pmats);CHKERRQ(ierr);
reuse = MAT_INITIAL_MATRIX;
} else {
reuse = MAT_REUSE_MATRIX;
}
ierr = MatCreateRedundantMatrix(pc->pmat,red->psubcomm->n,PetscSubcommChild(red->psubcomm),reuse,&red->pmats);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr);
} else { /* !red->useparallelmat */
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr);
}
}
if (pc->setfromoptionscalled) {
ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr);
}
ierr = KSPSetUp(red->ksp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCISSetUp(PC pc)
{
PC_IS *pcis = (PC_IS*)(pc->data);
Mat_IS *matis;
PetscErrorCode ierr;
PetscBool flg,issbaij;
Vec counter;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)pc->pmat,MATIS,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONG,"Preconditioner type of Neumann Neumman requires matrix of type MATIS");
matis = (Mat_IS*)pc->pmat->data;
pcis->pure_neumann = matis->pure_neumann;
/* get info on mapping */
ierr = PetscObjectReference((PetscObject)matis->mapping);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&pcis->mapping);CHKERRQ(ierr);
pcis->mapping = matis->mapping;
ierr = ISLocalToGlobalMappingGetSize(pcis->mapping,&pcis->n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetInfo(pcis->mapping,&(pcis->n_neigh),&(pcis->neigh),&(pcis->n_shared),&(pcis->shared));CHKERRQ(ierr);
/* Creating local and global index sets for interior and inteface nodes. */
{
PetscInt n_I;
PetscInt *idx_I_local,*idx_B_local,*idx_I_global,*idx_B_global;
PetscInt *array;
PetscInt i,j;
/* Identifying interior and interface nodes, in local numbering */
ierr = PetscMalloc1(pcis->n,&array);CHKERRQ(ierr);
ierr = PetscMemzero(array,pcis->n*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0;i<pcis->n_neigh;i++)
for (j=0;j<pcis->n_shared[i];j++)
array[pcis->shared[i][j]] += 1;
ierr = PetscMalloc1(pcis->n,&idx_I_local);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n,&idx_B_local);CHKERRQ(ierr);
for (i=0, pcis->n_B=0, n_I=0; i<pcis->n; i++) {
if (!array[i]) {
idx_I_local[n_I] = i;
n_I++;
} else {
idx_B_local[pcis->n_B] = i;
pcis->n_B++;
}
}
/* Getting the global numbering */
idx_B_global = idx_I_local + n_I; /* Just avoiding allocating extra memory, since we have vacant space */
idx_I_global = idx_B_local + pcis->n_B;
ierr = ISLocalToGlobalMappingApply(pcis->mapping,pcis->n_B,idx_B_local,idx_B_global);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(pcis->mapping,n_I, idx_I_local,idx_I_global);CHKERRQ(ierr);
/* Creating the index sets. */
ierr = ISCreateGeneral(PETSC_COMM_SELF,pcis->n_B,idx_B_local,PETSC_COPY_VALUES, &pcis->is_B_local);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,pcis->n_B,idx_B_global,PETSC_COPY_VALUES,&pcis->is_B_global);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n_I,idx_I_local,PETSC_COPY_VALUES, &pcis->is_I_local);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n_I,idx_I_global,PETSC_COPY_VALUES,&pcis->is_I_global);CHKERRQ(ierr);
/* Freeing memory and restoring arrays */
ierr = PetscFree(idx_B_local);CHKERRQ(ierr);
ierr = PetscFree(idx_I_local);CHKERRQ(ierr);
ierr = PetscFree(array);CHKERRQ(ierr);
}
/*
Extracting the blocks A_II, A_BI, A_IB and A_BB from A. If the numbering
is such that interior nodes come first than the interface ones, we have
[ | ]
[ A_II | A_IB ]
A = [ | ]
[-----------+------]
[ A_BI | A_BB ]
*/
ierr = MatGetSubMatrix(matis->A,pcis->is_I_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_II);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_B_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_BB);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
if (!issbaij) {
ierr = MatGetSubMatrix(matis->A,pcis->is_I_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_IB);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_B_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_BI);CHKERRQ(ierr);
} else {
Mat newmat;
ierr = MatConvert(matis->A,MATSEQBAIJ,MAT_INITIAL_MATRIX,&newmat);CHKERRQ(ierr);
ierr = MatGetSubMatrix(newmat,pcis->is_I_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_IB);CHKERRQ(ierr);
ierr = MatGetSubMatrix(newmat,pcis->is_B_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_BI);CHKERRQ(ierr);
ierr = MatDestroy(&newmat);CHKERRQ(ierr);
}
/*
Creating work vectors and arrays
*/
ierr = VecDuplicate(matis->x,&pcis->vec1_N);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_N,&pcis->vec2_N);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pcis->n-pcis->n_B,&pcis->vec1_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec2_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec3_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec4_D);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pcis->n_B,&pcis->vec1_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcis->vec2_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcis->vec3_B);CHKERRQ(ierr);
ierr = MatCreateVecs(pc->pmat,&pcis->vec1_global,0);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n,&pcis->work_N);CHKERRQ(ierr);
/* Creating the scatter contexts */
ierr = VecScatterCreate(pcis->vec1_global,pcis->is_I_global,pcis->vec1_D,(IS)0,&pcis->global_to_D);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,pcis->is_B_local,pcis->vec1_B,(IS)0,&pcis->N_to_B);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_global,pcis->is_B_global,pcis->vec1_B,(IS)0,&pcis->global_to_B);CHKERRQ(ierr);
/* Creating scaling "matrix" D */
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_is_use_stiffness_scaling",&pcis->use_stiffness_scaling,NULL);CHKERRQ(ierr);
if (!pcis->D) {
ierr = VecDuplicate(pcis->vec1_B,&pcis->D);CHKERRQ(ierr);
if (!pcis->use_stiffness_scaling) {
ierr = VecSet(pcis->D,pcis->scaling_factor);CHKERRQ(ierr);
} else {
ierr = MatGetDiagonal(matis->A,pcis->vec1_N);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
}
ierr = VecCopy(pcis->D,pcis->vec1_B);CHKERRQ(ierr);
ierr = MatCreateVecs(pc->pmat,&counter,0);CHKERRQ(ierr); /* temporary auxiliar vector */
ierr = VecSet(counter,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,counter,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,counter,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,counter,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,counter,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPointwiseDivide(pcis->D,pcis->D,pcis->vec1_B);CHKERRQ(ierr);
ierr = VecDestroy(&counter);CHKERRQ(ierr);
/* See historical note 01, at the bottom of this file. */
/*
Creating the KSP contexts for the local Dirichlet and Neumann problems.
*/
if (pcis->computesolvers) {
PC pc_ctx;
/* Dirichlet */
ierr = KSPCreate(PETSC_COMM_SELF,&pcis->ksp_D);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcis->ksp_D,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(pcis->ksp_D,pcis->A_II,pcis->A_II);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcis->ksp_D,"is_localD_");CHKERRQ(ierr);
ierr = KSPGetPC(pcis->ksp_D,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCLU);CHKERRQ(ierr);
ierr = KSPSetType(pcis->ksp_D,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcis->ksp_D);CHKERRQ(ierr);
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcis->ksp_D);CHKERRQ(ierr);
/* Neumann */
ierr = KSPCreate(PETSC_COMM_SELF,&pcis->ksp_N);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcis->ksp_N,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(pcis->ksp_N,matis->A,matis->A);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcis->ksp_N,"is_localN_");CHKERRQ(ierr);
ierr = KSPGetPC(pcis->ksp_N,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCLU);CHKERRQ(ierr);
ierr = KSPSetType(pcis->ksp_N,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcis->ksp_N);CHKERRQ(ierr);
{
PetscBool damp_fixed = PETSC_FALSE,
remove_nullspace_fixed = PETSC_FALSE,
set_damping_factor_floating = PETSC_FALSE,
not_damp_floating = PETSC_FALSE,
not_remove_nullspace_floating = PETSC_FALSE;
PetscReal fixed_factor,
floating_factor;
ierr = PetscOptionsGetReal(((PetscObject)pc_ctx)->prefix,"-pc_is_damp_fixed",&fixed_factor,&damp_fixed);CHKERRQ(ierr);
if (!damp_fixed) fixed_factor = 0.0;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_damp_fixed",&damp_fixed,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_remove_nullspace_fixed",&remove_nullspace_fixed,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(((PetscObject)pc_ctx)->prefix,"-pc_is_set_damping_factor_floating",
&floating_factor,&set_damping_factor_floating);CHKERRQ(ierr);
if (!set_damping_factor_floating) floating_factor = 0.0;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_set_damping_factor_floating",&set_damping_factor_floating,NULL);CHKERRQ(ierr);
if (!set_damping_factor_floating) floating_factor = 1.e-12;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_not_damp_floating",¬_damp_floating,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_not_remove_nullspace_floating",¬_remove_nullspace_floating,NULL);CHKERRQ(ierr);
if (pcis->pure_neumann) { /* floating subdomain */
if (!(not_damp_floating)) {
ierr = PCFactorSetShiftType(pc_ctx,MAT_SHIFT_NONZERO);CHKERRQ(ierr);
ierr = PCFactorSetShiftAmount(pc_ctx,floating_factor);CHKERRQ(ierr);
}
if (!(not_remove_nullspace_floating)) {
MatNullSpace nullsp;
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nullsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(pcis->ksp_N,nullsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullsp);CHKERRQ(ierr);
}
} else { /* fixed subdomain */
if (damp_fixed) {
ierr = PCFactorSetShiftType(pc_ctx,MAT_SHIFT_NONZERO);CHKERRQ(ierr);
ierr = PCFactorSetShiftAmount(pc_ctx,floating_factor);CHKERRQ(ierr);
}
if (remove_nullspace_fixed) {
MatNullSpace nullsp;
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nullsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(pcis->ksp_N,nullsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullsp);CHKERRQ(ierr);
}
}
}
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcis->ksp_N);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode port_lsd_bfbt(void)
{
Mat A;
Vec x,b;
KSP ksp_A;
PC pc_A;
IS isu,isp;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = LoadTestMatrices(&A,&x,&b,&isu,&isp);CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp_A);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(ksp_A,"fc_");CHKERRQ(ierr);
ierr = KSPSetOperators(ksp_A,A,A);CHKERRQ(ierr);
ierr = KSPGetPC(ksp_A,&pc_A);CHKERRQ(ierr);
ierr = PCSetType(pc_A,PCFIELDSPLIT);CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc_A,2);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc_A,"velocity",isu);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc_A,"pressure",isp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp_A);CHKERRQ(ierr);
ierr = KSPSolve(ksp_A,b,x);CHKERRQ(ierr);
/* Pull u,p out of x */
{
PetscInt loc;
PetscReal max,norm;
PetscScalar sum;
Vec uvec,pvec;
VecScatter uscat,pscat;
Mat A11,A22;
/* grab matrices and create the compatable u,p vectors */
ierr = MatGetSubMatrix(A,isu,isu,MAT_INITIAL_MATRIX,&A11);CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isp,isp,MAT_INITIAL_MATRIX,&A22);CHKERRQ(ierr);
ierr = MatCreateVecs(A11,&uvec,NULL);CHKERRQ(ierr);
ierr = MatCreateVecs(A22,&pvec,NULL);CHKERRQ(ierr);
/* perform the scatter from x -> (u,p) */
ierr = VecScatterCreate(x,isu,uvec,NULL,&uscat);CHKERRQ(ierr);
ierr = VecScatterBegin(uscat,x,uvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(uscat,x,uvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterCreate(x,isp,pvec,NULL,&pscat);CHKERRQ(ierr);
ierr = VecScatterBegin(pscat,x,pvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(pscat,x,pvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- vector vector values --\n");
ierr = VecMin(uvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(u) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(uvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(u) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(uvec,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(u) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(uvec,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(u) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- pressure vector values --\n");
ierr = VecMin(pvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(pvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(pvec,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(p) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(pvec,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(p) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- Full vector values --\n");
ierr = VecMin(x,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(u,p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(x,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(u,p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(u,p) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(x,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(u,p) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
ierr = VecScatterDestroy(&uscat);CHKERRQ(ierr);
ierr = VecScatterDestroy(&pscat);CHKERRQ(ierr);
ierr = VecDestroy(&uvec);CHKERRQ(ierr);
ierr = VecDestroy(&pvec);CHKERRQ(ierr);
ierr = MatDestroy(&A11);CHKERRQ(ierr);
ierr = MatDestroy(&A22);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp_A);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = ISDestroy(&isu);CHKERRQ(ierr);
ierr = ISDestroy(&isp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
EXTERN_C_END
/* -------------------------------------------------------------------------- */
/*
PCNNCreateCoarseMatrix -
*/
#undef __FUNCT__
#define __FUNCT__ "PCNNCreateCoarseMatrix"
PetscErrorCode PCNNCreateCoarseMatrix (PC pc)
{
MPI_Request *send_request, *recv_request;
PetscErrorCode ierr;
PetscInt i, j, k;
PetscScalar* mat; /* Sub-matrix with this subdomain's contribution to the coarse matrix */
PetscScalar** DZ_OUT; /* proc[k].DZ_OUT[i][] = bit of vector to be sent from processor k to processor i */
/* aliasing some names */
PC_IS* pcis = (PC_IS*)(pc->data);
PC_NN* pcnn = (PC_NN*)pc->data;
PetscInt n_neigh = pcis->n_neigh;
PetscInt* neigh = pcis->neigh;
PetscInt* n_shared = pcis->n_shared;
PetscInt** shared = pcis->shared;
PetscScalar** DZ_IN; /* Must be initialized after memory allocation. */
PetscFunctionBegin;
/* Allocate memory for mat (the +1 is to handle the case n_neigh equal to zero) */
ierr = PetscMalloc((n_neigh*n_neigh+1)*sizeof(PetscScalar),&mat);CHKERRQ(ierr);
/* Allocate memory for DZ */
/* Notice that DZ_OUT[0] is allocated some space that is never used. */
/* This is just in order to DZ_OUT and DZ_IN to have exactly the same form. */
{
PetscInt size_of_Z = 0;
ierr = PetscMalloc ((n_neigh+1)*sizeof(PetscScalar*),&pcnn->DZ_IN);CHKERRQ(ierr);
DZ_IN = pcnn->DZ_IN;
ierr = PetscMalloc ((n_neigh+1)*sizeof(PetscScalar*),&DZ_OUT);CHKERRQ(ierr);
for (i=0; i<n_neigh; i++) {
size_of_Z += n_shared[i];
}
ierr = PetscMalloc ((size_of_Z+1)*sizeof(PetscScalar),&DZ_IN[0]);CHKERRQ(ierr);
ierr = PetscMalloc ((size_of_Z+1)*sizeof(PetscScalar),&DZ_OUT[0]);CHKERRQ(ierr);
}
for (i=1; i<n_neigh; i++) {
DZ_IN[i] = DZ_IN [i-1] + n_shared[i-1];
DZ_OUT[i] = DZ_OUT[i-1] + n_shared[i-1];
}
/* Set the values of DZ_OUT, in order to send this info to the neighbours */
/* First, set the auxiliary array pcis->work_N. */
ierr = PCISScatterArrayNToVecB(pcis->work_N,pcis->D,INSERT_VALUES,SCATTER_REVERSE,pc);CHKERRQ(ierr);
for (i=1; i<n_neigh; i++){
for (j=0; j<n_shared[i]; j++) {
DZ_OUT[i][j] = pcis->work_N[shared[i][j]];
}
}
/* Non-blocking send/receive the common-interface chunks of scaled nullspaces */
/* Notice that send_request[] and recv_request[] could have one less element. */
/* We make them longer to have request[i] corresponding to neigh[i]. */
{
PetscMPIInt tag;
ierr = PetscObjectGetNewTag((PetscObject)pc,&tag);CHKERRQ(ierr);
ierr = PetscMalloc((2*(n_neigh)+1)*sizeof(MPI_Request),&send_request);CHKERRQ(ierr);
recv_request = send_request + (n_neigh);
for (i=1; i<n_neigh; i++) {
ierr = MPI_Isend((void*)(DZ_OUT[i]),n_shared[i],MPIU_SCALAR,neigh[i],tag,((PetscObject)pc)->comm,&(send_request[i]));CHKERRQ(ierr);
ierr = MPI_Irecv((void*)(DZ_IN [i]),n_shared[i],MPIU_SCALAR,neigh[i],tag,((PetscObject)pc)->comm,&(recv_request[i]));CHKERRQ(ierr);
}
}
/* Set DZ_IN[0][] (recall that neigh[0]==rank, always) */
for(j=0; j<n_shared[0]; j++) {
DZ_IN[0][j] = pcis->work_N[shared[0][j]];
}
/* Start computing with local D*Z while communication goes on. */
/* Apply Schur complement. The result is "stored" in vec (more */
/* precisely, vec points to the result, stored in pc_nn->vec1_B) */
/* and also scattered to pcnn->work_N. */
ierr = PCNNApplySchurToChunk(pc,n_shared[0],shared[0],DZ_IN[0],pcis->work_N,pcis->vec1_B,
pcis->vec2_B,pcis->vec1_D,pcis->vec2_D);CHKERRQ(ierr);
/* Compute the first column, while completing the receiving. */
for (i=0; i<n_neigh; i++) {
MPI_Status stat;
PetscMPIInt ind=0;
if (i>0) { ierr = MPI_Waitany(n_neigh-1,recv_request+1,&ind,&stat);CHKERRQ(ierr); ind++;}
mat[ind*n_neigh+0] = 0.0;
for (k=0; k<n_shared[ind]; k++) {
mat[ind*n_neigh+0] += DZ_IN[ind][k] * pcis->work_N[shared[ind][k]];
}
}
/* Compute the remaining of the columns */
for (j=1; j<n_neigh; j++) {
ierr = PCNNApplySchurToChunk(pc,n_shared[j],shared[j],DZ_IN[j],pcis->work_N,pcis->vec1_B,
pcis->vec2_B,pcis->vec1_D,pcis->vec2_D);CHKERRQ(ierr);
for (i=0; i<n_neigh; i++) {
mat[i*n_neigh+j] = 0.0;
for (k=0; k<n_shared[i]; k++) {
mat[i*n_neigh+j] += DZ_IN[i][k] * pcis->work_N[shared[i][k]];
}
}
}
/* Complete the sending. */
if (n_neigh>1) {
MPI_Status *stat;
ierr = PetscMalloc((n_neigh-1)*sizeof(MPI_Status),&stat);CHKERRQ(ierr);
if (n_neigh-1) {ierr = MPI_Waitall(n_neigh-1,&(send_request[1]),stat);CHKERRQ(ierr);}
ierr = PetscFree(stat);CHKERRQ(ierr);
}
/* Free the memory for the MPI requests */
ierr = PetscFree(send_request);CHKERRQ(ierr);
/* Free the memory for DZ_OUT */
if (DZ_OUT) {
ierr = PetscFree(DZ_OUT[0]);CHKERRQ(ierr);
ierr = PetscFree(DZ_OUT);CHKERRQ(ierr);
}
{
PetscMPIInt size;
ierr = MPI_Comm_size(((PetscObject)pc)->comm,&size);CHKERRQ(ierr);
/* Create the global coarse vectors (rhs and solution). */
ierr = VecCreateMPI(((PetscObject)pc)->comm,1,size,&(pcnn->coarse_b));CHKERRQ(ierr);
ierr = VecDuplicate(pcnn->coarse_b,&(pcnn->coarse_x));CHKERRQ(ierr);
/* Create and set the global coarse AIJ matrix. */
ierr = MatCreate(((PetscObject)pc)->comm,&(pcnn->coarse_mat));CHKERRQ(ierr);
ierr = MatSetSizes(pcnn->coarse_mat,1,1,size,size);CHKERRQ(ierr);
ierr = MatSetType(pcnn->coarse_mat,MATAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(pcnn->coarse_mat,1,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(pcnn->coarse_mat,1,PETSC_NULL,1,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSetValues(pcnn->coarse_mat,n_neigh,neigh,n_neigh,neigh,mat,ADD_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(pcnn->coarse_mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (pcnn->coarse_mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
{
PetscMPIInt rank;
PetscScalar one = 1.0;
ierr = MPI_Comm_rank(((PetscObject)pc)->comm,&rank);CHKERRQ(ierr);
/* "Zero out" rows of not-purely-Neumann subdomains */
if (pcis->pure_neumann) { /* does NOT zero the row; create an empty index set. The reason is that MatZeroRows() is collective. */
ierr = MatZeroRows(pcnn->coarse_mat,0,PETSC_NULL,one,0,0);CHKERRQ(ierr);
} else { /* here it DOES zero the row, since it's not a floating subdomain. */
PetscInt row = (PetscInt) rank;
ierr = MatZeroRows(pcnn->coarse_mat,1,&row,one,0,0);CHKERRQ(ierr);
}
}
/* Create the coarse linear solver context */
{
PC pc_ctx, inner_pc;
KSP inner_ksp;
ierr = KSPCreate(((PetscObject)pc)->comm,&pcnn->ksp_coarse);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcnn->ksp_coarse,(PetscObject)pc,2);CHKERRQ(ierr);
ierr = KSPSetOperators(pcnn->ksp_coarse,pcnn->coarse_mat,pcnn->coarse_mat,SAME_PRECONDITIONER);CHKERRQ(ierr);
ierr = KSPGetPC(pcnn->ksp_coarse,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCREDUNDANT);CHKERRQ(ierr);
ierr = KSPSetType(pcnn->ksp_coarse,KSPPREONLY);CHKERRQ(ierr);
ierr = PCRedundantGetKSP(pc_ctx,&inner_ksp);CHKERRQ(ierr);
ierr = KSPGetPC(inner_ksp,&inner_pc);CHKERRQ(ierr);
ierr = PCSetType(inner_pc,PCLU);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcnn->ksp_coarse,"nn_coarse_");CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcnn->ksp_coarse);CHKERRQ(ierr);
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcnn->ksp_coarse);CHKERRQ(ierr);
}
/* Free the memory for mat */
ierr = PetscFree(mat);CHKERRQ(ierr);
/* for DEBUGGING, save the coarse matrix to a file. */
{
PetscBool flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-pc_nn_save_coarse_matrix",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer viewer;
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,"coarse.m",&viewer);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);CHKERRQ(ierr);
ierr = MatView(pcnn->coarse_mat,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
}
/* Set the variable pcnn->factor_coarse_rhs. */
pcnn->factor_coarse_rhs = (pcis->pure_neumann) ? 1.0 : 0.0;
/* See historical note 02, at the bottom of this file. */
PetscFunctionReturn(0);
}
int main(int argc, char **argv){
PetscErrorCode ierr;
int nx = 63, ny = 63;
DM dm;
PetscBool flg;
Mat A;
Vec u, b;
KSP solver;
PC pc;
double norm;
PetscInt stage;
ierr = PetscInitialize(&argc, &argv, NULL, NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL, "-nx", &nx, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL, "-ny", &ny, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL, "-assemble", &flg);CHKERRQ(ierr);
ierr = PetscLogStageRegister("preparing",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Domain creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = createDomain(&dm, nx, ny);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("matrix creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = createMat(dm, &A, flg);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("Vector creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = DMCreateGlobalVector(dm, &b);CHKERRQ(ierr);
ierr = VecDuplicate(b, &u);CHKERRQ(ierr);
ierr = PetscLogStageRegister("Domain initialisation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = init2d(dm, b);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("solver creation",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD, &solver);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(solver, "poisson_");CHKERRQ(ierr);
ierr = KSPSetOperators(solver, A, A, DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPSetType(solver, KSPCG);
ierr = KSPGetPC(solver, &pc);CHKERRQ(ierr);
ierr = PCSetType(pc, PCNONE);CHKERRQ(ierr);
ierr = KSPSetFromOptions(solver);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStageRegister("Solving",&stage); CHKERRQ(ierr);
ierr = PetscLogStagePush(stage);CHKERRQ(ierr);
ierr = KSPSolve(solver, b, u);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
//ierr = VecView(u, PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
//sleep(10);
VecDestroy(&u);
VecDestroy(&b);
MatDestroy(&A);
DMDestroy(&dm);
KSPDestroy(&solver);
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode PCSetUp_Redundant(PC pc)
{
PC_Redundant *red = (PC_Redundant*)pc->data;
PetscErrorCode ierr;
PetscInt mstart,mend,mlocal,m,mlocal_sub,rstart_sub,rend_sub,mloc_sub;
PetscMPIInt size;
MatReuse reuse = MAT_INITIAL_MATRIX;
MatStructure str = DIFFERENT_NONZERO_PATTERN;
MPI_Comm comm = ((PetscObject)pc)->comm,subcomm;
Vec vec;
PetscMPIInt subsize,subrank;
const char *prefix;
const PetscInt *range;
PetscFunctionBegin;
ierr = MatGetVecs(pc->pmat,&vec,0);CHKERRQ(ierr);
ierr = VecGetSize(vec,&m);CHKERRQ(ierr);
if (!pc->setupcalled) {
if (!red->psubcomm) {
ierr = PetscSubcommCreate(comm,red->nsubcomm,&red->psubcomm);CHKERRQ(ierr);
ierr = PetscLogObjectMemory(pc,sizeof(PetscSubcomm));CHKERRQ(ierr);
/* create a new PC that processors in each subcomm have copy of */
subcomm = red->psubcomm->comm;
ierr = KSPCreate(subcomm,&red->ksp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = PetscLogObjectParent(pc,red->ksp);CHKERRQ(ierr);
ierr = KSPSetType(red->ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(red->ksp,&red->pc);CHKERRQ(ierr);
ierr = PCSetType(red->pc,PCLU);CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(red->ksp,prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(red->ksp,"redundant_");CHKERRQ(ierr);
} else {
subcomm = red->psubcomm->comm;
}
/* create working vectors xsub/ysub and xdup/ydup */
ierr = VecGetLocalSize(vec,&mlocal);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(vec,&mstart,&mend);CHKERRQ(ierr);
/* get local size of xsub/ysub */
ierr = MPI_Comm_size(subcomm,&subsize);CHKERRQ(ierr);
ierr = MPI_Comm_rank(subcomm,&subrank);CHKERRQ(ierr);
ierr = MatGetOwnershipRanges(pc->pmat,&range);CHKERRQ(ierr);
rstart_sub = range[red->psubcomm->n*subrank]; /* rstart in xsub/ysub */
if (subrank+1 < subsize){
rend_sub = range[red->psubcomm->n*(subrank+1)];
} else {
rend_sub = m;
}
mloc_sub = rend_sub - rstart_sub;
ierr = VecCreateMPI(subcomm,mloc_sub,PETSC_DECIDE,&red->ysub);CHKERRQ(ierr);
/* create xsub with empty local arrays, because xdup's arrays will be placed into it */
ierr = VecCreateMPIWithArray(subcomm,mloc_sub,PETSC_DECIDE,PETSC_NULL,&red->xsub);CHKERRQ(ierr);
/* create xdup and ydup. ydup has empty local arrays because ysub's arrays will be place into it.
Note: we use communicator dupcomm, not ((PetscObject)pc)->comm! */
ierr = VecCreateMPI(red->psubcomm->dupparent,mloc_sub,PETSC_DECIDE,&red->xdup);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(red->psubcomm->dupparent,mloc_sub,PETSC_DECIDE,PETSC_NULL,&red->ydup);CHKERRQ(ierr);
/* create vec scatters */
if (!red->scatterin){
IS is1,is2;
PetscInt *idx1,*idx2,i,j,k;
ierr = PetscMalloc2(red->psubcomm->n*mlocal,PetscInt,&idx1,red->psubcomm->n*mlocal,PetscInt,&idx2);CHKERRQ(ierr);
j = 0;
for (k=0; k<red->psubcomm->n; k++){
for (i=mstart; i<mend; i++){
idx1[j] = i;
idx2[j++] = i + m*k;
}
}
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(vec,is1,red->xdup,is2,&red->scatterin);CHKERRQ(ierr);
ierr = ISDestroy(is1);CHKERRQ(ierr);
ierr = ISDestroy(is2);CHKERRQ(ierr);
ierr = ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*m,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(comm,mlocal,mstart,1,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(red->xdup,is1,vec,is2,&red->scatterout);CHKERRQ(ierr);
ierr = ISDestroy(is1);CHKERRQ(ierr);
ierr = ISDestroy(is2);CHKERRQ(ierr);
ierr = PetscFree2(idx1,idx2);CHKERRQ(ierr);
}
}
ierr = VecDestroy(vec);CHKERRQ(ierr);
/* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size == 1) {
red->useparallelmat = PETSC_FALSE;
}
if (red->useparallelmat) {
if (pc->setupcalled == 1 && pc->flag == DIFFERENT_NONZERO_PATTERN) {
/* destroy old matrices */
if (red->pmats) {
ierr = MatDestroy(red->pmats);CHKERRQ(ierr);
}
} else if (pc->setupcalled == 1) {
reuse = MAT_REUSE_MATRIX;
str = SAME_NONZERO_PATTERN;
}
/* grab the parallel matrix and put it into processors of a subcomminicator */
/*--------------------------------------------------------------------------*/
ierr = VecGetLocalSize(red->ysub,&mlocal_sub);CHKERRQ(ierr);
ierr = MatGetRedundantMatrix(pc->pmat,red->psubcomm->n,red->psubcomm->comm,mlocal_sub,reuse,&red->pmats);CHKERRQ(ierr);
/* tell PC of the subcommunicator its operators */
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats,str);CHKERRQ(ierr);
} else {
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat,pc->flag);CHKERRQ(ierr);
}
if (pc->setfromoptionscalled){
ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr);
}
ierr = KSPSetUp(red->ksp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PCMGSetLevels - Sets the number of levels to use with MG.
Must be called before any other MG routine.
Logically Collective on PC
Input Parameters:
+ pc - the preconditioner context
. levels - the number of levels
- comms - optional communicators for each level; this is to allow solving the coarser problems
on smaller sets of processors. Use NULL_OBJECT for default in Fortran
Level: intermediate
Notes:
If the number of levels is one then the multigrid uses the -mg_levels prefix
for setting the level options rather than the -mg_coarse prefix.
.keywords: MG, set, levels, multigrid
.seealso: PCMGSetType(), PCMGGetLevels()
@*/
PetscErrorCode PCMGSetLevels(PC pc,PetscInt levels,MPI_Comm *comms)
{
PetscErrorCode ierr;
PC_MG *mg = (PC_MG*)pc->data;
MPI_Comm comm;
PC_MG_Levels **mglevels = mg->levels;
PCMGType mgtype = mg->am;
PetscInt mgctype = (PetscInt) PC_MG_CYCLE_V;
PetscInt i;
PetscMPIInt size;
const char *prefix;
PC ipc;
PetscInt n;
PetscFunctionBegin;
PetscValidHeaderSpecific(pc,PC_CLASSID,1);
PetscValidLogicalCollectiveInt(pc,levels,2);
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
if (mg->nlevels == levels) PetscFunctionReturn(0);
if (mglevels) {
mgctype = mglevels[0]->cycles;
/* changing the number of levels so free up the previous stuff */
ierr = PCReset_MG(pc);CHKERRQ(ierr);
n = mglevels[0]->levels;
for (i=0; i<n; i++) {
if (mglevels[i]->smoothd != mglevels[i]->smoothu) {
ierr = KSPDestroy(&mglevels[i]->smoothd);CHKERRQ(ierr);
}
ierr = KSPDestroy(&mglevels[i]->smoothu);CHKERRQ(ierr);
ierr = PetscFree(mglevels[i]);CHKERRQ(ierr);
}
ierr = PetscFree(mg->levels);CHKERRQ(ierr);
}
mg->nlevels = levels;
ierr = PetscMalloc1(levels,&mglevels);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)pc,levels*(sizeof(PC_MG*)));CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
mg->stageApply = 0;
for (i=0; i<levels; i++) {
ierr = PetscNewLog(pc,&mglevels[i]);CHKERRQ(ierr);
mglevels[i]->level = i;
mglevels[i]->levels = levels;
mglevels[i]->cycles = mgctype;
mg->default_smoothu = 2;
mg->default_smoothd = 2;
mglevels[i]->eventsmoothsetup = 0;
mglevels[i]->eventsmoothsolve = 0;
mglevels[i]->eventresidual = 0;
mglevels[i]->eventinterprestrict = 0;
if (comms) comm = comms[i];
ierr = KSPCreate(comm,&mglevels[i]->smoothd);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(mglevels[i]->smoothd,pc->erroriffailure);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)mglevels[i]->smoothd,(PetscObject)pc,levels-i);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(mglevels[i]->smoothd,prefix);CHKERRQ(ierr);
ierr = PetscObjectComposedDataSetInt((PetscObject) mglevels[i]->smoothd, PetscMGLevelId, mglevels[i]->level);CHKERRQ(ierr);
if (i || levels == 1) {
char tprefix[128];
ierr = KSPSetType(mglevels[i]->smoothd,KSPCHEBYSHEV);CHKERRQ(ierr);
ierr = KSPSetConvergenceTest(mglevels[i]->smoothd,KSPConvergedSkip,NULL,NULL);CHKERRQ(ierr);
ierr = KSPSetNormType(mglevels[i]->smoothd,KSP_NORM_NONE);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[i]->smoothd,&ipc);CHKERRQ(ierr);
ierr = PCSetType(ipc,PCSOR);CHKERRQ(ierr);
ierr = KSPSetTolerances(mglevels[i]->smoothd,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT, mg->default_smoothd);CHKERRQ(ierr);
sprintf(tprefix,"mg_levels_%d_",(int)i);
ierr = KSPAppendOptionsPrefix(mglevels[i]->smoothd,tprefix);CHKERRQ(ierr);
} else {
ierr = KSPAppendOptionsPrefix(mglevels[0]->smoothd,"mg_coarse_");CHKERRQ(ierr);
/* coarse solve is (redundant) LU by default; set shifttype NONZERO to avoid annoying zero-pivot in LU preconditioner */
ierr = KSPSetType(mglevels[0]->smoothd,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[0]->smoothd,&ipc);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size > 1) {
ierr = PCSetType(ipc,PCREDUNDANT);CHKERRQ(ierr);
} else {
ierr = PCSetType(ipc,PCLU);CHKERRQ(ierr);
}
ierr = PCFactorSetShiftType(ipc,MAT_SHIFT_INBLOCKS);CHKERRQ(ierr);
}
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)mglevels[i]->smoothd);CHKERRQ(ierr);
mglevels[i]->smoothu = mglevels[i]->smoothd;
mg->rtol = 0.0;
mg->abstol = 0.0;
mg->dtol = 0.0;
mg->ttol = 0.0;
mg->cyclesperpcapply = 1;
}
mg->levels = mglevels;
ierr = PCMGSetType(pc,mgtype);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETScKrylovLinearSolver::initializeSolverState(const SAMRAIVectorReal<NDIM, double>& x,
const SAMRAIVectorReal<NDIM, double>& b)
{
IBTK_TIMER_START(t_initialize_solver_state);
int ierr;
// Rudimentary error checking.
#if !defined(NDEBUG)
if (x.getNumberOfComponents() != b.getNumberOfComponents())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " vectors must have the same number of components"
<< std::endl);
}
const Pointer<PatchHierarchy<NDIM> >& patch_hierarchy = x.getPatchHierarchy();
if (patch_hierarchy != b.getPatchHierarchy())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " vectors must have the same hierarchy" << std::endl);
}
const int coarsest_ln = x.getCoarsestLevelNumber();
if (coarsest_ln < 0)
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " coarsest level number must not be negative"
<< std::endl);
}
if (coarsest_ln != b.getCoarsestLevelNumber())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " vectors must have same coarsest level number"
<< std::endl);
}
const int finest_ln = x.getFinestLevelNumber();
if (finest_ln < coarsest_ln)
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " finest level number must be >= coarsest level number"
<< std::endl);
}
if (finest_ln != b.getFinestLevelNumber())
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " vectors must have same finest level number"
<< std::endl);
}
for (int ln = coarsest_ln; ln <= finest_ln; ++ln)
{
if (!patch_hierarchy->getPatchLevel(ln))
{
TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
<< " hierarchy level " << ln << " does not exist"
<< std::endl);
}
}
#endif
// Deallocate the solver state if the solver is already initialized.
if (d_is_initialized)
{
d_reinitializing_solver = true;
deallocateSolverState();
}
// Create the KSP solver.
if (d_managing_petsc_ksp)
{
ierr = KSPCreate(d_petsc_comm, &d_petsc_ksp);
IBTK_CHKERRQ(ierr);
resetKSPOptions();
}
else if (!d_petsc_ksp)
{
TBOX_ERROR(d_object_name
<< "::initializeSolverState()\n"
<< " cannot initialize solver state for wrapped PETSc KSP object "
"if the wrapped object is NULL" << std::endl);
}
// Setup solution and rhs vectors.
d_x = x.cloneVector(x.getName());
d_petsc_x = PETScSAMRAIVectorReal::createPETScVector(d_x, d_petsc_comm);
d_b = b.cloneVector(b.getName());
d_petsc_b = PETScSAMRAIVectorReal::createPETScVector(d_b, d_petsc_comm);
// Initialize the linear operator and preconditioner objects.
if (d_A) d_A->initializeOperatorState(*d_x, *d_b);
if (d_managing_petsc_ksp || d_user_provided_mat) resetKSPOperators();
if (d_pc_solver) d_pc_solver->initializeSolverState(*d_x, *d_b);
if (d_managing_petsc_ksp || d_user_provided_pc) resetKSPPC();
// Set the KSP options from the PETSc options database.
if (d_options_prefix != "")
{
ierr = KSPSetOptionsPrefix(d_petsc_ksp, d_options_prefix.c_str());
IBTK_CHKERRQ(ierr);
}
ierr = KSPSetFromOptions(d_petsc_ksp);
IBTK_CHKERRQ(ierr);
// Reset the member state variables to correspond to the values used by the
// KSP object. (Command-line options always take precedence.)
const char* ksp_type;
ierr = KSPGetType(d_petsc_ksp, &ksp_type);
IBTK_CHKERRQ(ierr);
d_ksp_type = ksp_type;
PetscBool initial_guess_nonzero;
ierr = KSPGetInitialGuessNonzero(d_petsc_ksp, &initial_guess_nonzero);
IBTK_CHKERRQ(ierr);
d_initial_guess_nonzero = (initial_guess_nonzero == PETSC_TRUE);
ierr = KSPGetTolerances(
d_petsc_ksp, &d_rel_residual_tol, &d_abs_residual_tol, NULL, &d_max_iterations);
IBTK_CHKERRQ(ierr);
// Configure the nullspace object.
resetKSPNullspace();
// Indicate that the solver is initialized.
d_reinitializing_solver = false;
d_is_initialized = true;
IBTK_TIMER_STOP(t_initialize_solver_state);
return;
} // initializeSolverState
