/*@
KSPFGMRESModifyPCKSP - modifies the attributes of the
GMRES preconditioner. It serves as an example (not as something
useful!)
Input Parameters:
+ ksp - the ksp context being used.
. total_its - the total number of FGMRES iterations that have occurred.
. loc_its - the number of FGMRES iterations since last restart.
. res_norm - the current residual norm.
- dummy - context, not used here
Level: intermediate
Contributed by Allison Baker
This could be used as a template!
.seealso: KSPFGMRESSetModifyPC(), KSPFGMRESModifyPCKSP()
@*/
PetscErrorCode KSPFGMRESModifyPCKSP(KSP ksp,PetscInt total_its,PetscInt loc_its,PetscReal res_norm,void *dummy)
{
PC pc;
PetscErrorCode ierr;
PetscInt maxits;
KSP sub_ksp;
PetscReal rtol,abstol,dtol;
PetscBool isksp;
PetscFunctionBegin;
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)pc,PCKSP,&isksp);CHKERRQ(ierr);
if (isksp) {
ierr = PCKSPGetKSP(pc,&sub_ksp);CHKERRQ(ierr);
/* note that at this point you could check the type of KSP with KSPGetType() */
/* Now we can use functions such as KSPGMRESSetRestart() or
KSPGMRESSetOrthogonalization() or KSPSetTolerances() */
ierr = KSPGetTolerances(sub_ksp,&rtol,&abstol,&dtol,&maxits);CHKERRQ(ierr);
if (!loc_its) {
rtol = .1;
} else {
rtol *= .9;
}
ierr = KSPSetTolerances(sub_ksp,rtol,abstol,dtol,maxits);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@C
KSPMonitorDynamicTolerance - Recompute the inner tolerance in every
outer iteration in an adaptive way.
Collective on KSP
Input Parameters:
+ ksp - iterative context
. n - iteration number (not used)
. fnorm - the current residual norm
. dummy - some context as a C struct. fields:
coef: a scaling coefficient. default 1.0. can be passed through
-sub_ksp_dynamic_tolerance_param
bnrm: norm of the right-hand side. store it to avoid repeated calculation
Notes:
This may be useful for a flexibly preconditioner Krylov method to
control the accuracy of the inner solves needed to gaurantee the
convergence of the outer iterations.
Level: advanced
.keywords: KSP, inner tolerance
.seealso: KSPMonitorDynamicToleranceDestroy()
@*/
PetscErrorCode KSPMonitorDynamicTolerance(KSP ksp,PetscInt its,PetscReal fnorm,void *dummy)
{
PetscErrorCode ierr;
PC pc;
PetscReal outer_rtol, outer_abstol, outer_dtol, inner_rtol;
PetscInt outer_maxits,nksp,first,i;
KSPDynTolCtx *scale = (KSPDynTolCtx*)dummy;
KSP kspinner = NULL, *subksp = NULL;
PetscFunctionBegin;
ierr = KSPGetPC(ksp, &pc);
CHKERRQ(ierr);
/* compute inner_rtol */
if (scale->bnrm < 0.0) {
Vec b;
ierr = KSPGetRhs(ksp, &b);
CHKERRQ(ierr);
ierr = VecNorm(b, NORM_2, &(scale->bnrm));
CHKERRQ(ierr);
}
ierr = KSPGetTolerances(ksp, &outer_rtol, &outer_abstol, &outer_dtol, &outer_maxits);
CHKERRQ(ierr);
inner_rtol = PetscMin(scale->coef * scale->bnrm * outer_rtol / fnorm, 0.999);
/*ierr = PetscPrintf(PETSC_COMM_WORLD, " Inner rtol = %g\n", (double)inner_rtol);CHKERRQ(ierr);*/
/* if pc is ksp */
ierr = PCKSPGetKSP(pc, &kspinner);
CHKERRQ(ierr);
if (kspinner) {
ierr = KSPSetTolerances(kspinner, inner_rtol, outer_abstol, outer_dtol, outer_maxits);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* if pc is bjacobi */
ierr = PCBJacobiGetSubKSP(pc, &nksp, &first, &subksp);
CHKERRQ(ierr);
if (subksp) {
for (i=0; i<nksp; i++) {
ierr = KSPSetTolerances(subksp[i], inner_rtol, outer_abstol, outer_dtol, outer_maxits);
CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/* todo: dynamic tolerance may apply to other types of pc too */
PetscFunctionReturn(0);
}
PetscErrorCode KSPSolve_TSIRM(KSP ksp)
{
PetscErrorCode ierr;
KSP_TSIRM *tsirm = (KSP_TSIRM*)ksp->data;
KSP sub_ksp;
PC pc;
Mat AS;
Vec x,b;
PetscScalar *array;
PetscReal norm = 20;
PetscInt i,*ind_row,first_iteration = 1,its = 0,total = 0,col = 0;
PetscInt restart = 30;
KSP ksp_min; /* KSP for minimization */
PC pc_min; /* PC for minimization */
PetscFunctionBegin;
x = ksp->vec_sol; /* Solution vector */
b = ksp->vec_rhs; /* Right-hand side vector */
/* Row indexes (these indexes are global) */
ierr = PetscMalloc1(tsirm->Iend-tsirm->Istart,&ind_row);
CHKERRQ(ierr);
for (i=0; i<tsirm->Iend-tsirm->Istart; i++) ind_row[i] = i+tsirm->Istart;
/* Inner solver */
ierr = KSPGetPC(ksp,&pc);
CHKERRQ(ierr);
ierr = PCKSPGetKSP(pc,&sub_ksp);
CHKERRQ(ierr);
ierr = KSPSetTolerances(sub_ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,restart);
CHKERRQ(ierr);
/* previously it seemed good but with SNES it seems not good... */
ierr = KSP_MatMult(sub_ksp,tsirm->A,x,tsirm->r);
CHKERRQ(ierr);
ierr = VecAXPY(tsirm->r,-1,b);
CHKERRQ(ierr);
ierr = VecNorm(tsirm->r,NORM_2,&norm);
CHKERRQ(ierr);
ksp->its = 0;
ierr = KSPConvergedDefault(ksp,ksp->its,norm,&ksp->reason,ksp->cnvP);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(sub_ksp,PETSC_TRUE);
CHKERRQ(ierr);
do {
for (col=0; col<tsirm->size_ls && ksp->reason==0; col++) {
/* Solve (inner iteration) */
ierr = KSPSolve(sub_ksp,b,x);
CHKERRQ(ierr);
ierr = KSPGetIterationNumber(sub_ksp,&its);
CHKERRQ(ierr);
total += its;
/* Build S^T */
ierr = VecGetArray(x,&array);
CHKERRQ(ierr);
ierr = MatSetValues(tsirm->S,tsirm->Iend-tsirm->Istart,ind_row,1,&col,array,INSERT_VALUES);
CHKERRQ(ierr);
ierr = VecRestoreArray(x,&array);
CHKERRQ(ierr);
ierr = KSPGetResidualNorm(sub_ksp,&norm);
CHKERRQ(ierr);
ksp->rnorm = norm;
ksp->its ++;
ierr = KSPConvergedDefault(ksp,ksp->its,norm,&ksp->reason,ksp->cnvP);
CHKERRQ(ierr);
ierr = KSPMonitor(ksp,ksp->its,norm);
CHKERRQ(ierr);
}
/* Minimization step */
if (!ksp->reason) {
ierr = MatAssemblyBegin(tsirm->S,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(tsirm->S,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
if (first_iteration) {
ierr = MatMatMult(tsirm->A,tsirm->S,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&AS);
CHKERRQ(ierr);
first_iteration = 0;
} else {
ierr = MatMatMult(tsirm->A,tsirm->S,MAT_REUSE_MATRIX,PETSC_DEFAULT,&AS);
CHKERRQ(ierr);
}
/* CGLS or LSQR method to minimize the residuals*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp_min);
CHKERRQ(ierr);
if (tsirm->cgls) {
ierr = KSPSetType(ksp_min,KSPCGLS);
CHKERRQ(ierr);
} else {
ierr = KSPSetType(ksp_min,KSPLSQR);
CHKERRQ(ierr);
}
ierr = KSPSetOperators(ksp_min,AS,AS);
CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp_min,tsirm->tol_ls,PETSC_DEFAULT,PETSC_DEFAULT,tsirm->maxiter_ls);
CHKERRQ(ierr);
ierr = KSPGetPC(ksp_min,&pc_min);
CHKERRQ(ierr);
ierr = PCSetType(pc_min,PCNONE);
CHKERRQ(ierr);
ierr = KSPSolve(ksp_min,b,tsirm->Alpha);
CHKERRQ(ierr); /* Find Alpha such that ||AS Alpha = b|| */
ierr = KSPDestroy(&ksp_min);
CHKERRQ(ierr);
/* Apply minimization */
ierr = MatMult(tsirm->S,tsirm->Alpha,x);
CHKERRQ(ierr); /* x = S * Alpha */
}
} while (ksp->its<ksp->max_it && !ksp->reason);
ierr = MatDestroy(&AS);
CHKERRQ(ierr);
ierr = PetscFree(ind_row);
CHKERRQ(ierr);
ksp->its = total;
PetscFunctionReturn(0);
}
PetscErrorCode BSSCR_DRIVER_auglag( KSP ksp, Mat stokes_A, Vec stokes_x, Vec stokes_b, Mat approxS,
MatStokesBlockScaling BA, PetscTruth sym, KSP_BSSCR * bsscrp_self )
{
AugLagStokes_SLE * stokesSLE = (AugLagStokes_SLE*)bsscrp_self->solver->st_sle;
PetscTruth uzawastyle, KisJustK=PETSC_TRUE, restorek, change_A11rhspresolve;
PetscTruth usePreviousGuess, useNormInfStoppingConditions, useNormInfMonitor, found, forcecorrection;
PetscTruth change_backsolve, mg_active;
PetscErrorCode ierr;
//PetscInt monitor_index;
PetscInt max_it,min_it;
KSP ksp_inner, ksp_S, ksp_new_inner;
PC pc_S, pcInner;
Mat K,G,D,C, S, K2;// Korig;
Vec u,p,f,f2=0,f3=0,h, h_hat,t;
MGContext mgCtx;
double mgSetupTime, scrSolveTime, a11SingleSolveTime, penaltyNumber;// hFactor;
static int been_here = 0; /* Ha Ha Ha !! */
char name[PETSC_MAX_PATH_LEN];
char matname[PETSC_MAX_PATH_LEN];
char suffix[PETSC_MAX_PATH_LEN];
char str[PETSC_MAX_PATH_LEN];
PetscTruth flg, extractMats;
PetscLogDouble flopsA,flopsB;
/***************************************************************************************************************/
/***************************************************************************************************************/
//if( bsscrp_self->solver->st_sle->context->loadFromCheckPoint ){
// been_here=1;
//}
/* get sub matrix / vector objects */
/* note that here, the matrix D should always exist. It is set up in _StokesBlockKSPInterface_Solve in StokesBlockKSPInterface.c */
/* now extract K,G etc from a MatNest object */
MatNestGetSubMat( stokes_A, 0,0, &K );
MatNestGetSubMat( stokes_A, 0,1, &G );
MatNestGetSubMat( stokes_A, 1,0, &D );if(!D){ PetscPrintf( PETSC_COMM_WORLD, "D does not exist but should!!\n"); exit(1); }
MatNestGetSubMat( stokes_A, 1,1, &C );
VecNestGetSubVec( stokes_x, 0, &u );
VecNestGetSubVec( stokes_x, 1, &p );
VecNestGetSubVec( stokes_b, 0, &f );
VecNestGetSubVec( stokes_b, 1, &h );
PetscPrintf( PETSC_COMM_WORLD, "\n\n---------- AUGMENTED LAGRANGIAN K2 METHOD ---------\n\n" );
PetscPrintf( PETSC_COMM_WORLD, "----------- Penalty = %f\n\n", bsscrp_self->solver->penaltyNumber );
sprintf(suffix,"%s","x");
PetscOptionsGetString( PETSC_NULL, "-matsuffix", suffix, PETSC_MAX_PATH_LEN-1, &extractMats );
flg=0;
PetscOptionsGetString( PETSC_NULL, "-matdumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"K%s",suffix);
bsscr_dirwriteMat( K, matname,str, "Writing K matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"G%s",suffix);
bsscr_dirwriteMat( G, matname,str, "Writing G matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"D%s",suffix);
bsscr_dirwriteMat( D, matname,str, "Writing D matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"f%s",suffix);
bsscr_dirwriteVec( f, matname,str, "Writing f vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"h%s",suffix);
bsscr_dirwriteVec( h, matname,str, "Writing h vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"Shat%s",suffix);
bsscr_dirwriteMat( approxS, matname,str, "Writing Shat matrix in al Solver");
if(C){
sprintf(str,"%s/",name); sprintf(matname,"C%s",suffix);
bsscr_dirwriteMat( C, matname,str, "Writing C matrix in al Solver");
}
}
mg_active=PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_mg_active", &mg_active, &found );
bsscrp_self->solver->mg_active=mg_active;
penaltyNumber = bsscrp_self->solver->penaltyNumber;
//hFactor = stokesSLE->hFactor;
/***************************************************************************************************************/
/***************************************************************************************************************/
/****** GET K2 ****************************************************************************************/
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
flg=0;
PetscOptionsGetString( PETSC_NULL, "-matdumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"K2%s",suffix);
bsscr_dirwriteMat( bsscrp_self->K2, matname,str, "Writing K2 matrix in al Solver");
}
K2=bsscrp_self->K2;
scrSolveTime = MPI_Wtime();
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* K+p*K2 in time: %lf seconds\n\n", scrSolveTime);
KisJustK=PETSC_FALSE;
forcecorrection=PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL ,"-force_correction", &forcecorrection, &found );
if(forcecorrection){
if(bsscrp_self->f2 && forcecorrection){
f2=bsscrp_self->f2;
ierr=VecAXPY(f,penaltyNumber,f2);/* f <- f +a*f2 */
}else{
switch (bsscrp_self->k2type) {
case (K2_GG):
{
VecDuplicate( u, &f3 );
MatMult( G, h, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
}
break;
case (K2_GMG):
{
Mat M;
Vec Mdiag;
VecDuplicate( u, &f3 );
M = bsscrp_self->solver->mStiffMat->matrix;
MatGetVecs( M, &Mdiag, PETSC_NULL );
MatGetDiagonal( M, Mdiag );
VecReciprocal(Mdiag);
VecPointwiseMult(Mdiag,Mdiag,h);
MatMult( G, Mdiag, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
Stg_VecDestroy(&Mdiag);
}
break;
case (K2_DGMGD):
{
Mat M;
Vec Mdiag;
VecDuplicate( u, &f3 );
M = bsscrp_self->solver->mStiffMat->matrix;
MatGetVecs( M, &Mdiag, PETSC_NULL );
MatGetDiagonal( M, Mdiag );
VecReciprocal(Mdiag);
VecPointwiseMult(Mdiag,Mdiag,h);
MatMult( G, Mdiag, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
Stg_VecDestroy(&Mdiag);
}
break;
case (K2_NULL):
{
;
}
break;
case (K2_SLE):
{
;
}
break;
}
}
}
}
/* Create Schur complement matrix */
MatCreateSchurComplement(K,K,G,D,C, &S);
MatSchurComplementGetKSP( S, &ksp_inner);
KSPGetPC( ksp_inner, &pcInner );
/***************************************************************************************************************/
/***************************************************************************************************************/
/********* SET PREFIX FOR INNER/VELOCITY KSP *************************************************************/
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
Stg_KSPSetOperators(ksp_inner, K, K, DIFFERENT_NONZERO_PATTERN);
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest( ksp_inner );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor) KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfMonitor, PETSC_NULL, PETSC_NULL );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_to_norm_2_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor) KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/***************************************************************************************************************/
/***************************************************************************************************************/
/* If multigrid is enabled, set it now. */
change_A11rhspresolve = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-change_A11rhspresolve", &change_A11rhspresolve, &found );
if(bsscrp_self->solver->mg_active && !change_A11rhspresolve) { mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx ); }
/***************************************************************************************************************/
/***************************************************************************************************************/
/* create right hand side */
if(change_A11rhspresolve){
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix(ksp_new_inner, "rhsA11_");
MatSchurComplementSetKSP( S, ksp_new_inner );/* this call destroys the ksp_inner that is already set on S */
ksp_inner=ksp_new_inner;
KSPGetPC( ksp_inner, &pcInner );
KSPSetFromOptions(ksp_inner); /* make sure we are setting up our solver how we want it */
}
MatGetVecs( S, PETSC_NULL, &h_hat );
Vec f_tmp;
/* It may be the case that the current velocity solution might not be bad guess for f_tmp? <-- maybe not */
MatGetVecs( K, PETSC_NULL, &f_tmp );
scrSolveTime = MPI_Wtime();
KSPSolve(ksp_inner, f, f_tmp);
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve for RHS setup Finished in time: %lf seconds\n\n", scrSolveTime);
MatMult(D, f_tmp, h_hat);
VecAYPX(h_hat, -1.0, h); /* Computes y = x + alpha y. h_hat -> h - Gt*K^(-1)*f*/
Stg_VecDestroy(&f_tmp);
if(bsscrp_self->solver->mg_active && change_A11rhspresolve) {
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix( ksp_new_inner, "A11_" );
MatSchurComplementSetKSP( S, ksp_new_inner );
ksp_inner=ksp_new_inner;
//MatSchurSetKSP( S, ksp_inner );
KSPGetPC( ksp_inner, &pcInner );
KSPSetFromOptions( ksp_inner );
mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx );
}
/* create solver for S p = h_hat */
KSPCreate( PETSC_COMM_WORLD, &ksp_S );
KSPSetOptionsPrefix( ksp_S, "scr_");
/* By default use the UW approxS Schur preconditioner -- same as the one used by the Uzawa solver */
/* Note that if scaling is activated then the approxS matrix has been scaled already */
/* so no need to rebuild in the case of scaling as we have been doing */
if(!approxS){ PetscPrintf( PETSC_COMM_WORLD, "WARNING approxS is NULL\n"); }
Stg_KSPSetOperators( ksp_S, S, S, SAME_NONZERO_PATTERN );
KSPSetType( ksp_S, "cg" );
KSPGetPC( ksp_S, &pc_S );
BSSCR_BSSCR_StokesCreatePCSchur2( K,G,D,C,approxS, pc_S, sym, bsscrp_self );
flg=0;
PetscOptionsGetString( PETSC_NULL, "-NN", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
Mat Smat, Pmat;
//MatStructure mstruct;
Stg_PCGetOperators( pc_S, &Smat, &Pmat, NULL );
sprintf(str,"%s/",name); sprintf(matname,"Pmat%s",suffix);
bsscr_dirwriteMat( Pmat, matname,str, "Writing Pmat matrix in al Solver");
}
uzawastyle=PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-uzawa_style", &uzawastyle, &found );
if(uzawastyle){
/* now want to set up the ksp_S->pc to be of type ksp (gmres) by default to match Uzawa */
KSP pc_ksp;
KSPGetPC( ksp_S, &pc_S );
PCSetType(pc_S,PCKSP);
PCKSPGetKSP( pc_S, &pc_ksp);
KSPSetType(pc_ksp, "gmres" );
KSPSetOptionsPrefix( pc_ksp, "scrPCKSP_");
KSPSetFromOptions( pc_ksp );
}
KSPSetFromOptions( ksp_S );
/* Set specific monitor test */
KSPGetTolerances( ksp_S, PETSC_NULL, PETSC_NULL, PETSC_NULL, &max_it );
// Weirdness with petsc 3.2 here...look at it later
//BSSCR_KSPLogSetMonitor( ksp_S, max_it, &monitor_index );
/***************************************************************************************************************/
/* Pressure / Velocity Solve */
/***************************************************************************************************************/
PetscPrintf( PETSC_COMM_WORLD, "\t* Pressure / Velocity Solve \n");
/***************************************************************************************************************/
/***************************************************************************************************************/
usePreviousGuess = PETSC_FALSE;
if(been_here)
PetscOptionsGetTruth( PETSC_NULL, "-scr_use_previous_guess", &usePreviousGuess, &found );
if(usePreviousGuess) { /* Note this should actually look at checkpoint information */
KSPSetInitialGuessNonzero( ksp_S, PETSC_TRUE ); }
else {
KSPSetInitialGuessNonzero( ksp_S, PETSC_FALSE ); }
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* SET CONVERGENCE TESTS *************************************************************************/
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-scr_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest(ksp_S);
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_S, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* PRESSURE SOLVE ************************************************************************************/
PetscPrintf( PETSC_COMM_WORLD, "\t* KSPSolve( ksp_S, h_hat, p )\n");
/* if h_hat needs to be fixed up ..take out any nullspace vectors here */
/* we want to check that there is no "noise" in the null-space in the h vector */
/* this causes problems when we are trying to solve a Jacobian system when the Residual is almost converged */
if(bsscrp_self->check_pressureNS){
bsscrp_self->buildPNS(ksp);/* build and set nullspace vectors on bsscr - which is on ksp (function pointer is set in KSPSetUp_BSSCR) */
}
PetscScalar hnorm, gnorm;
MatNorm(G,NORM_INFINITY,&gnorm);
VecNorm(h_hat, NORM_2, &hnorm);
hnorm=hnorm/gnorm;
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,1.0/hnorm);
}
/* test to see if v or t are in nullspace of G and orthogonalize wrt h_hat if needed */
KSPRemovePressureNullspace_BSSCR(ksp, h_hat);
/* set convergence test to use min_it */
found = PETSC_FALSE;
min_it = 0;
PetscOptionsGetInt( PETSC_NULL,"-scr_ksp_set_min_it_converge", &min_it, &found);
if(found && min_it > 0){
BSSCR_KSPSetConvergenceMinIts(ksp_S, min_it, bsscrp_self);
}
/** Pressure Solve **/
PetscGetFlops(&flopsA);
scrSolveTime = MPI_Wtime();
KSPSolve( ksp_S, h_hat, p );
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscGetFlops(&flopsB);
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve( ksp_S, h_hat, p ) Solve Finished in time: %lf seconds\n\n", scrSolveTime);
bsscrp_self->solver->stats.pressure_time=scrSolveTime;
bsscrp_self->solver->stats.pressure_flops=(double)(flopsB-flopsA);
/***************************************/
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,hnorm);
VecScale(p,hnorm);
}
KSPRemovePressureNullspace_BSSCR(ksp, p);
/***************************************************************************************************************/
/***************************************************************************************************************/
/* restore K and f for the Velocity back solve */
found = PETSC_FALSE;
restorek = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-restore_K", &restorek, &found);
//PetscOptionsGetString( PETSC_NULL, "-restore_K", name, PETSC_MAX_PATH_LEN-1, &flg );
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
if(restorek){
penaltyNumber = -penaltyNumber;
if(f2) { ierr=VecAXPY(f,penaltyNumber,f2); }/* f <- f +a*f2 */
if(f3) { ierr=VecAXPY(f,penaltyNumber,f3); }/* f <- f +a*f3 */
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
//K=Korig;
Stg_KSPSetOperators(ksp_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KisJustK=PETSC_TRUE;
}
}
if(f3){ Stg_VecDestroy(&f3 ); } /* always destroy this local vector if was created */
/* obtain solution for u */
VecDuplicate( u, &t ); MatMult( G, p, t); VecAYPX( t, -1.0, f ); /*** t <- -t + f = f - G*p ***/
MatSchurComplementGetKSP( S, &ksp_inner );
a11SingleSolveTime = MPI_Wtime(); /* ---------------------------------- Final V Solve */
if(usePreviousGuess) KSPSetInitialGuessNonzero( ksp_inner, PETSC_TRUE );
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* VELOCITY SOLVE ************************************************************************************/
/** Easier to just create a new KSP here if we want to do backsolve diffferently. (getting petsc errors now when switching from fgmres) */
change_backsolve=PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-change_backsolve", &change_backsolve, &found );
if(change_backsolve){
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix(ksp_new_inner, "backsolveA11_");
KSPSetFromOptions(ksp_new_inner); /* make sure we are setting up our solver how we want it */
MatSchurComplementSetKSP( S, ksp_new_inner );/* need to give the Schur it's inner ksp back for when we destroy it at end */
ksp_inner=ksp_new_inner;
}
PetscGetFlops(&flopsA);
KSPSolve(ksp_inner, t, u); /* Solve, then restore default tolerance and initial guess */
PetscGetFlops(&flopsB);
bsscrp_self->solver->stats.velocity_backsolve_flops=(double)(flopsB-flopsA);
a11SingleSolveTime = MPI_Wtime() - a11SingleSolveTime; /* ------------------ Final V Solve */
bsscrp_self->solver->stats.velocity_backsolve_time=a11SingleSolveTime;
flg=0;
PetscOptionsGetString( PETSC_NULL, "-solutiondumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"p%s",suffix);
bsscr_dirwriteVec( p, matname,str, "Writing p vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"u%s",suffix);
bsscr_dirwriteVec( u, matname,str, "Writing u vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"h_hat%s",suffix);
bsscr_dirwriteVec( h_hat, matname,str, "Writing h_hat vector in al Solver");
}
found = PETSC_FALSE;
restorek = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-restore_K_after_solve", &restorek, &found);
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
if(restorek){
penaltyNumber = -penaltyNumber;
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
KisJustK=PETSC_TRUE;
}
}
/***************************************************************************************************************/
/***************************************************************************************************************/
/****** SOLUTION SUMMARY ******************************************************************************/
bsscr_summary(bsscrp_self,ksp_S,ksp_inner,K,K2,D,G,C,u,p,f,h,t,penaltyNumber,KisJustK, mgSetupTime, scrSolveTime, a11SingleSolveTime);
//bsscr_summary(bsscrp_self,ksp_S,ksp_inner,K,Korig,K2,D,G,C,u,p,f,h,t,penaltyNumber,KisJustK, mgSetupTime, scrSolveTime, a11SingleSolveTime);
/***************************************************************************************************************/
/***************************************************************************************************************/
Stg_VecDestroy(&t );
Stg_KSPDestroy(&ksp_S );
Stg_VecDestroy(&h_hat );
Stg_MatDestroy(&S );//This will destroy ksp_inner: also.. pcInner == pc_MG and is destroyed when ksp_inner is
been_here = 1;
PetscFunctionReturn(0);
}
