// updated
PetscErrorCode BSSCR_MatStokesMVBlockDefaultBuildScaling( MatStokesBlockScaling BA, Mat A, Vec b, Vec x, PetscTruth sym )
{
Mat K,G,D,C;
Vec rG;
PetscScalar rg2, rg, ra;
PetscInt N;
Vec rA, rC;
Vec L1,L2, R1,R2;
Mat S;
VecNestGetSubVec( BA->Lz, 0, &L1 );
VecNestGetSubVec( BA->Lz, 1, &L2 );
VecNestGetSubVec( BA->Rz, 0, &R1 );
VecNestGetSubVec( BA->Rz, 1, &R2 );
rA = L1;
rC = L2;
MatNestGetSubMat( A, 0,0, &K );
MatNestGetSubMat( A, 0,1, &G );
MatNestGetSubMat( A, 1,0, &D );
MatNestGetSubMat( A, 1,1, &C );
VecDuplicate( rA, &rG );
/* Get magnitude of K */
//px_MatGetAbsRowSum( K, rA );
//MatGetRowMax( K, rA, PETSC_NULL );
MatGetDiagonal( K, rA);
VecSqrt( rA );
VecReciprocal( rA );
/* VecDot( rA,rA, &ra ); */
/* VecGetSize( rA, &N ); */
/* ra = PetscSqrtScalar( ra/N ); */
/* Get magnitude of G */
//px_MatGetAbsRowSum( G, rG );
//MatGetRowMax( G, rG, PETSC_NULL );
Mat A21_cpy;
Mat Shat;
Vec diag; /* same as rA*rA */
MatGetVecs( K, &diag, PETSC_NULL );
MatGetDiagonal( K, diag );
VecReciprocal( diag );
if( sym )
#if( PETSC_VERSION_MAJOR <= 2 )
MatTranspose( G, &A21_cpy );
#else
MatTranspose( G, MAT_INITIAL_MATRIX, &A21_cpy );
#endif
else {
PetscErrorCode KSPBuildPressure_CB_Nullspace_BSSCR(KSP ksp)
{
KSP_BSSCR *bsscr = (KSP_BSSCR *)ksp->data;
FeEquationNumber *eq_num = bsscr->solver->st_sle->pSolnVec->feVariable->eqNum;
FeMesh *feMesh = bsscr->solver->st_sle->pSolnVec->feVariable->feMesh; /* is the pressure mesh */
unsigned ijk[3];
Vec t, v;
int numLocalNodes, globalNodeNumber, i, j, eq;
MatStokesBlockScaling BA = bsscr->BA;
PetscErrorCode ierr;
Mat Amat,Pmat, G;
MatStructure pflag;
PetscFunctionBegin;
/* get G matrix from Amat matrix operator on ksp */
ierr=Stg_PCGetOperators(ksp->pc,&Amat,&Pmat,&pflag);CHKERRQ(ierr);
MatNestGetSubMat( Amat, 0,1, &G );/* G should always exist */
/* now create Vecs t and v to match size of G: i.e. pressure */ /* NOTE: not using "h" vector from ksp->vec_rhs because this part of the block vector doesn't always exist */
MatGetVecs( G, &t, PETSC_NULL );/* t and v are destroyed in KSPDestroy_BSSCR */
MatGetVecs( G, &v, PETSC_NULL );/* t and v such that can do G*t */
numLocalNodes = Mesh_GetLocalSize( feMesh, MT_VERTEX); /* number of nodes on current proc not counting any shadow nodes */
for(j=0;j<numLocalNodes;j++){
i = globalNodeNumber = Mesh_DomainToGlobal( feMesh, MT_VERTEX, j);
RegularMeshUtils_Element_1DTo3D(feMesh, i, ijk);
eq = eq_num->destinationArray[j][0];/* get global equation number -- 2nd arg is always 0 because pressure has only one dof */
if(eq != -1){
if( (ijk[0]+ijk[1]+ijk[2])%2 ==0 ){
VecSetValue(t,eq,1.0,INSERT_VALUES);
}
else{
VecSetValue(v,eq,1.0,INSERT_VALUES); }}}
VecAssemblyBegin( t );
VecAssemblyEnd( t );
VecAssemblyBegin( v );
VecAssemblyEnd( v );
/* Scaling the null vectors here because it easier at the moment *//* maybe should do this in the original scaling function */
if( BA->scaling_exists == PETSC_TRUE ){
Vec R2;
/* Get the scalings out the block mat data */
VecNestGetSubVec( BA->Rz, 1, &R2 );
VecPointwiseDivide( t, t, R2); /* x <- x * 1/R2 */
VecPointwiseDivide( v, v, R2);
}
bsscr_writeVec( t, "t", "Writing t vector");
bsscr_writeVec( v, "v", "Writing v vector");
bsscr->t=t;
bsscr->v=v;
PetscFunctionReturn(0);
}
// updated
PetscErrorCode BSSCR_mat_mvblock_invert_scalings( MatStokesBlockScaling BA )
{
Vec L1,L2, R1,R2;
VecNestGetSubVec( BA->Lz, 0, &L1 );
VecNestGetSubVec( BA->Lz, 1, &L2 );
VecNestGetSubVec( BA->Rz, 0, &R1 );
VecNestGetSubVec( BA->Rz, 1, &R2 );
VecReciprocal(L1);
VecReciprocal(L2);
VecReciprocal(R1);
VecReciprocal(R2);
/* toggle inversion flag */
if( BA->scalings_have_been_inverted == PETSC_TRUE ) { BA->scalings_have_been_inverted = PETSC_FALSE; }
if( BA->scalings_have_been_inverted == PETSC_FALSE ) { BA->scalings_have_been_inverted = PETSC_TRUE; }
PetscFunctionReturn(0);
}
// updated
PetscErrorCode BSSCR_MatStokesMVBlock_ApplyScaling( MatStokesBlockScaling BA, Mat A, Vec b, Vec x, Mat S, PetscTruth sym )
{
Mat K,G,D,C;
Vec L1,L2, R1,R2;
Vec f,h, u,p;
/* Get the scalings out the block mat data */
VecNestGetSubVec( BA->Lz, 0, &L1 );
VecNestGetSubVec( BA->Lz, 1, &L2 );
VecNestGetSubVec( BA->Rz, 0, &R1 );
VecNestGetSubVec( BA->Rz, 1, &R2 );
/* get the subblock solution and rhs */
if( x != PETSC_NULL ) {
VecNestGetSubVec( x, 0, &u );
VecNestGetSubVec( x, 1, &p );
VecPointwiseDivide( u, u,R1); /* x <- x * 1/R1 */
VecPointwiseDivide( p, p,R2);
}
if( b != PETSC_NULL ) {
VecNestGetSubVec( b, 0, &f );
VecNestGetSubVec( b, 1, &h );
VecPointwiseMult( f, f,L1); /* f <- f * L1 */
VecPointwiseMult( h, h,L2);
}
/* Scale matrices */
MatNestGetSubMat( A, 0,0, &K );
MatNestGetSubMat( A, 0,1, &G );
MatNestGetSubMat( A, 1,0, &D );
MatNestGetSubMat( A, 1,1, &C );
if( K != PETSC_NULL ) { MatDiagonalScale( K, L1,R1 ); }
if( G != PETSC_NULL ) { MatDiagonalScale( G, L1,R2 ); }
if( D != PETSC_NULL && !sym ) { MatDiagonalScale( D, L2,R1 ); }
if( C != PETSC_NULL ) { MatDiagonalScale( C, L2,R2 ); }
if( S != PETSC_NULL ) { MatDiagonalScale( S, L2,R2 ); }
PetscFunctionReturn(0);
}
PetscErrorCode KSPBuildPressure_Const_Nullspace_BSSCR(KSP ksp)
{
KSP_BSSCR *bsscr = (KSP_BSSCR *)ksp->data;
FeEquationNumber *eq_num = bsscr->solver->st_sle->pSolnVec->feVariable->eqNum;
FeMesh *feMesh = bsscr->solver->st_sle->pSolnVec->feVariable->feMesh; /* is the pressure mesh */
unsigned ijk[3];
Vec t;
int numLocalNodes, globalNodeNumber, i, eq;
MatStokesBlockScaling BA = bsscr->BA;
PetscErrorCode ierr;
Mat Amat,Pmat, G;
MatStructure pflag;
PetscFunctionBegin;
/* get G matrix from Amat matrix operator on ksp */
ierr=Stg_PCGetOperators(ksp->pc,&Amat,&Pmat,&pflag);CHKERRQ(ierr);
MatNestGetSubMat( Amat, 0,1, &G );/* G should always exist */
/* now create Vec t to match size of G: i.e. pressure */ /* NOTE: not using "h" vector from ksp->vec_rhs because this part of the block vector doesn't always exist */
MatGetVecs( G, &t, PETSC_NULL );/* t is destroyed in KSPDestroy_BSSCR */
VecSet(t, 1.0);
VecAssemblyBegin( t );
VecAssemblyEnd( t );
/* Scaling the null vectors here because it easier at the moment */
if( BA->scaling_exists == PETSC_TRUE ){
Vec R2;
/* Get the scalings out the block mat data */
VecNestGetSubVec( BA->Rz, 1, &R2 );
VecPointwiseDivide( t, t, R2); /* x <- x * 1/R2 */
}
bsscr_writeVec( t, "t", "Writing t vector");
bsscr->t=t;
PetscFunctionReturn(0);
}
/*
Stokes output:
---------------------------------
Operator summary:
K
G
f,
h
u
p
---------------------------------
Solution summary:
max_u
min_u
average_u
|r_1|
|r_2|
---------------------------------
Solver summary:
name
---------------------------------
Petsc build summary:
*/
PetscErrorCode BSSCR_stokes_output( PetscViewer v, Mat stokes_A, Vec stokes_b, Vec stokes_x, KSP ksp, PetscInt monitor_index )
{
Mat K,G,D,C;
Vec f,h, u,p;
K = G = D = C = PETSC_NULL;
f = h = PETSC_NULL;
u = p = PETSC_NULL;
MatNestGetSubMat( stokes_A, 0,0, &K );
MatNestGetSubMat( stokes_A, 0,1, &G );
MatNestGetSubMat( stokes_A, 1,0, &D );
MatNestGetSubMat( stokes_A, 1,1, &C );
VecNestGetSubVec( stokes_x, 0, &u );
VecNestGetSubVec( stokes_x, 1, &p );
VecNestGetSubVec( stokes_b, 0, &f );
VecNestGetSubVec( stokes_b, 1, &h );
PetscViewerASCIIPrintf( v, "Stokes Output:\n");
PetscViewerASCIIPushTab( v );
/*--------------------------------------------------------------------------------------------*/
PetscViewerASCIIPrintf( v, "--------------------------------------------------\n");
PetscViewerASCIIPrintf( v, "Operator summary:\n");
PetscViewerASCIIPushTab( v );
if (K) {
BSSCR_MatInfoLog(v,K, "stokes_A11");
PetscViewerASCIIPrintf( v, "\n");
}
if (G) {
BSSCR_MatInfoLog(v,G, "stokes_A12");
PetscViewerASCIIPrintf( v, "\n");
}
if (D) {
BSSCR_MatInfoLog(v,D, "stokes_A21");
PetscViewerASCIIPrintf( v, "\n");
}
if (C) {
BSSCR_MatInfoLog(v,C, "stokes_A22");
PetscViewerASCIIPrintf( v, "\n");
}
if (f) {
BSSCR_VecInfoLog(v,f,"stokes_b1");
PetscViewerASCIIPrintf( v, "\n");
}
if (h) {
BSSCR_VecInfoLog(v,h,"stokes_b2");
PetscViewerASCIIPrintf( v, "\n");
}
PetscViewerASCIIPopTab( v );
/*--------------------------------------------------------------------------------------------*/
PetscViewerASCIIPrintf( v, "--------------------------------------------------\n");
PetscViewerASCIIPrintf( v, "Solution summary:\n");
PetscViewerASCIIPushTab( v );
if (u) {
BSSCR_VecInfoLog(v,u,"x1");
PetscViewerASCIIPrintf( v, "\n");
}
if (p) {
BSSCR_VecInfoLog(v,p,"x2");
PetscViewerASCIIPrintf( v, "\n");
}
{
PetscScalar s,sum;
PetscReal r,max,min;
PetscInt N, loc;
double r1,r2;
Vec K_d;
PetscInt loc_max, loc_min;
VecGetSize( u, &N );
VecMax( u, &loc, &r );
PetscViewerASCIIPrintf( v, "u_max: %1.12e [%d] \n", r, loc );
VecMin( u, &loc, &r );
PetscViewerASCIIPrintf( v, "u_min: %1.12e [%d] \n", r, loc );
VecDot( u,u, &s );
PetscViewerASCIIPrintf( v, "u_rms: %1.12e \n", sqrt( PetscRealPart(s) )/N );
VecDuplicate( u, &K_d );
MatGetDiagonal( K, K_d );
VecMax( K_d, &loc_max, &max );
VecMin( K_d, &loc_min, &min );
PetscViewerASCIIPrintf( v,"Algebraic contrast: max(K_d)=%.3e [%d] , min(K_d)=%.3e [%d] , max(K_d)/min(K_d) = %.8e \n", max,loc_max, min,loc_min, max/min );
MatGetRowMax( K, K_d, PETSC_NULL );
VecMax( K_d, &loc_max, &max );
MatGetRowMin( K, K_d, PETSC_NULL );
VecAbs( K_d );
VecMin( K_d, &loc_min, &min );
PetscViewerASCIIPrintf( v,"Algebraic contrast: max(K)=%.3e [%d] , |min(K)|=%.3e [%d] , max(K)/|min(K)| = %.8e \n", max,loc_max, min,loc_min, max/min );
Stg_VecDestroy(&K_d );
PetscViewerASCIIPrintf( v, "\n");
VecGetSize( p, &N );
VecMax( p, &loc, &r );
PetscViewerASCIIPrintf( v, "p_max: %1.12e [%d] \n", r, loc );
VecMin( p, &loc, &r );
PetscViewerASCIIPrintf( v, "p_min: %1.12e [%d] \n", r, loc );
VecDot( p,p, &s );
PetscViewerASCIIPrintf( v, "p_rms: %1.12e \n", sqrt( PetscRealPart(s) )/N );
VecSum( p, &sum );
PetscViewerASCIIPrintf( v, "sum(p): %1.12e \n", sum );
PetscViewerASCIIPrintf( v, "\n");
r1 = BSSCR_StokesMomentumResidual( K,G,f, u,p );
PetscViewerASCIIPrintf( v, "|r1| = %1.12e <momentum> \n", r1 );
r2 = BSSCR_StokesContinuityResidual( G,C,h, u,p );
PetscViewerASCIIPrintf( v, "|r2| = %1.12e <continuity> \n", r2 );
PetscViewerASCIIPrintf( v, "\n");
}
PetscViewerASCIIPopTab( v );
/*--------------------------------------------------------------------------------------------*/
if (ksp) {
PetscViewerASCIIPrintf( v, "--------------------------------------------------\n");
PetscViewerASCIIPrintf( v, "Solver summary:\n");
PetscViewerASCIIPushTab( v );
BSSCR_KSPLogSolve( v, monitor_index, ksp );
BSSCR_BSSCR_KSPLogSolveSummary( v, monitor_index, ksp );
PetscViewerASCIIPrintf( v, "\n");
PetscViewerASCIIPopTab( v );
}
/*--------------------------------------------------------------------------------------------*/
PetscViewerASCIIPrintf( v, "--------------------------------------------------\n");
PetscViewerASCIIPrintf( v, "Petsc build summary:\n");
PetscViewerASCIIPushTab( v );
BSSCR_GeneratePetscHeader_for_viewer( v );
PetscViewerASCIIPrintf( v, "\n");
PetscViewerASCIIPopTab( v );
/*--------------------------------------------------------------------------------------------*/
PetscViewerASCIIPopTab(v);
PetscFunctionReturn(0);
}
PetscErrorCode test_axpy_dot_max( void )
{
Vec x1,y1, x2,y2;
Vec tmp_buf[2];
Vec X, Y;
PetscReal real,real2;
PetscScalar scalar;
PetscInt index;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscPrintf( PETSC_COMM_WORLD, "\n\n============== %s ==============\n", PETSC_FUNCTION_NAME );
gen_test_vector( PETSC_COMM_WORLD, 4, 0, 1, &x1 );
gen_test_vector( PETSC_COMM_WORLD, 5, 10, 2, &x2 );
gen_test_vector( PETSC_COMM_WORLD, 4, 4, 3, &y1 );
gen_test_vector( PETSC_COMM_WORLD, 5, 5, 1, &y2 );
tmp_buf[0] = x1;
tmp_buf[1] = x2;
ierr = VecCreateNest(PETSC_COMM_WORLD,2,PETSC_NULL,tmp_buf,&X);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
ierr = VecDestroy(&x1);CHKERRQ(ierr);
ierr = VecDestroy(&x2);CHKERRQ(ierr);
tmp_buf[0] = y1;
tmp_buf[1] = y2;
ierr = VecCreateNest(PETSC_COMM_WORLD,2,PETSC_NULL,tmp_buf,&Y);CHKERRQ(ierr);
ierr = VecAssemblyBegin(Y);CHKERRQ(ierr);
ierr = VecAssemblyEnd(Y);CHKERRQ(ierr);
ierr = VecDestroy(&y1);CHKERRQ(ierr);
ierr = VecDestroy(&y2);CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "VecAXPY \n");
ierr = VecAXPY( Y, 1.0, X ); /* Y <- a X + Y */
ierr = VecNestGetSubVec( Y, 0, &y1 );CHKERRQ(ierr);
ierr = VecNestGetSubVec( Y, 1, &y2 );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "(1) y1 = \n" );
ierr = VecView( y1, PETSC_VIEWER_STDOUT_WORLD );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "(1) y2 = \n" );
ierr = VecView( y2, PETSC_VIEWER_STDOUT_WORLD );CHKERRQ(ierr);
ierr = VecDot( X,Y, &scalar );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "X.Y = %lf + %lfi \n", PetscRealPart(scalar), PetscImaginaryPart(scalar) );
ierr = VecDotNorm2( X,Y, &scalar, &real2 );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "X.Y = %lf + %lfi norm2(Y) = %lf\n", PetscRealPart(scalar), PetscImaginaryPart(scalar), real2);
ierr = VecAXPY( Y, 1.0, X ); /* Y <- a X + Y */
ierr = VecNestGetSubVec( Y, 0, &y1 );CHKERRQ(ierr);
ierr = VecNestGetSubVec( Y, 1, &y2 );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "(2) y1 = \n" );
ierr = VecView( y1, PETSC_VIEWER_STDOUT_WORLD );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "(2) y2 = \n" );
ierr = VecView( y2, PETSC_VIEWER_STDOUT_WORLD );CHKERRQ(ierr);
ierr = VecDot( X,Y, &scalar );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "X.Y = %lf + %lfi \n", PetscRealPart(scalar), PetscImaginaryPart(scalar) );
ierr = VecDotNorm2( X,Y, &scalar, &real2 );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "X.Y = %lf + %lfi norm2(Y) = %lf\n", PetscRealPart(scalar), PetscImaginaryPart(scalar), real2);
ierr = VecMax( X, &index, &real );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "(max-X) = %f : index = %d \n", real, index );
ierr = VecMin( X, &index, &real );CHKERRQ(ierr);
PetscPrintf( PETSC_COMM_WORLD, "(min-X) = %f : index = %d \n", real, index );
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&Y);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode BSSCR_DRIVER_flex( KSP ksp, Mat stokes_A, Vec stokes_x, Vec stokes_b, Mat approxS, KSP ksp_K,
MatStokesBlockScaling BA, PetscTruth sym, KSP_BSSCR * bsscrp_self )
{
char name[PETSC_MAX_PATH_LEN];
char ubefore[100];
char uafter[100];
char pbefore[100];
char pafter[100];
PetscTruth flg, flg2, truth, useAcceleratingSmoothingMG, useFancySmoothingMG;
PetscTruth usePreviousGuess, useNormInfStoppingConditions, useNormInfMonitor, found, extractMats;
Mat K,G,D,C;
Vec u,p,f,h;
Mat S;
Vec h_hat,t,t2,q,v;
KSP ksp_inner;
KSP ksp_S;
KSP ksp_cleaner;
KSPType ksp_inner_type;
PetscTruth has_cnst_nullspace;
PC pc_S, pc_MG, pcInner;
PetscInt monitor_index,max_it,min_it;
Vec nsp_vec = PETSC_NULL;
PetscReal scr_rtol;
PetscReal inner_rtol;
PetscReal vSolver_rtol;
PetscScalar uNormInf, pNormInf;
PetscScalar uNorm, pNorm, rNorm, fNorm;
PetscInt uSize, pSize;
PetscInt lmin,lmax;
PetscInt iterations;
PetscReal min,max;
PetscReal p_sum;
MGContext mgCtx;
PC shellPC;
double t0, t1;
double mgSetupTime, scrSolveTime, a11SingleSolveTime, solutionAnalysisTime;
Index nx,ny,nz;
PetscInt j,start,end;
static int been_here = 0; /* Ha Ha Ha !! */
/* get sub matrix / vector objects */
MatNestGetSubMat( stokes_A, 0,0, &K );
MatNestGetSubMat( stokes_A, 0,1, &G );
MatNestGetSubMat( stokes_A, 1,0, &D );
MatNestGetSubMat( stokes_A, 1,1, &C );
VecNestGetSubVec( stokes_x, 0, &u );
VecNestGetSubVec( stokes_x, 1, &p );
VecNestGetSubVec( stokes_b, 0, &f );
VecNestGetSubVec( stokes_b, 1, &h );
/* PetscPrintf( PETSC_COMM_WORLD, "\t Adress of stokes_x is %p\n", stokes_x); */
/* VecNorm( u, NORM_2, &uNorm ); */
/* PetscPrintf( PETSC_COMM_WORLD, "\t u Norm is %.6e in %s: address is %p\n",uNorm,__func__,u); */
/* VecNorm( p, NORM_2, &pNorm ); */
/* PetscPrintf( PETSC_COMM_WORLD, "\t p Norm is %.6e in %s: addres is %p\n",pNorm,__func__,p); */
/* Create Schur complement matrix */
//MatCreateSchurFromBlock( stokes_A, 0.0, "MatSchur_A11", &S );
MatCreateSchurComplement(K,K,G,D,C, &S);
/* configure inner solver */
if (ksp_K!=PETSC_NULL) {
MatSchurComplementSetKSP( S, ksp_K );
MatSchurComplementGetKSP( S, &ksp_inner );
}
else {
abort();
MatSchurComplementGetKSP( S, &ksp_inner );
KSPSetType( ksp_inner, "cg" );
}
KSPGetPC( ksp_inner, &pcInner );
/* If we're using multigrid, replace the preconditioner here
so we get the same options prefix. */
if(bsscrp_self->mg) {
mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx );
}
/* SETFROMOPTIONS MIGHT F**K MG UP */
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest( ksp_inner );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfMonitor, PETSC_NULL, PETSC_NULL );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_to_norm_2_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/* create right hand side */
/* h_hat = G'*inv(K)*f - h */
MatGetVecs(K,PETSC_NULL,&t);
MatGetVecs( S, PETSC_NULL, &h_hat );
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
KSPSolve(ksp_inner,f,t);/* t=f/K */
//bsscr_writeVec( t, "ts", "Writing t vector");
MatMult(D,t,h_hat);/* G'*t */
VecAXPY(h_hat, -1.0, h);/* h_hat = h_hat - h */
Stg_VecDestroy(&t);
//bsscr_writeVec( h_hat, "h_hat", "Writing h_hat Vector in Solver");
//MatSchurApplyReductionToVecFromBlock( S, stokes_b, h_hat );
/* create solver for S p = h_hat */
KSPCreate( PETSC_COMM_WORLD, &ksp_S );
KSPSetOptionsPrefix( ksp_S, "scr_");
Stg_KSPSetOperators( ksp_S, S,S, SAME_NONZERO_PATTERN );
KSPSetType( ksp_S, "cg" );
/* Build preconditioner for S */
KSPGetPC( ksp_S, &pc_S );
BSSCR_BSSCR_StokesCreatePCSchur2( K,G,D,C,approxS,pc_S,sym, bsscrp_self );
KSPSetFromOptions(ksp_S);
/* Set specific monitor test */
KSPGetTolerances( ksp_S, PETSC_NULL, PETSC_NULL, PETSC_NULL, &max_it );
//BSSCR_KSPLogSetMonitor( ksp_S, max_it, &monitor_index );
/* Pressure / Velocity Solve */
scrSolveTime = MPI_Wtime();
PetscPrintf( PETSC_COMM_WORLD, "\t* Pressure / Velocity Solve \n");
usePreviousGuess = PETSC_FALSE;
if(been_here)
PetscOptionsGetTruth( PETSC_NULL, "-scr_use_previous_guess", &usePreviousGuess, &found );
if(usePreviousGuess) { /* Note this should actually look at checkpoint information */
KSPSetInitialGuessNonzero( ksp_S, PETSC_TRUE );
}
else {
KSPSetInitialGuessNonzero( ksp_S, PETSC_FALSE );
}
//KSPSetRelativeRhsConvergenceTest( ksp_S );
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-scr_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest(ksp_S);
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_S, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
PetscPrintf( PETSC_COMM_WORLD, "\t* KSPSolve( ksp_S, h_hat, p )\n");
/* if h_hat needs to be fixed up ..take out any nullspace vectors here */
/* we want to check that there is no "noise" in the null-space in the h vector */
/* this causes problems when we are trying to solve a Jacobian system when the Residual is almost converged */
if(bsscrp_self->check_pressureNS){
bsscrp_self->buildPNS(ksp);/* build and set nullspace vectors on bsscr - which is on ksp (function pointer is set in KSPSetUp_BSSCR */
}
PetscScalar norm, a, a1, a2, hnorm, pnorm, gnorm;
MatNorm(G,NORM_INFINITY,&gnorm);
VecNorm(h_hat, NORM_2, &hnorm);
hnorm=hnorm/gnorm;
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,1.0/hnorm);
}
/* test to see if v or t are in nullspace of G and orthogonalize wrt h_hat if needed */
KSPRemovePressureNullspace_BSSCR(ksp, h_hat);
/***************************************/
/* set convergence test to use min_it */
found = PETSC_FALSE;
min_it = 0;
PetscOptionsGetInt( PETSC_NULL,"-scr_ksp_set_min_it_converge", &min_it, &found);
if(found && min_it > 0){
BSSCR_KSPSetConvergenceMinIts(ksp_S, min_it, bsscrp_self);
}
KSPSolve( ksp_S, h_hat, p );
sprintf(pafter,"psafter_%d",been_here);
bsscr_writeVec( p, pafter, "Writing p Vector in Solver");
/***************************************/
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,hnorm);
VecScale(p,hnorm);
}
KSPRemovePressureNullspace_BSSCR(ksp, p);
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve( ksp_S, h_hat, p ) Solve Finished in time: %lf seconds\n\n", scrSolveTime);
/* Resolve with this pressure to obtain solution for u */
/* obtain solution for u */
VecDuplicate( u, &t );
MatMult( G, p, t);
VecAYPX( t, -1.0, f ); /* t <- -t + f */
MatSchurComplementGetKSP( S, &ksp_inner );
a11SingleSolveTime = MPI_Wtime(); /* ---------------------------------- Final V Solve */
if(usePreviousGuess)
KSPSetInitialGuessNonzero( ksp_inner, PETSC_TRUE );
KSPSetOptionsPrefix( ksp_inner, "backsolveA11_" );
KSPSetFromOptions( ksp_inner );
KSPSolve( ksp_inner, t, u ); /* Solve, then restore default tolerance and initial guess */
a11SingleSolveTime = MPI_Wtime() - a11SingleSolveTime; /* ------------------ Final V Solve */
PetscPrintf( PETSC_COMM_WORLD, "\n\nSCR Solver Summary:\n\n");
if(bsscrp_self->mg)
PetscPrintf( PETSC_COMM_WORLD, " Multigrid setup: = %.4g secs \n", mgSetupTime);
KSPGetIterationNumber( ksp_S, &iterations);
PetscPrintf( PETSC_COMM_WORLD, " Pressure Solve: = %.4g secs / %d its\n", scrSolveTime, iterations);
KSPGetIterationNumber( ksp_inner, &iterations);
PetscPrintf( PETSC_COMM_WORLD, " Final V Solve: = %.4g secs / %d its\n\n", a11SingleSolveTime, iterations);
/* Analysis of solution:
This can be somewhat time consuming as it requires allocation / de-allocation,
computing vector norms etc. So we make it optional..
This should be put into a proper KSP monitor now?
*/
flg = PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_solution_summary", &flg, &found );
if(flg) {
solutionAnalysisTime = MPI_Wtime();
VecGetSize( u, &uSize );
VecGetSize( p, &pSize );
VecDuplicate( u, &t2 );
MatMult( K, u, t2);
VecAYPX( t2, -1.0, t ); /* t2 <- -t2 + t ... should be the formal residual vector */
VecNorm( t2, NORM_2, &rNorm );
VecNorm( f, NORM_2, &fNorm );
PetscPrintf( PETSC_COMM_WORLD, " |f - K u - G p|/|f| = %.6e\n", rNorm/fNorm );
VecDuplicate( p, &q );
MatMult( D, u, q );
VecNorm( u, NORM_2, &uNorm );
VecNorm( q, NORM_2, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_2/|u|_2 = %.6e\n", sqrt( (double) uSize / (double) pSize ) * rNorm / uNorm);
VecNorm( q, NORM_INFINITY, &rNorm );
PetscPrintf( PETSC_COMM_WORLD, " |G^T u|_infty/|u|_2 = %.6e\n", sqrt( (double) uSize ) * rNorm / uNorm);
VecNorm( u, NORM_INFINITY, &uNormInf );
VecNorm( u, NORM_2, &uNorm );
VecGetSize( u, &uSize );
VecNorm( p, NORM_INFINITY, &pNormInf );
VecNorm( p, NORM_2, &pNorm );
PetscPrintf( PETSC_COMM_WORLD, " |u|_{\\infty} = %.6e , u_rms = %.6e\n",
uNormInf, uNorm / sqrt( (double) uSize ) );
PetscPrintf( PETSC_COMM_WORLD, " |p|_{\\infty} = %.6e , p_rms = %.6e\n",
pNormInf, pNorm / sqrt( (double) pSize ) );
VecMax( u, &lmax, &max );
VecMin( u, &lmin, &min );
PetscPrintf( PETSC_COMM_WORLD, " min/max(u) = %.6e [%d] / %.6e [%d]\n",min,lmin,max,lmax);
VecMax( p, &lmax, &max );
VecMin( p, &lmin, &min );
PetscPrintf( PETSC_COMM_WORLD, " min/max(p) = %.6e [%d] / %.6e [%d]\n",min,lmin,max,lmax);
VecSum( p, &p_sum );
PetscPrintf( PETSC_COMM_WORLD, " \\sum_i p_i = %.6e \n", p_sum );
solutionAnalysisTime = MPI_Wtime() - solutionAnalysisTime;
PetscPrintf( PETSC_COMM_WORLD, "\n Time for this analysis = %.4g secs\n\n",solutionAnalysisTime);
Stg_VecDestroy(&t2 );
Stg_VecDestroy(&q );
}
if(bsscrp_self->mg) {
//MG_inner_solver_pcmg_shutdown( pcInner );
}
Stg_VecDestroy(&t );
// KSPLogDestroyMonitor( ksp_S );
Stg_KSPDestroy(&ksp_S );
//Stg_KSPDestroy(&ksp_inner );
Stg_VecDestroy(&h_hat );
Stg_MatDestroy(&S );
/* Destroy nullspace vector if it exists. */
if(nsp_vec)
Stg_VecDestroy(&nsp_vec);
//been_here = 1;
been_here++;
PetscFunctionReturn(0);
}
PetscErrorCode BSSCR_DRIVER_auglag( KSP ksp, Mat stokes_A, Vec stokes_x, Vec stokes_b, Mat approxS,
MatStokesBlockScaling BA, PetscTruth sym, KSP_BSSCR * bsscrp_self )
{
AugLagStokes_SLE * stokesSLE = (AugLagStokes_SLE*)bsscrp_self->solver->st_sle;
PetscTruth uzawastyle, KisJustK=PETSC_TRUE, restorek, change_A11rhspresolve;
PetscTruth usePreviousGuess, useNormInfStoppingConditions, useNormInfMonitor, found, forcecorrection;
PetscTruth change_backsolve, mg_active;
PetscErrorCode ierr;
//PetscInt monitor_index;
PetscInt max_it,min_it;
KSP ksp_inner, ksp_S, ksp_new_inner;
PC pc_S, pcInner;
Mat K,G,D,C, S, K2;// Korig;
Vec u,p,f,f2=0,f3=0,h, h_hat,t;
MGContext mgCtx;
double mgSetupTime, scrSolveTime, a11SingleSolveTime, penaltyNumber;// hFactor;
static int been_here = 0; /* Ha Ha Ha !! */
char name[PETSC_MAX_PATH_LEN];
char matname[PETSC_MAX_PATH_LEN];
char suffix[PETSC_MAX_PATH_LEN];
char str[PETSC_MAX_PATH_LEN];
PetscTruth flg, extractMats;
PetscLogDouble flopsA,flopsB;
/***************************************************************************************************************/
/***************************************************************************************************************/
//if( bsscrp_self->solver->st_sle->context->loadFromCheckPoint ){
// been_here=1;
//}
/* get sub matrix / vector objects */
/* note that here, the matrix D should always exist. It is set up in _StokesBlockKSPInterface_Solve in StokesBlockKSPInterface.c */
/* now extract K,G etc from a MatNest object */
MatNestGetSubMat( stokes_A, 0,0, &K );
MatNestGetSubMat( stokes_A, 0,1, &G );
MatNestGetSubMat( stokes_A, 1,0, &D );if(!D){ PetscPrintf( PETSC_COMM_WORLD, "D does not exist but should!!\n"); exit(1); }
MatNestGetSubMat( stokes_A, 1,1, &C );
VecNestGetSubVec( stokes_x, 0, &u );
VecNestGetSubVec( stokes_x, 1, &p );
VecNestGetSubVec( stokes_b, 0, &f );
VecNestGetSubVec( stokes_b, 1, &h );
PetscPrintf( PETSC_COMM_WORLD, "\n\n---------- AUGMENTED LAGRANGIAN K2 METHOD ---------\n\n" );
PetscPrintf( PETSC_COMM_WORLD, "----------- Penalty = %f\n\n", bsscrp_self->solver->penaltyNumber );
sprintf(suffix,"%s","x");
PetscOptionsGetString( PETSC_NULL, "-matsuffix", suffix, PETSC_MAX_PATH_LEN-1, &extractMats );
flg=0;
PetscOptionsGetString( PETSC_NULL, "-matdumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"K%s",suffix);
bsscr_dirwriteMat( K, matname,str, "Writing K matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"G%s",suffix);
bsscr_dirwriteMat( G, matname,str, "Writing G matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"D%s",suffix);
bsscr_dirwriteMat( D, matname,str, "Writing D matrix in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"f%s",suffix);
bsscr_dirwriteVec( f, matname,str, "Writing f vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"h%s",suffix);
bsscr_dirwriteVec( h, matname,str, "Writing h vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"Shat%s",suffix);
bsscr_dirwriteMat( approxS, matname,str, "Writing Shat matrix in al Solver");
if(C){
sprintf(str,"%s/",name); sprintf(matname,"C%s",suffix);
bsscr_dirwriteMat( C, matname,str, "Writing C matrix in al Solver");
}
}
mg_active=PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_mg_active", &mg_active, &found );
bsscrp_self->solver->mg_active=mg_active;
penaltyNumber = bsscrp_self->solver->penaltyNumber;
//hFactor = stokesSLE->hFactor;
/***************************************************************************************************************/
/***************************************************************************************************************/
/****** GET K2 ****************************************************************************************/
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
flg=0;
PetscOptionsGetString( PETSC_NULL, "-matdumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"K2%s",suffix);
bsscr_dirwriteMat( bsscrp_self->K2, matname,str, "Writing K2 matrix in al Solver");
}
K2=bsscrp_self->K2;
scrSolveTime = MPI_Wtime();
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* K+p*K2 in time: %lf seconds\n\n", scrSolveTime);
KisJustK=PETSC_FALSE;
forcecorrection=PETSC_TRUE;
PetscOptionsGetTruth( PETSC_NULL ,"-force_correction", &forcecorrection, &found );
if(forcecorrection){
if(bsscrp_self->f2 && forcecorrection){
f2=bsscrp_self->f2;
ierr=VecAXPY(f,penaltyNumber,f2);/* f <- f +a*f2 */
}else{
switch (bsscrp_self->k2type) {
case (K2_GG):
{
VecDuplicate( u, &f3 );
MatMult( G, h, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
}
break;
case (K2_GMG):
{
Mat M;
Vec Mdiag;
VecDuplicate( u, &f3 );
M = bsscrp_self->solver->mStiffMat->matrix;
MatGetVecs( M, &Mdiag, PETSC_NULL );
MatGetDiagonal( M, Mdiag );
VecReciprocal(Mdiag);
VecPointwiseMult(Mdiag,Mdiag,h);
MatMult( G, Mdiag, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
Stg_VecDestroy(&Mdiag);
}
break;
case (K2_DGMGD):
{
Mat M;
Vec Mdiag;
VecDuplicate( u, &f3 );
M = bsscrp_self->solver->mStiffMat->matrix;
MatGetVecs( M, &Mdiag, PETSC_NULL );
MatGetDiagonal( M, Mdiag );
VecReciprocal(Mdiag);
VecPointwiseMult(Mdiag,Mdiag,h);
MatMult( G, Mdiag, f3); ierr=VecAXPY(f,penaltyNumber,f3);/* f <- f +a*f2 */
Stg_VecDestroy(&Mdiag);
}
break;
case (K2_NULL):
{
;
}
break;
case (K2_SLE):
{
;
}
break;
}
}
}
}
/* Create Schur complement matrix */
MatCreateSchurComplement(K,K,G,D,C, &S);
MatSchurComplementGetKSP( S, &ksp_inner);
KSPGetPC( ksp_inner, &pcInner );
/***************************************************************************************************************/
/***************************************************************************************************************/
/********* SET PREFIX FOR INNER/VELOCITY KSP *************************************************************/
KSPSetOptionsPrefix( ksp_inner, "A11_" );
KSPSetFromOptions( ksp_inner );
Stg_KSPSetOperators(ksp_inner, K, K, DIFFERENT_NONZERO_PATTERN);
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-A11_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest( ksp_inner );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor) KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfMonitor, PETSC_NULL, PETSC_NULL );
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-A11_ksp_norm_inf_to_norm_2_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor) KSPMonitorSet( ksp_inner, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/***************************************************************************************************************/
/***************************************************************************************************************/
/* If multigrid is enabled, set it now. */
change_A11rhspresolve = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-change_A11rhspresolve", &change_A11rhspresolve, &found );
if(bsscrp_self->solver->mg_active && !change_A11rhspresolve) { mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx ); }
/***************************************************************************************************************/
/***************************************************************************************************************/
/* create right hand side */
if(change_A11rhspresolve){
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix(ksp_new_inner, "rhsA11_");
MatSchurComplementSetKSP( S, ksp_new_inner );/* this call destroys the ksp_inner that is already set on S */
ksp_inner=ksp_new_inner;
KSPGetPC( ksp_inner, &pcInner );
KSPSetFromOptions(ksp_inner); /* make sure we are setting up our solver how we want it */
}
MatGetVecs( S, PETSC_NULL, &h_hat );
Vec f_tmp;
/* It may be the case that the current velocity solution might not be bad guess for f_tmp? <-- maybe not */
MatGetVecs( K, PETSC_NULL, &f_tmp );
scrSolveTime = MPI_Wtime();
KSPSolve(ksp_inner, f, f_tmp);
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve for RHS setup Finished in time: %lf seconds\n\n", scrSolveTime);
MatMult(D, f_tmp, h_hat);
VecAYPX(h_hat, -1.0, h); /* Computes y = x + alpha y. h_hat -> h - Gt*K^(-1)*f*/
Stg_VecDestroy(&f_tmp);
if(bsscrp_self->solver->mg_active && change_A11rhspresolve) {
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix( ksp_new_inner, "A11_" );
MatSchurComplementSetKSP( S, ksp_new_inner );
ksp_inner=ksp_new_inner;
//MatSchurSetKSP( S, ksp_inner );
KSPGetPC( ksp_inner, &pcInner );
KSPSetFromOptions( ksp_inner );
mgSetupTime=setupMG( bsscrp_self, ksp_inner, pcInner, K, &mgCtx );
}
/* create solver for S p = h_hat */
KSPCreate( PETSC_COMM_WORLD, &ksp_S );
KSPSetOptionsPrefix( ksp_S, "scr_");
/* By default use the UW approxS Schur preconditioner -- same as the one used by the Uzawa solver */
/* Note that if scaling is activated then the approxS matrix has been scaled already */
/* so no need to rebuild in the case of scaling as we have been doing */
if(!approxS){ PetscPrintf( PETSC_COMM_WORLD, "WARNING approxS is NULL\n"); }
Stg_KSPSetOperators( ksp_S, S, S, SAME_NONZERO_PATTERN );
KSPSetType( ksp_S, "cg" );
KSPGetPC( ksp_S, &pc_S );
BSSCR_BSSCR_StokesCreatePCSchur2( K,G,D,C,approxS, pc_S, sym, bsscrp_self );
flg=0;
PetscOptionsGetString( PETSC_NULL, "-NN", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
Mat Smat, Pmat;
//MatStructure mstruct;
Stg_PCGetOperators( pc_S, &Smat, &Pmat, NULL );
sprintf(str,"%s/",name); sprintf(matname,"Pmat%s",suffix);
bsscr_dirwriteMat( Pmat, matname,str, "Writing Pmat matrix in al Solver");
}
uzawastyle=PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-uzawa_style", &uzawastyle, &found );
if(uzawastyle){
/* now want to set up the ksp_S->pc to be of type ksp (gmres) by default to match Uzawa */
KSP pc_ksp;
KSPGetPC( ksp_S, &pc_S );
PCSetType(pc_S,PCKSP);
PCKSPGetKSP( pc_S, &pc_ksp);
KSPSetType(pc_ksp, "gmres" );
KSPSetOptionsPrefix( pc_ksp, "scrPCKSP_");
KSPSetFromOptions( pc_ksp );
}
KSPSetFromOptions( ksp_S );
/* Set specific monitor test */
KSPGetTolerances( ksp_S, PETSC_NULL, PETSC_NULL, PETSC_NULL, &max_it );
// Weirdness with petsc 3.2 here...look at it later
//BSSCR_KSPLogSetMonitor( ksp_S, max_it, &monitor_index );
/***************************************************************************************************************/
/* Pressure / Velocity Solve */
/***************************************************************************************************************/
PetscPrintf( PETSC_COMM_WORLD, "\t* Pressure / Velocity Solve \n");
/***************************************************************************************************************/
/***************************************************************************************************************/
usePreviousGuess = PETSC_FALSE;
if(been_here)
PetscOptionsGetTruth( PETSC_NULL, "-scr_use_previous_guess", &usePreviousGuess, &found );
if(usePreviousGuess) { /* Note this should actually look at checkpoint information */
KSPSetInitialGuessNonzero( ksp_S, PETSC_TRUE ); }
else {
KSPSetInitialGuessNonzero( ksp_S, PETSC_FALSE ); }
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* SET CONVERGENCE TESTS *************************************************************************/
useNormInfStoppingConditions = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL ,"-scr_use_norm_inf_stopping_condition", &useNormInfStoppingConditions, &found );
if(useNormInfStoppingConditions)
BSSCR_KSPSetNormInfConvergenceTest(ksp_S);
useNormInfMonitor = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-scr_ksp_norm_inf_monitor", &useNormInfMonitor, &found );
if(useNormInfMonitor)
KSPMonitorSet( ksp_S, BSSCR_KSPNormInfToNorm2Monitor, PETSC_NULL, PETSC_NULL );
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* PRESSURE SOLVE ************************************************************************************/
PetscPrintf( PETSC_COMM_WORLD, "\t* KSPSolve( ksp_S, h_hat, p )\n");
/* if h_hat needs to be fixed up ..take out any nullspace vectors here */
/* we want to check that there is no "noise" in the null-space in the h vector */
/* this causes problems when we are trying to solve a Jacobian system when the Residual is almost converged */
if(bsscrp_self->check_pressureNS){
bsscrp_self->buildPNS(ksp);/* build and set nullspace vectors on bsscr - which is on ksp (function pointer is set in KSPSetUp_BSSCR) */
}
PetscScalar hnorm, gnorm;
MatNorm(G,NORM_INFINITY,&gnorm);
VecNorm(h_hat, NORM_2, &hnorm);
hnorm=hnorm/gnorm;
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,1.0/hnorm);
}
/* test to see if v or t are in nullspace of G and orthogonalize wrt h_hat if needed */
KSPRemovePressureNullspace_BSSCR(ksp, h_hat);
/* set convergence test to use min_it */
found = PETSC_FALSE;
min_it = 0;
PetscOptionsGetInt( PETSC_NULL,"-scr_ksp_set_min_it_converge", &min_it, &found);
if(found && min_it > 0){
BSSCR_KSPSetConvergenceMinIts(ksp_S, min_it, bsscrp_self);
}
/** Pressure Solve **/
PetscGetFlops(&flopsA);
scrSolveTime = MPI_Wtime();
KSPSolve( ksp_S, h_hat, p );
scrSolveTime = MPI_Wtime() - scrSolveTime;
PetscGetFlops(&flopsB);
PetscPrintf( PETSC_COMM_WORLD, "\n\t* KSPSolve( ksp_S, h_hat, p ) Solve Finished in time: %lf seconds\n\n", scrSolveTime);
bsscrp_self->solver->stats.pressure_time=scrSolveTime;
bsscrp_self->solver->stats.pressure_flops=(double)(flopsB-flopsA);
/***************************************/
if((hnorm < 1e-6) && (hnorm > 1e-20)){
VecScale(h_hat,hnorm);
VecScale(p,hnorm);
}
KSPRemovePressureNullspace_BSSCR(ksp, p);
/***************************************************************************************************************/
/***************************************************************************************************************/
/* restore K and f for the Velocity back solve */
found = PETSC_FALSE;
restorek = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-restore_K", &restorek, &found);
//PetscOptionsGetString( PETSC_NULL, "-restore_K", name, PETSC_MAX_PATH_LEN-1, &flg );
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
if(restorek){
penaltyNumber = -penaltyNumber;
if(f2) { ierr=VecAXPY(f,penaltyNumber,f2); }/* f <- f +a*f2 */
if(f3) { ierr=VecAXPY(f,penaltyNumber,f3); }/* f <- f +a*f3 */
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
//K=Korig;
Stg_KSPSetOperators(ksp_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KisJustK=PETSC_TRUE;
}
}
if(f3){ Stg_VecDestroy(&f3 ); } /* always destroy this local vector if was created */
/* obtain solution for u */
VecDuplicate( u, &t ); MatMult( G, p, t); VecAYPX( t, -1.0, f ); /*** t <- -t + f = f - G*p ***/
MatSchurComplementGetKSP( S, &ksp_inner );
a11SingleSolveTime = MPI_Wtime(); /* ---------------------------------- Final V Solve */
if(usePreviousGuess) KSPSetInitialGuessNonzero( ksp_inner, PETSC_TRUE );
/***************************************************************************************************************/
/***************************************************************************************************************/
/******* VELOCITY SOLVE ************************************************************************************/
/** Easier to just create a new KSP here if we want to do backsolve diffferently. (getting petsc errors now when switching from fgmres) */
change_backsolve=PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-change_backsolve", &change_backsolve, &found );
if(change_backsolve){
//Stg_KSPDestroy(&ksp_inner );
KSPCreate(PETSC_COMM_WORLD, &ksp_new_inner);
Stg_KSPSetOperators(ksp_new_inner, K, K, DIFFERENT_NONZERO_PATTERN);
KSPSetOptionsPrefix(ksp_new_inner, "backsolveA11_");
KSPSetFromOptions(ksp_new_inner); /* make sure we are setting up our solver how we want it */
MatSchurComplementSetKSP( S, ksp_new_inner );/* need to give the Schur it's inner ksp back for when we destroy it at end */
ksp_inner=ksp_new_inner;
}
PetscGetFlops(&flopsA);
KSPSolve(ksp_inner, t, u); /* Solve, then restore default tolerance and initial guess */
PetscGetFlops(&flopsB);
bsscrp_self->solver->stats.velocity_backsolve_flops=(double)(flopsB-flopsA);
a11SingleSolveTime = MPI_Wtime() - a11SingleSolveTime; /* ------------------ Final V Solve */
bsscrp_self->solver->stats.velocity_backsolve_time=a11SingleSolveTime;
flg=0;
PetscOptionsGetString( PETSC_NULL, "-solutiondumpdir", name, PETSC_MAX_PATH_LEN-1, &flg );
if(flg){
sprintf(str,"%s/",name); sprintf(matname,"p%s",suffix);
bsscr_dirwriteVec( p, matname,str, "Writing p vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"u%s",suffix);
bsscr_dirwriteVec( u, matname,str, "Writing u vector in al Solver");
sprintf(str,"%s/",name); sprintf(matname,"h_hat%s",suffix);
bsscr_dirwriteVec( h_hat, matname,str, "Writing h_hat vector in al Solver");
}
found = PETSC_FALSE;
restorek = PETSC_FALSE;
PetscOptionsGetTruth( PETSC_NULL, "-restore_K_after_solve", &restorek, &found);
if(penaltyNumber > 1e-10 && bsscrp_self->k2type){
if(restorek){
penaltyNumber = -penaltyNumber;
ierr=MatAXPY(K,penaltyNumber,K2,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);/* Computes K = penaltyNumber*K2 + K */
KisJustK=PETSC_TRUE;
}
}
/***************************************************************************************************************/
/***************************************************************************************************************/
/****** SOLUTION SUMMARY ******************************************************************************/
bsscr_summary(bsscrp_self,ksp_S,ksp_inner,K,K2,D,G,C,u,p,f,h,t,penaltyNumber,KisJustK, mgSetupTime, scrSolveTime, a11SingleSolveTime);
//bsscr_summary(bsscrp_self,ksp_S,ksp_inner,K,Korig,K2,D,G,C,u,p,f,h,t,penaltyNumber,KisJustK, mgSetupTime, scrSolveTime, a11SingleSolveTime);
/***************************************************************************************************************/
/***************************************************************************************************************/
Stg_VecDestroy(&t );
Stg_KSPDestroy(&ksp_S );
Stg_VecDestroy(&h_hat );
Stg_MatDestroy(&S );//This will destroy ksp_inner: also.. pcInner == pc_MG and is destroyed when ksp_inner is
been_here = 1;
PetscFunctionReturn(0);
}
