void
MultiGrid::coarsestSmooth (MultiFab& solL,
MultiFab& rhsL,
int level,
Real eps_rel,
Real eps_abs,
LinOp::BC_Mode bc_mode,
int local_usecg,
Real& cg_time)
{
BL_PROFILE("MultiGrid::coarsestSmooth()");
prepareForLevel(level);
if ( local_usecg == 0 )
{
Real error0 = 0;
if ( verbose > 0 )
{
error0 = errorEstimate(level, bc_mode);
if ( ParallelDescriptor::IOProcessor() )
std::cout << " Bottom Smoother: Initial error (error0) = "
<< error0 << '\n';
}
for (int i = finalSmooth(); i > 0; i--)
{
Lp.smooth(solL, rhsL, level, bc_mode);
if ( verbose > 1 || (i == 1 && verbose) )
{
Real error = errorEstimate(level, bc_mode);
const Real rel_error = (error0 != 0) ? error/error0 : 0;
if ( ParallelDescriptor::IOProcessor() )
std::cout << " Bottom Smoother: Iteration "
<< i
<< " error/error0 = "
<< rel_error << '\n';
}
}
}
else
{
bool use_mg_precond = false;
CGSolver cg(Lp, use_mg_precond, level);
cg.setMaxIter(maxiter_b);
const Real stime = ParallelDescriptor::second();
int ret = cg.solve(solL, rhsL, rtol_b, atol_b, bc_mode);
//
// The whole purpose of cg_time is to accumulate time spent in CGSolver.
//
cg_time += (ParallelDescriptor::second() - stime);
if ( ret != 0 )
{
if ( smooth_on_cg_unstable )
{
//
// If the CG solver returns a nonzero value indicating
// the problem is unstable. Assume this is not an accuracy
// issue and pound on it with the smoother.
// if ret == 8, then you have failure to converge
//
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
std::cout << "MultiGrid::coarsestSmooth(): CGSolver returns nonzero. Smoothing ...\n";
coarsestSmooth(solL, rhsL, level, eps_rel, eps_abs, bc_mode, 0, cg_time);
}
else
{
//
// cg failure probably indicates loss of precision accident.
// if ret == 8, then you have failure to converge
// setting solL to 0 should be ok.
//
solL.setVal(0);
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
std::cout << "MultiGrid::coarsestSmooth(): setting coarse corr to zero" << '\n';
}
}
}
for (int i = 0; i < nu_b; i++)
{
Lp.smooth(solL, rhsL, level, bc_mode);
}
}
}
void
MultiGrid::relax (MultiFab& solL,
MultiFab& rhsL,
int level,
Real eps_rel,
Real eps_abs,
LinOp::BC_Mode bc_mode,
Real& cg_time)
{
BL_PROFILE("MultiGrid::relax()");
//
// Recursively relax system. Equivalent to multigrid V-cycle.
// At coarsest grid, call coarsestSmooth.
//
if ( level < numlevels - 1 )
{
if ( verbose > 2 )
{
Real rnorm = errorEstimate(level, bc_mode);
if (ParallelDescriptor::IOProcessor())
{
std::cout << " AT LEVEL " << level << '\n';
std::cout << " DN:Norm before smooth " << rnorm << '\n';;
}
}
for (int i = preSmooth() ; i > 0 ; i--)
{
Lp.smooth(solL, rhsL, level, bc_mode);
}
Lp.residual(*res[level], rhsL, solL, level, bc_mode);
if ( verbose > 2 )
{
Real rnorm = norm_inf(*res[level]);
if (ParallelDescriptor::IOProcessor())
std::cout << " DN:Norm after smooth " << rnorm << '\n';
}
prepareForLevel(level+1);
average(*rhs[level+1], *res[level]);
cor[level+1]->setVal(0.0);
for (int i = cntRelax(); i > 0 ; i--)
{
relax(*cor[level+1],*rhs[level+1],level+1,eps_rel,eps_abs,bc_mode,cg_time);
}
interpolate(solL, *cor[level+1]);
if ( verbose > 2 )
{
Lp.residual(*res[level], rhsL, solL, level, bc_mode);
Real rnorm = norm_inf(*res[level]);
if ( ParallelDescriptor::IOProcessor() )
{
std::cout << " AT LEVEL " << level << '\n';
std::cout << " UP:Norm before smooth " << rnorm << '\n';
}
}
for (int i = postSmooth(); i > 0 ; i--)
{
Lp.smooth(solL, rhsL, level, bc_mode);
}
if ( verbose > 2 )
{
Lp.residual(*res[level], rhsL, solL, level, bc_mode);
Real rnorm = norm_inf(*res[level]);
if ( ParallelDescriptor::IOProcessor() )
std::cout << " UP:Norm after smooth " << rnorm << '\n';
}
}
else
{
if ( verbose > 2 )
{
Real rnorm = norm_inf(rhsL);
if ( ParallelDescriptor::IOProcessor() )
{
std::cout << " AT LEVEL " << level << '\n';
std::cout << " DN:Norm before bottom " << rnorm << '\n';
}
}
coarsestSmooth(solL, rhsL, level, eps_rel, eps_abs, bc_mode, usecg, cg_time);
if ( verbose > 2 )
{
Lp.residual(*res[level], rhsL, solL, level, bc_mode);
Real rnorm = norm_inf(*res[level]);
if ( ParallelDescriptor::IOProcessor() )
std::cout << " UP:Norm after bottom " << rnorm << '\n';
}
}
}
void ESDIRK::solveTimeStep( const scalar t0 )
{
if ( adaptiveTimeStepper->isPreviousStepAccepted() )
solver->nextTimeStep();
fsi::matrix solStages( nbStages, N ), F( nbStages, N );
F.setZero();
fsi::vector sol( N ), f( N ), qold( N ), result( N ), rhs( N );
solver->getSolution( sol, f );
solStages.row( 0 ) = sol;
qold = sol;
scalar t = t0;
solver->evaluateFunction( 0, sol, t, f );
F.row( 0 ) = f;
// Keep the solution of the first stage and function evaluate
// in memory in case the time step is rejected
fsi::vector solOld = sol;
fsi::vector fOld = f;
// Loop over the stages
solver->initTimeStep();
int iImplicitStage = 0;
for ( int j = 0; j < nbStages; j++ )
{
if ( !isStageImplicit( A( j, j ) ) )
continue;
t = t0 + C( j ) * dt;
Info << "\nTime = " << t << ", ESDIRK stage = " << j + 1 << "/" << nbStages << nl << endl;
rhs.setZero();
// Calculate sum of the stage residuals
for ( int iStage = 0; iStage < j; iStage++ )
rhs += A( j, iStage ) * F.row( iStage ).transpose();
rhs.array() *= dt;
solver->implicitSolve( false, iImplicitStage, 0, t, A( j, j ) * dt, qold, rhs, f, result );
assert( (1.0 / (A( j, j ) * dt) * (result - qold - rhs) - f).array().abs().maxCoeff() < 1.0e-8 );
solStages.row( j ) = result;
F.row( j ) = f;
iImplicitStage++;
}
if ( adaptiveTimeStepper->isEnabled() )
{
scalar newTimeStep = 0;
fsi::vector errorEstimate( N );
errorEstimate.setZero();
for ( int iStage = 0; iStage < nbStages; iStage++ )
errorEstimate += ( B( iStage ) - Bhat( iStage ) ) * F.row( iStage );
errorEstimate *= dt;
bool accepted = adaptiveTimeStepper->determineNewTimeStep( errorEstimate, result, dt, newTimeStep );
dt = newTimeStep;
if ( not accepted )
solver->setSolution( solOld, fOld );
}
if ( adaptiveTimeStepper->isAccepted() )
solver->finalizeTimeStep();
}
int
MultiGrid::solve_ (MultiFab& _sol,
Real eps_rel,
Real eps_abs,
LinOp::BC_Mode bc_mode,
Real bnorm,
Real resnorm0)
{
BL_PROFILE("MultiGrid::solve_()");
//
// If do_fixed_number_of_iters = 1, then do maxiter iterations without checking for convergence
//
// If do_fixed_number_of_iters = 0, then relax system maxiter times,
// and stop if relative error <= _eps_rel or if absolute err <= _abs_eps
//
const Real strt_time = ParallelDescriptor::second();
const int level = 0;
//
// We take the max of the norms of the initial RHS and the initial residual in order to capture both cases
//
Real norm_to_test_against;
bool using_bnorm;
if (bnorm >= resnorm0)
{
norm_to_test_against = bnorm;
using_bnorm = true;
} else {
norm_to_test_against = resnorm0;
using_bnorm = false;
}
int returnVal = 0;
Real error = resnorm0;
//
// Note: if eps_rel, eps_abs < 0 then that test is effectively bypassed
//
if ( ParallelDescriptor::IOProcessor() && eps_rel < 1.0e-16 && eps_rel > 0 )
{
std::cout << "MultiGrid: Tolerance "
<< eps_rel
<< " < 1e-16 is probably set too low" << '\n';
}
//
// We initially define norm_cor based on the initial solution only so we can use it in the very first iteration
// to decide whether the problem is already solved (this is relevant if the previous solve used was only solved
// according to the Anorm test and not the bnorm test).
//
Real norm_cor = norm_inf(*initialsolution,true);
ParallelDescriptor::ReduceRealMax(norm_cor);
int nit = 1;
const Real norm_Lp = Lp.norm(0, level);
Real cg_time = 0;
if ( use_Anorm_for_convergence == 1 )
{
//
// Don't need to go any further -- no iterations are required
//
if (error <= eps_abs || error < eps_rel*(norm_Lp*norm_cor+norm_to_test_against))
{
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
std::cout << " Problem is already converged -- no iterations required\n";
}
return 1;
}
for ( ;
( (error > eps_abs &&
error > eps_rel*(norm_Lp*norm_cor+norm_to_test_against)) ||
(do_fixed_number_of_iters == 1) )
&& nit <= maxiter;
++nit)
{
relax(*cor[level], *rhs[level], level, eps_rel, eps_abs, bc_mode, cg_time);
Real tmp[2] = { norm_inf(*cor[level],true), errorEstimate(level,bc_mode,true) };
ParallelDescriptor::ReduceRealMax(tmp,2);
norm_cor = tmp[0];
error = tmp[1];
if ( ParallelDescriptor::IOProcessor() && verbose > 1 )
{
const Real rel_error = error / norm_to_test_against;
Spacer(std::cout, level);
if (using_bnorm)
{
std::cout << "MultiGrid: Iteration "
<< nit
<< " resid/bnorm = "
<< rel_error << '\n';
} else {
std::cout << "MultiGrid: Iteration "
<< nit
<< " resid/resid0 = "
<< rel_error << '\n';
}
}
}
}
else
{
//
// Don't need to go any further -- no iterations are required
//
if (error <= eps_abs || error < eps_rel*norm_to_test_against)
{
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
std::cout << " Problem is already converged -- no iterations required\n";
}
return 1;
}
for ( ;
( (error > eps_abs &&
error > eps_rel*norm_to_test_against) ||
(do_fixed_number_of_iters == 1) )
&& nit <= maxiter;
++nit)
{
relax(*cor[level], *rhs[level], level, eps_rel, eps_abs, bc_mode, cg_time);
error = errorEstimate(level, bc_mode);
if ( ParallelDescriptor::IOProcessor() && verbose > 1 )
{
const Real rel_error = error / norm_to_test_against;
Spacer(std::cout, level);
if (using_bnorm)
{
std::cout << "MultiGrid: Iteration "
<< nit
<< " resid/bnorm = "
<< rel_error << '\n';
} else {
std::cout << "MultiGrid: Iteration "
<< nit
<< " resid/resid0 = "
<< rel_error << '\n';
}
}
}
}
Real run_time = (ParallelDescriptor::second() - strt_time);
if ( verbose > 0 )
{
if ( ParallelDescriptor::IOProcessor() )
{
const Real rel_error = error / norm_to_test_against;
Spacer(std::cout, level);
if (using_bnorm)
{
std::cout << "MultiGrid: Iteration "
<< nit-1
<< " resid/bnorm = "
<< rel_error << '\n';
} else {
std::cout << "MultiGrid: Iteration "
<< nit-1
<< " resid/resid0 = "
<< rel_error << '\n';
}
}
if ( verbose > 1 )
{
Real tmp[2] = { run_time, cg_time };
ParallelDescriptor::ReduceRealMax(tmp,2,ParallelDescriptor::IOProcessorNumber());
if ( ParallelDescriptor::IOProcessor() )
std::cout << ", Solve time: " << tmp[0] << ", CG time: " << tmp[1];
}
if ( ParallelDescriptor::IOProcessor() ) std::cout << '\n';
}
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
if ( do_fixed_number_of_iters == 1)
{
std::cout << " Did fixed number of iterations: " << maxiter << std::endl;
}
else if ( error < eps_rel*norm_to_test_against )
{
std::cout << " Converged res < eps_rel*max(bnorm,res_norm)\n";
}
else if ( (use_Anorm_for_convergence == 1) && (error < eps_rel*norm_Lp*norm_cor) )
{
std::cout << " Converged res < eps_rel*Anorm*sol\n";
}
else if ( error < eps_abs )
{
std::cout << " Converged res < eps_abs\n";
}
}
//
// Omit ghost update since maybe not initialized in calling routine.
// Add to boundary values stored in initialsolution.
//
_sol.copy(*cor[level]);
_sol.plus(*initialsolution,0,_sol.nComp(),0);
if ( use_Anorm_for_convergence == 1 )
{
if ( do_fixed_number_of_iters == 1 ||
error <= eps_rel*(norm_Lp*norm_cor+norm_to_test_against) ||
error <= eps_abs )
returnVal = 1;
}
else
{
if ( do_fixed_number_of_iters == 1 ||
error <= eps_rel*(norm_to_test_against) ||
error <= eps_abs )
returnVal = 1;
}
//
// Otherwise, failed to solve satisfactorily
//
return returnVal;
}
int
MCMultiGrid::solve_ (MultiFab& _sol,
Real eps_rel,
Real eps_abs,
MCBC_Mode bc_mode,
int level)
{
//
// Relax system maxiter times, stop if relative error <= _eps_rel or
// if absolute err <= _abs_eps
//
const Real strt_time = ParallelDescriptor::second();
//
// Elide a reduction by doing these together.
//
Real tmp[2] = { norm_inf(*rhs[level],true), errorEstimate(level,bc_mode,true) };
ParallelDescriptor::ReduceRealMax(tmp,2);
const Real norm_rhs = tmp[0];
const Real error0 = tmp[1];
int returnVal = 0;
Real error = error0;
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
Spacer(std::cout, level);
std::cout << "MCMultiGrid: Initial rhs = " << norm_rhs << '\n';
std::cout << "MCMultiGrid: Initial error (error0) = " << error0 << '\n';
}
if ( ParallelDescriptor::IOProcessor() && eps_rel < 1.0e-16 && eps_rel > 0 )
{
std::cout << "MCMultiGrid: Tolerance "
<< eps_rel
<< " < 1e-16 is probably set too low" << '\n';
}
//
// Initialize correction to zero at this level (auto-filled at levels below)
//
(*cor[level]).setVal(0.0);
//
// Note: if eps_rel, eps_abs < 0 then that test is effectively bypassed.
//
int nit = 1;
const Real new_error_0 = norm_rhs;
//const Real norm_Lp = Lp.norm(0, level);
for ( ;
error > eps_abs &&
error > eps_rel*norm_rhs &&
nit <= maxiter;
++nit)
{
relax(*cor[level], *rhs[level], level, eps_rel, eps_abs, bc_mode);
error = errorEstimate(level,bc_mode);
if ( ParallelDescriptor::IOProcessor() && verbose > 1 )
{
const Real rel_error = (error0 != 0) ? error/new_error_0 : 0;
Spacer(std::cout, level);
std::cout << "MCMultiGrid: Iteration "
<< nit
<< " error/error0 = "
<< rel_error << '\n';
}
}
Real run_time = (ParallelDescriptor::second() - strt_time);
if ( verbose > 0 )
{
if ( ParallelDescriptor::IOProcessor() )
{
const Real rel_error = (error0 != 0) ? error/error0 : 0;
Spacer(std::cout, level);
std::cout << "MCMultiGrid: Final Iter. "
<< nit-1
<< " error/error0 = "
<< rel_error;
}
if ( verbose > 1 )
{
ParallelDescriptor::ReduceRealMax(run_time);
if ( ParallelDescriptor::IOProcessor() )
std::cout << ", Solve time: " << run_time << '\n';
}
}
if ( ParallelDescriptor::IOProcessor() && (verbose > 0) )
{
if ( error < eps_rel*norm_rhs )
{
std::cout << " Converged res < eps_rel*bnorm\n";
}
else if ( error < eps_abs )
{
std::cout << " Converged res < eps_abs\n";
}
}
//
// Omit ghost update since maybe not initialized in calling routine.
// Add to boundary values stored in initialsolution.
//
_sol.copy(*cor[level]);
_sol.plus(*initialsolution,0,_sol.nComp(),0);
if ( error <= eps_rel*(norm_rhs) ||
error <= eps_abs )
returnVal = 1;
//
// Otherwise, failed to solve satisfactorily
//
return returnVal;
}
