void
TransientMultiApp::finishStep()
{
if (!_sub_cycling)
{
for (unsigned int i = 0; i < _my_num_apps; i++)
{
Transient * ex = _transient_executioners[i];
ex->endStep();
ex->postStep();
}
}
}
void
TransientMultiApp::advanceStep()
{
if (!_auto_advance && !_sub_cycling)
{
for (unsigned int i=0; i<_my_num_apps; i++)
{
/*FEProblemBase * problem =*/ appProblemBase(_first_local_app + i);
Transient * ex = _transient_executioners[i];
ex->endStep();
ex->postStep();
ex->incrementStepOrReject();
}
}
}
bool
TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
{
if (!_has_an_app)
return true;
_auto_advance = auto_advance;
_console << "Solving MultiApp " << name() << std::endl;
// "target_time" must always be in global time
target_time += _app.getGlobalTimeOffset();
Moose::ScopedCommSwapper swapper(_my_comm);
bool return_value = true;
// Make sure we swap back the communicator regardless of how this routine is exited
try
{
int rank;
int ierr;
ierr = MPI_Comm_rank(_orig_comm, &rank);
mooseCheckMPIErr(ierr);
for (unsigned int i = 0; i < _my_num_apps; i++)
{
FEProblemBase & problem = appProblemBase(_first_local_app + i);
Transient * ex = _transient_executioners[i];
// The App might have a different local time from the rest of the problem
Real app_time_offset = _apps[i]->getGlobalTimeOffset();
// Maybe this MultiApp was already solved
if ((ex->getTime() + app_time_offset + 2e-14 >= target_time) ||
(ex->getTime() >= ex->endTime()))
continue;
if (_sub_cycling)
{
Real time_old = ex->getTime() + app_time_offset;
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close();
// Save off the current auxiliary solution
transfer_old = solution;
transfer_old.close();
// Snag all of the local dof indices for all of these variables
AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
ConstElemRange & elem_range = *problem.mesh().getActiveLocalElementRange();
Threads::parallel_reduce(elem_range, aldit);
_transferred_dofs = aldit._all_dof_indices;
}
// Disable/enable output for sub cycling
problem.allowOutput(_output_sub_cycles); // disables all outputs, including console
problem.allowOutput<Console>(_print_sub_cycles); // re-enables Console to print, if desired
ex->setTargetTime(target_time - app_time_offset);
// unsigned int failures = 0;
bool at_steady = false;
if (_first && !_app.isRecovering())
problem.advanceState();
bool local_first = _first;
// Now do all of the solves we need
while ((!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time) ||
!ex->lastSolveConverged())
{
if (local_first != true)
ex->incrementStepOrReject();
local_first = false;
ex->preStep();
ex->computeDT();
if (_interpolate_transfers)
{
// See what time this executioner is going to go to.
Real future_time = ex->getTime() + app_time_offset + ex->getDT();
// How far along we are towards the target time:
Real step_percent = (future_time - time_old) / (target_time - time_old);
Real one_minus_step_percent = 1.0 - step_percent;
// Do the interpolation for each variable that was transferred to
FEProblemBase & problem = appProblemBase(_first_local_app + i);
AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close(); // Just to be sure
transfer.close();
transfer_old.close();
for (const auto & dof : _transferred_dofs)
{
solution.set(dof,
(transfer_old(dof) * one_minus_step_percent) +
(transfer(dof) * step_percent));
// solution.set(dof, transfer_old(dof));
// solution.set(dof, transfer(dof));
// solution.set(dof, 1);
}
solution.close();
}
ex->takeStep();
bool converged = ex->lastSolveConverged();
if (!converged)
{
mooseWarning(
"While sub_cycling ", name(), _first_local_app + i, " failed to converge!\n");
_failures++;
if (_failures > _max_failures)
{
std::stringstream oss;
oss << "While sub_cycling " << name() << _first_local_app << i << " REALLY failed!";
throw MultiAppSolveFailure(oss.str());
}
}
Real solution_change_norm = ex->getSolutionChangeNorm();
if (_detect_steady_state)
_console << "Solution change norm: " << solution_change_norm << std::endl;
if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
{
_console << "Detected Steady State! Fast-forwarding to " << target_time << std::endl;
at_steady = true;
// Indicate that the next output call (occurs in ex->endStep()) should output,
// regardless of intervals etc...
problem.forceOutput();
// Clean up the end
ex->endStep(target_time - app_time_offset);
ex->postStep();
}
else
{
ex->endStep();
ex->postStep();
}
}
// If we were looking for a steady state, but didn't reach one, we still need to output one
// more time, regardless of interval
if (!at_steady)
problem.outputStep(EXEC_FORCED);
} // sub_cycling
else if (_tolerate_failure)
{
ex->takeStep(dt);
ex->endStep(target_time - app_time_offset);
ex->postStep();
}
else
{
_console << "Solving Normal Step!" << std::endl;
if (_first && !_app.isRecovering())
problem.advanceState();
if (auto_advance)
problem.allowOutput(true);
ex->takeStep(dt);
if (auto_advance)
{
ex->endStep();
ex->postStep();
if (!ex->lastSolveConverged())
{
mooseWarning(name(), _first_local_app + i, " failed to converge!\n");
if (_catch_up)
{
_console << "Starting Catch Up!" << std::endl;
bool caught_up = false;
unsigned int catch_up_step = 0;
Real catch_up_dt = dt / 2;
while (!caught_up && catch_up_step < _max_catch_up_steps)
{
_console << "Solving " << name() << " catch up step " << catch_up_step << std::endl;
ex->incrementStepOrReject();
ex->computeDT();
ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
ex->endStep();
if (ex->lastSolveConverged())
{
if (ex->getTime() + app_time_offset +
(ex->timestepTol() * std::abs(ex->getTime())) >=
target_time)
{
problem.outputStep(EXEC_FORCED);
caught_up = true;
}
}
else
catch_up_dt /= 2.0;
ex->postStep();
catch_up_step++;
}
if (!caught_up)
throw MultiAppSolveFailure(name() + " Failed to catch up!\n");
}
}
}
else
{
if (!ex->lastSolveConverged())
{
// Even if we don't allow auto_advance - we can still catch up to the current time if
// possible
if (_catch_up)
{
_console << "Starting Catch Up!" << std::endl;
bool caught_up = false;
unsigned int catch_up_step = 0;
Real catch_up_dt = dt / 2;
// Note: this loop will _break_ if target_time is satisfied
while (catch_up_step < _max_catch_up_steps)
{
_console << "Solving " << name() << " catch up step " << catch_up_step << std::endl;
ex->incrementStepOrReject();
ex->computeDT();
ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
// This is required because we can't call endStep() yet
// (which normally increments time)
Real current_time = ex->getTime() + ex->getDT();
if (ex->lastSolveConverged())
{
if (current_time + app_time_offset +
(ex->timestepTol() * std::abs(current_time)) >=
target_time)
{
caught_up = true;
break; // break here so that we don't run endStep() or postStep() since this
// MultiApp should NOT be auto_advanced
}
}
else
catch_up_dt /= 2.0;
ex->endStep();
ex->postStep();
catch_up_step++;
}
if (!caught_up)
throw MultiAppSolveFailure(name() + " Failed to catch up!\n");
}
else
throw MultiAppSolveFailure(name() + " failed to converge");
}
}
}
// Re-enable all output (it may of been disabled by sub-cycling)
problem.allowOutput(true);
}
_first = false;
_console << "Successfully Solved MultiApp " << name() << "." << std::endl;
}
catch (MultiAppSolveFailure & e)
{
mooseWarning(e.what());
_console << "Failed to Solve MultiApp " << name() << ", attempting to recover." << std::endl;
return_value = false;
}
_transferred_vars.clear();
return return_value;
}