void
FullSolveMultiApp::solveStep(Real /*dt*/, Real /*target_time*/, bool auto_advance)
{
if (!auto_advance)
mooseError("FullSolveMultiApp is not compatible with auto_advance=false");
if (!_has_an_app)
return;
if (_solved)
return;
_console << "Fully Solving MultiApp " << name() << std::endl;
MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);
int rank;
int ierr;
ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);
for (unsigned int i=0; i<_my_num_apps; i++)
{
Executioner * ex = _executioners[i];
ex->execute();
}
// Swap back
Moose::swapLibMeshComm(swapped);
_solved = true;
_console << "Finished Solving MultiApp " << name() << std::endl;
}
MPI_Comm
swapLibMeshComm(MPI_Comm new_comm)
{
int ierr;
MPI_Comm old_comm = libMesh::COMM_WORLD;
libMesh::COMM_WORLD = new_comm;
libMesh::CommWorld = new_comm;
#ifdef LIBMESH_HAVE_PETSC
PETSC_COMM_WORLD = new_comm;
#endif //LIBMESH_HAVE_PETSC
int pid;
ierr = MPI_Comm_rank(new_comm, &pid); mooseCheckMPIErr(ierr);
int n_procs;
ierr = MPI_Comm_size(new_comm, &n_procs); mooseCheckMPIErr(ierr);
libMesh::libMeshPrivateData::_processor_id = pid;
libMesh::libMeshPrivateData::_n_processors = n_procs;
return old_comm;
}
void
TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
{
if (_sub_cycling && !auto_advance)
mooseError("TransientMultiApp with sub_cycling=true is not compatible with auto_advance=false");
if (_catch_up && !auto_advance)
mooseError("TransientMultiApp with catch_up=true is not compatible with auto_advance=false");
if (!_has_an_app)
return;
_auto_advance = auto_advance;
Moose::out << "Solving MultiApp " << _name << std::endl;
// "target_time" must always be in global time
target_time += _app.getGlobalTimeOffset();
MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);
int rank;
int ierr;
ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);
for (unsigned int i=0; i<_my_num_apps; i++)
{
FEProblem * problem = appProblem(_first_local_app + i);
OutputWarehouse & output_warehouse = _apps[i]->getOutputWarehouse();
Transient * ex = _transient_executioners[i];
// The App might have a different local time from the rest of the problem
Real app_time_offset = _apps[i]->getGlobalTimeOffset();
if ((ex->getTime() + app_time_offset) + 2e-14 >= target_time) // Maybe this MultiApp was already solved
continue;
if (_sub_cycling)
{
Real time_old = ex->getTime() + app_time_offset;
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close();
// Save off the current auxiliary solution
transfer_old = solution;
transfer_old.close();
// Snag all of the local dof indices for all of these variables
AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
ConstElemRange & elem_range = *problem->mesh().getActiveLocalElementRange();
Threads::parallel_reduce(elem_range, aldit);
_transferred_dofs = aldit._all_dof_indices;
}
if (_output_sub_cycles)
output_warehouse.allowOutput(true);
else
output_warehouse.allowOutput(false);
ex->setTargetTime(target_time-app_time_offset);
// unsigned int failures = 0;
bool at_steady = false;
// Now do all of the solves we need
while(true)
{
if (_first != true)
ex->incrementStepOrReject();
_first = false;
if (!(!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time))
break;
ex->computeDT();
if (_interpolate_transfers)
{
// See what time this executioner is going to go to.
Real future_time = ex->getTime() + app_time_offset + ex->getDT();
// How far along we are towards the target time:
Real step_percent = (future_time - time_old) / (target_time - time_old);
Real one_minus_step_percent = 1.0 - step_percent;
// Do the interpolation for each variable that was transferred to
FEProblem * problem = appProblem(_first_local_app + i);
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close(); // Just to be sure
transfer.close();
transfer_old.close();
std::set<dof_id_type>::iterator it = _transferred_dofs.begin();
std::set<dof_id_type>::iterator end = _transferred_dofs.end();
for(; it != end; ++it)
{
dof_id_type dof = *it;
solution.set(dof, (transfer_old(dof) * one_minus_step_percent) + (transfer(dof) * step_percent));
// solution.set(dof, transfer_old(dof));
// solution.set(dof, transfer(dof));
// solution.set(dof, 1);
}
solution.close();
}
ex->takeStep();
bool converged = ex->lastSolveConverged();
if (!converged)
{
mooseWarning("While sub_cycling "<<_name<<_first_local_app+i<<" failed to converge!"<<std::endl);
_failures++;
if (_failures > _max_failures)
mooseError("While sub_cycling "<<_name<<_first_local_app+i<<" REALLY failed!"<<std::endl);
}
Real solution_change_norm = ex->getSolutionChangeNorm();
if (_detect_steady_state)
Moose::out << "Solution change norm: " << solution_change_norm << std::endl;
if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
{
Moose::out << "Detected Steady State! Fast-forwarding to " << target_time << std::endl;
at_steady = true;
// Indicate that the next output call (occurs in ex->endStep()) should output, regarless of intervals etc...
output_warehouse.forceOutput();
// Clean up the end
ex->endStep(target_time-app_time_offset);
}
else
ex->endStep();
}
// If we were looking for a steady state, but didn't reach one, we still need to output one more time
if (!at_steady)
{
output_warehouse.forceOutput();
output_warehouse.outputStep();
}
}
else if (_tolerate_failure)
{
ex->takeStep(dt);
output_warehouse.forceOutput();
ex->endStep(target_time-app_time_offset);
}
else
{
Moose::out << "Solving Normal Step!" << std::endl;
if (auto_advance)
if (_first != true)
ex->incrementStepOrReject();
if (auto_advance)
output_warehouse.allowOutput(true);
ex->takeStep(dt);
if (auto_advance)
{
ex->endStep();
if (!ex->lastSolveConverged())
{
mooseWarning(_name << _first_local_app+i << " failed to converge!" << std::endl);
if (_catch_up)
{
Moose::out << "Starting Catch Up!" << std::endl;
bool caught_up = false;
unsigned int catch_up_step = 0;
Real catch_up_dt = dt/2;
while(!caught_up && catch_up_step < _max_catch_up_steps)
{
Moose::err << "Solving " << _name << "catch up step " << catch_up_step << std::endl;
ex->incrementStepOrReject();
ex->computeDT();
ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
if (ex->lastSolveConverged())
{
if (ex->getTime() + app_time_offset + ex->timestepTol()*std::abs(ex->getTime()) >= target_time)
{
output_warehouse.forceOutput();
output_warehouse.outputStep();
caught_up = true;
}
}
else
catch_up_dt /= 2.0;
ex->endStep();
catch_up_step++;
}
if (!caught_up)
mooseError(_name << " Failed to catch up!\n");
output_warehouse.allowOutput(true);
}
}
}
}
}
_first = false;
// Swap back
Moose::swapLibMeshComm(swapped);
_transferred_vars.clear();
Moose::out << "Finished Solving MultiApp " << _name << std::endl;
}
void
MultiApp::buildComm()
{
int ierr;
ierr = MPI_Comm_size(_orig_comm, &_orig_num_procs); mooseCheckMPIErr(ierr);
ierr = MPI_Comm_rank(_orig_comm, &_orig_rank); mooseCheckMPIErr(ierr);
struct utsname sysInfo;
uname(&sysInfo);
_node_name = sysInfo.nodename;
// If we have more apps than processors then we're just going to divide up the work
if (_total_num_apps >= (unsigned)_orig_num_procs)
{
_my_comm = MPI_COMM_SELF;
_my_rank = 0;
_my_num_apps = _total_num_apps/_orig_num_procs;
unsigned int jobs_left = _total_num_apps - (_my_num_apps * _orig_num_procs);
if (jobs_left != 0)
{
// Spread the remaining jobs out over the first set of processors
if ((unsigned)_orig_rank < jobs_left) // (these are the "jobs_left_pids" ie the pids that are snatching up extra jobs)
{
_my_num_apps += 1;
_first_local_app = _my_num_apps * _orig_rank;
}
else
{
unsigned int num_apps_in_jobs_left_pids = (_my_num_apps + 1) * jobs_left;
unsigned int distance_to_jobs_left_pids = _orig_rank - jobs_left;
_first_local_app = num_apps_in_jobs_left_pids + (_my_num_apps * distance_to_jobs_left_pids);
}
}
else
_first_local_app = _my_num_apps * _orig_rank;
return;
}
// In this case we need to divide up the processors that are going to work on each app
int rank;
ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);
unsigned int procs_per_app = _orig_num_procs / _total_num_apps;
if (_max_procs_per_app < procs_per_app)
procs_per_app = _max_procs_per_app;
int my_app = rank / procs_per_app;
unsigned int procs_for_my_app = procs_per_app;
if ((unsigned int) my_app > _total_num_apps-1 && procs_for_my_app == _max_procs_per_app)
{
// If we've already hit the max number of procs per app then this processor
// won't have an app at all
_my_num_apps = 0;
_has_an_app = false;
}
else if ((unsigned int) my_app >= _total_num_apps-1) // The last app will gain any left-over procs
{
my_app = _total_num_apps - 1;
// procs_for_my_app += _orig_num_procs % _total_num_apps;
_first_local_app = my_app;
_my_num_apps = 1;
}
else
{
_first_local_app = my_app;
_my_num_apps = 1;
}
if (_has_an_app)
{
ierr = MPI_Comm_split(_orig_comm, _first_local_app, rank, &_my_comm); mooseCheckMPIErr(ierr);
ierr = MPI_Comm_rank(_my_comm, &_my_rank); mooseCheckMPIErr(ierr);
}
else
{
ierr = MPI_Comm_split(_orig_comm, MPI_UNDEFINED, rank, &_my_comm); mooseCheckMPIErr(ierr);
_my_rank = 0;
}
}
bool
TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
{
if (!_has_an_app)
return true;
_auto_advance = auto_advance;
_console << "Solving MultiApp " << name() << std::endl;
// "target_time" must always be in global time
target_time += _app.getGlobalTimeOffset();
Moose::ScopedCommSwapper swapper(_my_comm);
bool return_value = true;
// Make sure we swap back the communicator regardless of how this routine is exited
try
{
int rank;
int ierr;
ierr = MPI_Comm_rank(_orig_comm, &rank);
mooseCheckMPIErr(ierr);
for (unsigned int i = 0; i < _my_num_apps; i++)
{
FEProblemBase & problem = appProblemBase(_first_local_app + i);
Transient * ex = _transient_executioners[i];
// The App might have a different local time from the rest of the problem
Real app_time_offset = _apps[i]->getGlobalTimeOffset();
// Maybe this MultiApp was already solved
if ((ex->getTime() + app_time_offset + 2e-14 >= target_time) ||
(ex->getTime() >= ex->endTime()))
continue;
if (_sub_cycling)
{
Real time_old = ex->getTime() + app_time_offset;
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close();
// Save off the current auxiliary solution
transfer_old = solution;
transfer_old.close();
// Snag all of the local dof indices for all of these variables
AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
ConstElemRange & elem_range = *problem.mesh().getActiveLocalElementRange();
Threads::parallel_reduce(elem_range, aldit);
_transferred_dofs = aldit._all_dof_indices;
}
// Disable/enable output for sub cycling
problem.allowOutput(_output_sub_cycles); // disables all outputs, including console
problem.allowOutput<Console>(_print_sub_cycles); // re-enables Console to print, if desired
ex->setTargetTime(target_time - app_time_offset);
// unsigned int failures = 0;
bool at_steady = false;
if (_first && !_app.isRecovering())
problem.advanceState();
bool local_first = _first;
// Now do all of the solves we need
while ((!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time) ||
!ex->lastSolveConverged())
{
if (local_first != true)
ex->incrementStepOrReject();
local_first = false;
ex->preStep();
ex->computeDT();
if (_interpolate_transfers)
{
// See what time this executioner is going to go to.
Real future_time = ex->getTime() + app_time_offset + ex->getDT();
// How far along we are towards the target time:
Real step_percent = (future_time - time_old) / (target_time - time_old);
Real one_minus_step_percent = 1.0 - step_percent;
// Do the interpolation for each variable that was transferred to
FEProblemBase & problem = appProblemBase(_first_local_app + i);
AuxiliarySystem & aux_system = problem.getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close(); // Just to be sure
transfer.close();
transfer_old.close();
for (const auto & dof : _transferred_dofs)
{
solution.set(dof,
(transfer_old(dof) * one_minus_step_percent) +
(transfer(dof) * step_percent));
// solution.set(dof, transfer_old(dof));
// solution.set(dof, transfer(dof));
// solution.set(dof, 1);
}
solution.close();
}
ex->takeStep();
bool converged = ex->lastSolveConverged();
if (!converged)
{
mooseWarning(
"While sub_cycling ", name(), _first_local_app + i, " failed to converge!\n");
_failures++;
if (_failures > _max_failures)
{
std::stringstream oss;
oss << "While sub_cycling " << name() << _first_local_app << i << " REALLY failed!";
throw MultiAppSolveFailure(oss.str());
}
}
Real solution_change_norm = ex->getSolutionChangeNorm();
if (_detect_steady_state)
_console << "Solution change norm: " << solution_change_norm << std::endl;
if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
{
_console << "Detected Steady State! Fast-forwarding to " << target_time << std::endl;
at_steady = true;
// Indicate that the next output call (occurs in ex->endStep()) should output,
// regardless of intervals etc...
problem.forceOutput();
// Clean up the end
ex->endStep(target_time - app_time_offset);
ex->postStep();
}
else
{
ex->endStep();
ex->postStep();
}
}
// If we were looking for a steady state, but didn't reach one, we still need to output one
// more time, regardless of interval
if (!at_steady)
problem.outputStep(EXEC_FORCED);
} // sub_cycling
else if (_tolerate_failure)
{
ex->takeStep(dt);
ex->endStep(target_time - app_time_offset);
ex->postStep();
}
else
{
_console << "Solving Normal Step!" << std::endl;
if (_first && !_app.isRecovering())
problem.advanceState();
if (auto_advance)
problem.allowOutput(true);
ex->takeStep(dt);
if (auto_advance)
{
ex->endStep();
ex->postStep();
if (!ex->lastSolveConverged())
{
mooseWarning(name(), _first_local_app + i, " failed to converge!\n");
if (_catch_up)
{
_console << "Starting Catch Up!" << std::endl;
bool caught_up = false;
unsigned int catch_up_step = 0;
Real catch_up_dt = dt / 2;
while (!caught_up && catch_up_step < _max_catch_up_steps)
{
_console << "Solving " << name() << " catch up step " << catch_up_step << std::endl;
ex->incrementStepOrReject();
ex->computeDT();
ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
ex->endStep();
if (ex->lastSolveConverged())
{
if (ex->getTime() + app_time_offset +
(ex->timestepTol() * std::abs(ex->getTime())) >=
target_time)
{
problem.outputStep(EXEC_FORCED);
caught_up = true;
}
}
else
catch_up_dt /= 2.0;
ex->postStep();
catch_up_step++;
}
if (!caught_up)
throw MultiAppSolveFailure(name() + " Failed to catch up!\n");
}
}
}
else
{
if (!ex->lastSolveConverged())
{
// Even if we don't allow auto_advance - we can still catch up to the current time if
// possible
if (_catch_up)
{
_console << "Starting Catch Up!" << std::endl;
bool caught_up = false;
unsigned int catch_up_step = 0;
Real catch_up_dt = dt / 2;
// Note: this loop will _break_ if target_time is satisfied
while (catch_up_step < _max_catch_up_steps)
{
_console << "Solving " << name() << " catch up step " << catch_up_step << std::endl;
ex->incrementStepOrReject();
ex->computeDT();
ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
// This is required because we can't call endStep() yet
// (which normally increments time)
Real current_time = ex->getTime() + ex->getDT();
if (ex->lastSolveConverged())
{
if (current_time + app_time_offset +
(ex->timestepTol() * std::abs(current_time)) >=
target_time)
{
caught_up = true;
break; // break here so that we don't run endStep() or postStep() since this
// MultiApp should NOT be auto_advanced
}
}
else
catch_up_dt /= 2.0;
ex->endStep();
ex->postStep();
catch_up_step++;
}
if (!caught_up)
throw MultiAppSolveFailure(name() + " Failed to catch up!\n");
}
else
throw MultiAppSolveFailure(name() + " failed to converge");
}
}
}
// Re-enable all output (it may of been disabled by sub-cycling)
problem.allowOutput(true);
}
_first = false;
_console << "Successfully Solved MultiApp " << name() << "." << std::endl;
}
catch (MultiAppSolveFailure & e)
{
mooseWarning(e.what());
_console << "Failed to Solve MultiApp " << name() << ", attempting to recover." << std::endl;
return_value = false;
}
_transferred_vars.clear();
return return_value;
}