void
TransientMultiApp::setupApp(unsigned int i, Real /*time*/, bool output_initial) // FIXME: Should we be passing time?
{
MooseApp * app = _apps[i];
Transient * ex = dynamic_cast<Transient *>(app->getExecutioner());
if (!ex)
mooseError("MultiApp " << _name << " is not using a Transient Executioner!");
// Get the FEProblem and OutputWarehouse for the current MultiApp
FEProblem * problem = appProblem(_first_local_app + i);
OutputWarehouse & output_warehouse = _apps[i]->getOutputWarehouse();
if (!output_initial)
{
ex->outputInitial(false);//\todo{Remove; handled within ex->init()}
output_warehouse.allowOutput(false);
}
// Set the file numbers of the i-th app to that of the parent app
output_warehouse.setFileNumbers(app->getOutputFileNumbers());
// Call initialization method of Executioner (Note, this preforms the output of the initial time step, if desired)
ex->init();
// Enable output after setup
output_warehouse.allowOutput(true);
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
// We'll store a copy of the auxiliary system's solution at the old time in here
libmesh_aux_system.add_vector("transfer_old", false);
// This will be where we'll transfer the value to for the "target" time
libmesh_aux_system.add_vector("transfer", false);
}
ex->preExecute();
problem->copyOldSolutions();
_transient_executioners[i] = ex;
if (_detect_steady_state || _tolerate_failure)
{
_apps[i]->getOutputWarehouse().allowOutput(false);
ex->allowOutput(false);
}
}
void
TransientMultiApp::solveStep(Real dt, Real target_time)
{
if (!_has_an_app)
return;
Moose::out << "Solving MultiApp " << _name << std::endl;
// "target_time" must always be in global time
target_time += _app.getGlobalTimeOffset();
MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);
int rank;
int ierr;
ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);
for(unsigned int i=0; i<_my_num_apps; i++)
{
FEProblem * problem = appProblem(_first_local_app + i);
OutputWarehouse & output_warehouse = _apps[i]->getOutputWarehouse();
output_warehouse.timestepSetup();
Transient * ex = _transient_executioners[i];
// The App might have a different local time from the rest of the problem
Real app_time_offset = _apps[i]->getGlobalTimeOffset();
if ((ex->getTime() + app_time_offset) + 2e-14 >= target_time) // Maybe this MultiApp was already solved
continue;
if (_sub_cycling)
{
Real time_old = ex->getTime() + app_time_offset;
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close();
// Save off the current auxiliary solution
transfer_old = solution;
transfer_old.close();
// Snag all of the local dof indices for all of these variables
AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
ConstElemRange & elem_range = *problem->mesh().getActiveLocalElementRange();
Threads::parallel_reduce(elem_range, aldit);
_transferred_dofs = aldit._all_dof_indices;
}
/// \todo{remove ex->allowOutput()}
if (_output_sub_cycles)
{
ex->allowOutput(true);
output_warehouse.allowOutput(true);
}
else
{
ex->allowOutput(false);
output_warehouse.allowOutput(false);
}
ex->setTargetTime(target_time-app_time_offset);
// unsigned int failures = 0;
bool at_steady = false;
// Now do all of the solves we need
while(true)
{
if (_first != true)
ex->incrementStepOrReject();
_first = false;
if (!(!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time))
break;
ex->computeDT();
if (_interpolate_transfers)
{
// See what time this executioner is going to go to.
Real future_time = ex->getTime() + app_time_offset + ex->getDT();
// How far along we are towards the target time:
Real step_percent = (future_time - time_old) / (target_time - time_old);
Real one_minus_step_percent = 1.0 - step_percent;
// Do the interpolation for each variable that was transferred to
FEProblem * problem = appProblem(_first_local_app + i);
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close(); // Just to be sure
transfer.close();
transfer_old.close();
std::set<dof_id_type>::iterator it = _transferred_dofs.begin();
std::set<dof_id_type>::iterator end = _transferred_dofs.end();
for(; it != end; ++it)
{
dof_id_type dof = *it;
solution.set(dof, (transfer_old(dof) * one_minus_step_percent) + (transfer(dof) * step_percent));
// solution.set(dof, transfer_old(dof));
// solution.set(dof, transfer(dof));
// solution.set(dof, 1);
}
solution.close();
}
ex->takeStep();
bool converged = ex->lastSolveConverged();
if (!converged)
{
mooseWarning("While sub_cycling "<<_name<<_first_local_app+i<<" failed to converge!"<<std::endl);
_failures++;
if (_failures > _max_failures)
mooseError("While sub_cycling "<<_name<<_first_local_app+i<<" REALLY failed!"<<std::endl);
}
Real solution_change_norm = ex->getSolutionChangeNorm();
if (_detect_steady_state)
Moose::out << "Solution change norm: " << solution_change_norm << std::endl;
if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
{
Moose::out << "Detected Steady State! Fast-forwarding to " << target_time << std::endl;
at_steady = true;
// Set the time for the problem to the target time we were looking for
ex->setTime(target_time-app_time_offset);
// Force it to output right now \todo{Remove}
ex->forceOutput();
// Indicate that the next output call (occurs in ex->endStep()) should output, regarless of intervals etc...
output_warehouse.forceOutput();
// Clean up the end
ex->endStep();
}
else
ex->endStep();
}
// If we were looking for a steady state, but didn't reach one, we still need to output one more time
if (!at_steady)
{
output_warehouse.forceOutput();
output_warehouse.outputStep();
ex->forceOutput(); // \todo{Remove}
}
}
else if (_tolerate_failure)
{
ex->takeStep(dt);
ex->setTime(target_time-app_time_offset);
ex->forceOutput(); // \todo{Remove}
output_warehouse.forceOutput();
ex->endStep();
}
else
{
Moose::out << "Solving Normal Step!" << std::endl;
if (_first != true)
ex->incrementStepOrReject();
output_warehouse.allowOutput(true);
ex->takeStep(dt);
ex->endStep();
if (!ex->lastSolveConverged())
{
mooseWarning(_name << _first_local_app+i << " failed to converge!" << std::endl);
if (_catch_up)
{
Moose::out << "Starting Catch Up!" << std::endl;
bool caught_up = false;
unsigned int catch_up_step = 0;
Real catch_up_dt = dt/2;
ex->allowOutput(false); // Don't output while catching up \todo{Remove}
// output_warehouse.allowOutput(false);
while(!caught_up && catch_up_step < _max_catch_up_steps)
{
Moose::err << "Solving " << _name << "catch up step " << catch_up_step << std::endl;
ex->incrementStepOrReject();
ex->computeDT();
ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
if (ex->lastSolveConverged())
{
if (ex->getTime() + app_time_offset + ex->timestepTol()*std::abs(ex->getTime()) >= target_time)
{
ex->forceOutput(); // This is here so that it is called before endStep() // \todo{Remove}
output_warehouse.forceOutput();
output_warehouse.outputStep();
caught_up = true;
}
}
else
catch_up_dt /= 2.0;
//output_warehouse.forceOutput();
ex->endStep(); // This is here so it is called after forceOutput()
catch_up_step++;
}
if (!caught_up)
mooseError(_name << " Failed to catch up!\n");
output_warehouse.allowOutput(true);
ex->allowOutput(true); // \todo{Remove}
}
}
}
}
_first = false;
// Swap back
Moose::swapLibMeshComm(swapped);
_transferred_vars.clear();
Moose::out << "Finished Solving MultiApp " << _name << std::endl;
}