void
TransientMultiApp::setupApp(unsigned int i, Real /*time*/) // FIXME: Should we be passing time?
{
MooseApp * app = _apps[i];
Transient * ex = dynamic_cast<Transient *>(app->getExecutioner());
if (!ex)
mooseError("MultiApp " << _name << " is not using a Transient Executioner!");
// Get the FEProblem and OutputWarehouse for the current MultiApp
FEProblem * problem = appProblem(_first_local_app + i);
OutputWarehouse & output_warehouse = app->getOutputWarehouse();
// Update the file numbers for the outputs from the parent application
output_warehouse.setFileNumbers(_app.getOutputFileNumbers());
// Call initialization method of Executioner (Note, this preforms the output of the initial time step, if desired)
ex->init();
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
// We'll store a copy of the auxiliary system's solution at the old time in here
libmesh_aux_system.add_vector("transfer_old", false);
// This will be where we'll transfer the value to for the "target" time
libmesh_aux_system.add_vector("transfer", false);
}
ex->preExecute();
problem->advanceState();
_transient_executioners[i] = ex;
}
void
MultiAppProjectionTransfer::projectSolution(FEProblem & to_problem, unsigned int app)
{
EquationSystems & proj_es = to_problem.es();
LinearImplicitSystem & ls = *_proj_sys[app];
// activate the current transfer
proj_es.parameters.set<MultiAppProjectionTransfer *>("transfer") = this;
proj_es.parameters.set<unsigned int>("app") = app;
// TODO: specify solver params in an input file
// solver tolerance
Real tol = proj_es.parameters.get<Real>("linear solver tolerance");
proj_es.parameters.set<Real>("linear solver tolerance") = 1e-10; // set our tolerance
// solve it
ls.solve();
proj_es.parameters.set<Real>("linear solver tolerance") = tol; // restore the original tolerance
// copy projected solution into target es
MeshBase & to_mesh = proj_es.get_mesh();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
System & to_sys = to_var.sys().system();
NumericVector<Number> * to_solution = to_sys.solution.get();
{
MeshBase::const_node_iterator it = to_mesh.local_nodes_begin();
const MeshBase::const_node_iterator end_it = to_mesh.local_nodes_end();
for ( ; it != end_it; ++it)
{
const Node * node = *it;
if (node->n_comp(to_sys.number(), to_var.number()) > 0)
{
const dof_id_type proj_index = node->dof_number(ls.number(), _proj_var_num, 0);
const dof_id_type to_index = node->dof_number(to_sys.number(), to_var.number(), 0);
to_solution->set(to_index, (*ls.solution)(proj_index));
}
}
}
{
MeshBase::const_element_iterator it = to_mesh.active_local_elements_begin();
const MeshBase::const_element_iterator end_it = to_mesh.active_local_elements_end();
for ( ; it != end_it; ++it)
{
const Elem * elem = *it;
if (elem->n_comp(to_sys.number(), to_var.number()) > 0)
{
const dof_id_type proj_index = elem->dof_number(ls.number(), _proj_var_num, 0);
const dof_id_type to_index = elem->dof_number(to_sys.number(), to_var.number(), 0);
to_solution->set(to_index, (*ls.solution)(proj_index));
}
}
}
to_solution->close();
to_sys.update();
}
void
Executioner::addAttributeReporter(const std::string & name, Real & attribute, const std::string execute_on)
{
FEProblem * problem = parameters().getCheckedPointerParam<FEProblem *>("_fe_problem", "Failed to retrieve FEProblem when adding a attribute reporter in Executioner");
InputParameters params = _app.getFactory().getValidParams("ExecutionerAttributeReporter");
params.set<Real *>("value") = &attribute;
if (!execute_on.empty())
params.set<MultiMooseEnum>("execute_on") = execute_on;
problem->addPostprocessor("ExecutionerAttributeReporter", name, params);
}
RandomInterface::RandomInterface(const std::string & name, InputParameters & parameters,
FEProblem & problem, THREAD_ID tid, bool is_nodal) :
_random_data(NULL),
_generator(NULL),
_ri_problem(problem),
_ri_name(name),
_master_seed(parameters.get<unsigned int>("seed")),
_is_nodal(is_nodal),
_reset_on(EXEC_RESIDUAL),
_curr_node(problem.assembly(tid).node()),
_curr_element(problem.assembly(tid).elem())
{
}
FunctionPeriodicBoundary::FunctionPeriodicBoundary(FEProblem & feproblem, std::vector<std::string> fn_names) :
_dim(fn_names.size()),
_tr_x(&feproblem.getFunction(fn_names[0])),
_tr_y(fn_names.size() > 1 ? &feproblem.getFunction(fn_names[1]) : NULL),
_tr_z(fn_names.size() > 2 ? &feproblem.getFunction(fn_names[2]) : NULL)
{
// Make certain the the dimensions agree
if (_dim != feproblem.mesh().dimension())
mooseError("Transform function has to have the same dimension as the problem being solved.");
// Initialize the functions (i.e., call thier initialSetup methods)
init();
}
void
TransientMultiApp::setupApp(unsigned int i, Real /*time*/, bool output_initial) // FIXME: Should we be passing time?
{
MooseApp * app = _apps[i];
Transient * ex = dynamic_cast<Transient *>(app->getExecutioner());
if (!ex)
mooseError("MultiApp " << _name << " is not using a Transient Executioner!");
// Get the FEProblem and OutputWarehouse for the current MultiApp
FEProblem * problem = appProblem(_first_local_app + i);
OutputWarehouse & output_warehouse = _apps[i]->getOutputWarehouse();
if (!output_initial)
{
ex->outputInitial(false);//\todo{Remove; handled within ex->init()}
output_warehouse.allowOutput(false);
}
// Set the file numbers of the i-th app to that of the parent app
output_warehouse.setFileNumbers(app->getOutputFileNumbers());
// Call initialization method of Executioner (Note, this preforms the output of the initial time step, if desired)
ex->init();
// Enable output after setup
output_warehouse.allowOutput(true);
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
// We'll store a copy of the auxiliary system's solution at the old time in here
libmesh_aux_system.add_vector("transfer_old", false);
// This will be where we'll transfer the value to for the "target" time
libmesh_aux_system.add_vector("transfer", false);
}
ex->preExecute();
problem->copyOldSolutions();
_transient_executioners[i] = ex;
if (_detect_steady_state || _tolerate_failure)
{
_apps[i]->getOutputWarehouse().allowOutput(false);
ex->allowOutput(false);
}
}
MeshTools::BoundingBox
MultiApp::getBoundingBox(unsigned int app)
{
if (!_has_an_app)
mooseError("No app for " << name() << " on processor " << _orig_rank);
FEProblem * problem = appProblem(app);
MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);
MooseMesh & mesh = problem->mesh();
MeshTools::BoundingBox bbox = MeshTools::bounding_box(mesh);
Moose::swapLibMeshComm(swapped);
Point min = bbox.min();
Point max = bbox.max();
Point inflation_amount = (max-min)*_inflation;
Point inflated_min = min - inflation_amount;
Point inflated_max = max + inflation_amount;
// This is where the app is located. We need to shift by this amount.
Point p = position(app);
Point shifted_min = inflated_min;
Point shifted_max = inflated_max;
// If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
// FIXME: Assuming all subdomains are the same coordinate system type!
if (problem->getCoordSystem(*(problem->mesh().meshSubdomains().begin())) == Moose::COORD_RZ)
{
shifted_min(0) = -inflated_max(0);
shifted_min(1) = inflated_min(1);
shifted_min(2) = -inflated_max(0);
shifted_max(0) = inflated_max(0);
shifted_max(1) = inflated_max(1);
shifted_max(2) = inflated_max(0);
}
// Shift them to the position they're supposed to be
shifted_min += p;
shifted_max += p;
return MeshTools::BoundingBox(shifted_min, shifted_max);
}
ComputeElemDampingThread::ComputeElemDampingThread(FEProblem & feproblem) :
ThreadedElementLoop<ConstElemRange>(feproblem),
_damping(1.0),
_nl(feproblem.getNonlinearSystem()),
_element_dampers(_nl.getElementDamperWarehouse())
{
}
ComputeNodalDampingThread::ComputeNodalDampingThread(FEProblem & feproblem) :
ThreadedNodeLoop<ConstNodeRange, ConstNodeRange::const_iterator>(feproblem),
_damping(1.0),
_nl(feproblem.getNonlinearSystem()),
_nodal_dampers(_nl.getNodalDamperWarehouse())
{
}
std::string
outputExecutionInformation(MooseApp & app, FEProblem & problem)
{
std::stringstream oss;
oss << std::left;
Executioner * exec = app.getExecutioner();
oss << "Execution Information:\n"
<< std::setw(console_field_width) << " Executioner: " << demangle(typeid(*exec).name()) << '\n';
std::string time_stepper = exec->getTimeStepperName();
if (time_stepper != "")
oss << std::setw(console_field_width) << " TimeStepper: " << time_stepper << '\n';
oss << std::setw(console_field_width) << " Solver Mode: " << Moose::stringify<Moose::SolveType>(problem.solverParams()._type) << '\n';
const std::string & pc_desc = problem.getPetscOptions().pc_description;
if (!pc_desc.empty())
oss << std::setw(console_field_width) << " Preconditioner: " << pc_desc << '\n';
oss << '\n';
return oss.str();
}
std::string outputLegacyInformation(MooseApp & /*app*/, FEProblem & problem)
{
std::stringstream oss;
oss << std::left;
if (problem.legacyUoAuxComputation() || problem.legacyUoInitialization())
{
oss << COLOR_RED << "LEGACY MODES ENABLED:" << COLOR_DEFAULT << '\n';
if (problem.legacyUoAuxComputation())
oss << COLOR_RED << " Computing EXEC_LINEAR AuxKernel types when any UserObject type is executed." << COLOR_DEFAULT << '\n';
if (problem.legacyUoInitialization())
oss << COLOR_RED << " Computing all UserObjects during initial setup." << COLOR_DEFAULT << '\n';
}
return oss.str();
}
ComputeNodalKernelBcsThread::ComputeNodalKernelBcsThread(FEProblem & fe_problem,
const MooseObjectWarehouse<NodalKernel> & nodal_kernels) :
ThreadedNodeLoop<ConstBndNodeRange, ConstBndNodeRange::const_iterator>(fe_problem),
_aux_sys(fe_problem.getAuxiliarySystem()),
_nodal_kernels(nodal_kernels),
_num_cached(0)
{
}
ComputeNodalKernelJacobiansThread::ComputeNodalKernelJacobiansThread(FEProblem & fe_problem,
const MooseObjectWarehouse<NodalKernel> & nodal_kernels,
SparseMatrix<Number> & jacobian) :
ThreadedNodeLoop<ConstNodeRange, ConstNodeRange::const_iterator>(fe_problem),
_aux_sys(fe_problem.getAuxiliarySystem()),
_nodal_kernels(nodal_kernels),
_jacobian(jacobian),
_num_cached(0)
{
}
ComputeJacobianBlockThread::ComputeJacobianBlockThread(FEProblem & fe_problem, libMesh::System & precond_system, SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar) :
_fe_problem(fe_problem),
_precond_system(precond_system),
_nl(_fe_problem.getNonlinearSystem()),
_mesh(fe_problem.mesh()),
_jacobian(jacobian),
_ivar(ivar),
_jvar(jvar)
{
}
void
petscSetOptions(FEProblem & problem)
{
// Reference to the options stored in FEPRoblem
PetscOptions & petsc = problem.getPetscOptions();
if (petsc.inames.size() != petsc.values.size())
mooseError("PETSc names and options are not the same length");
#if PETSC_VERSION_LESS_THAN(3,7,0)
PetscOptionsClear();
#else
PetscOptionsClear(PETSC_NULL);
#endif
setSolverOptions(problem.solverParams());
// Add any additional options specified in the input file
for (const auto & flag : petsc.flags)
setSinglePetscOption(flag.c_str());
for (unsigned int i=0; i<petsc.inames.size(); ++i)
setSinglePetscOption(petsc.inames[i], petsc.values[i]);
// set up DM which is required if use a field split preconditioner
if (problem.getNonlinearSystem().haveFieldSplitPreconditioner())
petscSetupDM(problem.getNonlinearSystem());
// commandline options always win
// the options from a user commandline will overwrite the existing ones if any conflicts
{ // Get any options specified on the command-line
int argc;
char ** args;
PetscGetArgs(&argc, &args);
#if PETSC_VERSION_LESS_THAN(3,7,0)
PetscOptionsInsert(&argc, &args, NULL);
#else
PetscOptionsInsert(PETSC_NULL, &argc, &args, NULL);
#endif
}
}
std::string
outputMeshInformation(FEProblem & problem, bool verbose)
{
std::stringstream oss;
oss << std::left;
MooseMesh & moose_mesh = problem.mesh();
MeshBase & mesh = moose_mesh.getMesh();
if (verbose)
{
oss << "Mesh: " << '\n'
<< std::setw(console_field_width) << " Distribution: " << (moose_mesh.isParallelMesh() ? "parallel" : "serial")
<< (moose_mesh.isDistributionForced() ? " (forced) " : "") << '\n'
<< std::setw(console_field_width) << " Mesh Dimension: " << mesh.mesh_dimension() << '\n'
<< std::setw(console_field_width) << " Spatial Dimension: " << mesh.spatial_dimension() << '\n';
}
oss << std::setw(console_field_width) << " Nodes:" << '\n'
<< std::setw(console_field_width) << " Total:" << mesh.n_nodes() << '\n'
<< std::setw(console_field_width) << " Local:" << mesh.n_local_nodes() << '\n'
<< std::setw(console_field_width) << " Elems:" << '\n'
<< std::setw(console_field_width) << " Total:" << mesh.n_elem() << '\n'
<< std::setw(console_field_width) << " Local:" << mesh.n_local_elem() << '\n';
if (verbose)
{
oss << std::setw(console_field_width) << " Num Subdomains: " << static_cast<std::size_t>(mesh.n_subdomains()) << '\n'
<< std::setw(console_field_width) << " Num Partitions: " << static_cast<std::size_t>(mesh.n_partitions()) << '\n';
if (problem.n_processors() > 1 && moose_mesh.partitionerName() != "")
oss << std::setw(console_field_width) << " Partitioner: " << moose_mesh.partitionerName()
<< (moose_mesh.isPartitionerForced() ? " (forced) " : "")
<< '\n';
}
oss << '\n';
return oss.str();
}
void
petscSetOptions(FEProblem & problem)
{
// Reference to the options stored in FEPRoblem
PetscOptions & petsc = problem.getPetscOptions();
if (petsc.inames.size() != petsc.values.size())
mooseError("PETSc names and options are not the same length");
PetscOptionsClear();
{ // Get any options specified on the command-line
int argc;
char ** args;
PetscGetArgs(&argc, &args);
PetscOptionsInsert(&argc, &args, NULL);
}
setSolverOptions(problem.solverParams());
// Add any additional options specified in the input file
for (MooseEnumIterator it = petsc.flags.begin(); it != petsc.flags.end(); ++it)
PetscOptionsSetValue(it->c_str(), PETSC_NULL);
for (unsigned int i=0; i<petsc.inames.size(); ++i)
PetscOptionsSetValue(petsc.inames[i].c_str(), petsc.values[i].c_str());
SolverParams& solver_params = problem.solverParams();
if (solver_params._type != Moose::ST_JFNK &&
solver_params._type != Moose::ST_FD &&
!problem.getNonlinearSystem().haveFiniteDifferencedPreconditioner() &&
problem.getNonlinearSystem().haveDecomposition())
{
// Set up DM only if have a decomposition. Additionally, turn DM OFF if not using FD-based solvers,
// (both -snes_mf and -snes_fd) and FDP. This is all rather crufty, but what's a good generic rule here?
// In principle at least, splits should be able to work with ST_FD (-snes_fd) and FDP (a coloring-based
// version of -snes_fd), but one has to be careful about the initialization order so as not to override
// SNESComputeJacobianDefaultColor() set up by FDP, for instance. However, it's unlikely that splits
// will be used when running an FD solver (debugging).
problem.getNonlinearSystem().setupDecomposition();
petscSetupDM(problem.getNonlinearSystem());
} else {
// Otherwise turn off the decomposition
std::vector<std::string> nosplits;
problem.getNonlinearSystem().setDecomposition(nosplits);
}
}
void
CreateExecutionerAction::storeCommonExecutionerParams(FEProblem & fe_problem, InputParameters & params)
{
// Note: Options set in the Preconditioner block will override those set in the Executioner block
if (params.isParamValid("solve_type"))
{
// Extract the solve type
const std::string & solve_type = params.get<MooseEnum>("solve_type");
fe_problem.solverParams()._type = Moose::stringToEnum<Moose::SolveType>(solve_type);
}
MooseEnum line_search = params.get<MooseEnum>("line_search");
if (fe_problem.solverParams()._line_search == Moose::LS_INVALID || line_search != "default")
fe_problem.solverParams()._line_search = Moose::stringToEnum<Moose::LineSearchType>(line_search);
#ifdef LIBMESH_HAVE_PETSC
MultiMooseEnum petsc_options = params.get<MultiMooseEnum>("petsc_options");
std::vector<std::string> petsc_options_iname = params.get<std::vector<std::string> >("petsc_options_iname");
std::vector<std::string> petsc_options_value = params.get<std::vector<std::string> >("petsc_options_value");
fe_problem.storePetscOptions(petsc_options, petsc_options_iname, petsc_options_value);
#endif //LIBMESH_HAVE_PETSC
}
void petscSetupDampers(NonlinearImplicitSystem& sys)
{
FEProblem * problem = sys.get_equation_systems().parameters.get<FEProblem *>("_fe_problem");
NonlinearSystem & nl = problem->getNonlinearSystem();
PetscNonlinearSolver<Number> * petsc_solver = dynamic_cast<PetscNonlinearSolver<Number> *>(nl.sys().nonlinear_solver.get());
SNES snes = petsc_solver->snes();
#if PETSC_VERSION_LESS_THAN(3,3,0)
// PETSc 3.2.x-
SNESLineSearchSetPostCheck(snes, dampedCheck, problem);
#else
// PETSc 3.3.0+
SNESLineSearch linesearch;
#if PETSC_VERSION_LESS_THAN(3,4,0)
PetscErrorCode ierr = SNESGetSNESLineSearch(snes, &linesearch);
#else
PetscErrorCode ierr = SNESGetLineSearch(snes, &linesearch);
#endif
CHKERRABORT(problem->comm().get(),ierr);
ierr = SNESLineSearchSetPostCheck(linesearch, dampedCheck, problem);
CHKERRABORT(problem->comm().get(),ierr);
#endif
}
DisplacedProblem::DisplacedProblem(FEProblem & mproblem, MooseMesh & displaced_mesh, InputParameters params) :
SubProblem(mproblem.name() + "_disp", params),
_mproblem(mproblem),
_mesh(displaced_mesh),
_eq(displaced_mesh),
_ref_mesh(_mproblem.mesh()),
_displacements(params.get<std::vector<std::string> >("displacements")),
_displaced_nl(*this, _mproblem.getNonlinearSystem(), _mproblem.getNonlinearSystem().name() + "_displaced", Moose::VAR_NONLINEAR),
_displaced_aux(*this, _mproblem.getAuxiliarySystem(), _mproblem.getAuxiliarySystem().name() + "_displaced", Moose::VAR_AUXILIARY),
_geometric_search_data(_mproblem, _mesh)
{
unsigned int n_threads = libMesh::n_threads();
_assembly.resize(n_threads);
for (unsigned int i = 0; i < n_threads; ++i)
_assembly[i] = new Assembly(_displaced_nl, _mproblem.couplingMatrix(), i);
}
void
petscSetDefaults(FEProblem & problem)
{
// dig out Petsc solver
NonlinearSystem & nl = problem.getNonlinearSystem();
PetscNonlinearSolver<Number> * petsc_solver = dynamic_cast<PetscNonlinearSolver<Number> *>(nl.sys().nonlinear_solver.get());
SNES snes = petsc_solver->snes();
KSP ksp;
SNESGetKSP(snes, &ksp);
PCSide pcside = getPetscPCSide(nl.getPCSide());
#if PETSC_VERSION_LESS_THAN(3,2,0)
// PETSc 3.1.x-
KSPSetPreconditionerSide(ksp, pcside);
#else
// PETSc 3.2.x+
KSPSetPCSide(ksp, pcside);
#endif
SNESSetMaxLinearSolveFailures(snes, 1000000);
#if PETSC_VERSION_LESS_THAN(3,0,0)
// PETSc 2.3.3-
KSPSetConvergenceTest(ksp, petscConverged, &problem);
SNESSetConvergenceTest(snes, petscNonlinearConverged, &problem);
#else
// PETSc 3.0.0+
// In 3.0.0, the context pointer must actually be used, and the
// final argument to KSPSetConvergenceTest() is a pointer to a
// routine for destroying said private data context. In this case,
// we use the default context provided by PETSc in addition to
// a few other tests.
{
PetscErrorCode ierr = KSPSetConvergenceTest(ksp,
petscConverged,
&problem,
PETSC_NULL);
CHKERRABORT(nl.comm().get(),ierr);
ierr = SNESSetConvergenceTest(snes,
petscNonlinearConverged,
&problem,
PETSC_NULL);
CHKERRABORT(nl.comm().get(),ierr);
}
#endif
}
void
CoupledExecutioner::projectVariables(FEProblem & fep)
{
std::string dest = _fep_mapping[&fep];
if (_var_mapping.find(dest) == _var_mapping.end())
return;
std::vector<ProjInfo *> & proj_list = _var_mapping[dest];
for (std::vector<ProjInfo *>::iterator it = proj_list.begin(); it != proj_list.end(); ++it)
{
ProjInfo * pi = *it;
unsigned int isrc = _name_index[pi->src];
FEProblem * src_fep = _fe_problems[isrc];
MooseVariable & src_mv = src_fep->getVariable(0, pi->src_var);
SystemBase & src_sys = src_mv.sys();
MooseVariable & dest_mv = fep.getVariable(0, pi->dest_var);
SystemBase & dest_sys = dest_mv.sys();
// get dof indices for source variable
unsigned int src_vn = src_sys.system().variable_number(src_mv.name());
std::set<dof_id_type> src_var_indices;
src_sys.system().local_dof_indices(src_vn, src_var_indices);
// get dof indices for destination variable
unsigned int dest_vn = dest_sys.system().variable_number(dest_mv.name());
std::set<dof_id_type> dest_var_indices;
dest_sys.system().local_dof_indices(dest_vn, dest_var_indices);
// NOTE: this is not very robust code. It relies on the same node numbering and that inserting values in the set in the same order will result
// in a std::set with the same ordering, i.e. items in src_var_indices and dest_var_indices correspond to each other.
// copy the values from src solution vector to dest solution vector
std::set<dof_id_type>::iterator src_it = src_var_indices.begin();
std::set<dof_id_type>::iterator src_it_end = src_var_indices.end();
std::set<dof_id_type>::iterator dest_it = dest_var_indices.begin();
for (; src_it != src_it_end; ++src_it, ++dest_it)
dest_sys.solution().set(*dest_it, src_sys.solution()(*src_it));
dest_sys.solution().close();
dest_sys.update();
}
}
Adaptivity::Adaptivity(FEProblem & subproblem) :
ConsoleStreamInterface(subproblem.getMooseApp()),
_subproblem(subproblem),
_mesh(_subproblem.mesh()),
_mesh_refinement_on(false),
_initial_steps(0),
_steps(0),
_print_mesh_changed(false),
_t(_subproblem.time()),
_step(_subproblem.timeStep()),
_interval(1),
_start_time(-std::numeric_limits<Real>::max()),
_stop_time(std::numeric_limits<Real>::max()),
_cycles_per_step(1),
_use_new_system(false),
_max_h_level(0),
_recompute_markers_during_cycles(false)
{
}
Adaptivity::Adaptivity(FEProblem & subproblem) :
ConsoleStreamInterface(subproblem.getMooseApp()),
_subproblem(subproblem),
_mesh(_subproblem.mesh()),
_mesh_refinement_on(false),
_mesh_refinement(NULL),
_error_estimator(NULL),
_error(NULL),
_displaced_problem(_subproblem.getDisplacedProblem()),
_displaced_mesh_refinement(NULL),
_initial_steps(0),
_steps(0),
_print_mesh_changed(false),
_t(_subproblem.time()),
_start_time(-std::numeric_limits<Real>::max()),
_stop_time(std::numeric_limits<Real>::max()),
_cycles_per_step(1),
_use_new_system(false),
_max_h_level(0)
{
}
std::string
outputAuxiliarySystemInformation(FEProblem & problem)
{
return outputSystemInformationHelper(problem.getAuxiliarySystem().system());
}
std::string
outputNonlinearSystemInformation(FEProblem & problem)
{
return outputSystemInformationHelper(problem.getNonlinearSystem().system());
}
void
TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
{
if (_sub_cycling && !auto_advance)
mooseError("TransientMultiApp with sub_cycling=true is not compatible with auto_advance=false");
if (_catch_up && !auto_advance)
mooseError("TransientMultiApp with catch_up=true is not compatible with auto_advance=false");
if (!_has_an_app)
return;
_auto_advance = auto_advance;
Moose::out << "Solving MultiApp " << _name << std::endl;
// "target_time" must always be in global time
target_time += _app.getGlobalTimeOffset();
MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);
int rank;
int ierr;
ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);
for (unsigned int i=0; i<_my_num_apps; i++)
{
FEProblem * problem = appProblem(_first_local_app + i);
OutputWarehouse & output_warehouse = _apps[i]->getOutputWarehouse();
Transient * ex = _transient_executioners[i];
// The App might have a different local time from the rest of the problem
Real app_time_offset = _apps[i]->getGlobalTimeOffset();
if ((ex->getTime() + app_time_offset) + 2e-14 >= target_time) // Maybe this MultiApp was already solved
continue;
if (_sub_cycling)
{
Real time_old = ex->getTime() + app_time_offset;
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close();
// Save off the current auxiliary solution
transfer_old = solution;
transfer_old.close();
// Snag all of the local dof indices for all of these variables
AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
ConstElemRange & elem_range = *problem->mesh().getActiveLocalElementRange();
Threads::parallel_reduce(elem_range, aldit);
_transferred_dofs = aldit._all_dof_indices;
}
if (_output_sub_cycles)
output_warehouse.allowOutput(true);
else
output_warehouse.allowOutput(false);
ex->setTargetTime(target_time-app_time_offset);
// unsigned int failures = 0;
bool at_steady = false;
// Now do all of the solves we need
while(true)
{
if (_first != true)
ex->incrementStepOrReject();
_first = false;
if (!(!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time))
break;
ex->computeDT();
if (_interpolate_transfers)
{
// See what time this executioner is going to go to.
Real future_time = ex->getTime() + app_time_offset + ex->getDT();
// How far along we are towards the target time:
Real step_percent = (future_time - time_old) / (target_time - time_old);
Real one_minus_step_percent = 1.0 - step_percent;
// Do the interpolation for each variable that was transferred to
FEProblem * problem = appProblem(_first_local_app + i);
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close(); // Just to be sure
transfer.close();
transfer_old.close();
std::set<dof_id_type>::iterator it = _transferred_dofs.begin();
std::set<dof_id_type>::iterator end = _transferred_dofs.end();
for(; it != end; ++it)
{
dof_id_type dof = *it;
solution.set(dof, (transfer_old(dof) * one_minus_step_percent) + (transfer(dof) * step_percent));
// solution.set(dof, transfer_old(dof));
// solution.set(dof, transfer(dof));
// solution.set(dof, 1);
}
solution.close();
}
ex->takeStep();
bool converged = ex->lastSolveConverged();
if (!converged)
{
mooseWarning("While sub_cycling "<<_name<<_first_local_app+i<<" failed to converge!"<<std::endl);
_failures++;
if (_failures > _max_failures)
mooseError("While sub_cycling "<<_name<<_first_local_app+i<<" REALLY failed!"<<std::endl);
}
Real solution_change_norm = ex->getSolutionChangeNorm();
if (_detect_steady_state)
Moose::out << "Solution change norm: " << solution_change_norm << std::endl;
if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
{
Moose::out << "Detected Steady State! Fast-forwarding to " << target_time << std::endl;
at_steady = true;
// Indicate that the next output call (occurs in ex->endStep()) should output, regarless of intervals etc...
output_warehouse.forceOutput();
// Clean up the end
ex->endStep(target_time-app_time_offset);
}
else
ex->endStep();
}
// If we were looking for a steady state, but didn't reach one, we still need to output one more time
if (!at_steady)
{
output_warehouse.forceOutput();
output_warehouse.outputStep();
}
}
else if (_tolerate_failure)
{
ex->takeStep(dt);
output_warehouse.forceOutput();
ex->endStep(target_time-app_time_offset);
}
else
{
Moose::out << "Solving Normal Step!" << std::endl;
if (auto_advance)
if (_first != true)
ex->incrementStepOrReject();
if (auto_advance)
output_warehouse.allowOutput(true);
ex->takeStep(dt);
if (auto_advance)
{
ex->endStep();
if (!ex->lastSolveConverged())
{
mooseWarning(_name << _first_local_app+i << " failed to converge!" << std::endl);
if (_catch_up)
{
Moose::out << "Starting Catch Up!" << std::endl;
bool caught_up = false;
unsigned int catch_up_step = 0;
Real catch_up_dt = dt/2;
while(!caught_up && catch_up_step < _max_catch_up_steps)
{
Moose::err << "Solving " << _name << "catch up step " << catch_up_step << std::endl;
ex->incrementStepOrReject();
ex->computeDT();
ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
if (ex->lastSolveConverged())
{
if (ex->getTime() + app_time_offset + ex->timestepTol()*std::abs(ex->getTime()) >= target_time)
{
output_warehouse.forceOutput();
output_warehouse.outputStep();
caught_up = true;
}
}
else
catch_up_dt /= 2.0;
ex->endStep();
catch_up_step++;
}
if (!caught_up)
mooseError(_name << " Failed to catch up!\n");
output_warehouse.allowOutput(true);
}
}
}
}
}
_first = false;
// Swap back
Moose::swapLibMeshComm(swapped);
_transferred_vars.clear();
Moose::out << "Finished Solving MultiApp " << _name << std::endl;
}
void
storePetscOptions(FEProblem & fe_problem, const InputParameters & params)
{
// Note: Options set in the Preconditioner block will override those set in the Executioner block
if (params.isParamValid("solve_type"))
{
// Extract the solve type
const std::string & solve_type = params.get<MooseEnum>("solve_type");
fe_problem.solverParams()._type = Moose::stringToEnum<Moose::SolveType>(solve_type);
}
if (params.isParamValid("line_search"))
{
MooseEnum line_search = params.get<MooseEnum>("line_search");
if (fe_problem.solverParams()._line_search == Moose::LS_INVALID || line_search != "default")
fe_problem.solverParams()._line_search = Moose::stringToEnum<Moose::LineSearchType>(line_search);
}
// The parameters contained in the Action
const MultiMooseEnum & petsc_options = params.get<MultiMooseEnum>("petsc_options");
const MultiMooseEnum & petsc_options_inames = params.get<MultiMooseEnum>("petsc_options_iname");
const std::vector<std::string> & petsc_options_values = params.get<std::vector<std::string> >("petsc_options_value");
// A reference to the PetscOptions object that contains the settings that will be used in the solve
Moose::PetscSupport::PetscOptions & po = fe_problem.getPetscOptions();
// Update the PETSc single flags
for (const auto & option : petsc_options)
{
/**
* "-log_summary" cannot be used in the input file. This option needs to be set when PETSc is initialized
* which happens before the parser is even created. We'll throw an error if somebody attempts to add this option later.
*/
if (option == "-log_summary")
mooseError("The PETSc option \"-log_summary\" can only be used on the command line. Please remove it from the input file");
// Warn about superseded PETSc options (Note: -snes is not a REAL option, but people used it in their input files)
else
{
std::string help_string;
if (option == "-snes" || option == "-snes_mf" || option == "-snes_mf_operator")
help_string = "Please set the solver type through \"solve_type\".";
else if (option == "-ksp_monitor")
help_string = "Please use \"Outputs/print_linear_residuals=true\"";
if (help_string != "")
mooseWarning("The PETSc option " << option << " should not be used directly in a MOOSE input file. " << help_string);
}
// Update the stored items, but do not create duplicates
if (find(po.flags.begin(), po.flags.end(), option) == po.flags.end())
po.flags.push_back(option);
}
// Check that the name value pairs are sized correctly
if (petsc_options_inames.size() != petsc_options_values.size())
mooseError("PETSc names and options are not the same length");
// Setup the name value pairs
bool boomeramg_found = false;
bool strong_threshold_found = false;
std::string pc_description = "";
for (unsigned int i = 0; i < petsc_options_inames.size(); i++)
{
// Do not add duplicate settings
if (find(po.inames.begin(), po.inames.end(), petsc_options_inames[i]) == po.inames.end())
{
po.inames.push_back(petsc_options_inames[i]);
po.values.push_back(petsc_options_values[i]);
// Look for a pc description
if (petsc_options_inames[i] == "-pc_type" || petsc_options_inames[i] == "-pc_sub_type" || petsc_options_inames[i] == "-pc_hypre_type")
pc_description += petsc_options_values[i] + ' ';
// This special case is common enough that we'd like to handle it for the user.
if (petsc_options_inames[i] == "-pc_hypre_type" && petsc_options_values[i] == "boomeramg")
boomeramg_found = true;
if (petsc_options_inames[i] == "-pc_hypre_boomeramg_strong_threshold")
strong_threshold_found = true;
}
else
{
for (unsigned int j = 0; j < po.inames.size(); j++)
if (po.inames[j] == petsc_options_inames[i])
po.values[j] = petsc_options_values[i];
}
}
// When running a 3D mesh with boomeramg, it is almost always best to supply a strong threshold value
// We will provide that for the user here if they haven't supplied it themselves.
if (boomeramg_found && !strong_threshold_found && fe_problem.mesh().dimension() == 3)
{
po.inames.push_back("-pc_hypre_boomeramg_strong_threshold");
po.values.push_back("0.7");
pc_description += "strong_threshold: 0.7 (auto)";
}
// Set Preconditioner description
po.pc_description = pc_description;
}
