MeshTools::BoundingBox
MultiApp::getBoundingBox(unsigned int app)
{
if (!_has_an_app)
mooseError("No app for " << name() << " on processor " << _orig_rank);
FEProblem * problem = appProblem(app);
MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);
MooseMesh & mesh = problem->mesh();
MeshTools::BoundingBox bbox = MeshTools::bounding_box(mesh);
Moose::swapLibMeshComm(swapped);
Point min = bbox.min();
Point max = bbox.max();
Point inflation_amount = (max-min)*_inflation;
Point inflated_min = min - inflation_amount;
Point inflated_max = max + inflation_amount;
// This is where the app is located. We need to shift by this amount.
Point p = position(app);
Point shifted_min = inflated_min;
Point shifted_max = inflated_max;
// If the problem is RZ then we're going to invent a box that would cover the whole "3D" app
// FIXME: Assuming all subdomains are the same coordinate system type!
if (problem->getCoordSystem(*(problem->mesh().meshSubdomains().begin())) == Moose::COORD_RZ)
{
shifted_min(0) = -inflated_max(0);
shifted_min(1) = inflated_min(1);
shifted_min(2) = -inflated_max(0);
shifted_max(0) = inflated_max(0);
shifted_max(1) = inflated_max(1);
shifted_max(2) = inflated_max(0);
}
// Shift them to the position they're supposed to be
shifted_min += p;
shifted_max += p;
return MeshTools::BoundingBox(shifted_min, shifted_max);
}
ComputeJacobianBlockThread::ComputeJacobianBlockThread(FEProblem & fe_problem, libMesh::System & precond_system, SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar) :
_fe_problem(fe_problem),
_precond_system(precond_system),
_nl(_fe_problem.getNonlinearSystem()),
_mesh(fe_problem.mesh()),
_jacobian(jacobian),
_ivar(ivar),
_jvar(jvar)
{
}
FunctionPeriodicBoundary::FunctionPeriodicBoundary(FEProblem & feproblem, std::vector<std::string> fn_names) :
_dim(fn_names.size()),
_tr_x(&feproblem.getFunction(fn_names[0])),
_tr_y(fn_names.size() > 1 ? &feproblem.getFunction(fn_names[1]) : NULL),
_tr_z(fn_names.size() > 2 ? &feproblem.getFunction(fn_names[2]) : NULL)
{
// Make certain the the dimensions agree
if (_dim != feproblem.mesh().dimension())
mooseError("Transform function has to have the same dimension as the problem being solved.");
// Initialize the functions (i.e., call thier initialSetup methods)
init();
}
std::string
outputMeshInformation(FEProblem & problem, bool verbose)
{
std::stringstream oss;
oss << std::left;
MooseMesh & moose_mesh = problem.mesh();
MeshBase & mesh = moose_mesh.getMesh();
if (verbose)
{
oss << "Mesh: " << '\n'
<< std::setw(console_field_width) << " Distribution: " << (moose_mesh.isParallelMesh() ? "parallel" : "serial")
<< (moose_mesh.isDistributionForced() ? " (forced) " : "") << '\n'
<< std::setw(console_field_width) << " Mesh Dimension: " << mesh.mesh_dimension() << '\n'
<< std::setw(console_field_width) << " Spatial Dimension: " << mesh.spatial_dimension() << '\n';
}
oss << std::setw(console_field_width) << " Nodes:" << '\n'
<< std::setw(console_field_width) << " Total:" << mesh.n_nodes() << '\n'
<< std::setw(console_field_width) << " Local:" << mesh.n_local_nodes() << '\n'
<< std::setw(console_field_width) << " Elems:" << '\n'
<< std::setw(console_field_width) << " Total:" << mesh.n_elem() << '\n'
<< std::setw(console_field_width) << " Local:" << mesh.n_local_elem() << '\n';
if (verbose)
{
oss << std::setw(console_field_width) << " Num Subdomains: " << static_cast<std::size_t>(mesh.n_subdomains()) << '\n'
<< std::setw(console_field_width) << " Num Partitions: " << static_cast<std::size_t>(mesh.n_partitions()) << '\n';
if (problem.n_processors() > 1 && moose_mesh.partitionerName() != "")
oss << std::setw(console_field_width) << " Partitioner: " << moose_mesh.partitionerName()
<< (moose_mesh.isPartitionerForced() ? " (forced) " : "")
<< '\n';
}
oss << '\n';
return oss.str();
}
void
TransientMultiApp::solveStep(Real dt, Real target_time, bool auto_advance)
{
if (_sub_cycling && !auto_advance)
mooseError("TransientMultiApp with sub_cycling=true is not compatible with auto_advance=false");
if (_catch_up && !auto_advance)
mooseError("TransientMultiApp with catch_up=true is not compatible with auto_advance=false");
if (!_has_an_app)
return;
_auto_advance = auto_advance;
Moose::out << "Solving MultiApp " << _name << std::endl;
// "target_time" must always be in global time
target_time += _app.getGlobalTimeOffset();
MPI_Comm swapped = Moose::swapLibMeshComm(_my_comm);
int rank;
int ierr;
ierr = MPI_Comm_rank(_orig_comm, &rank); mooseCheckMPIErr(ierr);
for (unsigned int i=0; i<_my_num_apps; i++)
{
FEProblem * problem = appProblem(_first_local_app + i);
OutputWarehouse & output_warehouse = _apps[i]->getOutputWarehouse();
Transient * ex = _transient_executioners[i];
// The App might have a different local time from the rest of the problem
Real app_time_offset = _apps[i]->getGlobalTimeOffset();
if ((ex->getTime() + app_time_offset) + 2e-14 >= target_time) // Maybe this MultiApp was already solved
continue;
if (_sub_cycling)
{
Real time_old = ex->getTime() + app_time_offset;
if (_interpolate_transfers)
{
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close();
// Save off the current auxiliary solution
transfer_old = solution;
transfer_old.close();
// Snag all of the local dof indices for all of these variables
AllLocalDofIndicesThread aldit(libmesh_aux_system, _transferred_vars);
ConstElemRange & elem_range = *problem->mesh().getActiveLocalElementRange();
Threads::parallel_reduce(elem_range, aldit);
_transferred_dofs = aldit._all_dof_indices;
}
if (_output_sub_cycles)
output_warehouse.allowOutput(true);
else
output_warehouse.allowOutput(false);
ex->setTargetTime(target_time-app_time_offset);
// unsigned int failures = 0;
bool at_steady = false;
// Now do all of the solves we need
while(true)
{
if (_first != true)
ex->incrementStepOrReject();
_first = false;
if (!(!at_steady && ex->getTime() + app_time_offset + 2e-14 < target_time))
break;
ex->computeDT();
if (_interpolate_transfers)
{
// See what time this executioner is going to go to.
Real future_time = ex->getTime() + app_time_offset + ex->getDT();
// How far along we are towards the target time:
Real step_percent = (future_time - time_old) / (target_time - time_old);
Real one_minus_step_percent = 1.0 - step_percent;
// Do the interpolation for each variable that was transferred to
FEProblem * problem = appProblem(_first_local_app + i);
AuxiliarySystem & aux_system = problem->getAuxiliarySystem();
System & libmesh_aux_system = aux_system.system();
NumericVector<Number> & solution = *libmesh_aux_system.solution;
NumericVector<Number> & transfer = libmesh_aux_system.get_vector("transfer");
NumericVector<Number> & transfer_old = libmesh_aux_system.get_vector("transfer_old");
solution.close(); // Just to be sure
transfer.close();
transfer_old.close();
std::set<dof_id_type>::iterator it = _transferred_dofs.begin();
std::set<dof_id_type>::iterator end = _transferred_dofs.end();
for(; it != end; ++it)
{
dof_id_type dof = *it;
solution.set(dof, (transfer_old(dof) * one_minus_step_percent) + (transfer(dof) * step_percent));
// solution.set(dof, transfer_old(dof));
// solution.set(dof, transfer(dof));
// solution.set(dof, 1);
}
solution.close();
}
ex->takeStep();
bool converged = ex->lastSolveConverged();
if (!converged)
{
mooseWarning("While sub_cycling "<<_name<<_first_local_app+i<<" failed to converge!"<<std::endl);
_failures++;
if (_failures > _max_failures)
mooseError("While sub_cycling "<<_name<<_first_local_app+i<<" REALLY failed!"<<std::endl);
}
Real solution_change_norm = ex->getSolutionChangeNorm();
if (_detect_steady_state)
Moose::out << "Solution change norm: " << solution_change_norm << std::endl;
if (converged && _detect_steady_state && solution_change_norm < _steady_state_tol)
{
Moose::out << "Detected Steady State! Fast-forwarding to " << target_time << std::endl;
at_steady = true;
// Indicate that the next output call (occurs in ex->endStep()) should output, regarless of intervals etc...
output_warehouse.forceOutput();
// Clean up the end
ex->endStep(target_time-app_time_offset);
}
else
ex->endStep();
}
// If we were looking for a steady state, but didn't reach one, we still need to output one more time
if (!at_steady)
{
output_warehouse.forceOutput();
output_warehouse.outputStep();
}
}
else if (_tolerate_failure)
{
ex->takeStep(dt);
output_warehouse.forceOutput();
ex->endStep(target_time-app_time_offset);
}
else
{
Moose::out << "Solving Normal Step!" << std::endl;
if (auto_advance)
if (_first != true)
ex->incrementStepOrReject();
if (auto_advance)
output_warehouse.allowOutput(true);
ex->takeStep(dt);
if (auto_advance)
{
ex->endStep();
if (!ex->lastSolveConverged())
{
mooseWarning(_name << _first_local_app+i << " failed to converge!" << std::endl);
if (_catch_up)
{
Moose::out << "Starting Catch Up!" << std::endl;
bool caught_up = false;
unsigned int catch_up_step = 0;
Real catch_up_dt = dt/2;
while(!caught_up && catch_up_step < _max_catch_up_steps)
{
Moose::err << "Solving " << _name << "catch up step " << catch_up_step << std::endl;
ex->incrementStepOrReject();
ex->computeDT();
ex->takeStep(catch_up_dt); // Cut the timestep in half to try two half-step solves
if (ex->lastSolveConverged())
{
if (ex->getTime() + app_time_offset + ex->timestepTol()*std::abs(ex->getTime()) >= target_time)
{
output_warehouse.forceOutput();
output_warehouse.outputStep();
caught_up = true;
}
}
else
catch_up_dt /= 2.0;
ex->endStep();
catch_up_step++;
}
if (!caught_up)
mooseError(_name << " Failed to catch up!\n");
output_warehouse.allowOutput(true);
}
}
}
}
}
_first = false;
// Swap back
Moose::swapLibMeshComm(swapped);
_transferred_vars.clear();
Moose::out << "Finished Solving MultiApp " << _name << std::endl;
}
void
storePetscOptions(FEProblem & fe_problem, const InputParameters & params)
{
// Note: Options set in the Preconditioner block will override those set in the Executioner block
if (params.isParamValid("solve_type"))
{
// Extract the solve type
const std::string & solve_type = params.get<MooseEnum>("solve_type");
fe_problem.solverParams()._type = Moose::stringToEnum<Moose::SolveType>(solve_type);
}
if (params.isParamValid("line_search"))
{
MooseEnum line_search = params.get<MooseEnum>("line_search");
if (fe_problem.solverParams()._line_search == Moose::LS_INVALID || line_search != "default")
fe_problem.solverParams()._line_search = Moose::stringToEnum<Moose::LineSearchType>(line_search);
}
// The parameters contained in the Action
const MultiMooseEnum & petsc_options = params.get<MultiMooseEnum>("petsc_options");
const MultiMooseEnum & petsc_options_inames = params.get<MultiMooseEnum>("petsc_options_iname");
const std::vector<std::string> & petsc_options_values = params.get<std::vector<std::string> >("petsc_options_value");
// A reference to the PetscOptions object that contains the settings that will be used in the solve
Moose::PetscSupport::PetscOptions & po = fe_problem.getPetscOptions();
// Update the PETSc single flags
for (const auto & option : petsc_options)
{
/**
* "-log_summary" cannot be used in the input file. This option needs to be set when PETSc is initialized
* which happens before the parser is even created. We'll throw an error if somebody attempts to add this option later.
*/
if (option == "-log_summary")
mooseError("The PETSc option \"-log_summary\" can only be used on the command line. Please remove it from the input file");
// Warn about superseded PETSc options (Note: -snes is not a REAL option, but people used it in their input files)
else
{
std::string help_string;
if (option == "-snes" || option == "-snes_mf" || option == "-snes_mf_operator")
help_string = "Please set the solver type through \"solve_type\".";
else if (option == "-ksp_monitor")
help_string = "Please use \"Outputs/print_linear_residuals=true\"";
if (help_string != "")
mooseWarning("The PETSc option " << option << " should not be used directly in a MOOSE input file. " << help_string);
}
// Update the stored items, but do not create duplicates
if (find(po.flags.begin(), po.flags.end(), option) == po.flags.end())
po.flags.push_back(option);
}
// Check that the name value pairs are sized correctly
if (petsc_options_inames.size() != petsc_options_values.size())
mooseError("PETSc names and options are not the same length");
// Setup the name value pairs
bool boomeramg_found = false;
bool strong_threshold_found = false;
std::string pc_description = "";
for (unsigned int i = 0; i < petsc_options_inames.size(); i++)
{
// Do not add duplicate settings
if (find(po.inames.begin(), po.inames.end(), petsc_options_inames[i]) == po.inames.end())
{
po.inames.push_back(petsc_options_inames[i]);
po.values.push_back(petsc_options_values[i]);
// Look for a pc description
if (petsc_options_inames[i] == "-pc_type" || petsc_options_inames[i] == "-pc_sub_type" || petsc_options_inames[i] == "-pc_hypre_type")
pc_description += petsc_options_values[i] + ' ';
// This special case is common enough that we'd like to handle it for the user.
if (petsc_options_inames[i] == "-pc_hypre_type" && petsc_options_values[i] == "boomeramg")
boomeramg_found = true;
if (petsc_options_inames[i] == "-pc_hypre_boomeramg_strong_threshold")
strong_threshold_found = true;
}
else
{
for (unsigned int j = 0; j < po.inames.size(); j++)
if (po.inames[j] == petsc_options_inames[i])
po.values[j] = petsc_options_values[i];
}
}
// When running a 3D mesh with boomeramg, it is almost always best to supply a strong threshold value
// We will provide that for the user here if they haven't supplied it themselves.
if (boomeramg_found && !strong_threshold_found && fe_problem.mesh().dimension() == 3)
{
po.inames.push_back("-pc_hypre_boomeramg_strong_threshold");
po.values.push_back("0.7");
pc_description += "strong_threshold: 0.7 (auto)";
}
// Set Preconditioner description
po.pc_description = pc_description;
}
