void
MoosePreconditioner::copyVarValues(MeshBase & mesh,
const unsigned int from_system,
const unsigned int from_var,
const NumericVector<Number> & from_vector,
const unsigned int to_system,
const unsigned int to_var,
NumericVector<Number> & to_vector)
{
{
MeshBase::node_iterator it = mesh.local_nodes_begin();
MeshBase::node_iterator it_end = mesh.local_nodes_end();
for (; it != it_end; ++it)
{
Node * node = *it;
unsigned int n_comp = node->n_comp(from_system, from_var);
mooseAssert(node->n_comp(from_system, from_var) == node->n_comp(to_system, to_var),
"Number of components does not match in each system");
for (unsigned int i = 0; i < n_comp; i++)
{
dof_id_type from_dof = node->dof_number(from_system, from_var, i);
dof_id_type to_dof = node->dof_number(to_system, to_var, i);
to_vector.set(to_dof, from_vector(from_dof));
}
}
}
{
MeshBase::element_iterator it = mesh.local_elements_begin();
MeshBase::element_iterator it_end = mesh.local_elements_end();
for (; it != it_end; ++it)
{
Elem * elem = *it;
unsigned int n_comp = elem->n_comp(from_system, from_var);
mooseAssert(elem->n_comp(from_system, from_var) == elem->n_comp(to_system, to_var),
"Number of components does not match in each system");
for (unsigned int i = 0; i < n_comp; i++)
{
dof_id_type from_dof = elem->dof_number(from_system, from_var, i);
dof_id_type to_dof = elem->dof_number(to_system, to_var, i);
to_vector.set(to_dof, from_vector(from_dof));
}
}
}
}
void
MultiAppMeshFunctionTransfer::execute()
{
Moose::out << "Beginning MeshFunctionTransfer " << name() << std::endl;
getAppInfo();
/**
* For every combination of global "from" problem and local "to" problem, find
* which "from" bounding boxes overlap with which "to" elements. Keep track
* of which processors own bounding boxes that overlap with which elements.
* Build vectors of node locations/element centroids to send to other
* processors for mesh function evaluations.
*/
// Get the bounding boxes for the "from" domains.
std::vector<MeshTools::BoundingBox> bboxes = getFromBoundingBoxes();
// Figure out how many "from" domains each processor owns.
std::vector<unsigned int> froms_per_proc = getFromsPerProc();
std::vector<std::vector<Point> > outgoing_points(n_processors());
std::vector<std::map<std::pair<unsigned int, unsigned int>, unsigned int> > point_index_map(n_processors());
// point_index_map[i_to, element_id] = index
// outgoing_points[index] is the first quadrature point in element
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
MeshBase * to_mesh = & _to_meshes[i_to]->getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
// Loop over the "froms" on processor i_proc. If the node is found in
// any of the "froms", add that node to the vector that will be sent to
// i_proc.
unsigned int from0 = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool point_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && ! point_found; i_from++)
{
if (bboxes[i_from].contains_point(*node + _to_positions[i_to]))
{
std::pair<unsigned int, unsigned int> key(i_to, node->id());
point_index_map[i_proc][key] = outgoing_points[i_proc].size();
outgoing_points[i_proc].push_back(*node + _to_positions[i_to]);
point_found = true;
}
}
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
// Loop over the "froms" on processor i_proc. If the elem is found in
// any of the "froms", add that elem to the vector that will be sent to
// i_proc.
unsigned int from0 = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool point_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && ! point_found; i_from++)
{
if (bboxes[i_from].contains_point(centroid + _to_positions[i_to]))
{
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
point_index_map[i_proc][key] = outgoing_points[i_proc].size();
outgoing_points[i_proc].push_back(centroid + _to_positions[i_to]);
point_found = true;
}
}
}
}
}
}
/**
* Request point evaluations from other processors and handle requests sent to
* this processor.
*/
// Get the local bounding boxes.
std::vector<MeshTools::BoundingBox> local_bboxes(froms_per_proc[processor_id()]);
{
// Find the index to the first of this processor's local bounding boxes.
unsigned int local_start = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors() && i_proc != processor_id();
i_proc++)
{
local_start += froms_per_proc[i_proc];
}
// Extract the local bounding boxes.
for (unsigned int i_from = 0; i_from < froms_per_proc[processor_id()]; i_from++)
{
local_bboxes[i_from] = bboxes[local_start + i_from];
}
}
// Setup the local mesh functions.
std::vector<MooseSharedPointer<MeshFunction> > local_meshfuns;
for (unsigned int i_from = 0; i_from < _from_problems.size(); i_from++)
{
FEProblem & from_problem = *_from_problems[i_from];
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
System & from_sys = from_var.sys().system();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
MooseSharedPointer<MeshFunction> from_func;
//TODO: make MultiAppTransfer give me the right es
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_func.reset(new MeshFunction(from_problem.getDisplacedProblem()->es(),
*from_sys.current_local_solution, from_sys.get_dof_map(), from_var_num));
else
from_func.reset(new MeshFunction(from_problem.es(),
*from_sys.current_local_solution, from_sys.get_dof_map(), from_var_num));
from_func->init(Trees::ELEMENTS);
from_func->enable_out_of_mesh_mode(OutOfMeshValue);
local_meshfuns.push_back(from_func);
}
// Send points to other processors.
std::vector<std::vector<Real> > incoming_evals(n_processors());
std::vector<std::vector<unsigned int> > incoming_app_ids(n_processors());
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.send(i_proc, outgoing_points[i_proc]);
}
// Recieve points from other processors, evaluate mesh frunctions at those
// points, and send the values back.
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
std::vector<Point> incoming_points;
if (i_proc == processor_id())
incoming_points = outgoing_points[i_proc];
else
_communicator.receive(i_proc, incoming_points);
std::vector<Real> outgoing_evals(incoming_points.size(), OutOfMeshValue);
std::vector<unsigned int> outgoing_ids(incoming_points.size(), -1); // -1 = largest unsigned int
for (unsigned int i_pt = 0; i_pt < incoming_points.size(); i_pt++)
{
Point pt = incoming_points[i_pt];
// Loop until we've found the lowest-ranked app that actually contains
// the quadrature point.
for (unsigned int i_from = 0; i_from < _from_problems.size() && outgoing_evals[i_pt] == OutOfMeshValue; i_from++)
{
if (local_bboxes[i_from].contains_point(pt))
{
outgoing_evals[i_pt] = (* local_meshfuns[i_from])(pt - _from_positions[i_from]);
if (_direction == FROM_MULTIAPP)
outgoing_ids[i_pt] = _local2global_map[i_from];
}
}
}
if (i_proc == processor_id())
{
incoming_evals[i_proc] = outgoing_evals;
if (_direction == FROM_MULTIAPP)
incoming_app_ids[i_proc] = outgoing_ids;
}
else
{
_communicator.send(i_proc, outgoing_evals);
if (_direction == FROM_MULTIAPP)
_communicator.send(i_proc, outgoing_ids);
}
}
/**
* Gather all of the evaluations, pick out the best ones for each point, and
* apply them to the solution vector. When we are transferring from
* multiapps, there may be multiple overlapping apps for a particular point.
* In that case, we'll try to use the value from the app with the lowest id.
*/
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.receive(i_proc, incoming_evals[i_proc]);
if (_direction == FROM_MULTIAPP)
_communicator.receive(i_proc, incoming_app_ids[i_proc]);
}
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> * solution;
switch (_direction)
{
case TO_MULTIAPP:
solution = & getTransferVector(i_to, _to_var_name);
break;
case FROM_MULTIAPP:
solution = to_sys->solution.get();
break;
}
MeshBase * to_mesh = & _to_meshes[i_to]->getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
unsigned int lowest_app_rank = libMesh::invalid_uint;
Real best_val = 0.;
bool point_found = false;
for (unsigned int i_proc = 0; i_proc < incoming_evals.size(); i_proc++)
{
// Skip this proc if the node wasn't in it's bounding boxes.
std::pair<unsigned int, unsigned int> key(i_to, node->id());
if (point_index_map[i_proc].find(key) == point_index_map[i_proc].end())
continue;
unsigned int i_pt = point_index_map[i_proc][key];
// Ignore this proc if it's app has a higher rank than the
// previously found lowest app rank.
if (_direction == FROM_MULTIAPP)
{
if (incoming_app_ids[i_proc][i_pt] >= lowest_app_rank)
continue;
}
// Ignore this proc if the point was actually outside its meshes.
if (incoming_evals[i_proc][i_pt] == OutOfMeshValue)
continue;
best_val = incoming_evals[i_proc][i_pt];
point_found = true;
}
if (_error_on_miss && ! point_found)
mooseError("Point not found! " << *node + _to_positions[i_to]);
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
unsigned int lowest_app_rank = libMesh::invalid_uint;
Real best_val = 0;
bool point_found = false;
for (unsigned int i_proc = 0; i_proc < incoming_evals.size(); i_proc++)
{
// Skip this proc if the elem wasn't in it's bounding boxes.
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
if (point_index_map[i_proc].find(key) == point_index_map[i_proc].end())
continue;
unsigned int i_pt = point_index_map[i_proc][key];
// Ignore this proc if it's app has a higher rank than the
// previously found lowest app rank.
if (_direction == FROM_MULTIAPP)
{
if (incoming_app_ids[i_proc][i_pt] >= lowest_app_rank)
continue;
}
// Ignore this proc if the point was actually outside its meshes.
if (incoming_evals[i_proc][i_pt] == OutOfMeshValue)
continue;
best_val = incoming_evals[i_proc][i_pt];
point_found = true;
}
if (_error_on_miss && ! point_found)
mooseError("Point not found! " << elem->centroid() + _to_positions[i_to]);
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
solution->close();
to_sys->update();
}
_console << "Finished MeshFunctionTransfer " << name() << std::endl;
}
void
MultiAppUserObjectTransfer::execute()
{
_console << "Beginning MultiAppUserObjectTransfer " << name() << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i).es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i).getDisplacedProblem())
{
mesh = &_multi_app->appProblem(i).getDisplacedProblem()->mesh().getMesh();
}
else
mesh = &_multi_app->appProblem(i).mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
const UserObject & user_object = _multi_app->problem().getUserObjectBase(_user_object_name);
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = user_object.spatialValue(*node+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = user_object.spatialValue(centroid+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = _multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppUserObjectTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
_console << "Transferring to: " << to_var.name() << std::endl;
// EquationSystems & to_es = to_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * to_solution = to_sys.solution.get();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num).family == LAGRANGE;
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
Point app_position = _multi_app->position(i);
MeshTools::BoundingBox app_box = _multi_app->getBoundingBox(i);
const UserObject & user_object = _multi_app->appUserObjectBase(i, _user_object_name);
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this node
{
// See if this node falls in this bounding box
if (app_box.contains_point(*node))
{
dof_id_type dof = node->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = user_object.spatialValue(*node-app_position);
Moose::swapLibMeshComm(swapped);
to_solution->set(dof, from_value);
}
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
if (elem->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this elem
{
Point centroid = elem->centroid();
// See if this elem falls in this bounding box
if (app_box.contains_point(centroid))
{
dof_id_type dof = elem->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = user_object.spatialValue(centroid-app_position);
Moose::swapLibMeshComm(swapped);
to_solution->set(dof, from_value);
}
}
}
}
}
to_solution->close();
to_sys.update();
break;
}
}
_console << "Finished MultiAppUserObjectTransfer " << name() << std::endl;
}
void
MultiAppMeshFunctionTransfer::execute()
{
Moose::out << "Beginning MeshFunctionTransfer " << _name << std::endl;
switch(_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = *_multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppMeshFunctionTransfer only works with SerialMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
EquationSystems & from_es = from_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * serialized_solution = NumericVector<Number>::build().release();
serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_solution);
MeshFunction from_func(from_es, *serialized_solution, from_sys.get_dof_map(), from_var_num);
from_func.init(Trees::ELEMENTS);
from_func.enable_out_of_mesh_mode(NOTFOUND);
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i)->es(), _to_var_name);
if (!to_sys)
mooseError("Cannot find variable "<<_to_var_name<<" for "<<_name<<" Transfer");
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase & mesh = _multi_app->appProblem(i)->mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh.local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh.local_nodes_end();
for(; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = from_func(*node+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
if (from_value != NOTFOUND)
solution.set(dof, from_value);
else if (_error_on_miss)
mooseError("Point not found! " << *node+_multi_app->position(i) << std::endl);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh.local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh.local_elements_end();
for(; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = from_func(centroid+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
if (from_value != NOTFOUND)
solution.set(dof, from_value);
else if (_error_on_miss)
mooseError("Point not found! " << centroid+_multi_app->position(i) << std::endl);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
delete serialized_solution;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = *_multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppMeshFunctionTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
EquationSystems & to_es = to_sys.get_equation_systems();
NumericVector<Number> * to_solution = to_sys.solution.get();
MeshBase & to_mesh = to_es.get_mesh();
bool is_nodal = to_sys.variable_type(to_var_num).family == LAGRANGE;
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppMeshFunctionTransfer only works with SerialMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
EquationSystems & from_es = from_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * serialized_from_solution = NumericVector<Number>::build().release();
serialized_from_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_from_solution);
MeshBase & from_mesh = from_es.get_mesh();
MeshTools::BoundingBox app_box = MeshTools::processor_bounding_box(from_mesh, libMesh::processor_id());
Point app_position = _multi_app->position(i);
MeshFunction from_func(from_es, *serialized_from_solution, from_sys.get_dof_map(), from_var_num);
from_func.init(Trees::ELEMENTS);
from_func.enable_out_of_mesh_mode(NOTFOUND);
Moose::swapLibMeshComm(swapped);
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh.nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh.nodes_end();
for(; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this node
{
// See if this node falls in this bounding box
if (app_box.contains_point(*node-app_position))
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = from_func(*node-app_position);
Moose::swapLibMeshComm(swapped);
if (from_value != NOTFOUND)
to_solution->set(dof, from_value);
else if (_error_on_miss)
mooseError("Point not found! " << *node-app_position <<std::endl);
}
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh.elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh.elements_end();
for(; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
if (elem->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this elem
{
Point centroid = elem->centroid();
// See if this elem falls in this bounding box
if (app_box.contains_point(centroid-app_position))
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = from_func(centroid-app_position);
Moose::swapLibMeshComm(swapped);
if (from_value != NOTFOUND)
to_solution->set(dof, from_value);
else if (_error_on_miss)
mooseError("Point not found! " << centroid-app_position << std::endl);
}
}
}
}
delete serialized_from_solution;
}
to_solution->close();
to_sys.update();
break;
}
}
Moose::out << "Finished MeshFunctionTransfer " << _name << std::endl;
}
void
MultiAppInterpolationTransfer::execute()
{
_console << "Beginning InterpolationTransfer " << _name << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = *_multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
bool from_is_nodal = from_sys.variable_type(from_var_num).family == LAGRANGE;
// EquationSystems & from_es = from_sys.get_equation_systems();
NumericVector<Number> & from_solution = *from_sys.solution;
InverseDistanceInterpolation<LIBMESH_DIM> * idi;
switch (_interp_type)
{
case 0:
idi = new InverseDistanceInterpolation<LIBMESH_DIM>(from_sys.comm(), _num_points, _power);
break;
case 1:
idi = new RadialBasisInterpolation<LIBMESH_DIM>(from_sys.comm(), _radius);
break;
default:
mooseError("Unknown interpolation type!");
}
std::vector<Point> &src_pts (idi->get_source_points());
std::vector<Number> &src_vals (idi->get_source_vals());
std::vector<std::string> field_vars;
field_vars.push_back(_to_var_name);
idi->set_field_variables(field_vars);
std::vector<std::string> vars;
vars.push_back(_to_var_name);
if (from_is_nodal)
{
MeshBase::const_node_iterator from_nodes_it = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
for (; from_nodes_it != from_nodes_end; ++from_nodes_it)
{
Node * from_node = *from_nodes_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(*from_node);
src_vals.push_back(from_solution(from_dof));
}
}
else
{
MeshBase::const_element_iterator from_elements_it = from_mesh->local_elements_begin();
MeshBase::const_element_iterator from_elements_end = from_mesh->local_elements_end();
for (; from_elements_it != from_elements_end; ++from_elements_it)
{
Elem * from_elem = *from_elements_it;
// Assuming CONSTANT MONOMIAL
dof_id_type from_dof = from_elem->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(from_elem->centroid());
src_vals.push_back(from_solution(from_dof));
}
}
// We have only set local values - prepare for use by gathering remote gata
idi->prepare_for_use();
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i)->es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i)->getDisplacedProblem())
mesh = &_multi_app->appProblem(i)->getDisplacedProblem()->mesh().getMesh();
else
mesh = &_multi_app->appProblem(i)->mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
Point actual_position = *node+_multi_app->position(i);
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(actual_position);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution.set(dof, value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
Point actual_position = centroid+_multi_app->position(i);
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(actual_position);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
solution.set(dof, value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
delete idi;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = *_multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
NumericVector<Real> & to_solution = *to_sys.solution;
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppInterpolationTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
// EquationSystems & to_es = to_sys.get_equation_systems();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num).family == LAGRANGE;
InverseDistanceInterpolation<LIBMESH_DIM> * idi;
switch (_interp_type)
{
case 0:
idi = new InverseDistanceInterpolation<LIBMESH_DIM>(to_sys.comm(), _num_points, _power);
break;
case 1:
idi = new RadialBasisInterpolation<LIBMESH_DIM>(to_sys.comm(), _radius);
break;
default:
mooseError("Unknown interpolation type!");
}
std::vector<Point> &src_pts (idi->get_source_points());
std::vector<Number> &src_vals (idi->get_source_vals());
std::vector<std::string> field_vars;
field_vars.push_back(_to_var_name);
idi->set_field_variables(field_vars);
std::vector<std::string> vars;
vars.push_back(_to_var_name);
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
bool from_is_nodal = from_sys.variable_type(from_var_num).family == LAGRANGE;
// EquationSystems & from_es = from_sys.get_equation_systems();
NumericVector<Number> & from_solution = *from_sys.solution;
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
Point app_position = _multi_app->position(i);
if (from_is_nodal)
{
MeshBase::const_node_iterator from_nodes_it = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
for (; from_nodes_it != from_nodes_end; ++from_nodes_it)
{
Node * from_node = *from_nodes_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(*from_node+app_position);
src_vals.push_back(from_solution(from_dof));
}
}
else
{
MeshBase::const_element_iterator from_elements_it = from_mesh->local_elements_begin();
MeshBase::const_element_iterator from_elements_end = from_mesh->local_elements_end();
for (; from_elements_it != from_elements_end; ++from_elements_it)
{
Elem * from_element = *from_elements_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_element->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(from_element->centroid()+app_position);
src_vals.push_back(from_solution(from_dof));
}
}
Moose::swapLibMeshComm(swapped);
}
// We have only set local values - prepare for use by gathering remote gata
idi->prepare_for_use();
// Now do the interpolation to the target system
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this node
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(*node);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(to_sys_num, to_var_num, 0);
to_solution.set(dof, value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
if (elem->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this elem
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(centroid);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
dof_id_type dof = elem->dof_number(to_sys_num, to_var_num, 0);
to_solution.set(dof, value);
}
}
}
to_solution.close();
to_sys.update();
delete idi;
break;
}
}
_console << "Finished InterpolationTransfer " << _name << std::endl;
}
void
MultiAppNearestNodeTransfer::execute()
{
_console << "Beginning NearestNodeTransfer " << name() << std::endl;
getAppInfo();
// Get the bounding boxes for the "from" domains.
std::vector<MeshTools::BoundingBox> bboxes = getFromBoundingBoxes();
// Figure out how many "from" domains each processor owns.
std::vector<unsigned int> froms_per_proc = getFromsPerProc();
////////////////////
// For every point in the local "to" domain, figure out which "from" domains
// might contain it's nearest neighbor, and send that point to the processors
// that own those "from" domains.
//
// How do we know which "from" domains might contain the nearest neighbor, you
// ask? Well, consider two "from" domains, A and B. If every point in A is
// closer than every point in B, then we know that B cannot possibly contain
// the nearest neighbor. Hence, we'll only check A for the nearest neighbor.
// We'll use the functions bboxMaxDistance and bboxMinDistance to figure out
// if every point in A is closer than every point in B.
////////////////////
// outgoing_qps = nodes/centroids we'll send to other processors.
std::vector<std::vector<Point> > outgoing_qps(n_processors());
// When we get results back, node_index_map will tell us which results go with
// which points
std::vector<std::map<std::pair<unsigned int, unsigned int>, unsigned int> > node_index_map(n_processors());
if (! _neighbors_cached)
{
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
MeshBase * to_mesh = & _to_meshes[i_to]->getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
std::vector<Node *> target_local_nodes;
if (isParamValid("target_boundary"))
{
BoundaryID target_bnd_id = _to_meshes[i_to]->getBoundaryID(getParam<BoundaryName>("target_boundary"));
ConstBndNodeRange & bnd_nodes = *(_to_meshes[i_to])->getBoundaryNodeRange();
for (const auto & bnode : bnd_nodes)
if (bnode->_bnd_id == target_bnd_id && bnode->_node->processor_id() == processor_id())
target_local_nodes.push_back(bnode->_node);
}
else
{
target_local_nodes.resize(to_mesh->n_local_nodes());
MeshBase::const_node_iterator nodes_begin = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator nodes_end = to_mesh->local_nodes_end();
unsigned int i = 0;
for (MeshBase::const_node_iterator nodes_it = nodes_begin; nodes_it != nodes_end; ++nodes_it, ++i)
target_local_nodes[i] = *nodes_it;
}
for (const auto & node : target_local_nodes)
{
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
// Find which bboxes might have the nearest node to this point.
Real nearest_max_distance = std::numeric_limits<Real>::max();
for (const auto & bbox : bboxes)
{
Real distance = bboxMaxDistance(*node, bbox);
if (distance < nearest_max_distance)
nearest_max_distance = distance;
}
unsigned int from0 = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool qp_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && ! qp_found; i_from++)
{
Real distance = bboxMinDistance(*node, bboxes[i_from]);
if (distance < nearest_max_distance || bboxes[i_from].contains_point(*node))
{
std::pair<unsigned int, unsigned int> key(i_to, node->id());
node_index_map[i_proc][key] = outgoing_qps[i_proc].size();
outgoing_qps[i_proc].push_back(*node + _to_positions[i_to]);
qp_found = true;
}
}
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
// Find which bboxes might have the nearest node to this point.
Real nearest_max_distance = std::numeric_limits<Real>::max();
for (const auto & bbox : bboxes)
{
Real distance = bboxMaxDistance(centroid, bbox);
if (distance < nearest_max_distance)
nearest_max_distance = distance;
}
unsigned int from0 = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool qp_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && ! qp_found; i_from++)
{
Real distance = bboxMinDistance(centroid, bboxes[i_from]);
if (distance < nearest_max_distance || bboxes[i_from].contains_point(centroid))
{
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
node_index_map[i_proc][key] = outgoing_qps[i_proc].size();
outgoing_qps[i_proc].push_back(centroid + _to_positions[i_to]);
qp_found = true;
}
}
}
}
}
}
}
////////////////////
// Send local node/centroid positions off to the other processors and take
// care of points sent to this processor. We'll need to check the points
// against all of the "from" domains that this processor owns. For each
// point, we'll find the nearest node, then we'll send the value at that node
// and the distance between the node and the point back to the processor that
// requested that point.
////////////////////
std::vector<std::vector<Real> > incoming_evals(n_processors());
std::vector<Parallel::Request> send_qps(n_processors());
std::vector<Parallel::Request> send_evals(n_processors());
if (! _neighbors_cached)
{
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.send(i_proc, outgoing_qps[i_proc], send_qps[i_proc]);
}
// Build an array of pointers to all of this processor's local nodes. We
// need to do this to avoid the expense of using LibMesh iterators. This
// step also takes care of limiting the search to boundary nodes, if
// applicable.
std::vector< std::vector<Node *> > local_nodes(froms_per_proc[processor_id()]);
for (unsigned int i = 0; i < froms_per_proc[processor_id()]; i++)
{
getLocalNodes(_from_meshes[i], local_nodes[i]);
}
if (_fixed_meshes)
{
_cached_froms.resize(n_processors());
_cached_dof_ids.resize(n_processors());
}
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
std::vector<Point> incoming_qps;
if (i_proc == processor_id())
incoming_qps = outgoing_qps[i_proc];
else
_communicator.receive(i_proc, incoming_qps);
if (_fixed_meshes)
{
_cached_froms[i_proc].resize(incoming_qps.size());
_cached_dof_ids[i_proc].resize(incoming_qps.size());
}
std::vector<Real> outgoing_evals(2 * incoming_qps.size());
for (unsigned int qp = 0; qp < incoming_qps.size(); qp++)
{
Point qpt = incoming_qps[qp];
outgoing_evals[2*qp] = std::numeric_limits<Real>::max();
for (unsigned int i_local_from = 0; i_local_from < froms_per_proc[processor_id()]; i_local_from++)
{
MooseVariable & from_var = _from_problems[i_local_from]->getVariable(0, _from_var_name);
System & from_sys = from_var.sys().system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
for (unsigned int i_node = 0; i_node < local_nodes[i_local_from].size(); i_node++)
{
Real current_distance = (qpt - *(local_nodes[i_local_from][i_node]) - _from_positions[i_local_from]).norm();
if (current_distance < outgoing_evals[2*qp])
{
// Assuming LAGRANGE!
if (local_nodes[i_local_from][i_node]->n_dofs(from_sys_num, from_var_num) > 0)
{
dof_id_type from_dof = local_nodes[i_local_from][i_node]->dof_number(from_sys_num, from_var_num, 0);
outgoing_evals[2*qp] = current_distance;
outgoing_evals[2*qp + 1] = (*from_sys.solution)(from_dof);
if (_fixed_meshes)
{
// Cache the nearest nodes.
_cached_froms[i_proc][qp] = i_local_from;
_cached_dof_ids[i_proc][qp] = from_dof;
}
}
}
}
}
}
if (i_proc == processor_id())
incoming_evals[i_proc] = outgoing_evals;
else
_communicator.send(i_proc, outgoing_evals, send_evals[i_proc]);
}
}
else // We've cached the nearest nodes.
{
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
std::vector<Real> outgoing_evals(_cached_froms[i_proc].size());
for (unsigned int qp = 0; qp < outgoing_evals.size(); qp++)
{
MooseVariable & from_var = _from_problems[_cached_froms[i_proc][qp]]->getVariable(0, _from_var_name);
System & from_sys = from_var.sys().system();
dof_id_type from_dof = _cached_dof_ids[i_proc][qp];
//outgoing_evals[qp] = (*from_sys.solution)(_cached_dof_ids[i_proc][qp]);
outgoing_evals[qp] = (*from_sys.solution)(from_dof);
}
if (i_proc == processor_id())
incoming_evals[i_proc] = outgoing_evals;
else
_communicator.send(i_proc, outgoing_evals, send_evals[i_proc]);
}
}
////////////////////
// Gather all of the evaluations, find the nearest one for each node/element,
// and apply the values.
////////////////////
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.receive(i_proc, incoming_evals[i_proc]);
}
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> * solution = nullptr;
switch (_direction)
{
case TO_MULTIAPP:
solution = & getTransferVector(i_to, _to_var_name);
break;
case FROM_MULTIAPP:
solution = to_sys->solution.get();
break;
}
MeshBase * to_mesh = & _to_meshes[i_to]->getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
std::vector<Node *> target_local_nodes;
if (isParamValid("target_boundary"))
{
BoundaryID target_bnd_id = _to_meshes[i_to]->getBoundaryID(getParam<BoundaryName>("target_boundary"));
ConstBndNodeRange & bnd_nodes = *(_to_meshes[i_to])->getBoundaryNodeRange();
for (const auto & bnode : bnd_nodes)
if (bnode->_bnd_id == target_bnd_id && bnode->_node->processor_id() == processor_id())
target_local_nodes.push_back(bnode->_node);
}
else
{
target_local_nodes.resize(to_mesh->n_local_nodes());
MeshBase::const_node_iterator nodes_begin = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator nodes_end = to_mesh->local_nodes_end();
unsigned int i = 0;
for (MeshBase::const_node_iterator nodes_it = nodes_begin; nodes_it != nodes_end; ++nodes_it, ++i)
target_local_nodes[i] = *nodes_it;
}
for (const auto & node : target_local_nodes)
{
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
Real best_val = 0;
if (! _neighbors_cached)
{
Real min_dist = std::numeric_limits<Real>::max();
for (unsigned int i_from = 0; i_from < incoming_evals.size(); i_from++)
{
std::pair<unsigned int, unsigned int> key(i_to, node->id());
if (node_index_map[i_from].find(key) == node_index_map[i_from].end())
continue;
unsigned int qp_ind = node_index_map[i_from][key];
if (incoming_evals[i_from][2*qp_ind] >= min_dist)
continue;
min_dist = incoming_evals[i_from][2*qp_ind];
best_val = incoming_evals[i_from][2*qp_ind + 1];
if (_fixed_meshes)
{
// Cache these indices.
_cached_from_inds[node->id()] = i_from;
_cached_qp_inds[node->id()] = qp_ind;
}
}
}
else
{
best_val = incoming_evals[_cached_from_inds[node->id()]][_cached_qp_inds[node->id()]];
}
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
Real best_val = 0;
if (! _neighbors_cached)
{
Real min_dist = std::numeric_limits<Real>::max();
for (unsigned int i_from = 0; i_from < incoming_evals.size(); i_from++)
{
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
if (node_index_map[i_from].find(key) == node_index_map[i_from].end())
continue;
unsigned int qp_ind = node_index_map[i_from][key];
if (incoming_evals[i_from][2*qp_ind] >= min_dist)
continue;
min_dist = incoming_evals[i_from][2*qp_ind];
best_val = incoming_evals[i_from][2*qp_ind + 1];
if (_fixed_meshes)
{
// Cache these indices.
_cached_from_inds[elem->id()] = i_from;
_cached_qp_inds[elem->id()] = qp_ind;
}
}
}
else
{
best_val = incoming_evals[_cached_from_inds[elem->id()]][_cached_qp_inds[elem->id()]];
}
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
solution->close();
to_sys->update();
}
if (_fixed_meshes)
_neighbors_cached = true;
// Make sure all our sends succeeded.
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
send_qps[i_proc].wait();
send_evals[i_proc].wait();
}
_console << "Finished NearestNodeTransfer " << name() << std::endl;
}
void
MultiAppNearestNodeTransfer::execute()
{
Moose::out << "Beginning NearestNodeTransfer " << _name << std::endl;
switch(_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = *_multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
{
mooseError("Cannot use a NearestNode transfer from a displaced mesh to a MultiApp!");
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
}
else
from_mesh = &from_problem.mesh().getMesh();
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * serialized_solution = NumericVector<Number>::build().release();
serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_solution);
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i)->es(), _to_var_name);
if (!to_sys)
mooseError("Cannot find variable "<<_to_var_name<<" for "<<_name<<" Transfer");
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i)->getDisplacedProblem())
mesh = &_multi_app->appProblem(i)->getDisplacedProblem()->mesh().getMesh();
else
mesh = &_multi_app->appProblem(i)->mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for(; node_it != node_end; ++node_it)
{
Node * node = *node_it;
Point actual_position = *node+_multi_app->position(i);
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real distance = 0; // Just to satisfy the last argument
MeshBase::const_node_iterator from_nodes_begin = from_mesh->nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(node->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
_node_map[node->id()] = nearest_node;
_distance_map[node->id()] = distance;
}
else
{
nearest_node = _node_map[node->id()];
//distance = _distance_map[node->id()];
}
}
else
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
// Assuming LAGRANGE!
dof_id_type from_dof = nearest_node->dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*serialized_solution)(from_dof);
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for(; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
Point actual_position = centroid+_multi_app->position(i);
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real distance = 0; // Just to satisfy the last argument
MeshBase::const_node_iterator from_nodes_begin = from_mesh->nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(elem->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
_node_map[elem->id()] = nearest_node;
_distance_map[elem->id()] = distance;
}
else
{
nearest_node = _node_map[elem->id()];
//distance = _distance_map[elem->id()];
}
}
else
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
// Assuming LAGRANGE!
dof_id_type from_dof = nearest_node->dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*serialized_solution)(from_dof);
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
delete serialized_solution;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = *_multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
NumericVector<Real> & to_solution = *to_sys.solution;
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
// EquationSystems & to_es = to_sys.get_equation_systems();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num) == FEType();
dof_id_type n_nodes = to_mesh->n_nodes();
dof_id_type n_elems = to_mesh->n_elem();
///// All of the following are indexed off to_node->id() or to_elem->id() /////
// Minimum distances from each node in the "to" mesh to a node in
std::vector<Real> min_distances;
// The node ids in the "from" mesh that this processor has found to be the minimum distances to the "to" nodes
std::vector<dof_id_type> min_nodes;
// After the call to maxloc() this will tell us which processor actually has the minimum
std::vector<unsigned int> min_procs;
// The global multiapp ID that this processor found had the minimum distance node in it.
std::vector<unsigned int> min_apps;
if (is_nodal)
{
min_distances.resize(n_nodes, std::numeric_limits<Real>::max());
min_nodes.resize(n_nodes);
min_procs.resize(n_nodes);
min_apps.resize(n_nodes);
}
else
{
min_distances.resize(n_elems, std::numeric_limits<Real>::max());
min_nodes.resize(n_elems);
min_procs.resize(n_elems);
min_apps.resize(n_elems);
}
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
// unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
MeshTools::BoundingBox app_box = MeshTools::processor_bounding_box(*from_mesh, libMesh::processor_id());
Point app_position = _multi_app->position(i);
Moose::swapLibMeshComm(swapped);
if (is_nodal)
{
MeshBase::const_node_iterator to_node_it = to_mesh->nodes_begin();
MeshBase::const_node_iterator to_node_end = to_mesh->nodes_end();
for(; to_node_it != to_node_end; ++to_node_it)
{
Node * to_node = *to_node_it;
unsigned int to_node_id = to_node->id();
Real current_distance = 0;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
MeshBase::const_node_iterator from_nodes_begin = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(to_node->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(*to_node-app_position, current_distance, from_nodes_begin, from_nodes_end);
_node_map[to_node->id()] = nearest_node;
_distance_map[to_node->id()] = current_distance;
}
else
{
nearest_node = _node_map[to_node->id()];
current_distance = _distance_map[to_node->id()];
}
}
else
nearest_node = getNearestNode(*to_node-app_position, current_distance, from_nodes_begin, from_nodes_end);
Moose::swapLibMeshComm(swapped);
// TODO: Logic bug when we are using caching. "current_distance" is set by a call to getNearestNode which is
// skipped in that case. We shouldn't be relying on it or stuffing it in another data structure
if (current_distance < min_distances[to_node->id()])
{
min_distances[to_node_id] = current_distance;
min_nodes[to_node_id] = nearest_node->id();
min_apps[to_node_id] = i;
}
}
}
else // Elemental
{
MeshBase::const_element_iterator to_elem_it = to_mesh->elements_begin();
MeshBase::const_element_iterator to_elem_end = to_mesh->elements_end();
for(; to_elem_it != to_elem_end; ++to_elem_it)
{
Elem * to_elem = *to_elem_it;
unsigned int to_elem_id = to_elem->id();
Point actual_position = to_elem->centroid()-app_position;
Real current_distance = 0;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
MeshBase::const_node_iterator from_nodes_begin = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(to_elem->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, current_distance, from_nodes_begin, from_nodes_end);
_node_map[to_elem->id()] = nearest_node;
_distance_map[to_elem->id()] = current_distance;
}
else
{
nearest_node = _node_map[to_elem->id()];
current_distance = _distance_map[to_elem->id()];
}
}
else
nearest_node = getNearestNode(actual_position, current_distance, from_nodes_begin, from_nodes_end);
Moose::swapLibMeshComm(swapped);
// TODO: Logic bug when we are using caching. "current_distance" is set by a call to getNearestNode which is
// skipped in that case. We shouldn't be relying on it or stuffing it in another data structure
if (current_distance < min_distances[to_elem->id()])
{
min_distances[to_elem_id] = current_distance;
min_nodes[to_elem_id] = nearest_node->id();
min_apps[to_elem_id] = i;
}
}
}
}
/*
// We're going to need serialized solution vectors for each app
// We could try to only do it for the apps that have mins in them...
// but it's tough because this is a collective operation... so that would have to be coordinated
std::vector<NumericVector<Number> *> serialized_from_solutions(_multi_app->numGlobalApps());
if (_multi_app->hasApp())
{
// Swap
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
//Create a serialized version of the solution vector
serialized_from_solutions[i] = NumericVector<Number>::build().release();
serialized_from_solutions[i]->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_from_solutions[i]);
}
// Swap back
Moose::swapLibMeshComm(swapped);
}
*/
// We've found the nearest nodes for this processor. We need to see which processor _actually_ found the nearest though
Parallel::minloc(min_distances, min_procs);
// Now loop through min_procs and see if _this_ processor had the actual minimum for any nodes.
// If it did then we're going to go get the value from that nearest node and transfer its value
processor_id_type proc_id = libMesh::processor_id();
for(unsigned int j=0; j<min_procs.size(); j++)
{
if (min_procs[j] == proc_id) // This means that this processor really did find the minumum so we need to transfer the value
{
// The zero only works for LAGRANGE!
dof_id_type to_dof = 0;
if (is_nodal)
{
Node & to_node = to_mesh->node(j);
to_dof = to_node.dof_number(to_sys_num, to_var_num, 0);
}
else
{
Elem & to_elem = *to_mesh->elem(j);
to_dof = to_elem.dof_number(to_sys_num, to_var_num, 0);
}
// The app that has the nearest node in it
unsigned int from_app_num = min_apps[j];
mooseAssert(_multi_app->hasLocalApp(from_app_num), "Something went very wrong!");
// Swap
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(from_app_num);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
Node & from_node = from_mesh->node(min_nodes[j]);
// Assuming LAGRANGE!
dof_id_type from_dof = from_node.dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*from_sys.solution)(from_dof);
// Swap back
Moose::swapLibMeshComm(swapped);
to_solution.set(to_dof, from_value);
}
}
to_solution.close();
to_sys.update();
break;
}
}
Moose::out << "Finished NearestNodeTransfer " << _name << std::endl;
}
