void
MoosePreconditioner::copyVarValues(MeshBase & mesh,
const unsigned int from_system,
const unsigned int from_var,
const NumericVector<Number> & from_vector,
const unsigned int to_system,
const unsigned int to_var,
NumericVector<Number> & to_vector)
{
{
MeshBase::node_iterator it = mesh.local_nodes_begin();
MeshBase::node_iterator it_end = mesh.local_nodes_end();
for (; it != it_end; ++it)
{
Node * node = *it;
unsigned int n_comp = node->n_comp(from_system, from_var);
mooseAssert(node->n_comp(from_system, from_var) == node->n_comp(to_system, to_var),
"Number of components does not match in each system");
for (unsigned int i = 0; i < n_comp; i++)
{
dof_id_type from_dof = node->dof_number(from_system, from_var, i);
dof_id_type to_dof = node->dof_number(to_system, to_var, i);
to_vector.set(to_dof, from_vector(from_dof));
}
}
}
{
MeshBase::element_iterator it = mesh.local_elements_begin();
MeshBase::element_iterator it_end = mesh.local_elements_end();
for (; it != it_end; ++it)
{
Elem * elem = *it;
unsigned int n_comp = elem->n_comp(from_system, from_var);
mooseAssert(elem->n_comp(from_system, from_var) == elem->n_comp(to_system, to_var),
"Number of components does not match in each system");
for (unsigned int i = 0; i < n_comp; i++)
{
dof_id_type from_dof = elem->dof_number(from_system, from_var, i);
dof_id_type to_dof = elem->dof_number(to_system, to_var, i);
to_vector.set(to_dof, from_vector(from_dof));
}
}
}
}
void
MultiAppUserObjectTransfer::execute()
{
_console << "Beginning MultiAppUserObjectTransfer " << name() << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i).es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i).getDisplacedProblem())
{
mesh = &_multi_app->appProblem(i).getDisplacedProblem()->mesh().getMesh();
}
else
mesh = &_multi_app->appProblem(i).mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
const UserObject & user_object = _multi_app->problem().getUserObjectBase(_user_object_name);
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = user_object.spatialValue(*node+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = user_object.spatialValue(centroid+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = _multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppUserObjectTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
_console << "Transferring to: " << to_var.name() << std::endl;
// EquationSystems & to_es = to_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * to_solution = to_sys.solution.get();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num).family == LAGRANGE;
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
Point app_position = _multi_app->position(i);
MeshTools::BoundingBox app_box = _multi_app->getBoundingBox(i);
const UserObject & user_object = _multi_app->appUserObjectBase(i, _user_object_name);
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this node
{
// See if this node falls in this bounding box
if (app_box.contains_point(*node))
{
dof_id_type dof = node->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = user_object.spatialValue(*node-app_position);
Moose::swapLibMeshComm(swapped);
to_solution->set(dof, from_value);
}
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
if (elem->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this elem
{
Point centroid = elem->centroid();
// See if this elem falls in this bounding box
if (app_box.contains_point(centroid))
{
dof_id_type dof = elem->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = user_object.spatialValue(centroid-app_position);
Moose::swapLibMeshComm(swapped);
to_solution->set(dof, from_value);
}
}
}
}
}
to_solution->close();
to_sys.update();
break;
}
}
_console << "Finished MultiAppUserObjectTransfer " << name() << std::endl;
}
void
MultiAppInterpolationTransfer::execute()
{
_console << "Beginning InterpolationTransfer " << _name << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = *_multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
bool from_is_nodal = from_sys.variable_type(from_var_num).family == LAGRANGE;
// EquationSystems & from_es = from_sys.get_equation_systems();
NumericVector<Number> & from_solution = *from_sys.solution;
InverseDistanceInterpolation<LIBMESH_DIM> * idi;
switch (_interp_type)
{
case 0:
idi = new InverseDistanceInterpolation<LIBMESH_DIM>(from_sys.comm(), _num_points, _power);
break;
case 1:
idi = new RadialBasisInterpolation<LIBMESH_DIM>(from_sys.comm(), _radius);
break;
default:
mooseError("Unknown interpolation type!");
}
std::vector<Point> &src_pts (idi->get_source_points());
std::vector<Number> &src_vals (idi->get_source_vals());
std::vector<std::string> field_vars;
field_vars.push_back(_to_var_name);
idi->set_field_variables(field_vars);
std::vector<std::string> vars;
vars.push_back(_to_var_name);
if (from_is_nodal)
{
MeshBase::const_node_iterator from_nodes_it = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
for (; from_nodes_it != from_nodes_end; ++from_nodes_it)
{
Node * from_node = *from_nodes_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(*from_node);
src_vals.push_back(from_solution(from_dof));
}
}
else
{
MeshBase::const_element_iterator from_elements_it = from_mesh->local_elements_begin();
MeshBase::const_element_iterator from_elements_end = from_mesh->local_elements_end();
for (; from_elements_it != from_elements_end; ++from_elements_it)
{
Elem * from_elem = *from_elements_it;
// Assuming CONSTANT MONOMIAL
dof_id_type from_dof = from_elem->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(from_elem->centroid());
src_vals.push_back(from_solution(from_dof));
}
}
// We have only set local values - prepare for use by gathering remote gata
idi->prepare_for_use();
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i)->es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i)->getDisplacedProblem())
mesh = &_multi_app->appProblem(i)->getDisplacedProblem()->mesh().getMesh();
else
mesh = &_multi_app->appProblem(i)->mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
Point actual_position = *node+_multi_app->position(i);
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(actual_position);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution.set(dof, value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
Point actual_position = centroid+_multi_app->position(i);
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(actual_position);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
solution.set(dof, value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
delete idi;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = *_multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
NumericVector<Real> & to_solution = *to_sys.solution;
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppInterpolationTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
// EquationSystems & to_es = to_sys.get_equation_systems();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num).family == LAGRANGE;
InverseDistanceInterpolation<LIBMESH_DIM> * idi;
switch (_interp_type)
{
case 0:
idi = new InverseDistanceInterpolation<LIBMESH_DIM>(to_sys.comm(), _num_points, _power);
break;
case 1:
idi = new RadialBasisInterpolation<LIBMESH_DIM>(to_sys.comm(), _radius);
break;
default:
mooseError("Unknown interpolation type!");
}
std::vector<Point> &src_pts (idi->get_source_points());
std::vector<Number> &src_vals (idi->get_source_vals());
std::vector<std::string> field_vars;
field_vars.push_back(_to_var_name);
idi->set_field_variables(field_vars);
std::vector<std::string> vars;
vars.push_back(_to_var_name);
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
bool from_is_nodal = from_sys.variable_type(from_var_num).family == LAGRANGE;
// EquationSystems & from_es = from_sys.get_equation_systems();
NumericVector<Number> & from_solution = *from_sys.solution;
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
Point app_position = _multi_app->position(i);
if (from_is_nodal)
{
MeshBase::const_node_iterator from_nodes_it = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
for (; from_nodes_it != from_nodes_end; ++from_nodes_it)
{
Node * from_node = *from_nodes_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(*from_node+app_position);
src_vals.push_back(from_solution(from_dof));
}
}
else
{
MeshBase::const_element_iterator from_elements_it = from_mesh->local_elements_begin();
MeshBase::const_element_iterator from_elements_end = from_mesh->local_elements_end();
for (; from_elements_it != from_elements_end; ++from_elements_it)
{
Elem * from_element = *from_elements_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_element->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(from_element->centroid()+app_position);
src_vals.push_back(from_solution(from_dof));
}
}
Moose::swapLibMeshComm(swapped);
}
// We have only set local values - prepare for use by gathering remote gata
idi->prepare_for_use();
// Now do the interpolation to the target system
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this node
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(*node);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(to_sys_num, to_var_num, 0);
to_solution.set(dof, value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
if (elem->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this elem
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(centroid);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
dof_id_type dof = elem->dof_number(to_sys_num, to_var_num, 0);
to_solution.set(dof, value);
}
}
}
to_solution.close();
to_sys.update();
delete idi;
break;
}
}
_console << "Finished InterpolationTransfer " << _name << std::endl;
}
void
MultiAppCopyTransfer::execute()
{
_console << "Beginning CopyTransfer " << name() << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = _multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
MeshBase * from_mesh = NULL;
from_mesh = &from_problem.mesh().getMesh();
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppCopyTransfer only works with ReplicatedMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
//Create a serialized version of the solution vector
// NumericVector<Number> * serialized_solution = NumericVector<Number>::build(from_sys.comm()).release();
// serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
// from_sys.solution->localize(*serialized_solution);
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i).es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
mesh = &_multi_app->appProblem(i).mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
unsigned int node_id = node->id();
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Node * from_node = from_mesh->node_ptr(node_id);
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
//Real from_value = (*serialized_solution)(from_dof);
Real from_value = (*from_sys.solution)(from_dof);
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
else // Elemental
{
mooseError("MultiAppCopyTransfer can only be used on nodal variables");
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
// delete serialized_solution;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = _multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
NumericVector<Real> & to_solution = *to_sys.solution;
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppCopyTransfer only works with ReplicatedMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
MeshBase * to_mesh = NULL;
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num) == FEType();
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = _multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppCopyTransfer only works with ReplicatedMesh!");
//Create a serialized version of the solution vector
// NumericVector<Number> * serialized_solution = NumericVector<Number>::build(from_sys.comm()).release();
// serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
// from_sys.solution->localize(*serialized_solution);
MeshBase * from_mesh = &from_problem.mesh().getMesh();
Moose::swapLibMeshComm(swapped);
if (is_nodal)
{
MeshBase::const_node_iterator to_node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator to_node_end = to_mesh->local_nodes_end();
for (; to_node_it != to_node_end; ++to_node_it)
{
Node * to_node = *to_node_it;
unsigned int to_node_id = to_node->id();
// The zero only works for LAGRANGE!
dof_id_type to_dof = to_node->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Node * from_node = from_mesh->node_ptr(to_node_id);
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
//Real from_value = (*serialized_solution)(from_dof);
Real from_value = (*from_sys.solution)(from_dof);
// Swap back
Moose::swapLibMeshComm(swapped);
to_solution.set(to_dof, from_value);
}
}
else // Elemental
{
mooseError("MultiAppCopyTransfer can only be used on nodal variables");
}
// delete serialized_solution;
}
to_solution.close();
to_sys.update();
break;
}
}
_console << "Finished CopyTransfer " << name() << std::endl;
}
void
MultiAppNearestNodeTransfer::execute()
{
Moose::out << "Beginning NearestNodeTransfer " << _name << std::endl;
switch(_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = *_multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
{
mooseError("Cannot use a NearestNode transfer from a displaced mesh to a MultiApp!");
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
}
else
from_mesh = &from_problem.mesh().getMesh();
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * serialized_solution = NumericVector<Number>::build().release();
serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_solution);
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i)->es(), _to_var_name);
if (!to_sys)
mooseError("Cannot find variable "<<_to_var_name<<" for "<<_name<<" Transfer");
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i)->getDisplacedProblem())
mesh = &_multi_app->appProblem(i)->getDisplacedProblem()->mesh().getMesh();
else
mesh = &_multi_app->appProblem(i)->mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for(; node_it != node_end; ++node_it)
{
Node * node = *node_it;
Point actual_position = *node+_multi_app->position(i);
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real distance = 0; // Just to satisfy the last argument
MeshBase::const_node_iterator from_nodes_begin = from_mesh->nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(node->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
_node_map[node->id()] = nearest_node;
_distance_map[node->id()] = distance;
}
else
{
nearest_node = _node_map[node->id()];
//distance = _distance_map[node->id()];
}
}
else
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
// Assuming LAGRANGE!
dof_id_type from_dof = nearest_node->dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*serialized_solution)(from_dof);
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for(; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
Point actual_position = centroid+_multi_app->position(i);
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real distance = 0; // Just to satisfy the last argument
MeshBase::const_node_iterator from_nodes_begin = from_mesh->nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(elem->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
_node_map[elem->id()] = nearest_node;
_distance_map[elem->id()] = distance;
}
else
{
nearest_node = _node_map[elem->id()];
//distance = _distance_map[elem->id()];
}
}
else
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
// Assuming LAGRANGE!
dof_id_type from_dof = nearest_node->dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*serialized_solution)(from_dof);
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
delete serialized_solution;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = *_multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
NumericVector<Real> & to_solution = *to_sys.solution;
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
// EquationSystems & to_es = to_sys.get_equation_systems();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num) == FEType();
dof_id_type n_nodes = to_mesh->n_nodes();
dof_id_type n_elems = to_mesh->n_elem();
///// All of the following are indexed off to_node->id() or to_elem->id() /////
// Minimum distances from each node in the "to" mesh to a node in
std::vector<Real> min_distances;
// The node ids in the "from" mesh that this processor has found to be the minimum distances to the "to" nodes
std::vector<dof_id_type> min_nodes;
// After the call to maxloc() this will tell us which processor actually has the minimum
std::vector<unsigned int> min_procs;
// The global multiapp ID that this processor found had the minimum distance node in it.
std::vector<unsigned int> min_apps;
if (is_nodal)
{
min_distances.resize(n_nodes, std::numeric_limits<Real>::max());
min_nodes.resize(n_nodes);
min_procs.resize(n_nodes);
min_apps.resize(n_nodes);
}
else
{
min_distances.resize(n_elems, std::numeric_limits<Real>::max());
min_nodes.resize(n_elems);
min_procs.resize(n_elems);
min_apps.resize(n_elems);
}
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
// unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
MeshTools::BoundingBox app_box = MeshTools::processor_bounding_box(*from_mesh, libMesh::processor_id());
Point app_position = _multi_app->position(i);
Moose::swapLibMeshComm(swapped);
if (is_nodal)
{
MeshBase::const_node_iterator to_node_it = to_mesh->nodes_begin();
MeshBase::const_node_iterator to_node_end = to_mesh->nodes_end();
for(; to_node_it != to_node_end; ++to_node_it)
{
Node * to_node = *to_node_it;
unsigned int to_node_id = to_node->id();
Real current_distance = 0;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
MeshBase::const_node_iterator from_nodes_begin = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(to_node->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(*to_node-app_position, current_distance, from_nodes_begin, from_nodes_end);
_node_map[to_node->id()] = nearest_node;
_distance_map[to_node->id()] = current_distance;
}
else
{
nearest_node = _node_map[to_node->id()];
current_distance = _distance_map[to_node->id()];
}
}
else
nearest_node = getNearestNode(*to_node-app_position, current_distance, from_nodes_begin, from_nodes_end);
Moose::swapLibMeshComm(swapped);
// TODO: Logic bug when we are using caching. "current_distance" is set by a call to getNearestNode which is
// skipped in that case. We shouldn't be relying on it or stuffing it in another data structure
if (current_distance < min_distances[to_node->id()])
{
min_distances[to_node_id] = current_distance;
min_nodes[to_node_id] = nearest_node->id();
min_apps[to_node_id] = i;
}
}
}
else // Elemental
{
MeshBase::const_element_iterator to_elem_it = to_mesh->elements_begin();
MeshBase::const_element_iterator to_elem_end = to_mesh->elements_end();
for(; to_elem_it != to_elem_end; ++to_elem_it)
{
Elem * to_elem = *to_elem_it;
unsigned int to_elem_id = to_elem->id();
Point actual_position = to_elem->centroid()-app_position;
Real current_distance = 0;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
MeshBase::const_node_iterator from_nodes_begin = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(to_elem->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, current_distance, from_nodes_begin, from_nodes_end);
_node_map[to_elem->id()] = nearest_node;
_distance_map[to_elem->id()] = current_distance;
}
else
{
nearest_node = _node_map[to_elem->id()];
current_distance = _distance_map[to_elem->id()];
}
}
else
nearest_node = getNearestNode(actual_position, current_distance, from_nodes_begin, from_nodes_end);
Moose::swapLibMeshComm(swapped);
// TODO: Logic bug when we are using caching. "current_distance" is set by a call to getNearestNode which is
// skipped in that case. We shouldn't be relying on it or stuffing it in another data structure
if (current_distance < min_distances[to_elem->id()])
{
min_distances[to_elem_id] = current_distance;
min_nodes[to_elem_id] = nearest_node->id();
min_apps[to_elem_id] = i;
}
}
}
}
/*
// We're going to need serialized solution vectors for each app
// We could try to only do it for the apps that have mins in them...
// but it's tough because this is a collective operation... so that would have to be coordinated
std::vector<NumericVector<Number> *> serialized_from_solutions(_multi_app->numGlobalApps());
if (_multi_app->hasApp())
{
// Swap
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
//Create a serialized version of the solution vector
serialized_from_solutions[i] = NumericVector<Number>::build().release();
serialized_from_solutions[i]->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_from_solutions[i]);
}
// Swap back
Moose::swapLibMeshComm(swapped);
}
*/
// We've found the nearest nodes for this processor. We need to see which processor _actually_ found the nearest though
Parallel::minloc(min_distances, min_procs);
// Now loop through min_procs and see if _this_ processor had the actual minimum for any nodes.
// If it did then we're going to go get the value from that nearest node and transfer its value
processor_id_type proc_id = libMesh::processor_id();
for(unsigned int j=0; j<min_procs.size(); j++)
{
if (min_procs[j] == proc_id) // This means that this processor really did find the minumum so we need to transfer the value
{
// The zero only works for LAGRANGE!
dof_id_type to_dof = 0;
if (is_nodal)
{
Node & to_node = to_mesh->node(j);
to_dof = to_node.dof_number(to_sys_num, to_var_num, 0);
}
else
{
Elem & to_elem = *to_mesh->elem(j);
to_dof = to_elem.dof_number(to_sys_num, to_var_num, 0);
}
// The app that has the nearest node in it
unsigned int from_app_num = min_apps[j];
mooseAssert(_multi_app->hasLocalApp(from_app_num), "Something went very wrong!");
// Swap
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(from_app_num);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
Node & from_node = from_mesh->node(min_nodes[j]);
// Assuming LAGRANGE!
dof_id_type from_dof = from_node.dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*from_sys.solution)(from_dof);
// Swap back
Moose::swapLibMeshComm(swapped);
to_solution.set(to_dof, from_value);
}
}
to_solution.close();
to_sys.update();
break;
}
}
Moose::out << "Finished NearestNodeTransfer " << _name << std::endl;
}