void
MultiAppDTKInterpolationTransfer::execute()
{
// Every processor has to be involved with every transfer because the "master" domain is on all
// processors
// However, each processor might or might not have the particular multiapp on it. When that
// happens
// we need to pass NULL in for the variable and the MPI_Comm for that piece of the transfer
for (unsigned int i = 0; i < _multi_app->numGlobalApps(); i++)
{
switch (_direction)
{
case TO_MULTIAPP:
{
System * from_sys = find_sys(_multi_app->problemBase().es(), _from_var_name);
System * to_sys = NULL;
if (_multi_app->hasLocalApp(i))
to_sys = find_sys(_multi_app->appProblemBase(i).es(), _to_var_name);
_helper.transferWithOffset(
0,
i,
&from_sys->variable(from_sys->variable_number(_from_var_name)),
to_sys ? &to_sys->variable(to_sys->variable_number(_to_var_name)) : NULL,
_master_position,
_multi_app->position(i),
const_cast<libMesh::Parallel::communicator *>(&_communicator.get()),
to_sys ? &_multi_app->comm() : NULL);
break;
}
case FROM_MULTIAPP:
{
System * from_sys = NULL;
System * to_sys = find_sys(_multi_app->problemBase().es(), _to_var_name);
if (_multi_app->hasLocalApp(i))
from_sys = find_sys(_multi_app->appProblemBase(i).es(), _from_var_name);
_helper.transferWithOffset(
i,
0,
from_sys ? &from_sys->variable(from_sys->variable_number(_from_var_name)) : NULL,
&to_sys->variable(to_sys->variable_number(_to_var_name)),
_multi_app->position(i),
_master_position,
from_sys ? &_multi_app->comm() : NULL,
const_cast<libMesh::Parallel::communicator *>(&_communicator.get()));
break;
}
_multi_app->problemBase().es().update();
if (_multi_app->hasLocalApp(i))
_multi_app->appProblemBase(i).es().update();
}
}
}
void
DTKAdapter::update_variable_values(std::string var_name)
{
System * sys = find_sys(var_name);
unsigned int var_num = sys->variable_number(var_name);
Teuchos::RCP<FieldContainerType> values = values_to_fill[var_name]->field();
unsigned int i=0;
// Loop over the values (one for each node) and assign the value of this variable at each node
for(FieldContainerType::iterator it=values->begin(); it != values->end(); ++it)
{
unsigned int node_num = vertices[i];
const Node & node = mesh.node(node_num);
if(node.processor_id() == sys->processor_id())
{
// The 0 is for the component... this only works for LAGRANGE!
dof_id_type dof = node.dof_number(sys->number(), var_num, 0);
sys->solution->set(dof, *it);
}
i++;
}
sys->solution->close();
}
void
MultiAppTransfer::variableIntegrityCheck(const AuxVariableName & var_name) const
{
for (unsigned int i = 0; i < _multi_app->numGlobalApps(); i++)
if (_multi_app->hasLocalApp(i) && !find_sys(_multi_app->appProblem(i)->es(), var_name))
mooseError("Cannot find variable " << var_name << " for " << name() << " Transfer");
}
void
DTKInterpolationAdapter::update_variable_values(std::string var_name, Teuchos::ArrayView<GlobalOrdinal> missed_points)
{
MPI_Comm old_comm = Moose::swapLibMeshComm(*comm->getRawMpiComm());
System * sys = find_sys(var_name);
unsigned int var_num = sys->variable_number(var_name);
bool is_nodal = sys->variable_type(var_num).family == LAGRANGE;
Teuchos::RCP<FieldContainerType> values = values_to_fill[var_name]->field();
// Create a vector containing true or false for each point saying whether it was missed or not
// We're only going to update values for points that were not missed
std::vector<bool> missed(values->size(), false);
for (Teuchos::ArrayView<const GlobalOrdinal>::const_iterator i=missed_points.begin();
i != missed_points.end();
++i)
missed[*i] = true;
unsigned int i=0;
// Loop over the values (one for each node) and assign the value of this variable at each node
for (FieldContainerType::iterator it=values->begin(); it != values->end(); ++it)
{
// If this point "missed" then skip it
if (missed[i])
{
i++;
continue;
}
const DofObject * dof_object = NULL;
if (is_nodal)
dof_object = mesh.node_ptr(vertices[i]);
else
dof_object = mesh.elem(elements[i]);
if (dof_object->processor_id() == mesh.processor_id())
{
// The 0 is for the component... this only works for LAGRANGE!
dof_id_type dof = dof_object->dof_number(sys->number(), var_num, 0);
sys->solution->set(dof, *it);
}
i++;
}
sys->solution->close();
// Swap back
Moose::swapLibMeshComm(old_comm);
}
DTKInterpolationAdapter::RCP_Evaluator
DTKInterpolationAdapter::get_variable_evaluator(std::string var_name)
{
if (evaluators.find(var_name) == evaluators.end()) // We haven't created an evaluator for the variable yet
{
System * sys = find_sys(var_name);
// Create the FieldEvaluator
evaluators[var_name] = Teuchos::rcp(new DTKInterpolationEvaluator(*sys, var_name, _offset));
}
return evaluators[var_name];
}
Teuchos::RCP<DataTransferKit::FieldManager<DTKInterpolationAdapter::FieldContainerType> >
DTKInterpolationAdapter::get_values_to_fill(std::string var_name)
{
if (values_to_fill.find(var_name) == values_to_fill.end())
{
System * sys = find_sys(var_name);
unsigned int var_num = sys->variable_number(var_name);
bool is_nodal = sys->variable_type(var_num).family == LAGRANGE;
Teuchos::ArrayRCP<double> data_space;
if (is_nodal)
data_space = Teuchos::ArrayRCP<double>(vertices.size());
else
data_space = Teuchos::ArrayRCP<double>(elements.size());
Teuchos::RCP<FieldContainerType> field_container = Teuchos::rcp(new FieldContainerType(data_space, 1));
values_to_fill[var_name] = Teuchos::rcp(new DataTransferKit::FieldManager<FieldContainerType>(field_container, comm));
}
return values_to_fill[var_name];
}
static
int
do_loads(int filec, char *filev[], struct membuf *mem,
int basic_txt_start, int sys_token,
int *basic_var_startp, int *runp, int trim_sys)
{
int run = -1;
int min_start = 65537;
int max_end = -1;
int basic_code = 0;
int i;
unsigned char *p;
struct load_info info[1];
membuf_clear(mem);
membuf_append(mem, NULL, 65536);
p = membuf_get(mem);
for (i = 0; i < filec; ++i)
{
info->basic_txt_start = basic_txt_start;
load_located(filev[i], p, info);
run = info->run;
if(run != -1 && runp != NULL)
{
LOG(LOG_DEBUG, ("Propagating found run address $%04X.\n",
info->run));
*runp = info->run;
}
/* do we expect any basic file? */
if(basic_txt_start >= 0)
{
if(info->basic_var_start >= 0)
{
basic_code = 1;
if(basic_var_startp != NULL)
{
*basic_var_startp = info->basic_var_start;
}
if(runp != NULL && run == -1)
{
/* only if we didn't get run address from load_located
* (run is not -1 if we did) */
int stub_len;
run = find_sys(p + basic_txt_start, sys_token, &stub_len);
*runp = run;
if (trim_sys &&
basic_txt_start == info->start &&
min_start >= info->start)
{
if (run >= info->start &&
run < info->start + stub_len)
{
/* the run address points into the sys stub,
trim up to it but no further */
info->start = run;
}
else
{
/* trim the sys stub*/
info->start += stub_len;
}
}
}
}
}
if (info->start < min_start)
{
min_start = info->start;
}
if (info->end > max_end)
{
max_end = info->end;
}
}
if(basic_txt_start >= 0 && !basic_code && run == -1)
{
/* no program loaded to the basic start */
LOG(LOG_ERROR, ("\nError: nothing loaded at the start of basic "
"text address ($%04X).\n",
basic_txt_start));
exit(1);
}
/* if we have a basic code loaded and we are doing a proper basic start
* (the caller don't expect a sys address so runp is NULL */
if(basic_code && runp == NULL)
{
int valuepos = basic_txt_start - 1;
/* the byte immediatley preceeding the basic start must be 0
* for basic to function properly. */
if(min_start > valuepos)
{
/* It not covered by the files to crunch. Since the
* default fill value is 0 we don't need to set it but we
* need to include that location in the crunch as well. */
min_start = valuepos;
}
else
{
int value = p[valuepos];
/* it has been covered by at least one file. Let's check
* if it is zero. */
if(value != 0)
{
/* Hm, its not, danger Will Robinson! */
LOG(LOG_WARNING,
("Warning, basic will probably not work since the value of"
" the location \npreceeding the basic start ($%04X)"
" is not 0 but %d.\n", valuepos, value));
}
}
}
/* move memory to beginning of buffer */
membuf_truncate(mem, max_end);
membuf_trim(mem, min_start);
return min_start;
}
void
MultiAppPostprocessorInterpolationTransfer::execute()
{
switch (_direction)
{
case TO_MULTIAPP:
{
mooseError("Can't interpolate to a MultiApp!!");
break;
}
case FROM_MULTIAPP:
{
InverseDistanceInterpolation<LIBMESH_DIM> * idi;
switch (_interp_type)
{
case 0:
idi = new InverseDistanceInterpolation<LIBMESH_DIM>(_communicator, _num_points, _power);
break;
case 1:
idi = new RadialBasisInterpolation<LIBMESH_DIM>(_communicator, _radius);
break;
default:
mooseError("Unknown interpolation type!");
}
std::vector<Point> &src_pts (idi->get_source_points());
std::vector<Number> &src_vals (idi->get_source_vals());
std::vector<std::string> field_vars;
field_vars.push_back(_to_var_name);
idi->set_field_variables(field_vars);
{
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i) && _multi_app->isRootProcessor())
{
src_pts.push_back(_multi_app->position(i));
src_vals.push_back(_multi_app->appPostprocessorValue(i,_postprocessor));
}
}
}
// We have only set local values - prepare for use by gathering remote gata
idi->prepare_for_use();
// Loop over the master nodes and set the value of the variable
{
System * to_sys = find_sys(_multi_app->problem()->es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = *to_sys->solution;
MooseMesh & mesh = _multi_app->problem()->mesh();
std::vector<std::string> vars;
vars.push_back(_to_var_name);
MeshBase::const_node_iterator node_it = mesh.localNodesBegin();
MeshBase::const_node_iterator node_end = mesh.localNodesEnd();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(*node);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution.set(dof, value);
}
}
solution.close();
}
_multi_app->problem()->es().update();
delete idi;
break;
}
}
}
void
MultiAppMeshFunctionTransfer::execute()
{
Moose::out << "Beginning MeshFunctionTransfer " << name() << std::endl;
getAppInfo();
/**
* For every combination of global "from" problem and local "to" problem, find
* which "from" bounding boxes overlap with which "to" elements. Keep track
* of which processors own bounding boxes that overlap with which elements.
* Build vectors of node locations/element centroids to send to other
* processors for mesh function evaluations.
*/
// Get the bounding boxes for the "from" domains.
std::vector<MeshTools::BoundingBox> bboxes = getFromBoundingBoxes();
// Figure out how many "from" domains each processor owns.
std::vector<unsigned int> froms_per_proc = getFromsPerProc();
std::vector<std::vector<Point> > outgoing_points(n_processors());
std::vector<std::map<std::pair<unsigned int, unsigned int>, unsigned int> > point_index_map(n_processors());
// point_index_map[i_to, element_id] = index
// outgoing_points[index] is the first quadrature point in element
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
MeshBase * to_mesh = & _to_meshes[i_to]->getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
// Loop over the "froms" on processor i_proc. If the node is found in
// any of the "froms", add that node to the vector that will be sent to
// i_proc.
unsigned int from0 = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool point_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && ! point_found; i_from++)
{
if (bboxes[i_from].contains_point(*node + _to_positions[i_to]))
{
std::pair<unsigned int, unsigned int> key(i_to, node->id());
point_index_map[i_proc][key] = outgoing_points[i_proc].size();
outgoing_points[i_proc].push_back(*node + _to_positions[i_to]);
point_found = true;
}
}
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
// Loop over the "froms" on processor i_proc. If the elem is found in
// any of the "froms", add that elem to the vector that will be sent to
// i_proc.
unsigned int from0 = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool point_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && ! point_found; i_from++)
{
if (bboxes[i_from].contains_point(centroid + _to_positions[i_to]))
{
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
point_index_map[i_proc][key] = outgoing_points[i_proc].size();
outgoing_points[i_proc].push_back(centroid + _to_positions[i_to]);
point_found = true;
}
}
}
}
}
}
/**
* Request point evaluations from other processors and handle requests sent to
* this processor.
*/
// Get the local bounding boxes.
std::vector<MeshTools::BoundingBox> local_bboxes(froms_per_proc[processor_id()]);
{
// Find the index to the first of this processor's local bounding boxes.
unsigned int local_start = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors() && i_proc != processor_id();
i_proc++)
{
local_start += froms_per_proc[i_proc];
}
// Extract the local bounding boxes.
for (unsigned int i_from = 0; i_from < froms_per_proc[processor_id()]; i_from++)
{
local_bboxes[i_from] = bboxes[local_start + i_from];
}
}
// Setup the local mesh functions.
std::vector<MooseSharedPointer<MeshFunction> > local_meshfuns;
for (unsigned int i_from = 0; i_from < _from_problems.size(); i_from++)
{
FEProblem & from_problem = *_from_problems[i_from];
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
System & from_sys = from_var.sys().system();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
MooseSharedPointer<MeshFunction> from_func;
//TODO: make MultiAppTransfer give me the right es
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_func.reset(new MeshFunction(from_problem.getDisplacedProblem()->es(),
*from_sys.current_local_solution, from_sys.get_dof_map(), from_var_num));
else
from_func.reset(new MeshFunction(from_problem.es(),
*from_sys.current_local_solution, from_sys.get_dof_map(), from_var_num));
from_func->init(Trees::ELEMENTS);
from_func->enable_out_of_mesh_mode(OutOfMeshValue);
local_meshfuns.push_back(from_func);
}
// Send points to other processors.
std::vector<std::vector<Real> > incoming_evals(n_processors());
std::vector<std::vector<unsigned int> > incoming_app_ids(n_processors());
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.send(i_proc, outgoing_points[i_proc]);
}
// Recieve points from other processors, evaluate mesh frunctions at those
// points, and send the values back.
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
std::vector<Point> incoming_points;
if (i_proc == processor_id())
incoming_points = outgoing_points[i_proc];
else
_communicator.receive(i_proc, incoming_points);
std::vector<Real> outgoing_evals(incoming_points.size(), OutOfMeshValue);
std::vector<unsigned int> outgoing_ids(incoming_points.size(), -1); // -1 = largest unsigned int
for (unsigned int i_pt = 0; i_pt < incoming_points.size(); i_pt++)
{
Point pt = incoming_points[i_pt];
// Loop until we've found the lowest-ranked app that actually contains
// the quadrature point.
for (unsigned int i_from = 0; i_from < _from_problems.size() && outgoing_evals[i_pt] == OutOfMeshValue; i_from++)
{
if (local_bboxes[i_from].contains_point(pt))
{
outgoing_evals[i_pt] = (* local_meshfuns[i_from])(pt - _from_positions[i_from]);
if (_direction == FROM_MULTIAPP)
outgoing_ids[i_pt] = _local2global_map[i_from];
}
}
}
if (i_proc == processor_id())
{
incoming_evals[i_proc] = outgoing_evals;
if (_direction == FROM_MULTIAPP)
incoming_app_ids[i_proc] = outgoing_ids;
}
else
{
_communicator.send(i_proc, outgoing_evals);
if (_direction == FROM_MULTIAPP)
_communicator.send(i_proc, outgoing_ids);
}
}
/**
* Gather all of the evaluations, pick out the best ones for each point, and
* apply them to the solution vector. When we are transferring from
* multiapps, there may be multiple overlapping apps for a particular point.
* In that case, we'll try to use the value from the app with the lowest id.
*/
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.receive(i_proc, incoming_evals[i_proc]);
if (_direction == FROM_MULTIAPP)
_communicator.receive(i_proc, incoming_app_ids[i_proc]);
}
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> * solution;
switch (_direction)
{
case TO_MULTIAPP:
solution = & getTransferVector(i_to, _to_var_name);
break;
case FROM_MULTIAPP:
solution = to_sys->solution.get();
break;
}
MeshBase * to_mesh = & _to_meshes[i_to]->getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
unsigned int lowest_app_rank = libMesh::invalid_uint;
Real best_val = 0.;
bool point_found = false;
for (unsigned int i_proc = 0; i_proc < incoming_evals.size(); i_proc++)
{
// Skip this proc if the node wasn't in it's bounding boxes.
std::pair<unsigned int, unsigned int> key(i_to, node->id());
if (point_index_map[i_proc].find(key) == point_index_map[i_proc].end())
continue;
unsigned int i_pt = point_index_map[i_proc][key];
// Ignore this proc if it's app has a higher rank than the
// previously found lowest app rank.
if (_direction == FROM_MULTIAPP)
{
if (incoming_app_ids[i_proc][i_pt] >= lowest_app_rank)
continue;
}
// Ignore this proc if the point was actually outside its meshes.
if (incoming_evals[i_proc][i_pt] == OutOfMeshValue)
continue;
best_val = incoming_evals[i_proc][i_pt];
point_found = true;
}
if (_error_on_miss && ! point_found)
mooseError("Point not found! " << *node + _to_positions[i_to]);
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
unsigned int lowest_app_rank = libMesh::invalid_uint;
Real best_val = 0;
bool point_found = false;
for (unsigned int i_proc = 0; i_proc < incoming_evals.size(); i_proc++)
{
// Skip this proc if the elem wasn't in it's bounding boxes.
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
if (point_index_map[i_proc].find(key) == point_index_map[i_proc].end())
continue;
unsigned int i_pt = point_index_map[i_proc][key];
// Ignore this proc if it's app has a higher rank than the
// previously found lowest app rank.
if (_direction == FROM_MULTIAPP)
{
if (incoming_app_ids[i_proc][i_pt] >= lowest_app_rank)
continue;
}
// Ignore this proc if the point was actually outside its meshes.
if (incoming_evals[i_proc][i_pt] == OutOfMeshValue)
continue;
best_val = incoming_evals[i_proc][i_pt];
point_found = true;
}
if (_error_on_miss && ! point_found)
mooseError("Point not found! " << elem->centroid() + _to_positions[i_to]);
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
solution->close();
to_sys->update();
}
_console << "Finished MeshFunctionTransfer " << name() << std::endl;
}
void
MultiAppVariableValueSampleTransfer::execute()
{
_console << "Beginning VariableValueSampleTransfer " << name() << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = _multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
SubProblem & from_sub_problem = from_system_base.subproblem();
MooseMesh & from_mesh = from_problem.mesh();
std::unique_ptr<PointLocatorBase> pl = from_mesh.getPointLocator();
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
Real value = -std::numeric_limits<Real>::max();
{ // Get the value of the variable at the point where this multiapp is in the master domain
Point multi_app_position = _multi_app->position(i);
std::vector<Point> point_vec(1, multi_app_position);
// First find the element the hit lands in
const Elem * elem = (*pl)(multi_app_position);
if (elem && elem->processor_id() == from_mesh.processor_id())
{
from_sub_problem.reinitElemPhys(elem, point_vec, 0);
mooseAssert(from_var.sln().size() == 1, "No values in u!");
value = from_var.sln()[0];
}
_communicator.max(value);
if (value == -std::numeric_limits<Real>::max())
mooseError("Transfer failed to sample point value at point: " << multi_app_position);
}
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i).es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MooseMesh & mesh = _multi_app->appProblem(i).mesh();
MeshBase::const_node_iterator node_it = mesh.localNodesBegin();
MeshBase::const_node_iterator node_end = mesh.localNodesEnd();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution.set(dof, value);
}
}
solution.close();
_multi_app->appProblem(i).es().update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
break;
}
case FROM_MULTIAPP:
{
mooseError("Doesn't make sense to transfer a sampled variable's value from a MultiApp!!");
break;
}
}
_console << "Finished VariableValueSampleTransfer " << name() << std::endl;
}
void
MultiAppUserObjectTransfer::execute()
{
_console << "Beginning MultiAppUserObjectTransfer " << name() << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i).es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i).getDisplacedProblem())
{
mesh = &_multi_app->appProblem(i).getDisplacedProblem()->mesh().getMesh();
}
else
mesh = &_multi_app->appProblem(i).mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
const UserObject & user_object = _multi_app->problem().getUserObjectBase(_user_object_name);
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = user_object.spatialValue(*node+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = user_object.spatialValue(centroid+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = _multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppUserObjectTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
_console << "Transferring to: " << to_var.name() << std::endl;
// EquationSystems & to_es = to_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * to_solution = to_sys.solution.get();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num).family == LAGRANGE;
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
Point app_position = _multi_app->position(i);
MeshTools::BoundingBox app_box = _multi_app->getBoundingBox(i);
const UserObject & user_object = _multi_app->appUserObjectBase(i, _user_object_name);
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this node
{
// See if this node falls in this bounding box
if (app_box.contains_point(*node))
{
dof_id_type dof = node->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = user_object.spatialValue(*node-app_position);
Moose::swapLibMeshComm(swapped);
to_solution->set(dof, from_value);
}
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
if (elem->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this elem
{
Point centroid = elem->centroid();
// See if this elem falls in this bounding box
if (app_box.contains_point(centroid))
{
dof_id_type dof = elem->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = user_object.spatialValue(centroid-app_position);
Moose::swapLibMeshComm(swapped);
to_solution->set(dof, from_value);
}
}
}
}
}
to_solution->close();
to_sys.update();
break;
}
}
_console << "Finished MultiAppUserObjectTransfer " << name() << std::endl;
}
void
MultiAppMeshFunctionTransfer::execute()
{
Moose::out << "Beginning MeshFunctionTransfer " << _name << std::endl;
switch(_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = *_multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppMeshFunctionTransfer only works with SerialMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
EquationSystems & from_es = from_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * serialized_solution = NumericVector<Number>::build().release();
serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_solution);
MeshFunction from_func(from_es, *serialized_solution, from_sys.get_dof_map(), from_var_num);
from_func.init(Trees::ELEMENTS);
from_func.enable_out_of_mesh_mode(NOTFOUND);
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i)->es(), _to_var_name);
if (!to_sys)
mooseError("Cannot find variable "<<_to_var_name<<" for "<<_name<<" Transfer");
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase & mesh = _multi_app->appProblem(i)->mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh.local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh.local_nodes_end();
for(; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = from_func(*node+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
if (from_value != NOTFOUND)
solution.set(dof, from_value);
else if (_error_on_miss)
mooseError("Point not found! " << *node+_multi_app->position(i) << std::endl);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh.local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh.local_elements_end();
for(; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real from_value = from_func(centroid+_multi_app->position(i));
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
if (from_value != NOTFOUND)
solution.set(dof, from_value);
else if (_error_on_miss)
mooseError("Point not found! " << centroid+_multi_app->position(i) << std::endl);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
delete serialized_solution;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = *_multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppMeshFunctionTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
EquationSystems & to_es = to_sys.get_equation_systems();
NumericVector<Number> * to_solution = to_sys.solution.get();
MeshBase & to_mesh = to_es.get_mesh();
bool is_nodal = to_sys.variable_type(to_var_num).family == LAGRANGE;
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppMeshFunctionTransfer only works with SerialMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
EquationSystems & from_es = from_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * serialized_from_solution = NumericVector<Number>::build().release();
serialized_from_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_from_solution);
MeshBase & from_mesh = from_es.get_mesh();
MeshTools::BoundingBox app_box = MeshTools::processor_bounding_box(from_mesh, libMesh::processor_id());
Point app_position = _multi_app->position(i);
MeshFunction from_func(from_es, *serialized_from_solution, from_sys.get_dof_map(), from_var_num);
from_func.init(Trees::ELEMENTS);
from_func.enable_out_of_mesh_mode(NOTFOUND);
Moose::swapLibMeshComm(swapped);
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh.nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh.nodes_end();
for(; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this node
{
// See if this node falls in this bounding box
if (app_box.contains_point(*node-app_position))
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = from_func(*node-app_position);
Moose::swapLibMeshComm(swapped);
if (from_value != NOTFOUND)
to_solution->set(dof, from_value);
else if (_error_on_miss)
mooseError("Point not found! " << *node-app_position <<std::endl);
}
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh.elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh.elements_end();
for(; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
if (elem->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this elem
{
Point centroid = elem->centroid();
// See if this elem falls in this bounding box
if (app_box.contains_point(centroid-app_position))
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Real from_value = from_func(centroid-app_position);
Moose::swapLibMeshComm(swapped);
if (from_value != NOTFOUND)
to_solution->set(dof, from_value);
else if (_error_on_miss)
mooseError("Point not found! " << centroid-app_position << std::endl);
}
}
}
}
delete serialized_from_solution;
}
to_solution->close();
to_sys.update();
break;
}
}
Moose::out << "Finished MeshFunctionTransfer " << _name << std::endl;
}
void
MultiAppInterpolationTransfer::execute()
{
_console << "Beginning InterpolationTransfer " << _name << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = *_multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
bool from_is_nodal = from_sys.variable_type(from_var_num).family == LAGRANGE;
// EquationSystems & from_es = from_sys.get_equation_systems();
NumericVector<Number> & from_solution = *from_sys.solution;
InverseDistanceInterpolation<LIBMESH_DIM> * idi;
switch (_interp_type)
{
case 0:
idi = new InverseDistanceInterpolation<LIBMESH_DIM>(from_sys.comm(), _num_points, _power);
break;
case 1:
idi = new RadialBasisInterpolation<LIBMESH_DIM>(from_sys.comm(), _radius);
break;
default:
mooseError("Unknown interpolation type!");
}
std::vector<Point> &src_pts (idi->get_source_points());
std::vector<Number> &src_vals (idi->get_source_vals());
std::vector<std::string> field_vars;
field_vars.push_back(_to_var_name);
idi->set_field_variables(field_vars);
std::vector<std::string> vars;
vars.push_back(_to_var_name);
if (from_is_nodal)
{
MeshBase::const_node_iterator from_nodes_it = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
for (; from_nodes_it != from_nodes_end; ++from_nodes_it)
{
Node * from_node = *from_nodes_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(*from_node);
src_vals.push_back(from_solution(from_dof));
}
}
else
{
MeshBase::const_element_iterator from_elements_it = from_mesh->local_elements_begin();
MeshBase::const_element_iterator from_elements_end = from_mesh->local_elements_end();
for (; from_elements_it != from_elements_end; ++from_elements_it)
{
Elem * from_elem = *from_elements_it;
// Assuming CONSTANT MONOMIAL
dof_id_type from_dof = from_elem->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(from_elem->centroid());
src_vals.push_back(from_solution(from_dof));
}
}
// We have only set local values - prepare for use by gathering remote gata
idi->prepare_for_use();
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i)->es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i)->getDisplacedProblem())
mesh = &_multi_app->appProblem(i)->getDisplacedProblem()->mesh().getMesh();
else
mesh = &_multi_app->appProblem(i)->mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
Point actual_position = *node+_multi_app->position(i);
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(actual_position);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution.set(dof, value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
Point actual_position = centroid+_multi_app->position(i);
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(actual_position);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
solution.set(dof, value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
delete idi;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = *_multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
NumericVector<Real> & to_solution = *to_sys.solution;
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppInterpolationTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
// EquationSystems & to_es = to_sys.get_equation_systems();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num).family == LAGRANGE;
InverseDistanceInterpolation<LIBMESH_DIM> * idi;
switch (_interp_type)
{
case 0:
idi = new InverseDistanceInterpolation<LIBMESH_DIM>(to_sys.comm(), _num_points, _power);
break;
case 1:
idi = new RadialBasisInterpolation<LIBMESH_DIM>(to_sys.comm(), _radius);
break;
default:
mooseError("Unknown interpolation type!");
}
std::vector<Point> &src_pts (idi->get_source_points());
std::vector<Number> &src_vals (idi->get_source_vals());
std::vector<std::string> field_vars;
field_vars.push_back(_to_var_name);
idi->set_field_variables(field_vars);
std::vector<std::string> vars;
vars.push_back(_to_var_name);
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
bool from_is_nodal = from_sys.variable_type(from_var_num).family == LAGRANGE;
// EquationSystems & from_es = from_sys.get_equation_systems();
NumericVector<Number> & from_solution = *from_sys.solution;
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
Point app_position = _multi_app->position(i);
if (from_is_nodal)
{
MeshBase::const_node_iterator from_nodes_it = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
for (; from_nodes_it != from_nodes_end; ++from_nodes_it)
{
Node * from_node = *from_nodes_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(*from_node+app_position);
src_vals.push_back(from_solution(from_dof));
}
}
else
{
MeshBase::const_element_iterator from_elements_it = from_mesh->local_elements_begin();
MeshBase::const_element_iterator from_elements_end = from_mesh->local_elements_end();
for (; from_elements_it != from_elements_end; ++from_elements_it)
{
Elem * from_element = *from_elements_it;
// Assuming LAGRANGE!
dof_id_type from_dof = from_element->dof_number(from_sys_num, from_var_num, 0);
src_pts.push_back(from_element->centroid()+app_position);
src_vals.push_back(from_solution(from_dof));
}
}
Moose::swapLibMeshComm(swapped);
}
// We have only set local values - prepare for use by gathering remote gata
idi->prepare_for_use();
// Now do the interpolation to the target system
if (is_nodal)
{
MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
if (node->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this node
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(*node);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(to_sys_num, to_var_num, 0);
to_solution.set(dof, value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end();
for (; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
if (elem->n_dofs(to_sys_num, to_var_num) > 0) // If this variable has dofs at this elem
{
std::vector<Point> pts;
std::vector<Number> vals;
pts.push_back(centroid);
vals.resize(1);
idi->interpolate_field_data(vars, pts, vals);
Real value = vals.front();
dof_id_type dof = elem->dof_number(to_sys_num, to_var_num, 0);
to_solution.set(dof, value);
}
}
}
to_solution.close();
to_sys.update();
delete idi;
break;
}
}
_console << "Finished InterpolationTransfer " << _name << std::endl;
}
void
MultiAppNearestNodeTransfer::execute()
{
_console << "Beginning NearestNodeTransfer " << name() << std::endl;
getAppInfo();
// Get the bounding boxes for the "from" domains.
std::vector<BoundingBox> bboxes;
if (isParamValid("source_boundary"))
bboxes = getFromBoundingBoxes(
_from_meshes[0]->getBoundaryID(getParam<BoundaryName>("source_boundary")));
else
bboxes = getFromBoundingBoxes();
// Figure out how many "from" domains each processor owns.
std::vector<unsigned int> froms_per_proc = getFromsPerProc();
////////////////////
// For every point in the local "to" domain, figure out which "from" domains
// might contain it's nearest neighbor, and send that point to the processors
// that own those "from" domains.
//
// How do we know which "from" domains might contain the nearest neighbor, you
// ask? Well, consider two "from" domains, A and B. If every point in A is
// closer than every point in B, then we know that B cannot possibly contain
// the nearest neighbor. Hence, we'll only check A for the nearest neighbor.
// We'll use the functions bboxMaxDistance and bboxMinDistance to figure out
// if every point in A is closer than every point in B.
////////////////////
// outgoing_qps = nodes/centroids we'll send to other processors.
std::vector<std::vector<Point>> outgoing_qps(n_processors());
// When we get results back, node_index_map will tell us which results go with
// which points
std::vector<std::map<std::pair<unsigned int, unsigned int>, unsigned int>> node_index_map(
n_processors());
if (!_neighbors_cached)
{
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
MeshBase * to_mesh = &_to_meshes[i_to]->getMesh();
bool is_to_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_to_nodal)
{
std::vector<Node *> target_local_nodes;
if (isParamValid("target_boundary"))
{
BoundaryID target_bnd_id =
_to_meshes[i_to]->getBoundaryID(getParam<BoundaryName>("target_boundary"));
ConstBndNodeRange & bnd_nodes = *(_to_meshes[i_to])->getBoundaryNodeRange();
for (const auto & bnode : bnd_nodes)
if (bnode->_bnd_id == target_bnd_id && bnode->_node->processor_id() == processor_id())
target_local_nodes.push_back(bnode->_node);
}
else
{
target_local_nodes.resize(to_mesh->n_local_nodes());
unsigned int i = 0;
for (auto & node : to_mesh->local_node_ptr_range())
target_local_nodes[i++] = node;
}
// For error checking: keep track of all target_local_nodes
// which are successfully mapped to at least one domain where
// the nearest neighbor might be found.
std::set<Node *> local_nodes_found;
for (const auto & node : target_local_nodes)
{
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
// Find which bboxes might have the nearest node to this point.
Real nearest_max_distance = std::numeric_limits<Real>::max();
for (const auto & bbox : bboxes)
{
Real distance = bboxMaxDistance(*node, bbox);
if (distance < nearest_max_distance)
nearest_max_distance = distance;
}
unsigned int from0 = 0;
for (processor_id_type i_proc = 0; i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool qp_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && !qp_found;
i_from++)
{
Real distance = bboxMinDistance(*node, bboxes[i_from]);
if (distance <= nearest_max_distance || bboxes[i_from].contains_point(*node))
{
std::pair<unsigned int, unsigned int> key(i_to, node->id());
node_index_map[i_proc][key] = outgoing_qps[i_proc].size();
outgoing_qps[i_proc].push_back(*node + _to_positions[i_to]);
qp_found = true;
local_nodes_found.insert(node);
}
}
}
}
// By the time we get to here, we should have found at least
// one candidate BoundingBox for every node in the
// target_local_nodes array that has dofs for the current
// variable in the current System.
for (const auto & node : target_local_nodes)
if (node->n_dofs(sys_num, var_num) && !local_nodes_found.count(node))
mooseError("In ",
name(),
": No candidate BoundingBoxes found for node ",
node->id(),
" at position ",
*node);
}
else // Elemental
{
// For error checking: keep track of all local elements
// which are successfully mapped to at least one domain where
// the nearest neighbor might be found.
std::set<Elem *> local_elems_found;
for (auto & elem : as_range(to_mesh->local_elements_begin(), to_mesh->local_elements_end()))
{
Point centroid = elem->centroid();
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
// Find which bboxes might have the nearest node to this point.
Real nearest_max_distance = std::numeric_limits<Real>::max();
for (const auto & bbox : bboxes)
{
Real distance = bboxMaxDistance(centroid, bbox);
if (distance < nearest_max_distance)
nearest_max_distance = distance;
}
unsigned int from0 = 0;
for (processor_id_type i_proc = 0; i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool qp_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && !qp_found;
i_from++)
{
Real distance = bboxMinDistance(centroid, bboxes[i_from]);
if (distance <= nearest_max_distance || bboxes[i_from].contains_point(centroid))
{
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
node_index_map[i_proc][key] = outgoing_qps[i_proc].size();
outgoing_qps[i_proc].push_back(centroid + _to_positions[i_to]);
qp_found = true;
local_elems_found.insert(elem);
}
}
}
}
// Verify that we found at least one candidate bounding
// box for each local element with dofs for the current
// variable in the current System.
for (auto & elem : as_range(to_mesh->local_elements_begin(), to_mesh->local_elements_end()))
if (elem->n_dofs(sys_num, var_num) && !local_elems_found.count(elem))
mooseError("In ",
name(),
": No candidate BoundingBoxes found for Elem ",
elem->id(),
", centroid = ",
elem->centroid());
}
}
}
////////////////////
// Send local node/centroid positions off to the other processors and take
// care of points sent to this processor. We'll need to check the points
// against all of the "from" domains that this processor owns. For each
// point, we'll find the nearest node, then we'll send the value at that node
// and the distance between the node and the point back to the processor that
// requested that point.
////////////////////
std::vector<std::vector<Real>> incoming_evals(n_processors());
std::vector<Parallel::Request> send_qps(n_processors());
std::vector<Parallel::Request> send_evals(n_processors());
// Create these here so that they live the entire life of this function
// and are NOT reused per processor.
std::vector<std::vector<Real>> processor_outgoing_evals(n_processors());
if (!_neighbors_cached)
{
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.send(i_proc, outgoing_qps[i_proc], send_qps[i_proc]);
}
// Build an array of pointers to all of this processor's local entities (nodes or
// elements). We need to do this to avoid the expense of using LibMesh iterators.
// This step also takes care of limiting the search to boundary nodes, if
// applicable.
std::vector<std::vector<std::pair<Point, DofObject *>>> local_entities(
froms_per_proc[processor_id()]);
// Local array of all from Variable references
std::vector<std::reference_wrapper<MooseVariableFEBase>> _from_vars;
for (unsigned int i = 0; i < froms_per_proc[processor_id()]; i++)
{
MooseVariableFEBase & from_var = _from_problems[i]->getVariable(
0, _from_var_name, Moose::VarKindType::VAR_ANY, Moose::VarFieldType::VAR_FIELD_STANDARD);
bool is_to_nodal = from_var.feType().family == LAGRANGE;
_from_vars.emplace_back(from_var);
getLocalEntities(_from_meshes[i], local_entities[i], is_to_nodal);
}
if (_fixed_meshes)
{
_cached_froms.resize(n_processors());
_cached_dof_ids.resize(n_processors());
}
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
// We either use our own outgoing_qps or receive them from
// another processor.
std::vector<Point> incoming_qps;
if (i_proc == processor_id())
incoming_qps = outgoing_qps[i_proc];
else
_communicator.receive(i_proc, incoming_qps);
if (_fixed_meshes)
{
_cached_froms[i_proc].resize(incoming_qps.size());
_cached_dof_ids[i_proc].resize(incoming_qps.size());
}
std::vector<Real> & outgoing_evals = processor_outgoing_evals[i_proc];
// Resize this vector to two times the size of the incoming_qps
// vector because we are going to store both the value from the nearest
// local node *and* the distance between the incoming_qp and that node
// for later comparison purposes.
outgoing_evals.resize(2 * incoming_qps.size());
for (unsigned int qp = 0; qp < incoming_qps.size(); qp++)
{
const Point & qpt = incoming_qps[qp];
outgoing_evals[2 * qp] = std::numeric_limits<Real>::max();
for (unsigned int i_local_from = 0; i_local_from < froms_per_proc[processor_id()];
i_local_from++)
{
MooseVariableFEBase & from_var = _from_vars[i_local_from];
System & from_sys = from_var.sys().system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
for (unsigned int i_node = 0; i_node < local_entities[i_local_from].size(); i_node++)
{
// Compute distance between the current incoming_qp to local node i_node.
Real current_distance =
(qpt - local_entities[i_local_from][i_node].first - _from_positions[i_local_from])
.norm();
// If an incoming_qp is equally close to two or more local nodes, then
// the first one we test will "win", even though any of the others could
// also potentially be chosen instead... there's no way to decide among
// the set of all equidistant points.
//
// outgoing_evals[2 * qp] is the current closest distance between a local point and
// the incoming_qp.
if (current_distance < outgoing_evals[2 * qp])
{
// Assuming LAGRANGE!
if (local_entities[i_local_from][i_node].second->n_dofs(from_sys_num, from_var_num) >
0)
{
dof_id_type from_dof = local_entities[i_local_from][i_node].second->dof_number(
from_sys_num, from_var_num, 0);
// The indexing of the outgoing_evals vector looks
// like [(distance, value), (distance, value), ...]
// for each incoming_qp. We only keep the value from
// the node with the smallest distance to the
// incoming_qp, and then we compare across all
// processors later and pick the closest one.
outgoing_evals[2 * qp] = current_distance;
outgoing_evals[2 * qp + 1] = (*from_sys.solution)(from_dof);
if (_fixed_meshes)
{
// Cache the nearest nodes.
_cached_froms[i_proc][qp] = i_local_from;
_cached_dof_ids[i_proc][qp] = from_dof;
}
}
}
}
}
}
if (i_proc == processor_id())
incoming_evals[i_proc] = outgoing_evals;
else
_communicator.send(i_proc, outgoing_evals, send_evals[i_proc]);
}
}
else // We've cached the nearest nodes.
{
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
std::vector<Real> & outgoing_evals = processor_outgoing_evals[i_proc];
outgoing_evals.resize(_cached_froms[i_proc].size());
for (unsigned int qp = 0; qp < outgoing_evals.size(); qp++)
{
MooseVariableFEBase & from_var = _from_problems[_cached_froms[i_proc][qp]]->getVariable(
0,
_from_var_name,
Moose::VarKindType::VAR_ANY,
Moose::VarFieldType::VAR_FIELD_STANDARD);
System & from_sys = from_var.sys().system();
dof_id_type from_dof = _cached_dof_ids[i_proc][qp];
// outgoing_evals[qp] = (*from_sys.solution)(_cached_dof_ids[i_proc][qp]);
outgoing_evals[qp] = (*from_sys.solution)(from_dof);
}
if (i_proc == processor_id())
incoming_evals[i_proc] = outgoing_evals;
else
_communicator.send(i_proc, outgoing_evals, send_evals[i_proc]);
}
}
////////////////////
// Gather all of the evaluations, find the nearest one for each node/element,
// and apply the values.
////////////////////
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.receive(i_proc, incoming_evals[i_proc]);
}
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> * solution = nullptr;
switch (_direction)
{
case TO_MULTIAPP:
solution = &getTransferVector(i_to, _to_var_name);
break;
case FROM_MULTIAPP:
solution = to_sys->solution.get();
break;
default:
mooseError("Unknown direction");
}
const MeshBase & to_mesh = _to_meshes[i_to]->getMesh();
bool is_to_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_to_nodal)
{
std::vector<Node *> target_local_nodes;
if (isParamValid("target_boundary"))
{
BoundaryID target_bnd_id =
_to_meshes[i_to]->getBoundaryID(getParam<BoundaryName>("target_boundary"));
ConstBndNodeRange & bnd_nodes = *(_to_meshes[i_to])->getBoundaryNodeRange();
for (const auto & bnode : bnd_nodes)
if (bnode->_bnd_id == target_bnd_id && bnode->_node->processor_id() == processor_id())
target_local_nodes.push_back(bnode->_node);
}
else
{
target_local_nodes.resize(to_mesh.n_local_nodes());
unsigned int i = 0;
for (auto & node : to_mesh.local_node_ptr_range())
target_local_nodes[i++] = node;
}
for (const auto & node : target_local_nodes)
{
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
// If nothing is in the node_index_map for a given local node,
// it will get the value 0.
Real best_val = 0;
if (!_neighbors_cached)
{
// Search through all the incoming evaluation points from
// different processors for the one with the closest
// point. If there are multiple values from other processors
// which are equidistant, the first one we check will "win".
Real min_dist = std::numeric_limits<Real>::max();
for (unsigned int i_from = 0; i_from < incoming_evals.size(); i_from++)
{
std::pair<unsigned int, unsigned int> key(i_to, node->id());
if (node_index_map[i_from].find(key) == node_index_map[i_from].end())
continue;
unsigned int qp_ind = node_index_map[i_from][key];
if (incoming_evals[i_from][2 * qp_ind] >= min_dist)
continue;
// If we made it here, we are going set a new value and
// distance because we found one that was closer.
min_dist = incoming_evals[i_from][2 * qp_ind];
best_val = incoming_evals[i_from][2 * qp_ind + 1];
if (_fixed_meshes)
{
// Cache these indices.
_cached_from_inds[node->id()] = i_from;
_cached_qp_inds[node->id()] = qp_ind;
}
}
}
else
{
best_val = incoming_evals[_cached_from_inds[node->id()]][_cached_qp_inds[node->id()]];
}
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
else // Elemental
{
for (auto & elem : to_mesh.active_local_element_ptr_range())
{
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
Real best_val = 0;
if (!_neighbors_cached)
{
Real min_dist = std::numeric_limits<Real>::max();
for (unsigned int i_from = 0; i_from < incoming_evals.size(); i_from++)
{
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
if (node_index_map[i_from].find(key) == node_index_map[i_from].end())
continue;
unsigned int qp_ind = node_index_map[i_from][key];
if (incoming_evals[i_from][2 * qp_ind] >= min_dist)
continue;
min_dist = incoming_evals[i_from][2 * qp_ind];
best_val = incoming_evals[i_from][2 * qp_ind + 1];
if (_fixed_meshes)
{
// Cache these indices.
_cached_from_inds[elem->id()] = i_from;
_cached_qp_inds[elem->id()] = qp_ind;
}
}
}
else
{
best_val = incoming_evals[_cached_from_inds[elem->id()]][_cached_qp_inds[elem->id()]];
}
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
solution->close();
to_sys->update();
}
if (_fixed_meshes)
_neighbors_cached = true;
// Make sure all our sends succeeded.
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
send_qps[i_proc].wait();
send_evals[i_proc].wait();
}
_console << "Finished NearestNodeTransfer " << name() << std::endl;
}
void
MultiAppCopyTransfer::execute()
{
_console << "Beginning CopyTransfer " << name() << std::endl;
switch (_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = _multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
MeshBase * from_mesh = NULL;
from_mesh = &from_problem.mesh().getMesh();
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppCopyTransfer only works with ReplicatedMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
//Create a serialized version of the solution vector
// NumericVector<Number> * serialized_solution = NumericVector<Number>::build(from_sys.comm()).release();
// serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
// from_sys.solution->localize(*serialized_solution);
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i).es(), _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
mesh = &_multi_app->appProblem(i).mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for (; node_it != node_end; ++node_it)
{
Node * node = *node_it;
unsigned int node_id = node->id();
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Node * from_node = from_mesh->node_ptr(node_id);
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
//Real from_value = (*serialized_solution)(from_dof);
Real from_value = (*from_sys.solution)(from_dof);
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
else // Elemental
{
mooseError("MultiAppCopyTransfer can only be used on nodal variables");
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
// delete serialized_solution;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = _multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
NumericVector<Real> & to_solution = *to_sys.solution;
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppCopyTransfer only works with ReplicatedMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
MeshBase * to_mesh = NULL;
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num) == FEType();
for (unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = _multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppCopyTransfer only works with ReplicatedMesh!");
//Create a serialized version of the solution vector
// NumericVector<Number> * serialized_solution = NumericVector<Number>::build(from_sys.comm()).release();
// serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
// from_sys.solution->localize(*serialized_solution);
MeshBase * from_mesh = &from_problem.mesh().getMesh();
Moose::swapLibMeshComm(swapped);
if (is_nodal)
{
MeshBase::const_node_iterator to_node_it = to_mesh->local_nodes_begin();
MeshBase::const_node_iterator to_node_end = to_mesh->local_nodes_end();
for (; to_node_it != to_node_end; ++to_node_it)
{
Node * to_node = *to_node_it;
unsigned int to_node_id = to_node->id();
// The zero only works for LAGRANGE!
dof_id_type to_dof = to_node->dof_number(to_sys_num, to_var_num, 0);
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
Node * from_node = from_mesh->node_ptr(to_node_id);
// Assuming LAGRANGE!
dof_id_type from_dof = from_node->dof_number(from_sys_num, from_var_num, 0);
//Real from_value = (*serialized_solution)(from_dof);
Real from_value = (*from_sys.solution)(from_dof);
// Swap back
Moose::swapLibMeshComm(swapped);
to_solution.set(to_dof, from_value);
}
}
else // Elemental
{
mooseError("MultiAppCopyTransfer can only be used on nodal variables");
}
// delete serialized_solution;
}
to_solution.close();
to_sys.update();
break;
}
}
_console << "Finished CopyTransfer " << name() << std::endl;
}
void
MultiAppNearestNodeTransfer::execute()
{
_console << "Beginning NearestNodeTransfer " << name() << std::endl;
getAppInfo();
// Get the bounding boxes for the "from" domains.
std::vector<BoundingBox> bboxes = getFromBoundingBoxes();
// Figure out how many "from" domains each processor owns.
std::vector<unsigned int> froms_per_proc = getFromsPerProc();
////////////////////
// For every point in the local "to" domain, figure out which "from" domains
// might contain it's nearest neighbor, and send that point to the processors
// that own those "from" domains.
//
// How do we know which "from" domains might contain the nearest neighbor, you
// ask? Well, consider two "from" domains, A and B. If every point in A is
// closer than every point in B, then we know that B cannot possibly contain
// the nearest neighbor. Hence, we'll only check A for the nearest neighbor.
// We'll use the functions bboxMaxDistance and bboxMinDistance to figure out
// if every point in A is closer than every point in B.
////////////////////
// outgoing_qps = nodes/centroids we'll send to other processors.
std::vector<std::vector<Point>> outgoing_qps(n_processors());
// When we get results back, node_index_map will tell us which results go with
// which points
std::vector<std::map<std::pair<unsigned int, unsigned int>, unsigned int>> node_index_map(
n_processors());
if (!_neighbors_cached)
{
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
MeshBase * to_mesh = &_to_meshes[i_to]->getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
std::vector<Node *> target_local_nodes;
if (isParamValid("target_boundary"))
{
BoundaryID target_bnd_id =
_to_meshes[i_to]->getBoundaryID(getParam<BoundaryName>("target_boundary"));
ConstBndNodeRange & bnd_nodes = *(_to_meshes[i_to])->getBoundaryNodeRange();
for (const auto & bnode : bnd_nodes)
if (bnode->_bnd_id == target_bnd_id && bnode->_node->processor_id() == processor_id())
target_local_nodes.push_back(bnode->_node);
}
else
{
target_local_nodes.resize(to_mesh->n_local_nodes());
unsigned int i = 0;
for (auto & node : to_mesh->local_node_ptr_range())
target_local_nodes[i++] = node;
}
for (const auto & node : target_local_nodes)
{
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
// Find which bboxes might have the nearest node to this point.
Real nearest_max_distance = std::numeric_limits<Real>::max();
for (const auto & bbox : bboxes)
{
Real distance = bboxMaxDistance(*node, bbox);
if (distance < nearest_max_distance)
nearest_max_distance = distance;
}
unsigned int from0 = 0;
for (processor_id_type i_proc = 0; i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool qp_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && !qp_found;
i_from++)
{
Real distance = bboxMinDistance(*node, bboxes[i_from]);
if (distance < nearest_max_distance || bboxes[i_from].contains_point(*node))
{
std::pair<unsigned int, unsigned int> key(i_to, node->id());
node_index_map[i_proc][key] = outgoing_qps[i_proc].size();
outgoing_qps[i_proc].push_back(*node + _to_positions[i_to]);
qp_found = true;
}
}
}
}
}
else // Elemental
{
for (auto & elem : as_range(to_mesh->local_elements_begin(), to_mesh->local_elements_end()))
{
Point centroid = elem->centroid();
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
// Find which bboxes might have the nearest node to this point.
Real nearest_max_distance = std::numeric_limits<Real>::max();
for (const auto & bbox : bboxes)
{
Real distance = bboxMaxDistance(centroid, bbox);
if (distance < nearest_max_distance)
nearest_max_distance = distance;
}
unsigned int from0 = 0;
for (processor_id_type i_proc = 0; i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
bool qp_found = false;
for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && !qp_found;
i_from++)
{
Real distance = bboxMinDistance(centroid, bboxes[i_from]);
if (distance < nearest_max_distance || bboxes[i_from].contains_point(centroid))
{
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
node_index_map[i_proc][key] = outgoing_qps[i_proc].size();
outgoing_qps[i_proc].push_back(centroid + _to_positions[i_to]);
qp_found = true;
}
}
}
}
}
}
}
////////////////////
// Send local node/centroid positions off to the other processors and take
// care of points sent to this processor. We'll need to check the points
// against all of the "from" domains that this processor owns. For each
// point, we'll find the nearest node, then we'll send the value at that node
// and the distance between the node and the point back to the processor that
// requested that point.
////////////////////
std::vector<std::vector<Real>> incoming_evals(n_processors());
std::vector<Parallel::Request> send_qps(n_processors());
std::vector<Parallel::Request> send_evals(n_processors());
// Create these here so that they live the entire life of this function
// and are NOT reused per processor.
std::vector<std::vector<Real>> processor_outgoing_evals(n_processors());
if (!_neighbors_cached)
{
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.send(i_proc, outgoing_qps[i_proc], send_qps[i_proc]);
}
// Build an array of pointers to all of this processor's local nodes. We
// need to do this to avoid the expense of using LibMesh iterators. This
// step also takes care of limiting the search to boundary nodes, if
// applicable.
std::vector<std::vector<Node *>> local_nodes(froms_per_proc[processor_id()]);
for (unsigned int i = 0; i < froms_per_proc[processor_id()]; i++)
{
getLocalNodes(_from_meshes[i], local_nodes[i]);
}
if (_fixed_meshes)
{
_cached_froms.resize(n_processors());
_cached_dof_ids.resize(n_processors());
}
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
std::vector<Point> incoming_qps;
if (i_proc == processor_id())
incoming_qps = outgoing_qps[i_proc];
else
_communicator.receive(i_proc, incoming_qps);
if (_fixed_meshes)
{
_cached_froms[i_proc].resize(incoming_qps.size());
_cached_dof_ids[i_proc].resize(incoming_qps.size());
}
std::vector<Real> & outgoing_evals = processor_outgoing_evals[i_proc];
outgoing_evals.resize(2 * incoming_qps.size());
for (unsigned int qp = 0; qp < incoming_qps.size(); qp++)
{
Point qpt = incoming_qps[qp];
outgoing_evals[2 * qp] = std::numeric_limits<Real>::max();
for (unsigned int i_local_from = 0; i_local_from < froms_per_proc[processor_id()];
i_local_from++)
{
MooseVariableFEBase & from_var =
_from_problems[i_local_from]->getVariable(0,
_from_var_name,
Moose::VarKindType::VAR_ANY,
Moose::VarFieldType::VAR_FIELD_STANDARD);
System & from_sys = from_var.sys().system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
for (unsigned int i_node = 0; i_node < local_nodes[i_local_from].size(); i_node++)
{
Real current_distance =
(qpt - *(local_nodes[i_local_from][i_node]) - _from_positions[i_local_from]).norm();
if (current_distance < outgoing_evals[2 * qp])
{
// Assuming LAGRANGE!
if (local_nodes[i_local_from][i_node]->n_dofs(from_sys_num, from_var_num) > 0)
{
dof_id_type from_dof =
local_nodes[i_local_from][i_node]->dof_number(from_sys_num, from_var_num, 0);
outgoing_evals[2 * qp] = current_distance;
outgoing_evals[2 * qp + 1] = (*from_sys.solution)(from_dof);
if (_fixed_meshes)
{
// Cache the nearest nodes.
_cached_froms[i_proc][qp] = i_local_from;
_cached_dof_ids[i_proc][qp] = from_dof;
}
}
}
}
}
}
if (i_proc == processor_id())
incoming_evals[i_proc] = outgoing_evals;
else
_communicator.send(i_proc, outgoing_evals, send_evals[i_proc]);
}
}
else // We've cached the nearest nodes.
{
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
std::vector<Real> & outgoing_evals = processor_outgoing_evals[i_proc];
outgoing_evals.resize(_cached_froms[i_proc].size());
for (unsigned int qp = 0; qp < outgoing_evals.size(); qp++)
{
MooseVariableFEBase & from_var = _from_problems[_cached_froms[i_proc][qp]]->getVariable(
0,
_from_var_name,
Moose::VarKindType::VAR_ANY,
Moose::VarFieldType::VAR_FIELD_STANDARD);
System & from_sys = from_var.sys().system();
dof_id_type from_dof = _cached_dof_ids[i_proc][qp];
// outgoing_evals[qp] = (*from_sys.solution)(_cached_dof_ids[i_proc][qp]);
outgoing_evals[qp] = (*from_sys.solution)(from_dof);
}
if (i_proc == processor_id())
incoming_evals[i_proc] = outgoing_evals;
else
_communicator.send(i_proc, outgoing_evals, send_evals[i_proc]);
}
}
////////////////////
// Gather all of the evaluations, find the nearest one for each node/element,
// and apply the values.
////////////////////
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.receive(i_proc, incoming_evals[i_proc]);
}
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(*_to_es[i_to], _to_var_name);
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> * solution = nullptr;
switch (_direction)
{
case TO_MULTIAPP:
solution = &getTransferVector(i_to, _to_var_name);
break;
case FROM_MULTIAPP:
solution = to_sys->solution.get();
break;
default:
mooseError("Unknown direction");
}
MeshBase * to_mesh = &_to_meshes[i_to]->getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
std::vector<Node *> target_local_nodes;
if (isParamValid("target_boundary"))
{
BoundaryID target_bnd_id =
_to_meshes[i_to]->getBoundaryID(getParam<BoundaryName>("target_boundary"));
ConstBndNodeRange & bnd_nodes = *(_to_meshes[i_to])->getBoundaryNodeRange();
for (const auto & bnode : bnd_nodes)
if (bnode->_bnd_id == target_bnd_id && bnode->_node->processor_id() == processor_id())
target_local_nodes.push_back(bnode->_node);
}
else
{
target_local_nodes.resize(to_mesh->n_local_nodes());
unsigned int i = 0;
for (auto & node : to_mesh->local_node_ptr_range())
target_local_nodes[i++] = node;
}
for (const auto & node : target_local_nodes)
{
// Skip this node if the variable has no dofs at it.
if (node->n_dofs(sys_num, var_num) < 1)
continue;
Real best_val = 0;
if (!_neighbors_cached)
{
Real min_dist = std::numeric_limits<Real>::max();
for (unsigned int i_from = 0; i_from < incoming_evals.size(); i_from++)
{
std::pair<unsigned int, unsigned int> key(i_to, node->id());
if (node_index_map[i_from].find(key) == node_index_map[i_from].end())
continue;
unsigned int qp_ind = node_index_map[i_from][key];
if (incoming_evals[i_from][2 * qp_ind] >= min_dist)
continue;
min_dist = incoming_evals[i_from][2 * qp_ind];
best_val = incoming_evals[i_from][2 * qp_ind + 1];
if (_fixed_meshes)
{
// Cache these indices.
_cached_from_inds[node->id()] = i_from;
_cached_qp_inds[node->id()] = qp_ind;
}
}
}
else
{
best_val = incoming_evals[_cached_from_inds[node->id()]][_cached_qp_inds[node->id()]];
}
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
else // Elemental
{
for (auto & elem : as_range(to_mesh->local_elements_begin(), to_mesh->local_elements_end()))
{
// Skip this element if the variable has no dofs at it.
if (elem->n_dofs(sys_num, var_num) < 1)
continue;
Real best_val = 0;
if (!_neighbors_cached)
{
Real min_dist = std::numeric_limits<Real>::max();
for (unsigned int i_from = 0; i_from < incoming_evals.size(); i_from++)
{
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
if (node_index_map[i_from].find(key) == node_index_map[i_from].end())
continue;
unsigned int qp_ind = node_index_map[i_from][key];
if (incoming_evals[i_from][2 * qp_ind] >= min_dist)
continue;
min_dist = incoming_evals[i_from][2 * qp_ind];
best_val = incoming_evals[i_from][2 * qp_ind + 1];
if (_fixed_meshes)
{
// Cache these indices.
_cached_from_inds[elem->id()] = i_from;
_cached_qp_inds[elem->id()] = qp_ind;
}
}
}
else
{
best_val = incoming_evals[_cached_from_inds[elem->id()]][_cached_qp_inds[elem->id()]];
}
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
solution->set(dof, best_val);
}
}
solution->close();
to_sys->update();
}
if (_fixed_meshes)
_neighbors_cached = true;
// Make sure all our sends succeeded.
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
send_qps[i_proc].wait();
send_evals[i_proc].wait();
}
_console << "Finished NearestNodeTransfer " << name() << std::endl;
}
void
MultiAppNearestNodeTransfer::execute()
{
Moose::out << "Beginning NearestNodeTransfer " << _name << std::endl;
switch(_direction)
{
case TO_MULTIAPP:
{
FEProblem & from_problem = *_multi_app->problem();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
{
mooseError("Cannot use a NearestNode transfer from a displaced mesh to a MultiApp!");
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
}
else
from_mesh = &from_problem.mesh().getMesh();
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
//Create a serialized version of the solution vector
NumericVector<Number> * serialized_solution = NumericVector<Number>::build().release();
serialized_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_solution);
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (_multi_app->hasLocalApp(i))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
// Loop over the master nodes and set the value of the variable
System * to_sys = find_sys(_multi_app->appProblem(i)->es(), _to_var_name);
if (!to_sys)
mooseError("Cannot find variable "<<_to_var_name<<" for "<<_name<<" Transfer");
unsigned int sys_num = to_sys->number();
unsigned int var_num = to_sys->variable_number(_to_var_name);
NumericVector<Real> & solution = _multi_app->appTransferVector(i, _to_var_name);
MeshBase * mesh = NULL;
if (_displaced_target_mesh && _multi_app->appProblem(i)->getDisplacedProblem())
mesh = &_multi_app->appProblem(i)->getDisplacedProblem()->mesh().getMesh();
else
mesh = &_multi_app->appProblem(i)->mesh().getMesh();
bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE;
if (is_nodal)
{
MeshBase::const_node_iterator node_it = mesh->local_nodes_begin();
MeshBase::const_node_iterator node_end = mesh->local_nodes_end();
for(; node_it != node_end; ++node_it)
{
Node * node = *node_it;
Point actual_position = *node+_multi_app->position(i);
if (node->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this node
{
// The zero only works for LAGRANGE!
dof_id_type dof = node->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real distance = 0; // Just to satisfy the last argument
MeshBase::const_node_iterator from_nodes_begin = from_mesh->nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(node->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
_node_map[node->id()] = nearest_node;
_distance_map[node->id()] = distance;
}
else
{
nearest_node = _node_map[node->id()];
//distance = _distance_map[node->id()];
}
}
else
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
// Assuming LAGRANGE!
dof_id_type from_dof = nearest_node->dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*serialized_solution)(from_dof);
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
else // Elemental
{
MeshBase::const_element_iterator elem_it = mesh->local_elements_begin();
MeshBase::const_element_iterator elem_end = mesh->local_elements_end();
for(; elem_it != elem_end; ++elem_it)
{
Elem * elem = *elem_it;
Point centroid = elem->centroid();
Point actual_position = centroid+_multi_app->position(i);
if (elem->n_dofs(sys_num, var_num) > 0) // If this variable has dofs at this elem
{
// The zero only works for LAGRANGE!
dof_id_type dof = elem->dof_number(sys_num, var_num, 0);
// Swap back
Moose::swapLibMeshComm(swapped);
Real distance = 0; // Just to satisfy the last argument
MeshBase::const_node_iterator from_nodes_begin = from_mesh->nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(elem->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
_node_map[elem->id()] = nearest_node;
_distance_map[elem->id()] = distance;
}
else
{
nearest_node = _node_map[elem->id()];
//distance = _distance_map[elem->id()];
}
}
else
nearest_node = getNearestNode(actual_position, distance, from_nodes_begin, from_nodes_end);
// Assuming LAGRANGE!
dof_id_type from_dof = nearest_node->dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*serialized_solution)(from_dof);
// Swap again
swapped = Moose::swapLibMeshComm(_multi_app->comm());
solution.set(dof, from_value);
}
}
}
solution.close();
to_sys->update();
// Swap back
Moose::swapLibMeshComm(swapped);
}
}
delete serialized_solution;
break;
}
case FROM_MULTIAPP:
{
FEProblem & to_problem = *_multi_app->problem();
MooseVariable & to_var = to_problem.getVariable(0, _to_var_name);
SystemBase & to_system_base = to_var.sys();
System & to_sys = to_system_base.system();
NumericVector<Real> & to_solution = *to_sys.solution;
unsigned int to_sys_num = to_sys.number();
// Only works with a serialized mesh to transfer to!
mooseAssert(to_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
unsigned int to_var_num = to_sys.variable_number(to_var.name());
// EquationSystems & to_es = to_sys.get_equation_systems();
MeshBase * to_mesh = NULL;
if (_displaced_target_mesh && to_problem.getDisplacedProblem())
to_mesh = &to_problem.getDisplacedProblem()->mesh().getMesh();
else
to_mesh = &to_problem.mesh().getMesh();
bool is_nodal = to_sys.variable_type(to_var_num) == FEType();
dof_id_type n_nodes = to_mesh->n_nodes();
dof_id_type n_elems = to_mesh->n_elem();
///// All of the following are indexed off to_node->id() or to_elem->id() /////
// Minimum distances from each node in the "to" mesh to a node in
std::vector<Real> min_distances;
// The node ids in the "from" mesh that this processor has found to be the minimum distances to the "to" nodes
std::vector<dof_id_type> min_nodes;
// After the call to maxloc() this will tell us which processor actually has the minimum
std::vector<unsigned int> min_procs;
// The global multiapp ID that this processor found had the minimum distance node in it.
std::vector<unsigned int> min_apps;
if (is_nodal)
{
min_distances.resize(n_nodes, std::numeric_limits<Real>::max());
min_nodes.resize(n_nodes);
min_procs.resize(n_nodes);
min_apps.resize(n_nodes);
}
else
{
min_distances.resize(n_elems, std::numeric_limits<Real>::max());
min_nodes.resize(n_elems);
min_procs.resize(n_elems);
min_apps.resize(n_elems);
}
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
// Only works with a serialized mesh to transfer from!
mooseAssert(from_sys.get_mesh().is_serial(), "MultiAppNearestNodeTransfer only works with SerialMesh!");
// unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
MeshTools::BoundingBox app_box = MeshTools::processor_bounding_box(*from_mesh, libMesh::processor_id());
Point app_position = _multi_app->position(i);
Moose::swapLibMeshComm(swapped);
if (is_nodal)
{
MeshBase::const_node_iterator to_node_it = to_mesh->nodes_begin();
MeshBase::const_node_iterator to_node_end = to_mesh->nodes_end();
for(; to_node_it != to_node_end; ++to_node_it)
{
Node * to_node = *to_node_it;
unsigned int to_node_id = to_node->id();
Real current_distance = 0;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
MeshBase::const_node_iterator from_nodes_begin = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(to_node->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(*to_node-app_position, current_distance, from_nodes_begin, from_nodes_end);
_node_map[to_node->id()] = nearest_node;
_distance_map[to_node->id()] = current_distance;
}
else
{
nearest_node = _node_map[to_node->id()];
current_distance = _distance_map[to_node->id()];
}
}
else
nearest_node = getNearestNode(*to_node-app_position, current_distance, from_nodes_begin, from_nodes_end);
Moose::swapLibMeshComm(swapped);
// TODO: Logic bug when we are using caching. "current_distance" is set by a call to getNearestNode which is
// skipped in that case. We shouldn't be relying on it or stuffing it in another data structure
if (current_distance < min_distances[to_node->id()])
{
min_distances[to_node_id] = current_distance;
min_nodes[to_node_id] = nearest_node->id();
min_apps[to_node_id] = i;
}
}
}
else // Elemental
{
MeshBase::const_element_iterator to_elem_it = to_mesh->elements_begin();
MeshBase::const_element_iterator to_elem_end = to_mesh->elements_end();
for(; to_elem_it != to_elem_end; ++to_elem_it)
{
Elem * to_elem = *to_elem_it;
unsigned int to_elem_id = to_elem->id();
Point actual_position = to_elem->centroid()-app_position;
Real current_distance = 0;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
MeshBase::const_node_iterator from_nodes_begin = from_mesh->local_nodes_begin();
MeshBase::const_node_iterator from_nodes_end = from_mesh->local_nodes_end();
Node * nearest_node = NULL;
if (_fixed_meshes)
{
if (_node_map.find(to_elem->id()) == _node_map.end()) // Haven't cached it yet
{
nearest_node = getNearestNode(actual_position, current_distance, from_nodes_begin, from_nodes_end);
_node_map[to_elem->id()] = nearest_node;
_distance_map[to_elem->id()] = current_distance;
}
else
{
nearest_node = _node_map[to_elem->id()];
current_distance = _distance_map[to_elem->id()];
}
}
else
nearest_node = getNearestNode(actual_position, current_distance, from_nodes_begin, from_nodes_end);
Moose::swapLibMeshComm(swapped);
// TODO: Logic bug when we are using caching. "current_distance" is set by a call to getNearestNode which is
// skipped in that case. We shouldn't be relying on it or stuffing it in another data structure
if (current_distance < min_distances[to_elem->id()])
{
min_distances[to_elem_id] = current_distance;
min_nodes[to_elem_id] = nearest_node->id();
min_apps[to_elem_id] = i;
}
}
}
}
/*
// We're going to need serialized solution vectors for each app
// We could try to only do it for the apps that have mins in them...
// but it's tough because this is a collective operation... so that would have to be coordinated
std::vector<NumericVector<Number> *> serialized_from_solutions(_multi_app->numGlobalApps());
if (_multi_app->hasApp())
{
// Swap
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
for(unsigned int i=0; i<_multi_app->numGlobalApps(); i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
FEProblem & from_problem = *_multi_app->appProblem(i);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
//Create a serialized version of the solution vector
serialized_from_solutions[i] = NumericVector<Number>::build().release();
serialized_from_solutions[i]->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_from_solutions[i]);
}
// Swap back
Moose::swapLibMeshComm(swapped);
}
*/
// We've found the nearest nodes for this processor. We need to see which processor _actually_ found the nearest though
Parallel::minloc(min_distances, min_procs);
// Now loop through min_procs and see if _this_ processor had the actual minimum for any nodes.
// If it did then we're going to go get the value from that nearest node and transfer its value
processor_id_type proc_id = libMesh::processor_id();
for(unsigned int j=0; j<min_procs.size(); j++)
{
if (min_procs[j] == proc_id) // This means that this processor really did find the minumum so we need to transfer the value
{
// The zero only works for LAGRANGE!
dof_id_type to_dof = 0;
if (is_nodal)
{
Node & to_node = to_mesh->node(j);
to_dof = to_node.dof_number(to_sys_num, to_var_num, 0);
}
else
{
Elem & to_elem = *to_mesh->elem(j);
to_dof = to_elem.dof_number(to_sys_num, to_var_num, 0);
}
// The app that has the nearest node in it
unsigned int from_app_num = min_apps[j];
mooseAssert(_multi_app->hasLocalApp(from_app_num), "Something went very wrong!");
// Swap
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(from_app_num);
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
SystemBase & from_system_base = from_var.sys();
System & from_sys = from_system_base.system();
unsigned int from_sys_num = from_sys.number();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
// EquationSystems & from_es = from_sys.get_equation_systems();
MeshBase * from_mesh = NULL;
if (_displaced_source_mesh && from_problem.getDisplacedProblem())
from_mesh = &from_problem.getDisplacedProblem()->mesh().getMesh();
else
from_mesh = &from_problem.mesh().getMesh();
Node & from_node = from_mesh->node(min_nodes[j]);
// Assuming LAGRANGE!
dof_id_type from_dof = from_node.dof_number(from_sys_num, from_var_num, 0);
Real from_value = (*from_sys.solution)(from_dof);
// Swap back
Moose::swapLibMeshComm(swapped);
to_solution.set(to_dof, from_value);
}
}
to_solution.close();
to_sys.update();
break;
}
}
Moose::out << "Finished NearestNodeTransfer " << _name << std::endl;
}
