void MultiAppProjectionTransfer::execute() { _console << "Beginning projection transfer " << name() << std::endl; getAppInfo(); //////////////////// // We are going to project the solutions by solving some linear systems. In // order to assemble the systems, we need to evaluate the "from" domain // solutions at quadrature points in the "to" domain. Some parallel // communication is necessary because each processor doesn't necessarily have // all the "from" information it needs to set its "to" values. We don't want // to use a bunch of big all-to-all broadcasts, so we'll use bounding boxes to // figure out which processors have the information we need and only // communicate with those processors. // // Each processor will // 1. Check its local quadrature points in the "to" domains to see which // "from" domains they might be in. // 2. Send quadrature points to the processors with "from" domains that might // contain those points. // 3. Recieve quadrature points from other processors, evaluate its mesh // functions at those points, and send the values back to the proper // processor // 4. Recieve mesh function evaluations from all relevant processors and // decide which one to use at every quadrature point (the lowest global app // index always wins) // 5. And use the mesh function evaluations to assemble and solve an L2 // projection system on its local elements. //////////////////// //////////////////// // For every combination of global "from" problem and local "to" problem, find // which "from" bounding boxes overlap with which "to" elements. Keep track // of which processors own bounding boxes that overlap with which elements. // Build vectors of quadrature points to send to other processors for mesh // function evaluations. //////////////////// // Get the bounding boxes for the "from" domains. std::vector<MeshTools::BoundingBox> bboxes = getFromBoundingBoxes(); // Figure out how many "from" domains each processor owns. std::vector<unsigned int> froms_per_proc = getFromsPerProc(); std::vector<std::vector<Point> > outgoing_qps(n_processors()); std::vector<std::map<std::pair<unsigned int, unsigned int>, unsigned int> > element_index_map(n_processors()); // element_index_map[i_to, element_id] = index // outgoing_qps[index] is the first quadrature point in element if (! _qps_cached) { for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++) { MeshBase & to_mesh = _to_meshes[i_to]->getMesh(); LinearImplicitSystem & system = * _proj_sys[i_to]; FEType fe_type = system.variable_type(0); std::unique_ptr<FEBase> fe(FEBase::build(to_mesh.mesh_dimension(), fe_type)); QGauss qrule(to_mesh.mesh_dimension(), fe_type.default_quadrature_order()); fe->attach_quadrature_rule(&qrule); const std::vector<Point> & xyz = fe->get_xyz(); MeshBase::const_element_iterator el = to_mesh.local_elements_begin(); const MeshBase::const_element_iterator end_el = to_mesh.local_elements_end(); unsigned int from0 = 0; for (processor_id_type i_proc = 0; i_proc < n_processors(); from0 += froms_per_proc[i_proc], i_proc++) { for (el = to_mesh.local_elements_begin(); el != end_el; el++) { const Elem* elem = *el; fe->reinit (elem); bool qp_hit = false; for (unsigned int i_from = 0; i_from < froms_per_proc[i_proc] && ! qp_hit; i_from++) { for (unsigned int qp = 0; qp < qrule.n_points() && ! qp_hit; qp ++) { Point qpt = xyz[qp]; if (bboxes[from0 + i_from].contains_point(qpt + _to_positions[i_to])) qp_hit = true; } } if (qp_hit) { // The selected processor's bounding box contains at least one // quadrature point from this element. Send all qps from this element // and remember where they are in the array using the map. std::pair<unsigned int, unsigned int> key(i_to, elem->id()); element_index_map[i_proc][key] = outgoing_qps[i_proc].size(); for (unsigned int qp = 0; qp < qrule.n_points(); qp ++) { Point qpt = xyz[qp]; outgoing_qps[i_proc].push_back(qpt + _to_positions[i_to]); } } } } } if (_fixed_meshes) _cached_index_map = element_index_map; } else { element_index_map = _cached_index_map; } //////////////////// // Request quadrature point evaluations from other processors and handle // requests sent to this processor. //////////////////// // Non-blocking send quadrature points to other processors. std::vector<Parallel::Request> send_qps(n_processors()); if (! _qps_cached) for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++) if (i_proc != processor_id()) _communicator.send(i_proc, outgoing_qps[i_proc], send_qps[i_proc]); // Get the local bounding boxes. std::vector<MeshTools::BoundingBox> local_bboxes(froms_per_proc[processor_id()]); { // Find the index to the first of this processor's local bounding boxes. unsigned int local_start = 0; for (processor_id_type i_proc = 0; i_proc < n_processors() && i_proc != processor_id(); i_proc++) local_start += froms_per_proc[i_proc]; // Extract the local bounding boxes. for (unsigned int i_from = 0; i_from < froms_per_proc[processor_id()]; i_from++) local_bboxes[i_from] = bboxes[local_start + i_from]; } // Setup the local mesh functions. std::vector<MeshFunction *> local_meshfuns(froms_per_proc[processor_id()], NULL); for (unsigned int i_from = 0; i_from < _from_problems.size(); i_from++) { FEProblemBase & from_problem = *_from_problems[i_from]; MooseVariable & from_var = from_problem.getVariable(0, _from_var_name); System & from_sys = from_var.sys().system(); unsigned int from_var_num = from_sys.variable_number(from_var.name()); MeshFunction * from_func = new MeshFunction(from_problem.es(), *from_sys.current_local_solution, from_sys.get_dof_map(), from_var_num); from_func->init(Trees::ELEMENTS); from_func->enable_out_of_mesh_mode(OutOfMeshValue); local_meshfuns[i_from] = from_func; } // Recieve quadrature points from other processors, evaluate mesh frunctions // at those points, and send the values back. std::vector<Parallel::Request> send_evals(n_processors()); std::vector<Parallel::Request> send_ids(n_processors()); std::vector<std::vector<Real> > outgoing_evals(n_processors()); std::vector<std::vector<unsigned int> > outgoing_ids(n_processors()); std::vector<std::vector<Real> > incoming_evals(n_processors()); std::vector<std::vector<unsigned int> > incoming_app_ids(n_processors()); for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++) { // Use the cached qps if they're available. std::vector<Point> incoming_qps; if (! _qps_cached) { if (i_proc == processor_id()) incoming_qps = outgoing_qps[i_proc]; else _communicator.receive(i_proc, incoming_qps); // Cache these qps for later if _fixed_meshes if (_fixed_meshes) _cached_qps[i_proc] = incoming_qps; } else { incoming_qps = _cached_qps[i_proc]; } outgoing_evals[i_proc].resize(incoming_qps.size(), OutOfMeshValue); if (_direction == FROM_MULTIAPP) outgoing_ids[i_proc].resize(incoming_qps.size(), libMesh::invalid_uint); for (unsigned int qp = 0; qp < incoming_qps.size(); qp++) { Point qpt = incoming_qps[qp]; // Loop until we've found the lowest-ranked app that actually contains // the quadrature point. for (unsigned int i_from = 0; i_from < _from_problems.size(); i_from++) { if (local_bboxes[i_from].contains_point(qpt)) { outgoing_evals[i_proc][qp] = (* local_meshfuns[i_from])(qpt - _from_positions[i_from]); if (_direction == FROM_MULTIAPP) outgoing_ids[i_proc][qp] = _local2global_map[i_from]; } } } if (i_proc == processor_id()) { incoming_evals[i_proc] = outgoing_evals[i_proc]; if (_direction == FROM_MULTIAPP) incoming_app_ids[i_proc] = outgoing_ids[i_proc]; } else { _communicator.send(i_proc, outgoing_evals[i_proc], send_evals[i_proc]); if (_direction == FROM_MULTIAPP) _communicator.send(i_proc, outgoing_ids[i_proc], send_ids[i_proc]); } } //////////////////// // Gather all of the qp evaluations and pick out the best ones for each qp. //////////////////// for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++) { if (i_proc == processor_id()) continue; _communicator.receive(i_proc, incoming_evals[i_proc]); if (_direction == FROM_MULTIAPP) _communicator.receive(i_proc, incoming_app_ids[i_proc]); } std::vector<std::vector<Real> > final_evals(_to_problems.size()); std::vector<std::map<unsigned int, unsigned int> > trimmed_element_maps(_to_problems.size()); for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++) { MeshBase & to_mesh = _to_meshes[i_to]->getMesh(); LinearImplicitSystem & system = * _proj_sys[i_to]; FEType fe_type = system.variable_type(0); std::unique_ptr<FEBase> fe(FEBase::build(to_mesh.mesh_dimension(), fe_type)); QGauss qrule(to_mesh.mesh_dimension(), fe_type.default_quadrature_order()); fe->attach_quadrature_rule(&qrule); const std::vector<Point> & xyz = fe->get_xyz(); MeshBase::const_element_iterator el = to_mesh.local_elements_begin(); const MeshBase::const_element_iterator end_el = to_mesh.local_elements_end(); for (el = to_mesh.active_local_elements_begin(); el != end_el; el++) { const Elem* elem = *el; fe->reinit (elem); bool element_is_evaled = false; std::vector<Real> evals(qrule.n_points(), 0.); for (unsigned int qp = 0; qp < qrule.n_points(); qp++) { Point qpt = xyz[qp]; unsigned int lowest_app_rank = libMesh::invalid_uint; for (unsigned int i_proc = 0; i_proc < n_processors(); i_proc++) { // Ignore the selected processor if the element wasn't found in it's // bounding box. std::map<std::pair<unsigned int, unsigned int>, unsigned int> & map = element_index_map[i_proc]; std::pair<unsigned int, unsigned int> key(i_to, elem->id()); if (map.find(key) == map.end()) continue; unsigned int qp0 = map[key]; // Ignore the selected processor if it's app has a higher rank than the // previously found lowest app rank. if (_direction == FROM_MULTIAPP) if (incoming_app_ids[i_proc][qp0 + qp] >= lowest_app_rank) continue; // Ignore the selected processor if the qp was actually outside the // processor's subapp's mesh. if (incoming_evals[i_proc][qp0 + qp] == OutOfMeshValue) continue; // This is the best meshfunction evaluation so far, save it. element_is_evaled = true; evals[qp] = incoming_evals[i_proc][qp0 + qp]; } } // If we found good evaluations for any of the qps in this element, save // those evaluations for later. if (element_is_evaled) { trimmed_element_maps[i_to][elem->id()] = final_evals[i_to].size(); for (unsigned int qp = 0; qp < qrule.n_points(); qp++) final_evals[i_to].push_back(evals[qp]); } } } //////////////////// // We now have just one or zero mesh function values at all of our local // quadrature points. Stash those values (and a map linking them to element // ids) in the equation systems parameters and project the solution. //////////////////// for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++) { _to_es[i_to]->parameters.set<std::vector<Real>*>("final_evals") = & final_evals[i_to]; _to_es[i_to]->parameters.set<std::map<unsigned int, unsigned int>*>("element_map") = & trimmed_element_maps[i_to]; projectSolution(i_to); _to_es[i_to]->parameters.set<std::vector<Real>*>("final_evals") = NULL; _to_es[i_to]->parameters.set<std::map<unsigned int, unsigned int>*>("element_map") = NULL; } for (unsigned int i = 0; i < _from_problems.size(); i++) delete local_meshfuns[i]; // Make sure all our sends succeeded. for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++) { if (i_proc == processor_id()) continue; if (! _qps_cached) send_qps[i_proc].wait(); send_evals[i_proc].wait(); if (_direction == FROM_MULTIAPP) send_ids[i_proc].wait(); } if (_fixed_meshes) _qps_cached = true; _console << "Finished projection transfer " << name() << std::endl; }
void MultiAppMeshFunctionTransfer::execute() { Moose::out << "Beginning MeshFunctionTransfer " << name() << std::endl; getAppInfo(); /** * For every combination of global "from" problem and local "to" problem, find * which "from" bounding boxes overlap with which "to" elements. Keep track * of which processors own bounding boxes that overlap with which elements. * Build vectors of node locations/element centroids to send to other * processors for mesh function evaluations. */ // Get the bounding boxes for the "from" domains. std::vector<MeshTools::BoundingBox> bboxes = getFromBoundingBoxes(); // Figure out how many "from" domains each processor owns. std::vector<unsigned int> froms_per_proc = getFromsPerProc(); std::vector<std::vector<Point> > outgoing_points(n_processors()); std::vector<std::map<std::pair<unsigned int, unsigned int>, unsigned int> > point_index_map(n_processors()); // point_index_map[i_to, element_id] = index // outgoing_points[index] is the first quadrature point in element for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++) { System * to_sys = find_sys(*_to_es[i_to], _to_var_name); unsigned int sys_num = to_sys->number(); unsigned int var_num = to_sys->variable_number(_to_var_name); MeshBase * to_mesh = & _to_meshes[i_to]->getMesh(); bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE; if (is_nodal) { MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin(); MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end(); for (; node_it != node_end; ++node_it) { Node * node = *node_it; // Skip this node if the variable has no dofs at it. if (node->n_dofs(sys_num, var_num) < 1) continue; // Loop over the "froms" on processor i_proc. If the node is found in // any of the "froms", add that node to the vector that will be sent to // i_proc. unsigned int from0 = 0; for (processor_id_type i_proc = 0; i_proc < n_processors(); from0 += froms_per_proc[i_proc], i_proc++) { bool point_found = false; for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && ! point_found; i_from++) { if (bboxes[i_from].contains_point(*node + _to_positions[i_to])) { std::pair<unsigned int, unsigned int> key(i_to, node->id()); point_index_map[i_proc][key] = outgoing_points[i_proc].size(); outgoing_points[i_proc].push_back(*node + _to_positions[i_to]); point_found = true; } } } } } else // Elemental { MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin(); MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end(); for (; elem_it != elem_end; ++elem_it) { Elem * elem = *elem_it; Point centroid = elem->centroid(); // Skip this element if the variable has no dofs at it. if (elem->n_dofs(sys_num, var_num) < 1) continue; // Loop over the "froms" on processor i_proc. If the elem is found in // any of the "froms", add that elem to the vector that will be sent to // i_proc. unsigned int from0 = 0; for (processor_id_type i_proc = 0; i_proc < n_processors(); from0 += froms_per_proc[i_proc], i_proc++) { bool point_found = false; for (unsigned int i_from = from0; i_from < from0 + froms_per_proc[i_proc] && ! point_found; i_from++) { if (bboxes[i_from].contains_point(centroid + _to_positions[i_to])) { std::pair<unsigned int, unsigned int> key(i_to, elem->id()); point_index_map[i_proc][key] = outgoing_points[i_proc].size(); outgoing_points[i_proc].push_back(centroid + _to_positions[i_to]); point_found = true; } } } } } } /** * Request point evaluations from other processors and handle requests sent to * this processor. */ // Get the local bounding boxes. std::vector<MeshTools::BoundingBox> local_bboxes(froms_per_proc[processor_id()]); { // Find the index to the first of this processor's local bounding boxes. unsigned int local_start = 0; for (processor_id_type i_proc = 0; i_proc < n_processors() && i_proc != processor_id(); i_proc++) { local_start += froms_per_proc[i_proc]; } // Extract the local bounding boxes. for (unsigned int i_from = 0; i_from < froms_per_proc[processor_id()]; i_from++) { local_bboxes[i_from] = bboxes[local_start + i_from]; } } // Setup the local mesh functions. std::vector<MooseSharedPointer<MeshFunction> > local_meshfuns; for (unsigned int i_from = 0; i_from < _from_problems.size(); i_from++) { FEProblem & from_problem = *_from_problems[i_from]; MooseVariable & from_var = from_problem.getVariable(0, _from_var_name); System & from_sys = from_var.sys().system(); unsigned int from_var_num = from_sys.variable_number(from_var.name()); MooseSharedPointer<MeshFunction> from_func; //TODO: make MultiAppTransfer give me the right es if (_displaced_source_mesh && from_problem.getDisplacedProblem()) from_func.reset(new MeshFunction(from_problem.getDisplacedProblem()->es(), *from_sys.current_local_solution, from_sys.get_dof_map(), from_var_num)); else from_func.reset(new MeshFunction(from_problem.es(), *from_sys.current_local_solution, from_sys.get_dof_map(), from_var_num)); from_func->init(Trees::ELEMENTS); from_func->enable_out_of_mesh_mode(OutOfMeshValue); local_meshfuns.push_back(from_func); } // Send points to other processors. std::vector<std::vector<Real> > incoming_evals(n_processors()); std::vector<std::vector<unsigned int> > incoming_app_ids(n_processors()); for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++) { if (i_proc == processor_id()) continue; _communicator.send(i_proc, outgoing_points[i_proc]); } // Recieve points from other processors, evaluate mesh frunctions at those // points, and send the values back. for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++) { std::vector<Point> incoming_points; if (i_proc == processor_id()) incoming_points = outgoing_points[i_proc]; else _communicator.receive(i_proc, incoming_points); std::vector<Real> outgoing_evals(incoming_points.size(), OutOfMeshValue); std::vector<unsigned int> outgoing_ids(incoming_points.size(), -1); // -1 = largest unsigned int for (unsigned int i_pt = 0; i_pt < incoming_points.size(); i_pt++) { Point pt = incoming_points[i_pt]; // Loop until we've found the lowest-ranked app that actually contains // the quadrature point. for (unsigned int i_from = 0; i_from < _from_problems.size() && outgoing_evals[i_pt] == OutOfMeshValue; i_from++) { if (local_bboxes[i_from].contains_point(pt)) { outgoing_evals[i_pt] = (* local_meshfuns[i_from])(pt - _from_positions[i_from]); if (_direction == FROM_MULTIAPP) outgoing_ids[i_pt] = _local2global_map[i_from]; } } } if (i_proc == processor_id()) { incoming_evals[i_proc] = outgoing_evals; if (_direction == FROM_MULTIAPP) incoming_app_ids[i_proc] = outgoing_ids; } else { _communicator.send(i_proc, outgoing_evals); if (_direction == FROM_MULTIAPP) _communicator.send(i_proc, outgoing_ids); } } /** * Gather all of the evaluations, pick out the best ones for each point, and * apply them to the solution vector. When we are transferring from * multiapps, there may be multiple overlapping apps for a particular point. * In that case, we'll try to use the value from the app with the lowest id. */ for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++) { if (i_proc == processor_id()) continue; _communicator.receive(i_proc, incoming_evals[i_proc]); if (_direction == FROM_MULTIAPP) _communicator.receive(i_proc, incoming_app_ids[i_proc]); } for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++) { System * to_sys = find_sys(*_to_es[i_to], _to_var_name); unsigned int sys_num = to_sys->number(); unsigned int var_num = to_sys->variable_number(_to_var_name); NumericVector<Real> * solution; switch (_direction) { case TO_MULTIAPP: solution = & getTransferVector(i_to, _to_var_name); break; case FROM_MULTIAPP: solution = to_sys->solution.get(); break; } MeshBase * to_mesh = & _to_meshes[i_to]->getMesh(); bool is_nodal = to_sys->variable_type(var_num).family == LAGRANGE; if (is_nodal) { MeshBase::const_node_iterator node_it = to_mesh->local_nodes_begin(); MeshBase::const_node_iterator node_end = to_mesh->local_nodes_end(); for (; node_it != node_end; ++node_it) { Node * node = *node_it; // Skip this node if the variable has no dofs at it. if (node->n_dofs(sys_num, var_num) < 1) continue; unsigned int lowest_app_rank = libMesh::invalid_uint; Real best_val = 0.; bool point_found = false; for (unsigned int i_proc = 0; i_proc < incoming_evals.size(); i_proc++) { // Skip this proc if the node wasn't in it's bounding boxes. std::pair<unsigned int, unsigned int> key(i_to, node->id()); if (point_index_map[i_proc].find(key) == point_index_map[i_proc].end()) continue; unsigned int i_pt = point_index_map[i_proc][key]; // Ignore this proc if it's app has a higher rank than the // previously found lowest app rank. if (_direction == FROM_MULTIAPP) { if (incoming_app_ids[i_proc][i_pt] >= lowest_app_rank) continue; } // Ignore this proc if the point was actually outside its meshes. if (incoming_evals[i_proc][i_pt] == OutOfMeshValue) continue; best_val = incoming_evals[i_proc][i_pt]; point_found = true; } if (_error_on_miss && ! point_found) mooseError("Point not found! " << *node + _to_positions[i_to]); dof_id_type dof = node->dof_number(sys_num, var_num, 0); solution->set(dof, best_val); } } else // Elemental { MeshBase::const_element_iterator elem_it = to_mesh->local_elements_begin(); MeshBase::const_element_iterator elem_end = to_mesh->local_elements_end(); for (; elem_it != elem_end; ++elem_it) { Elem * elem = *elem_it; // Skip this element if the variable has no dofs at it. if (elem->n_dofs(sys_num, var_num) < 1) continue; unsigned int lowest_app_rank = libMesh::invalid_uint; Real best_val = 0; bool point_found = false; for (unsigned int i_proc = 0; i_proc < incoming_evals.size(); i_proc++) { // Skip this proc if the elem wasn't in it's bounding boxes. std::pair<unsigned int, unsigned int> key(i_to, elem->id()); if (point_index_map[i_proc].find(key) == point_index_map[i_proc].end()) continue; unsigned int i_pt = point_index_map[i_proc][key]; // Ignore this proc if it's app has a higher rank than the // previously found lowest app rank. if (_direction == FROM_MULTIAPP) { if (incoming_app_ids[i_proc][i_pt] >= lowest_app_rank) continue; } // Ignore this proc if the point was actually outside its meshes. if (incoming_evals[i_proc][i_pt] == OutOfMeshValue) continue; best_val = incoming_evals[i_proc][i_pt]; point_found = true; } if (_error_on_miss && ! point_found) mooseError("Point not found! " << elem->centroid() + _to_positions[i_to]); dof_id_type dof = elem->dof_number(sys_num, var_num, 0); solution->set(dof, best_val); } } solution->close(); to_sys->update(); } _console << "Finished MeshFunctionTransfer " << name() << std::endl; }