//----------------------------------------------------------------------------- void DofMapBuilder::compute_node_ownership(boost::array<set, 3>& node_ownership, vec_map& shared_node_processes, DofMap& dofmap, const std::set<std::size_t>& global_dofs, const Mesh& mesh, std::shared_ptr<const Restriction> restriction, const map& restricted_nodes_inverse, std::size_t block_size) { log(TRACE, "Determining dof ownership for parallel dof map"); const MPI_Comm mpi_comm = mesh.mpi_comm(); const std::size_t N = dofmap.global_dimension(); dolfin_assert(N % block_size == 0); const std::size_t num_nodes = N/block_size; if (restriction) dolfin_assert(block_size == 1); // References to 'node' ownership sets set& owned_nodes = node_ownership[0]; set& shared_owned_nodes = node_ownership[1]; set& shared_unowned_nodes = node_ownership[2]; // Clear data structures owned_nodes.clear(); shared_owned_nodes.clear(); shared_unowned_nodes.clear(); // Data structures for computing ownership boost::unordered_map<std::size_t, std::size_t> node_vote; std::vector<std::size_t> facet_dofs(dofmap.num_facet_dofs()); // Communication buffer std::vector<std::size_t> send_buffer; // Extract the interior boundary BoundaryMesh boundary(mesh, "local"); // Create a random number generator for ownership 'voting' boost::mt19937 engine(MPI::rank(mpi_comm)); boost::uniform_int<> distribution(0, 100000000); boost::variate_generator<boost::mt19937&, boost::uniform_int<> > rng(engine, distribution); // Build set of dofs on process boundary (first assuming that all // are owned by this process) const MeshFunction<std::size_t>& cell_map = boundary.entity_map(boundary.topology().dim()); if (!cell_map.empty()) { for (CellIterator _f(boundary); !_f.end(); ++_f) { // Create a vote per facet const std::size_t facet_vote = rng(); // Get boundary facet Facet f(mesh, cell_map[*_f]); // Get cell to which facet belongs (pick first) Cell c(mesh, f.entities(mesh.topology().dim())[0]); // Skip cells not included in restriction if (restriction && !restriction->contains(c)) continue; // Tabulate dofs on cell const std::vector<dolfin::la_index>& cell_dofs = dofmap.cell_dofs(c.index()); // Tabulate which dofs are on the facet dofmap.tabulate_facet_dofs(facet_dofs, c.index(f)); // Insert shared nodes into set and assign a 'vote' dolfin_assert(dofmap.num_facet_dofs() % block_size == 0); for (std::size_t i = 0; i < dofmap.num_facet_dofs(); ++i) { // Get facet node size_t facet_node = cell_dofs[facet_dofs[i]] % num_nodes; // Map back to original (and common) numbering for restricted // space if (restriction) { const map_iterator it = restricted_nodes_inverse.find(facet_node); dolfin_assert(it != restricted_nodes_inverse.end()); facet_node = it->second; } // Add to list of shared nodes if (shared_owned_nodes.find(facet_node) == shared_owned_nodes.end()) { shared_owned_nodes.insert(facet_node); node_vote[facet_node] = facet_vote; send_buffer.push_back(facet_node); send_buffer.push_back(node_vote[facet_node]); } } } } // FIXME: The below algortihm can be improved (made more scalable) // by distributing (dof, process) pairs to 'owner' range owner, // then letting each process get the sharing process list. This // will avoid interleaving communication and computation. // Decide ownership of shared nodes const std::size_t num_prococesses = MPI::size(mpi_comm); const std::size_t process_number = MPI::rank(mpi_comm); std::vector<std::size_t> recv_buffer; for (std::size_t k = 1; k < num_prococesses; ++k) { const std::size_t src = (process_number - k + num_prococesses) % num_prococesses; const std::size_t dest = (process_number + k) % num_prococesses; MPI::send_recv(mpi_comm, send_buffer, dest, recv_buffer, src); for (std::size_t i = 0; i < recv_buffer.size(); i += 2) { const std::size_t received_node = recv_buffer[i]; const std::size_t received_vote = recv_buffer[i + 1]; if (shared_owned_nodes.find(received_node) != shared_owned_nodes.end()) { // Move dofs with higher ownership votes from shared to shared // but not owned if (received_vote < node_vote[received_node]) { shared_unowned_nodes.insert(received_node); shared_owned_nodes.erase(received_node); } else if (received_vote == node_vote[received_node] && process_number > src) { // If votes are equal, let lower rank process take ownership shared_unowned_nodes.insert(received_node); shared_owned_nodes.erase(received_node); } // Remember the sharing of the node shared_node_processes[received_node].push_back(src); } else if (shared_unowned_nodes.find(received_node) != shared_unowned_nodes.end()) { // Remember the sharing of the node shared_node_processes[received_node].push_back(src); } } } // Add/remove global dofs to/from relevant sets (last process owns // global dofs) if (process_number == num_prococesses - 1) { shared_owned_nodes.insert(global_dofs.begin(), global_dofs.end()); for (std::set<std::size_t>::const_iterator dof = global_dofs.begin(); dof != global_dofs.end(); ++dof) { set::const_iterator _dof = shared_unowned_nodes.find(*dof); if (_dof != shared_unowned_nodes.end()) shared_unowned_nodes.erase(_dof); } } else { shared_unowned_nodes.insert(global_dofs.begin(), global_dofs.end()); for (std::set<std::size_t>::const_iterator dof = global_dofs.begin(); dof != global_dofs.end(); ++dof) { set::const_iterator _dof = shared_owned_nodes.find(*dof); if (_dof != shared_owned_nodes.end()) shared_owned_nodes.erase(_dof); } } // Mark all shared-and-owned dofs as owned by the processes for (CellIterator cell(mesh); !cell.end(); ++cell) { const std::vector<dolfin::la_index>& cell_dofs = dofmap.cell_dofs(cell->index()); for (std::size_t i = 0; i < cell_dofs.size(); ++i) { // Get cell node size_t cell_node = cell_dofs[i] % num_nodes; // Map back to original (and common) numbering for restricted // space if (restriction) { const map_iterator it = restricted_nodes_inverse.find(cell_node); dolfin_assert(it != restricted_nodes_inverse.end()); cell_node = it->second; } // Mark dof as owned if not in unowned set if (shared_unowned_nodes.find(cell_node) == shared_unowned_nodes.end()) owned_nodes.insert(cell_node); } } // Check or set global dimension if (restriction) { // Global dimension for restricted space needs to be computed here // since it is not know by the UFC dof map. const std::size_t _owned_dim = owned_nodes.size(); const std::size_t _global_dimension = block_size*MPI::sum(mpi_comm, _owned_dim); dofmap._global_dimension = _global_dimension; } else { const std::size_t _owned_dim = owned_nodes.size(); dolfin_assert(block_size*MPI::sum(mpi_comm, _owned_dim) == dofmap.global_dimension()); } log(TRACE, "Finished determining dof ownership for parallel dof map"); }