//----------------------------------------------------------------------------- void DofMapBuilder::parallel_renumber( const boost::array<set, 3>& node_ownership, const vec_map& shared_node_processes, DofMap& dofmap, const std::set<std::size_t>& global_dofs, const Mesh& mesh, std::shared_ptr<const Restriction> restriction, const map& restricted_nodes_inverse, std::size_t block_size) { log(TRACE, "Renumber dofs for parallel dof map"); // MPI communicator const MPI_Comm mpi_comm = mesh.mpi_comm(); const std::size_t N = dofmap.global_dimension(); dolfin_assert(N % block_size == 0); const std::size_t num_nodes = N/block_size; // References to dof ownership sets const set& owned_nodes = node_ownership[0]; const set& shared_owned_nodes = node_ownership[1]; const set& shared_unowned_nodes = node_ownership[2]; // FIXME: Handle double-renumbered dof map if (!dofmap.ufc_map_to_dofmap.empty()) { dolfin_error("DofMapBuilder.cpp", "compute parallel renumbering of degrees of freedom", "The degree of freedom mapping cannot be renumbered twice"); } const std::vector<std::vector<dolfin::la_index> >& old_dofmap = dofmap._dofmap; std::vector<std::vector<dolfin::la_index> > new_dofmap(old_dofmap.size()); dolfin_assert(old_dofmap.size() == mesh.num_cells()); // Compute offset for owned and non-shared nodes const std::size_t process_offset = MPI::global_offset(mpi_comm, owned_nodes.size(), true); // Clear some data dofmap._off_process_owner.clear(); // Create graph Graph graph(owned_nodes.size()); // Build graph for re-ordering. Below block is scoped to clear working // data structures once graph is constructed. { // Create contiguous local numbering for locally owned dofs std::size_t my_counter = 0; boost::unordered_map<std::size_t, std::size_t> my_old_to_new_node_index; for (set_iterator owned_node = owned_nodes.begin(); owned_node != owned_nodes.end(); ++owned_node, my_counter++) { my_old_to_new_node_index[*owned_node] = my_counter; } // Build local graph, based on old dof map, with contiguous numbering for (std::size_t cell = 0; cell < old_dofmap.size(); ++cell) { // Cell dofmaps with old indices const std::vector<dolfin::la_index>& dofs0 = dofmap.cell_dofs(cell); const std::vector<dolfin::la_index>& dofs1 = dofmap.cell_dofs(cell); dolfin_assert(dofs0.size() % block_size == 0); const std::size_t nodes_per_cell = dofs0.size()/block_size; // Loop over each node in dofs0 std::vector<dolfin::la_index>::const_iterator dof0, dof1; for (std::size_t i = 0; i < nodes_per_cell; ++i) { if (global_dofs.find(dofs0[i]) != global_dofs.end()) continue; const std::size_t n0_old = dofs0[i] % num_nodes; // Get new node index from contiguous map boost::unordered_map<std::size_t, std::size_t>::const_iterator n0 = my_old_to_new_node_index.find(n0_old); if (n0 != my_old_to_new_node_index.end()) { const std::size_t n0_local = n0->second; dolfin_assert(n0_local < graph.size()); for (std::size_t j = 0; j < nodes_per_cell; ++j) { if (global_dofs.find(dofs0[j]) != global_dofs.end()) continue; const std::size_t n1_old = dofs1[j] % num_nodes; boost::unordered_map<std::size_t, std::size_t>::const_iterator n1 = my_old_to_new_node_index.find(n1_old); if (n1 != my_old_to_new_node_index.end()) { const std::size_t n1_local = n1->second; if (n0_local != n1_local) graph[n0_local].insert(n1_local); } } } } } } // Reorder nodes locally // Reorder block graph const std::string ordering_library = dolfin::parameters["dof_ordering_library"]; std::vector<std::size_t> node_remap; if (ordering_library == "Boost") node_remap = BoostGraphOrdering::compute_cuthill_mckee(graph, true); else if (ordering_library == "SCOTCH") node_remap = SCOTCH::compute_gps(graph); else if (ordering_library == "random") { // NOTE: Randomised dof ordering should only be used for // testing/benchmarking node_remap.resize(graph.size()); for (std::size_t i = 0; i < node_remap.size(); ++i) node_remap[i] = i; std::random_shuffle(node_remap.begin(), node_remap.end()); } else { dolfin_error("DofMapBuilder.cpp", "reorder degrees of freedom", "The requested ordering library '%s' is unknown", ordering_library.c_str()); } // Map from old to new index for dofs boost::unordered_map<std::size_t, std::size_t> old_to_new_node_index; // Renumber owned dofs and buffer nodes that are owned but shared with // another process std::size_t counter = 0; std::vector<std::size_t> send_buffer; for (set_iterator owned_node = owned_nodes.begin(); owned_node != owned_nodes.end(); ++owned_node, counter++) { // Set new node number old_to_new_node_index[*owned_node] = process_offset + node_remap[counter]; // Update UFC-to-renumbered map for new dof number for (std::size_t i = 0; i < block_size; ++i) { std::size_t ufc_dof_index = *owned_node + i*num_nodes; std::size_t new_dof_index = (process_offset + node_remap[counter])*block_size + i; dofmap.ufc_map_to_dofmap[ufc_dof_index] = new_dof_index; } // If this node is shared and owned, buffer old and new index for // sending if (shared_owned_nodes.find(*owned_node) != shared_owned_nodes.end()) { send_buffer.push_back(*owned_node); send_buffer.push_back(process_offset + node_remap[counter]); } } // FIXME: The below algortihm can be improved (made more scalable) // by distributing (dof, process) pairs to 'owner' range owner, // then letting each process get the sharing process list. This // will avoid interleaving communication and computation. // Exchange new node numbers for nodes that are shared const std::size_t num_processes = MPI::size(mpi_comm); const std::size_t process_number = MPI::rank(mpi_comm); std::vector<std::size_t> recv_buffer; for (std::size_t k = 1; k < num_processes; ++k) { const std::size_t src = (process_number - k + num_processes) % num_processes; const std::size_t dest = (process_number + k) % num_processes; MPI::send_recv(mpi_comm, send_buffer, dest, recv_buffer, src); // Add dofs renumbered by another process to the old-to-new map for (std::size_t i = 0; i < recv_buffer.size(); i += 2) { const std::size_t received_old_node_index = recv_buffer[i]; const std::size_t received_new_node_index = recv_buffer[i + 1]; // Check if this process has shared dof (and is not the owner) if (shared_unowned_nodes.find(received_old_node_index) != shared_unowned_nodes.end()) { // Add to old-to-new node map old_to_new_node_index[received_old_node_index] = received_new_node_index; // Store map from off-process dof to owner and update // UFC-to-renumbered map for (std::size_t i = 0; i < block_size; ++i) { std::size_t ufc_dof_index = received_old_node_index + i*num_nodes; std::size_t new_dof_index = received_new_node_index*block_size + i; dofmap._off_process_owner[new_dof_index] = src; // Update UFC-to-renumbered map dofmap.ufc_map_to_dofmap[ufc_dof_index] = new_dof_index; } } } } // FIXME: Should dofmap._shared_dofs be cleared? // Insert the shared-dof-to-process mapping into the dofmap, // renumbering as necessary for (vec_map::const_iterator it = shared_node_processes.begin(); it != shared_node_processes.end(); ++it) { // Check for shared node in old_to_new_node_index map boost::unordered_map<std::size_t, std::size_t>::const_iterator new_index = old_to_new_node_index.find(it->first); if (new_index == old_to_new_node_index.end()) { for (std::size_t i = 0; i < block_size; ++i) { const std::size_t dof = it->first*block_size + i; dofmap._shared_dofs.insert(std::make_pair(dof, it->second)); } } else { for (std::size_t i = 0; i < block_size; ++i) { const std::size_t dof = (new_index->second)*block_size + i; dofmap._shared_dofs.insert(std::make_pair(dof, it->second)); } } dofmap._neighbours.insert(it->second.begin(), it->second.end()); } // Build new dofmap for (CellIterator cell(mesh); !cell.end(); ++cell) { // Skip cells not included in restriction if (restriction && !restriction->contains(*cell)) continue; // Get cell index and dimension const std::size_t cell_index = cell->index(); const std::size_t cell_dimension = dofmap.cell_dimension(cell_index); // Resize cell map and insert dofs new_dofmap[cell_index].resize(cell_dimension); for (std::size_t i = 0; i < cell_dimension; ++i) { const std::size_t old_index = old_dofmap[cell_index][i]; const std::size_t old_node = old_index % num_nodes; const std::size_t new_node = old_to_new_node_index[old_node]; new_dofmap[cell_index][i] = new_node*block_size + old_index/num_nodes; } } // Set new dof map dofmap._dofmap = new_dofmap; // Set ownership range dofmap._ownership_range = std::make_pair(block_size*process_offset, block_size*(process_offset + owned_nodes.size())); log(TRACE, "Finished renumbering dofs for parallel dof map"); }