//-----------------------------------------------------------------------------
void DofMapBuilder::parallel_renumber(
const boost::array<set, 3>& node_ownership,
const vec_map& shared_node_processes,
DofMap& dofmap,
const std::set<std::size_t>& global_dofs,
const Mesh& mesh,
std::shared_ptr<const Restriction> restriction,
const map& restricted_nodes_inverse,
std::size_t block_size)
{
log(TRACE, "Renumber dofs for parallel dof map");
// MPI communicator
const MPI_Comm mpi_comm = mesh.mpi_comm();
const std::size_t N = dofmap.global_dimension();
dolfin_assert(N % block_size == 0);
const std::size_t num_nodes = N/block_size;
// References to dof ownership sets
const set& owned_nodes = node_ownership[0];
const set& shared_owned_nodes = node_ownership[1];
const set& shared_unowned_nodes = node_ownership[2];
// FIXME: Handle double-renumbered dof map
if (!dofmap.ufc_map_to_dofmap.empty())
{
dolfin_error("DofMapBuilder.cpp",
"compute parallel renumbering of degrees of freedom",
"The degree of freedom mapping cannot be renumbered twice");
}
const std::vector<std::vector<dolfin::la_index> >& old_dofmap
= dofmap._dofmap;
std::vector<std::vector<dolfin::la_index> > new_dofmap(old_dofmap.size());
dolfin_assert(old_dofmap.size() == mesh.num_cells());
// Compute offset for owned and non-shared nodes
const std::size_t process_offset
= MPI::global_offset(mpi_comm, owned_nodes.size(), true);
// Clear some data
dofmap._off_process_owner.clear();
// Create graph
Graph graph(owned_nodes.size());
// Build graph for re-ordering. Below block is scoped to clear working
// data structures once graph is constructed.
{
// Create contiguous local numbering for locally owned dofs
std::size_t my_counter = 0;
boost::unordered_map<std::size_t, std::size_t> my_old_to_new_node_index;
for (set_iterator owned_node = owned_nodes.begin();
owned_node != owned_nodes.end(); ++owned_node, my_counter++)
{
my_old_to_new_node_index[*owned_node] = my_counter;
}
// Build local graph, based on old dof map, with contiguous numbering
for (std::size_t cell = 0; cell < old_dofmap.size(); ++cell)
{
// Cell dofmaps with old indices
const std::vector<dolfin::la_index>& dofs0 = dofmap.cell_dofs(cell);
const std::vector<dolfin::la_index>& dofs1 = dofmap.cell_dofs(cell);
dolfin_assert(dofs0.size() % block_size == 0);
const std::size_t nodes_per_cell = dofs0.size()/block_size;
// Loop over each node in dofs0
std::vector<dolfin::la_index>::const_iterator dof0, dof1;
for (std::size_t i = 0; i < nodes_per_cell; ++i)
{
if (global_dofs.find(dofs0[i]) != global_dofs.end())
continue;
const std::size_t n0_old = dofs0[i] % num_nodes;
// Get new node index from contiguous map
boost::unordered_map<std::size_t, std::size_t>::const_iterator n0
= my_old_to_new_node_index.find(n0_old);
if (n0 != my_old_to_new_node_index.end())
{
const std::size_t n0_local = n0->second;
dolfin_assert(n0_local < graph.size());
for (std::size_t j = 0; j < nodes_per_cell; ++j)
{
if (global_dofs.find(dofs0[j]) != global_dofs.end())
continue;
const std::size_t n1_old = dofs1[j] % num_nodes;
boost::unordered_map<std::size_t, std::size_t>::const_iterator
n1 = my_old_to_new_node_index.find(n1_old);
if (n1 != my_old_to_new_node_index.end())
{
const std::size_t n1_local = n1->second;
if (n0_local != n1_local)
graph[n0_local].insert(n1_local);
}
}
}
}
}
}
// Reorder nodes locally
// Reorder block graph
const std::string ordering_library
= dolfin::parameters["dof_ordering_library"];
std::vector<std::size_t> node_remap;
if (ordering_library == "Boost")
node_remap = BoostGraphOrdering::compute_cuthill_mckee(graph, true);
else if (ordering_library == "SCOTCH")
node_remap = SCOTCH::compute_gps(graph);
else if (ordering_library == "random")
{
// NOTE: Randomised dof ordering should only be used for
// testing/benchmarking
node_remap.resize(graph.size());
for (std::size_t i = 0; i < node_remap.size(); ++i)
node_remap[i] = i;
std::random_shuffle(node_remap.begin(), node_remap.end());
}
else
{
dolfin_error("DofMapBuilder.cpp",
"reorder degrees of freedom",
"The requested ordering library '%s' is unknown",
ordering_library.c_str());
}
// Map from old to new index for dofs
boost::unordered_map<std::size_t, std::size_t> old_to_new_node_index;
// Renumber owned dofs and buffer nodes that are owned but shared with
// another process
std::size_t counter = 0;
std::vector<std::size_t> send_buffer;
for (set_iterator owned_node = owned_nodes.begin();
owned_node != owned_nodes.end(); ++owned_node, counter++)
{
// Set new node number
old_to_new_node_index[*owned_node] = process_offset + node_remap[counter];
// Update UFC-to-renumbered map for new dof number
for (std::size_t i = 0; i < block_size; ++i)
{
std::size_t ufc_dof_index = *owned_node + i*num_nodes;
std::size_t new_dof_index = (process_offset
+ node_remap[counter])*block_size + i;
dofmap.ufc_map_to_dofmap[ufc_dof_index] = new_dof_index;
}
// If this node is shared and owned, buffer old and new index for
// sending
if (shared_owned_nodes.find(*owned_node) != shared_owned_nodes.end())
{
send_buffer.push_back(*owned_node);
send_buffer.push_back(process_offset + node_remap[counter]);
}
}
// FIXME: The below algortihm can be improved (made more scalable)
// by distributing (dof, process) pairs to 'owner' range owner,
// then letting each process get the sharing process list. This
// will avoid interleaving communication and computation.
// Exchange new node numbers for nodes that are shared
const std::size_t num_processes = MPI::size(mpi_comm);
const std::size_t process_number = MPI::rank(mpi_comm);
std::vector<std::size_t> recv_buffer;
for (std::size_t k = 1; k < num_processes; ++k)
{
const std::size_t src
= (process_number - k + num_processes) % num_processes;
const std::size_t dest = (process_number + k) % num_processes;
MPI::send_recv(mpi_comm, send_buffer, dest, recv_buffer, src);
// Add dofs renumbered by another process to the old-to-new map
for (std::size_t i = 0; i < recv_buffer.size(); i += 2)
{
const std::size_t received_old_node_index = recv_buffer[i];
const std::size_t received_new_node_index = recv_buffer[i + 1];
// Check if this process has shared dof (and is not the owner)
if (shared_unowned_nodes.find(received_old_node_index)
!= shared_unowned_nodes.end())
{
// Add to old-to-new node map
old_to_new_node_index[received_old_node_index]
= received_new_node_index;
// Store map from off-process dof to owner and update
// UFC-to-renumbered map
for (std::size_t i = 0; i < block_size; ++i)
{
std::size_t ufc_dof_index = received_old_node_index + i*num_nodes;
std::size_t new_dof_index = received_new_node_index*block_size + i;
dofmap._off_process_owner[new_dof_index] = src;
// Update UFC-to-renumbered map
dofmap.ufc_map_to_dofmap[ufc_dof_index] = new_dof_index;
}
}
}
}
// FIXME: Should dofmap._shared_dofs be cleared?
// Insert the shared-dof-to-process mapping into the dofmap,
// renumbering as necessary
for (vec_map::const_iterator it = shared_node_processes.begin();
it != shared_node_processes.end(); ++it)
{
// Check for shared node in old_to_new_node_index map
boost::unordered_map<std::size_t, std::size_t>::const_iterator
new_index = old_to_new_node_index.find(it->first);
if (new_index == old_to_new_node_index.end())
{
for (std::size_t i = 0; i < block_size; ++i)
{
const std::size_t dof = it->first*block_size + i;
dofmap._shared_dofs.insert(std::make_pair(dof, it->second));
}
}
else
{
for (std::size_t i = 0; i < block_size; ++i)
{
const std::size_t dof = (new_index->second)*block_size + i;
dofmap._shared_dofs.insert(std::make_pair(dof, it->second));
}
}
dofmap._neighbours.insert(it->second.begin(), it->second.end());
}
// Build new dofmap
for (CellIterator cell(mesh); !cell.end(); ++cell)
{
// Skip cells not included in restriction
if (restriction && !restriction->contains(*cell))
continue;
// Get cell index and dimension
const std::size_t cell_index = cell->index();
const std::size_t cell_dimension = dofmap.cell_dimension(cell_index);
// Resize cell map and insert dofs
new_dofmap[cell_index].resize(cell_dimension);
for (std::size_t i = 0; i < cell_dimension; ++i)
{
const std::size_t old_index = old_dofmap[cell_index][i];
const std::size_t old_node = old_index % num_nodes;
const std::size_t new_node = old_to_new_node_index[old_node];
new_dofmap[cell_index][i] = new_node*block_size + old_index/num_nodes;
}
}
// Set new dof map
dofmap._dofmap = new_dofmap;
// Set ownership range
dofmap._ownership_range
= std::make_pair(block_size*process_offset,
block_size*(process_offset + owned_nodes.size()));
log(TRACE, "Finished renumbering dofs for parallel dof map");
}
