//-----------------------------------------------------------------------------
void DofMapBuilder::compute_node_ownership(boost::array<set, 3>& node_ownership,
vec_map& shared_node_processes,
DofMap& dofmap,
const std::set<std::size_t>& global_dofs,
const Mesh& mesh,
std::shared_ptr<const Restriction> restriction,
const map& restricted_nodes_inverse,
std::size_t block_size)
{
log(TRACE, "Determining dof ownership for parallel dof map");
const MPI_Comm mpi_comm = mesh.mpi_comm();
const std::size_t N = dofmap.global_dimension();
dolfin_assert(N % block_size == 0);
const std::size_t num_nodes = N/block_size;
if (restriction)
dolfin_assert(block_size == 1);
// References to 'node' ownership sets
set& owned_nodes = node_ownership[0];
set& shared_owned_nodes = node_ownership[1];
set& shared_unowned_nodes = node_ownership[2];
// Clear data structures
owned_nodes.clear();
shared_owned_nodes.clear();
shared_unowned_nodes.clear();
// Data structures for computing ownership
boost::unordered_map<std::size_t, std::size_t> node_vote;
std::vector<std::size_t> facet_dofs(dofmap.num_facet_dofs());
// Communication buffer
std::vector<std::size_t> send_buffer;
// Extract the interior boundary
BoundaryMesh boundary(mesh, "local");
// Create a random number generator for ownership 'voting'
boost::mt19937 engine(MPI::rank(mpi_comm));
boost::uniform_int<> distribution(0, 100000000);
boost::variate_generator<boost::mt19937&, boost::uniform_int<> >
rng(engine, distribution);
// Build set of dofs on process boundary (first assuming that all
// are owned by this process)
const MeshFunction<std::size_t>& cell_map
= boundary.entity_map(boundary.topology().dim());
if (!cell_map.empty())
{
for (CellIterator _f(boundary); !_f.end(); ++_f)
{
// Create a vote per facet
const std::size_t facet_vote = rng();
// Get boundary facet
Facet f(mesh, cell_map[*_f]);
// Get cell to which facet belongs (pick first)
Cell c(mesh, f.entities(mesh.topology().dim())[0]);
// Skip cells not included in restriction
if (restriction && !restriction->contains(c))
continue;
// Tabulate dofs on cell
const std::vector<dolfin::la_index>& cell_dofs
= dofmap.cell_dofs(c.index());
// Tabulate which dofs are on the facet
dofmap.tabulate_facet_dofs(facet_dofs, c.index(f));
// Insert shared nodes into set and assign a 'vote'
dolfin_assert(dofmap.num_facet_dofs() % block_size == 0);
for (std::size_t i = 0; i < dofmap.num_facet_dofs(); ++i)
{
// Get facet node
size_t facet_node = cell_dofs[facet_dofs[i]] % num_nodes;
// Map back to original (and common) numbering for restricted
// space
if (restriction)
{
const map_iterator it = restricted_nodes_inverse.find(facet_node);
dolfin_assert(it != restricted_nodes_inverse.end());
facet_node = it->second;
}
// Add to list of shared nodes
if (shared_owned_nodes.find(facet_node) == shared_owned_nodes.end())
{
shared_owned_nodes.insert(facet_node);
node_vote[facet_node] = facet_vote;
send_buffer.push_back(facet_node);
send_buffer.push_back(node_vote[facet_node]);
}
}
}
}
// FIXME: The below algortihm can be improved (made more scalable)
// by distributing (dof, process) pairs to 'owner' range owner,
// then letting each process get the sharing process list. This
// will avoid interleaving communication and computation.
// Decide ownership of shared nodes
const std::size_t num_prococesses = MPI::size(mpi_comm);
const std::size_t process_number = MPI::rank(mpi_comm);
std::vector<std::size_t> recv_buffer;
for (std::size_t k = 1; k < num_prococesses; ++k)
{
const std::size_t src
= (process_number - k + num_prococesses) % num_prococesses;
const std::size_t dest = (process_number + k) % num_prococesses;
MPI::send_recv(mpi_comm, send_buffer, dest, recv_buffer, src);
for (std::size_t i = 0; i < recv_buffer.size(); i += 2)
{
const std::size_t received_node = recv_buffer[i];
const std::size_t received_vote = recv_buffer[i + 1];
if (shared_owned_nodes.find(received_node) != shared_owned_nodes.end())
{
// Move dofs with higher ownership votes from shared to shared
// but not owned
if (received_vote < node_vote[received_node])
{
shared_unowned_nodes.insert(received_node);
shared_owned_nodes.erase(received_node);
}
else if (received_vote == node_vote[received_node]
&& process_number > src)
{
// If votes are equal, let lower rank process take ownership
shared_unowned_nodes.insert(received_node);
shared_owned_nodes.erase(received_node);
}
// Remember the sharing of the node
shared_node_processes[received_node].push_back(src);
}
else if (shared_unowned_nodes.find(received_node)
!= shared_unowned_nodes.end())
{
// Remember the sharing of the node
shared_node_processes[received_node].push_back(src);
}
}
}
// Add/remove global dofs to/from relevant sets (last process owns
// global dofs)
if (process_number == num_prococesses - 1)
{
shared_owned_nodes.insert(global_dofs.begin(), global_dofs.end());
for (std::set<std::size_t>::const_iterator dof = global_dofs.begin();
dof != global_dofs.end(); ++dof)
{
set::const_iterator _dof = shared_unowned_nodes.find(*dof);
if (_dof != shared_unowned_nodes.end())
shared_unowned_nodes.erase(_dof);
}
}
else
{
shared_unowned_nodes.insert(global_dofs.begin(), global_dofs.end());
for (std::set<std::size_t>::const_iterator dof = global_dofs.begin();
dof != global_dofs.end(); ++dof)
{
set::const_iterator _dof = shared_owned_nodes.find(*dof);
if (_dof != shared_owned_nodes.end())
shared_owned_nodes.erase(_dof);
}
}
// Mark all shared-and-owned dofs as owned by the processes
for (CellIterator cell(mesh); !cell.end(); ++cell)
{
const std::vector<dolfin::la_index>& cell_dofs
= dofmap.cell_dofs(cell->index());
for (std::size_t i = 0; i < cell_dofs.size(); ++i)
{
// Get cell node
size_t cell_node = cell_dofs[i] % num_nodes;
// Map back to original (and common) numbering for restricted
// space
if (restriction)
{
const map_iterator it = restricted_nodes_inverse.find(cell_node);
dolfin_assert(it != restricted_nodes_inverse.end());
cell_node = it->second;
}
// Mark dof as owned if not in unowned set
if (shared_unowned_nodes.find(cell_node) == shared_unowned_nodes.end())
owned_nodes.insert(cell_node);
}
}
// Check or set global dimension
if (restriction)
{
// Global dimension for restricted space needs to be computed here
// since it is not know by the UFC dof map.
const std::size_t _owned_dim = owned_nodes.size();
const std::size_t _global_dimension = block_size*MPI::sum(mpi_comm,
_owned_dim);
dofmap._global_dimension = _global_dimension;
}
else
{
const std::size_t _owned_dim = owned_nodes.size();
dolfin_assert(block_size*MPI::sum(mpi_comm, _owned_dim)
== dofmap.global_dimension());
}
log(TRACE, "Finished determining dof ownership for parallel dof map");
}
