//-----------------------------------------------------------------------------
DofMap::DofMap(std::unordered_map<std::size_t, std::size_t>& collapsed_map,
const DofMap& dofmap_view, const Mesh& mesh)
: _cell_dimension(0), _ufc_dofmap(dofmap_view._ufc_dofmap), _is_view(false),
_global_dimension(0), _ufc_offset(0), _multimesh_offset(0),
_index_map(new IndexMap(mesh.mpi_comm()))
{
dolfin_assert(_ufc_dofmap);
// Check for dimensional consistency between the dofmap and mesh
check_dimensional_consistency(*_ufc_dofmap, mesh);
// Check that mesh has been ordered
if (!mesh.ordered())
{
dolfin_error("DofMap.cpp",
"create mapping of degrees of freedom",
"Mesh is not ordered according to the UFC numbering convention. "
"Consider calling mesh.order()");
}
// Check dimensional consistency between UFC dofmap and the mesh
check_provided_entities(*_ufc_dofmap, mesh);
// Build new dof map
DofMapBuilder::build(*this, mesh, constrained_domain);
// Dimension sanity checks
dolfin_assert(dofmap_view._dofmap.size()
== mesh.num_cells()*dofmap_view._cell_dimension);
dolfin_assert(global_dimension() == dofmap_view.global_dimension());
dolfin_assert(_dofmap.size() == mesh.num_cells()*_cell_dimension);
// FIXME: Could we use a std::vector instead of std::map if the
// collapsed dof map is contiguous (0, . . . , n)?
// Build map from collapsed dof index to original dof index
collapsed_map.clear();
for (std::size_t i = 0; i < mesh.num_cells(); ++i)
{
const ArrayView<const dolfin::la_index> view_cell_dofs
= dofmap_view.cell_dofs(i);
const ArrayView<const dolfin::la_index> cell_dofs = this->cell_dofs(i);
dolfin_assert(view_cell_dofs.size() == cell_dofs.size());
for (std::size_t j = 0; j < view_cell_dofs.size(); ++j)
collapsed_map[cell_dofs[j]] = view_cell_dofs[j];
}
}
//-----------------------------------------------------------------------------
void DofMapBuilder::build(
DofMap& dofmap,
const Mesh& mesh,
std::shared_ptr<const std::map<unsigned int, std::map<unsigned int,
std::pair<unsigned int, unsigned int> > > > slave_master_entities,
std::shared_ptr<const Restriction> restriction)
{
// Start timer for dofmap initialization
Timer t0("Init dofmap");
// Check that mesh has been ordered
if (!mesh.ordered())
{
dolfin_error("DofMapBuiler.cpp",
"create mapping of degrees of freedom",
"Mesh is not ordered according to the UFC numbering convention. "
"Consider calling mesh.order()");
}
// Build dofmap based on UFC-provided map. This function does not
// set local_range
map restricted_dofs_inverse;
build_ufc_dofmap(dofmap, restricted_dofs_inverse, mesh,
slave_master_entities, restriction);
// Check if dofmap is distributed
const bool distributed = MPI::size(mesh.mpi_comm()) > 1;
// Build set of dofs that are not associated with a mesh entity
// (global dofs)
const std::set<std::size_t> global_dofs = compute_global_dofs(dofmap);
// Determine and set dof block size
dolfin_assert(dofmap._ufc_dofmap);
const std::size_t block_size = compute_blocksize(*dofmap._ufc_dofmap);
dofmap.block_size = block_size;
// Re-order dofmap when distributed for process locality and set
// local_range
if (distributed)
{
reorder_distributed(dofmap, mesh, restriction, restricted_dofs_inverse,
block_size, global_dofs);
}
else
{
// Optionally re-order local dofmap for spatial locality
const bool reorder = dolfin::parameters["reorder_dofs_serial"];
if (reorder)
reorder_local(dofmap, mesh, block_size, global_dofs);
// Set local dof ownbership range
dofmap._ownership_range = std::make_pair(0, dofmap.global_dimension());
}
}
void
Assembly::addJacobianNeighbor(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices, std::vector<dof_id_type> & neighbor_dof_indices)
{
DenseMatrix<Number> & kee = jacobianBlock(ivar, jvar);
DenseMatrix<Number> & ken = jacobianBlockNeighbor(Moose::ElementNeighbor, ivar, jvar);
DenseMatrix<Number> & kne = jacobianBlockNeighbor(Moose::NeighborElement, ivar, jvar);
DenseMatrix<Number> & knn = jacobianBlockNeighbor(Moose::NeighborNeighbor, ivar, jvar);
std::vector<dof_id_type> di(dof_indices);
std::vector<dof_id_type> dn(neighbor_dof_indices);
// stick it into the matrix
dof_map.constrain_element_matrix(kee, di, false);
dof_map.constrain_element_matrix(ken, di, dn, false);
dof_map.constrain_element_matrix(kne, dn, di, false);
dof_map.constrain_element_matrix(knn, dn, false);
Real scaling_factor = _sys.getVariable(_tid, ivar).scalingFactor();
if (scaling_factor != 1.0)
{
_tmp_Ke = ken;
_tmp_Ke *= scaling_factor;
jacobian.add_matrix(_tmp_Ke, di, dn);
_tmp_Ke = kne;
_tmp_Ke *= scaling_factor;
jacobian.add_matrix(_tmp_Ke, dn, di);
_tmp_Ke = knn;
_tmp_Ke *= scaling_factor;
jacobian.add_matrix(_tmp_Ke, dn);
}
else
{
jacobian.add_matrix(ken, di, dn);
jacobian.add_matrix(kne, dn, di);
jacobian.add_matrix(knn, dn);
}
}
//-----------------------------------------------------------------------------
DofMap::DofMap(boost::unordered_map<std::size_t, std::size_t>& collapsed_map,
const DofMap& dofmap_view, const Mesh& mesh)
: _ufc_dofmap(dofmap_view._ufc_dofmap), _global_dimension(0),
_ufc_offset(0)
{
dolfin_assert(_ufc_dofmap);
// Check for dimensional consistency between the dofmap and mesh
check_dimensional_consistency(*_ufc_dofmap, mesh);
// Check that mesh has been ordered
if (!mesh.ordered())
{
dolfin_error("DofMap.cpp",
"create mapping of degrees of freedom",
"Mesh is not ordered according to the UFC numbering convention. "
"Consider calling mesh.order()");
}
// Check dimensional consistency between UFC dofmap and the mesh
check_provided_entities(*_ufc_dofmap, mesh);
// Copy slave-master map (copy or share?)
if (dofmap_view.slave_master_mesh_entities)
slave_master_mesh_entities.reset(new std::map<unsigned int, std::map<unsigned int, std::pair<unsigned int, unsigned int> > >(*dofmap_view.slave_master_mesh_entities));
// Build new dof map
DofMapBuilder::build(*this, mesh, slave_master_mesh_entities, _restriction);
// Dimension sanity checks
dolfin_assert(dofmap_view._dofmap.size() == mesh.num_cells());
dolfin_assert(global_dimension() == dofmap_view.global_dimension());
dolfin_assert(_dofmap.size() == mesh.num_cells());
// FIXME: Could we use a std::vector instead of std::map if the
// collapsed dof map is contiguous (0, . . . , n)?
// Build map from collapsed dof index to original dof index
collapsed_map.clear();
for (std::size_t i = 0; i < mesh.num_cells(); ++i)
{
const std::vector<dolfin::la_index>& view_cell_dofs = dofmap_view._dofmap[i];
const std::vector<dolfin::la_index>& cell_dofs = _dofmap[i];
dolfin_assert(view_cell_dofs.size() == cell_dofs.size());
for (std::size_t j = 0; j < view_cell_dofs.size(); ++j)
collapsed_map[cell_dofs[j]] = view_cell_dofs[j];
}
}
void
Assembly::addJacobianBlock(SparseMatrix<Number> & jacobian, unsigned int ivar, unsigned int jvar, const DofMap & dof_map, std::vector<dof_id_type> & dof_indices)
{
DenseMatrix<Number> & ke = jacobianBlock(ivar, jvar);
// stick it into the matrix
std::vector<dof_id_type> di(dof_indices);
dof_map.constrain_element_matrix(ke, di, false);
Real scaling_factor = _sys.getVariable(_tid, ivar).scalingFactor();
if (scaling_factor != 1.0)
{
_tmp_Ke = ke;
_tmp_Ke *= scaling_factor;
jacobian.add_matrix(_tmp_Ke, di);
}
else
jacobian.add_matrix(ke, di);
}
//-----------------------------------------------------------------------------
void DofMapBuilder::parallel_renumber(
const boost::array<set, 3>& node_ownership,
const vec_map& shared_node_processes,
DofMap& dofmap,
const std::set<std::size_t>& global_dofs,
const Mesh& mesh,
std::shared_ptr<const Restriction> restriction,
const map& restricted_nodes_inverse,
std::size_t block_size)
{
log(TRACE, "Renumber dofs for parallel dof map");
// MPI communicator
const MPI_Comm mpi_comm = mesh.mpi_comm();
const std::size_t N = dofmap.global_dimension();
dolfin_assert(N % block_size == 0);
const std::size_t num_nodes = N/block_size;
// References to dof ownership sets
const set& owned_nodes = node_ownership[0];
const set& shared_owned_nodes = node_ownership[1];
const set& shared_unowned_nodes = node_ownership[2];
// FIXME: Handle double-renumbered dof map
if (!dofmap.ufc_map_to_dofmap.empty())
{
dolfin_error("DofMapBuilder.cpp",
"compute parallel renumbering of degrees of freedom",
"The degree of freedom mapping cannot be renumbered twice");
}
const std::vector<std::vector<dolfin::la_index> >& old_dofmap
= dofmap._dofmap;
std::vector<std::vector<dolfin::la_index> > new_dofmap(old_dofmap.size());
dolfin_assert(old_dofmap.size() == mesh.num_cells());
// Compute offset for owned and non-shared nodes
const std::size_t process_offset
= MPI::global_offset(mpi_comm, owned_nodes.size(), true);
// Clear some data
dofmap._off_process_owner.clear();
// Create graph
Graph graph(owned_nodes.size());
// Build graph for re-ordering. Below block is scoped to clear working
// data structures once graph is constructed.
{
// Create contiguous local numbering for locally owned dofs
std::size_t my_counter = 0;
boost::unordered_map<std::size_t, std::size_t> my_old_to_new_node_index;
for (set_iterator owned_node = owned_nodes.begin();
owned_node != owned_nodes.end(); ++owned_node, my_counter++)
{
my_old_to_new_node_index[*owned_node] = my_counter;
}
// Build local graph, based on old dof map, with contiguous numbering
for (std::size_t cell = 0; cell < old_dofmap.size(); ++cell)
{
// Cell dofmaps with old indices
const std::vector<dolfin::la_index>& dofs0 = dofmap.cell_dofs(cell);
const std::vector<dolfin::la_index>& dofs1 = dofmap.cell_dofs(cell);
dolfin_assert(dofs0.size() % block_size == 0);
const std::size_t nodes_per_cell = dofs0.size()/block_size;
// Loop over each node in dofs0
std::vector<dolfin::la_index>::const_iterator dof0, dof1;
for (std::size_t i = 0; i < nodes_per_cell; ++i)
{
if (global_dofs.find(dofs0[i]) != global_dofs.end())
continue;
const std::size_t n0_old = dofs0[i] % num_nodes;
// Get new node index from contiguous map
boost::unordered_map<std::size_t, std::size_t>::const_iterator n0
= my_old_to_new_node_index.find(n0_old);
if (n0 != my_old_to_new_node_index.end())
{
const std::size_t n0_local = n0->second;
dolfin_assert(n0_local < graph.size());
for (std::size_t j = 0; j < nodes_per_cell; ++j)
{
if (global_dofs.find(dofs0[j]) != global_dofs.end())
continue;
const std::size_t n1_old = dofs1[j] % num_nodes;
boost::unordered_map<std::size_t, std::size_t>::const_iterator
n1 = my_old_to_new_node_index.find(n1_old);
if (n1 != my_old_to_new_node_index.end())
{
const std::size_t n1_local = n1->second;
if (n0_local != n1_local)
graph[n0_local].insert(n1_local);
}
}
}
}
}
}
// Reorder nodes locally
// Reorder block graph
const std::string ordering_library
= dolfin::parameters["dof_ordering_library"];
std::vector<std::size_t> node_remap;
if (ordering_library == "Boost")
node_remap = BoostGraphOrdering::compute_cuthill_mckee(graph, true);
else if (ordering_library == "SCOTCH")
node_remap = SCOTCH::compute_gps(graph);
else if (ordering_library == "random")
{
// NOTE: Randomised dof ordering should only be used for
// testing/benchmarking
node_remap.resize(graph.size());
for (std::size_t i = 0; i < node_remap.size(); ++i)
node_remap[i] = i;
std::random_shuffle(node_remap.begin(), node_remap.end());
}
else
{
dolfin_error("DofMapBuilder.cpp",
"reorder degrees of freedom",
"The requested ordering library '%s' is unknown",
ordering_library.c_str());
}
// Map from old to new index for dofs
boost::unordered_map<std::size_t, std::size_t> old_to_new_node_index;
// Renumber owned dofs and buffer nodes that are owned but shared with
// another process
std::size_t counter = 0;
std::vector<std::size_t> send_buffer;
for (set_iterator owned_node = owned_nodes.begin();
owned_node != owned_nodes.end(); ++owned_node, counter++)
{
// Set new node number
old_to_new_node_index[*owned_node] = process_offset + node_remap[counter];
// Update UFC-to-renumbered map for new dof number
for (std::size_t i = 0; i < block_size; ++i)
{
std::size_t ufc_dof_index = *owned_node + i*num_nodes;
std::size_t new_dof_index = (process_offset
+ node_remap[counter])*block_size + i;
dofmap.ufc_map_to_dofmap[ufc_dof_index] = new_dof_index;
}
// If this node is shared and owned, buffer old and new index for
// sending
if (shared_owned_nodes.find(*owned_node) != shared_owned_nodes.end())
{
send_buffer.push_back(*owned_node);
send_buffer.push_back(process_offset + node_remap[counter]);
}
}
// FIXME: The below algortihm can be improved (made more scalable)
// by distributing (dof, process) pairs to 'owner' range owner,
// then letting each process get the sharing process list. This
// will avoid interleaving communication and computation.
// Exchange new node numbers for nodes that are shared
const std::size_t num_processes = MPI::size(mpi_comm);
const std::size_t process_number = MPI::rank(mpi_comm);
std::vector<std::size_t> recv_buffer;
for (std::size_t k = 1; k < num_processes; ++k)
{
const std::size_t src
= (process_number - k + num_processes) % num_processes;
const std::size_t dest = (process_number + k) % num_processes;
MPI::send_recv(mpi_comm, send_buffer, dest, recv_buffer, src);
// Add dofs renumbered by another process to the old-to-new map
for (std::size_t i = 0; i < recv_buffer.size(); i += 2)
{
const std::size_t received_old_node_index = recv_buffer[i];
const std::size_t received_new_node_index = recv_buffer[i + 1];
// Check if this process has shared dof (and is not the owner)
if (shared_unowned_nodes.find(received_old_node_index)
!= shared_unowned_nodes.end())
{
// Add to old-to-new node map
old_to_new_node_index[received_old_node_index]
= received_new_node_index;
// Store map from off-process dof to owner and update
// UFC-to-renumbered map
for (std::size_t i = 0; i < block_size; ++i)
{
std::size_t ufc_dof_index = received_old_node_index + i*num_nodes;
std::size_t new_dof_index = received_new_node_index*block_size + i;
dofmap._off_process_owner[new_dof_index] = src;
// Update UFC-to-renumbered map
dofmap.ufc_map_to_dofmap[ufc_dof_index] = new_dof_index;
}
}
}
}
// FIXME: Should dofmap._shared_dofs be cleared?
// Insert the shared-dof-to-process mapping into the dofmap,
// renumbering as necessary
for (vec_map::const_iterator it = shared_node_processes.begin();
it != shared_node_processes.end(); ++it)
{
// Check for shared node in old_to_new_node_index map
boost::unordered_map<std::size_t, std::size_t>::const_iterator
new_index = old_to_new_node_index.find(it->first);
if (new_index == old_to_new_node_index.end())
{
for (std::size_t i = 0; i < block_size; ++i)
{
const std::size_t dof = it->first*block_size + i;
dofmap._shared_dofs.insert(std::make_pair(dof, it->second));
}
}
else
{
for (std::size_t i = 0; i < block_size; ++i)
{
const std::size_t dof = (new_index->second)*block_size + i;
dofmap._shared_dofs.insert(std::make_pair(dof, it->second));
}
}
dofmap._neighbours.insert(it->second.begin(), it->second.end());
}
// Build new dofmap
for (CellIterator cell(mesh); !cell.end(); ++cell)
{
// Skip cells not included in restriction
if (restriction && !restriction->contains(*cell))
continue;
// Get cell index and dimension
const std::size_t cell_index = cell->index();
const std::size_t cell_dimension = dofmap.cell_dimension(cell_index);
// Resize cell map and insert dofs
new_dofmap[cell_index].resize(cell_dimension);
for (std::size_t i = 0; i < cell_dimension; ++i)
{
const std::size_t old_index = old_dofmap[cell_index][i];
const std::size_t old_node = old_index % num_nodes;
const std::size_t new_node = old_to_new_node_index[old_node];
new_dofmap[cell_index][i] = new_node*block_size + old_index/num_nodes;
}
}
// Set new dof map
dofmap._dofmap = new_dofmap;
// Set ownership range
dofmap._ownership_range
= std::make_pair(block_size*process_offset,
block_size*(process_offset + owned_nodes.size()));
log(TRACE, "Finished renumbering dofs for parallel dof map");
}
//-----------------------------------------------------------------------------
void DofMapBuilder::compute_node_ownership(boost::array<set, 3>& node_ownership,
vec_map& shared_node_processes,
DofMap& dofmap,
const std::set<std::size_t>& global_dofs,
const Mesh& mesh,
std::shared_ptr<const Restriction> restriction,
const map& restricted_nodes_inverse,
std::size_t block_size)
{
log(TRACE, "Determining dof ownership for parallel dof map");
const MPI_Comm mpi_comm = mesh.mpi_comm();
const std::size_t N = dofmap.global_dimension();
dolfin_assert(N % block_size == 0);
const std::size_t num_nodes = N/block_size;
if (restriction)
dolfin_assert(block_size == 1);
// References to 'node' ownership sets
set& owned_nodes = node_ownership[0];
set& shared_owned_nodes = node_ownership[1];
set& shared_unowned_nodes = node_ownership[2];
// Clear data structures
owned_nodes.clear();
shared_owned_nodes.clear();
shared_unowned_nodes.clear();
// Data structures for computing ownership
boost::unordered_map<std::size_t, std::size_t> node_vote;
std::vector<std::size_t> facet_dofs(dofmap.num_facet_dofs());
// Communication buffer
std::vector<std::size_t> send_buffer;
// Extract the interior boundary
BoundaryMesh boundary(mesh, "local");
// Create a random number generator for ownership 'voting'
boost::mt19937 engine(MPI::rank(mpi_comm));
boost::uniform_int<> distribution(0, 100000000);
boost::variate_generator<boost::mt19937&, boost::uniform_int<> >
rng(engine, distribution);
// Build set of dofs on process boundary (first assuming that all
// are owned by this process)
const MeshFunction<std::size_t>& cell_map
= boundary.entity_map(boundary.topology().dim());
if (!cell_map.empty())
{
for (CellIterator _f(boundary); !_f.end(); ++_f)
{
// Create a vote per facet
const std::size_t facet_vote = rng();
// Get boundary facet
Facet f(mesh, cell_map[*_f]);
// Get cell to which facet belongs (pick first)
Cell c(mesh, f.entities(mesh.topology().dim())[0]);
// Skip cells not included in restriction
if (restriction && !restriction->contains(c))
continue;
// Tabulate dofs on cell
const std::vector<dolfin::la_index>& cell_dofs
= dofmap.cell_dofs(c.index());
// Tabulate which dofs are on the facet
dofmap.tabulate_facet_dofs(facet_dofs, c.index(f));
// Insert shared nodes into set and assign a 'vote'
dolfin_assert(dofmap.num_facet_dofs() % block_size == 0);
for (std::size_t i = 0; i < dofmap.num_facet_dofs(); ++i)
{
// Get facet node
size_t facet_node = cell_dofs[facet_dofs[i]] % num_nodes;
// Map back to original (and common) numbering for restricted
// space
if (restriction)
{
const map_iterator it = restricted_nodes_inverse.find(facet_node);
dolfin_assert(it != restricted_nodes_inverse.end());
facet_node = it->second;
}
// Add to list of shared nodes
if (shared_owned_nodes.find(facet_node) == shared_owned_nodes.end())
{
shared_owned_nodes.insert(facet_node);
node_vote[facet_node] = facet_vote;
send_buffer.push_back(facet_node);
send_buffer.push_back(node_vote[facet_node]);
}
}
}
}
// FIXME: The below algortihm can be improved (made more scalable)
// by distributing (dof, process) pairs to 'owner' range owner,
// then letting each process get the sharing process list. This
// will avoid interleaving communication and computation.
// Decide ownership of shared nodes
const std::size_t num_prococesses = MPI::size(mpi_comm);
const std::size_t process_number = MPI::rank(mpi_comm);
std::vector<std::size_t> recv_buffer;
for (std::size_t k = 1; k < num_prococesses; ++k)
{
const std::size_t src
= (process_number - k + num_prococesses) % num_prococesses;
const std::size_t dest = (process_number + k) % num_prococesses;
MPI::send_recv(mpi_comm, send_buffer, dest, recv_buffer, src);
for (std::size_t i = 0; i < recv_buffer.size(); i += 2)
{
const std::size_t received_node = recv_buffer[i];
const std::size_t received_vote = recv_buffer[i + 1];
if (shared_owned_nodes.find(received_node) != shared_owned_nodes.end())
{
// Move dofs with higher ownership votes from shared to shared
// but not owned
if (received_vote < node_vote[received_node])
{
shared_unowned_nodes.insert(received_node);
shared_owned_nodes.erase(received_node);
}
else if (received_vote == node_vote[received_node]
&& process_number > src)
{
// If votes are equal, let lower rank process take ownership
shared_unowned_nodes.insert(received_node);
shared_owned_nodes.erase(received_node);
}
// Remember the sharing of the node
shared_node_processes[received_node].push_back(src);
}
else if (shared_unowned_nodes.find(received_node)
!= shared_unowned_nodes.end())
{
// Remember the sharing of the node
shared_node_processes[received_node].push_back(src);
}
}
}
// Add/remove global dofs to/from relevant sets (last process owns
// global dofs)
if (process_number == num_prococesses - 1)
{
shared_owned_nodes.insert(global_dofs.begin(), global_dofs.end());
for (std::set<std::size_t>::const_iterator dof = global_dofs.begin();
dof != global_dofs.end(); ++dof)
{
set::const_iterator _dof = shared_unowned_nodes.find(*dof);
if (_dof != shared_unowned_nodes.end())
shared_unowned_nodes.erase(_dof);
}
}
else
{
shared_unowned_nodes.insert(global_dofs.begin(), global_dofs.end());
for (std::set<std::size_t>::const_iterator dof = global_dofs.begin();
dof != global_dofs.end(); ++dof)
{
set::const_iterator _dof = shared_owned_nodes.find(*dof);
if (_dof != shared_owned_nodes.end())
shared_owned_nodes.erase(_dof);
}
}
// Mark all shared-and-owned dofs as owned by the processes
for (CellIterator cell(mesh); !cell.end(); ++cell)
{
const std::vector<dolfin::la_index>& cell_dofs
= dofmap.cell_dofs(cell->index());
for (std::size_t i = 0; i < cell_dofs.size(); ++i)
{
// Get cell node
size_t cell_node = cell_dofs[i] % num_nodes;
// Map back to original (and common) numbering for restricted
// space
if (restriction)
{
const map_iterator it = restricted_nodes_inverse.find(cell_node);
dolfin_assert(it != restricted_nodes_inverse.end());
cell_node = it->second;
}
// Mark dof as owned if not in unowned set
if (shared_unowned_nodes.find(cell_node) == shared_unowned_nodes.end())
owned_nodes.insert(cell_node);
}
}
// Check or set global dimension
if (restriction)
{
// Global dimension for restricted space needs to be computed here
// since it is not know by the UFC dof map.
const std::size_t _owned_dim = owned_nodes.size();
const std::size_t _global_dimension = block_size*MPI::sum(mpi_comm,
_owned_dim);
dofmap._global_dimension = _global_dimension;
}
else
{
const std::size_t _owned_dim = owned_nodes.size();
dolfin_assert(block_size*MPI::sum(mpi_comm, _owned_dim)
== dofmap.global_dimension());
}
log(TRACE, "Finished determining dof ownership for parallel dof map");
}
//-----------------------------------------------------------------------------
void DofMapBuilder::reorder_local(DofMap& dofmap, const Mesh& mesh,
std::size_t block_size,
const std::set<std::size_t>& global_dofs)
{
// Global dimension
const std::size_t N = dofmap.global_dimension();
// Create empty graph
dolfin_assert(N % block_size == 0);
const std::size_t num_nodes = N/block_size;
Graph graph(num_nodes);
// Build local graph for blocks
std::vector<dolfin::la_index> dofs_tmp;
for (CellIterator cell(mesh); !cell.end(); ++cell)
{
const std::vector<dolfin::la_index>& dofs0
= dofmap.cell_dofs(cell->index());
const std::vector<dolfin::la_index>& dofs1
= dofmap.cell_dofs(cell->index());
dolfin_assert(dofs0.size() % block_size == 0);
const std::size_t nodes_per_cell = dofs0.size()/block_size;
// Build vector of graph 'edges'
dofs_tmp.resize(nodes_per_cell);
std::size_t dof_counter = 0;
for (std::size_t i = 0; i < nodes_per_cell; ++i)
{
if (global_dofs.find(dofs0[i]) == global_dofs.end())
{
dofs_tmp[i] = dofs1[i] % num_nodes;
++dof_counter;
}
}
// Add graph 'edges'
for (std::size_t i = 0; i < nodes_per_cell; ++i)
{
if (global_dofs.find(dofs1[i]) == global_dofs.end())
{
graph[dofs0[i] % num_nodes].insert(dofs_tmp.begin(),
dofs_tmp.begin() + dof_counter);
}
}
}
// Reorder block graph
const std::string ordering_library
= dolfin::parameters["dof_ordering_library"];
std::vector<std::size_t> block_remap;
if (ordering_library == "Boost")
block_remap = BoostGraphOrdering::compute_cuthill_mckee(graph, true);
else if (ordering_library == "SCOTCH")
block_remap = SCOTCH::compute_gps(graph);
else if (ordering_library == "random")
{
// NOTE: Randomised dof ordering should only be used for
// testing/benchmarking
block_remap.resize(graph.size());
for (std::size_t i = 0; i < block_remap.size(); ++i)
block_remap[i] = i;
std::random_shuffle(block_remap.begin(), block_remap.end());
}
else
{
dolfin_error("DofMapBuilder.cpp",
"reorder degrees of freedom",
"The requested ordering library '%s' is unknown", ordering_library.c_str());
}
// Re-number dofs for each cell
std::vector<std::vector<dolfin::la_index> >::iterator cell_map;
std::vector<dolfin::la_index>::iterator dof;
for (cell_map = dofmap._dofmap.begin(); cell_map != dofmap._dofmap.end();
++cell_map)
{
for (dof = cell_map->begin(); dof != cell_map->end(); ++dof)
{
const std::size_t old_node = (*dof) % num_nodes;
const std::size_t new_node = block_remap[old_node];
*dof = new_node*block_size + (*dof)/num_nodes;
}
}
// Store re-ordering map (from UFC dofmap)
dolfin_assert(dofmap.ufc_map_to_dofmap.empty());
for (std::size_t i = 0; i < dofmap.global_dimension(); ++i)
{
const std::size_t old_node = i % num_nodes;
const std::size_t new_node = block_remap[old_node];
dofmap.ufc_map_to_dofmap[i] = new_node*block_size + i/num_nodes;
}
}