Exemplo n.º 1
0
//-----------------------------------------------------------------------------
void DofMapBuilder::build(
  DofMap& dofmap,
  const Mesh& mesh,
  std::shared_ptr<const std::map<unsigned int, std::map<unsigned int,
  std::pair<unsigned int, unsigned int> > > > slave_master_entities,
  std::shared_ptr<const Restriction> restriction)
{
  // Start timer for dofmap initialization
  Timer t0("Init dofmap");

  // Check that mesh has been ordered
  if (!mesh.ordered())
  {
     dolfin_error("DofMapBuiler.cpp",
                  "create mapping of degrees of freedom",
                  "Mesh is not ordered according to the UFC numbering convention. "
                  "Consider calling mesh.order()");
  }

  // Build dofmap based on UFC-provided map. This function does not
  // set local_range
  map restricted_dofs_inverse;
  build_ufc_dofmap(dofmap, restricted_dofs_inverse, mesh,
                   slave_master_entities, restriction);

  // Check if dofmap is distributed
  const bool distributed = MPI::size(mesh.mpi_comm()) > 1;

  // Build set of dofs that are not associated with a mesh entity
  // (global dofs)
  const std::set<std::size_t>  global_dofs = compute_global_dofs(dofmap);

  // Determine and set dof block size
  dolfin_assert(dofmap._ufc_dofmap);
  const std::size_t block_size = compute_blocksize(*dofmap._ufc_dofmap);
  dofmap.block_size = block_size;

  // Re-order dofmap when distributed for process locality and set
  // local_range
  if (distributed)
  {
    reorder_distributed(dofmap, mesh, restriction, restricted_dofs_inverse,
                        block_size, global_dofs);
  }
  else
  {
    // Optionally re-order local dofmap for spatial locality
    const bool reorder = dolfin::parameters["reorder_dofs_serial"];
    if (reorder)
      reorder_local(dofmap, mesh, block_size, global_dofs);

    // Set local dof ownbership range
    dofmap._ownership_range = std::make_pair(0, dofmap.global_dimension());
  }
}
Exemplo n.º 2
0
//-----------------------------------------------------------------------------
DofMap::DofMap(boost::unordered_map<std::size_t, std::size_t>& collapsed_map,
               const DofMap& dofmap_view, const Mesh& mesh)
   :  _ufc_dofmap(dofmap_view._ufc_dofmap), _global_dimension(0),
      _ufc_offset(0)
{
  dolfin_assert(_ufc_dofmap);

  // Check for dimensional consistency between the dofmap and mesh
  check_dimensional_consistency(*_ufc_dofmap, mesh);

  // Check that mesh has been ordered
  if (!mesh.ordered())
  {
     dolfin_error("DofMap.cpp",
                  "create mapping of degrees of freedom",
                  "Mesh is not ordered according to the UFC numbering convention. "
                  "Consider calling mesh.order()");
  }

  // Check dimensional consistency between UFC dofmap and the mesh
  check_provided_entities(*_ufc_dofmap, mesh);

  // Copy slave-master map (copy or share?)
  if (dofmap_view.slave_master_mesh_entities)
    slave_master_mesh_entities.reset(new std::map<unsigned int, std::map<unsigned int, std::pair<unsigned int, unsigned int> > >(*dofmap_view.slave_master_mesh_entities));

  // Build new dof map
  DofMapBuilder::build(*this, mesh, slave_master_mesh_entities, _restriction);

  // Dimension sanity checks
  dolfin_assert(dofmap_view._dofmap.size() == mesh.num_cells());
  dolfin_assert(global_dimension() == dofmap_view.global_dimension());
  dolfin_assert(_dofmap.size() == mesh.num_cells());

  // FIXME: Could we use a std::vector instead of std::map if the
  //        collapsed dof map is contiguous (0, . . . , n)?

  // Build map from collapsed dof index to original dof index
  collapsed_map.clear();
  for (std::size_t i = 0; i < mesh.num_cells(); ++i)
  {
    const std::vector<dolfin::la_index>& view_cell_dofs = dofmap_view._dofmap[i];
    const std::vector<dolfin::la_index>& cell_dofs = _dofmap[i];
    dolfin_assert(view_cell_dofs.size() == cell_dofs.size());

    for (std::size_t j = 0; j < view_cell_dofs.size(); ++j)
      collapsed_map[cell_dofs[j]] = view_cell_dofs[j];
  }
}
Exemplo n.º 3
0
//-----------------------------------------------------------------------------
DofMap::DofMap(std::unordered_map<std::size_t, std::size_t>& collapsed_map,
               const DofMap& dofmap_view, const Mesh& mesh)
  : _cell_dimension(0), _ufc_dofmap(dofmap_view._ufc_dofmap), _is_view(false),
    _global_dimension(0), _ufc_offset(0), _multimesh_offset(0),
    _index_map(new IndexMap(mesh.mpi_comm()))
{
  dolfin_assert(_ufc_dofmap);

  // Check for dimensional consistency between the dofmap and mesh
  check_dimensional_consistency(*_ufc_dofmap, mesh);

  // Check that mesh has been ordered
  if (!mesh.ordered())
  {
     dolfin_error("DofMap.cpp",
                  "create mapping of degrees of freedom",
                  "Mesh is not ordered according to the UFC numbering convention. "
                  "Consider calling mesh.order()");
  }

  // Check dimensional consistency between UFC dofmap and the mesh
  check_provided_entities(*_ufc_dofmap, mesh);

  // Build new dof map
  DofMapBuilder::build(*this, mesh, constrained_domain);

  // Dimension sanity checks
  dolfin_assert(dofmap_view._dofmap.size()
                == mesh.num_cells()*dofmap_view._cell_dimension);
  dolfin_assert(global_dimension() == dofmap_view.global_dimension());
  dolfin_assert(_dofmap.size() == mesh.num_cells()*_cell_dimension);

  // FIXME: Could we use a std::vector instead of std::map if the
  //        collapsed dof map is contiguous (0, . . . , n)?

  // Build map from collapsed dof index to original dof index
  collapsed_map.clear();
  for (std::size_t i = 0; i < mesh.num_cells(); ++i)
  {
    const ArrayView<const dolfin::la_index> view_cell_dofs
      = dofmap_view.cell_dofs(i);
    const ArrayView<const dolfin::la_index> cell_dofs = this->cell_dofs(i);
    dolfin_assert(view_cell_dofs.size() == cell_dofs.size());

    for (std::size_t j = 0; j < view_cell_dofs.size(); ++j)
      collapsed_map[cell_dofs[j]] = view_cell_dofs[j];
  }
}
Exemplo n.º 4
0
//-----------------------------------------------------------------------------
void DofMapBuilder::parallel_renumber(
  const boost::array<set, 3>& node_ownership,
  const vec_map& shared_node_processes,
  DofMap& dofmap,
  const std::set<std::size_t>& global_dofs,
  const Mesh& mesh,
  std::shared_ptr<const Restriction> restriction,
  const map& restricted_nodes_inverse,
  std::size_t block_size)
{
  log(TRACE, "Renumber dofs for parallel dof map");

  // MPI communicator
  const MPI_Comm mpi_comm = mesh.mpi_comm();

  const std::size_t N = dofmap.global_dimension();
  dolfin_assert(N % block_size == 0);
  const std::size_t num_nodes = N/block_size;

  // References to dof ownership sets
  const set& owned_nodes          = node_ownership[0];
  const set& shared_owned_nodes   = node_ownership[1];
  const set& shared_unowned_nodes = node_ownership[2];

  // FIXME: Handle double-renumbered dof map
  if (!dofmap.ufc_map_to_dofmap.empty())
  {
    dolfin_error("DofMapBuilder.cpp",
                 "compute parallel renumbering of degrees of freedom",
                 "The degree of freedom mapping cannot be renumbered twice");
  }

  const std::vector<std::vector<dolfin::la_index> >& old_dofmap
    = dofmap._dofmap;
  std::vector<std::vector<dolfin::la_index> > new_dofmap(old_dofmap.size());
  dolfin_assert(old_dofmap.size() == mesh.num_cells());

  // Compute offset for owned and non-shared nodes
  const std::size_t process_offset
    = MPI::global_offset(mpi_comm, owned_nodes.size(), true);

  // Clear some data
  dofmap._off_process_owner.clear();

  // Create graph
  Graph graph(owned_nodes.size());

  // Build graph for re-ordering. Below block is scoped to clear working
  // data structures once graph is constructed.
  {
    // Create contiguous local numbering for locally owned dofs
    std::size_t my_counter = 0;
    boost::unordered_map<std::size_t, std::size_t> my_old_to_new_node_index;
    for (set_iterator owned_node = owned_nodes.begin();
          owned_node != owned_nodes.end(); ++owned_node, my_counter++)
    {
      my_old_to_new_node_index[*owned_node] = my_counter;
    }

    // Build local graph, based on old dof map, with contiguous numbering
    for (std::size_t cell = 0; cell < old_dofmap.size(); ++cell)
    {
      // Cell dofmaps with old indices
      const std::vector<dolfin::la_index>& dofs0 = dofmap.cell_dofs(cell);
      const std::vector<dolfin::la_index>& dofs1 = dofmap.cell_dofs(cell);

      dolfin_assert(dofs0.size() % block_size == 0);
      const std::size_t nodes_per_cell = dofs0.size()/block_size;

      // Loop over each node in dofs0
      std::vector<dolfin::la_index>::const_iterator dof0, dof1;
      for (std::size_t i = 0; i < nodes_per_cell; ++i)
      {
	if (global_dofs.find(dofs0[i]) != global_dofs.end())
	  continue;

        const std::size_t n0_old = dofs0[i] % num_nodes;

        // Get new node index from contiguous map
        boost::unordered_map<std::size_t, std::size_t>::const_iterator n0
            = my_old_to_new_node_index.find(n0_old);
        if (n0 != my_old_to_new_node_index.end())
        {
          const std::size_t n0_local = n0->second;
          dolfin_assert(n0_local < graph.size());
          for (std::size_t j = 0; j < nodes_per_cell; ++j)
          {
	    if (global_dofs.find(dofs0[j]) != global_dofs.end())
	      continue;

            const std::size_t n1_old = dofs1[j] % num_nodes;
            boost::unordered_map<std::size_t, std::size_t>::const_iterator
              n1 = my_old_to_new_node_index.find(n1_old);
            if (n1 != my_old_to_new_node_index.end())
            {
              const std::size_t n1_local = n1->second;
              if (n0_local != n1_local)
                graph[n0_local].insert(n1_local);
            }
          }
        }
      }
    }
  }

  // Reorder nodes locally
  // Reorder block graph
  const std::string ordering_library
    = dolfin::parameters["dof_ordering_library"];
  std::vector<std::size_t> node_remap;
  if (ordering_library == "Boost")
    node_remap = BoostGraphOrdering::compute_cuthill_mckee(graph, true);
  else if (ordering_library == "SCOTCH")
    node_remap = SCOTCH::compute_gps(graph);
  else if (ordering_library == "random")
  {
    // NOTE: Randomised dof ordering should only be used for
    // testing/benchmarking
    node_remap.resize(graph.size());
    for (std::size_t i = 0; i < node_remap.size(); ++i)
      node_remap[i] = i;
    std::random_shuffle(node_remap.begin(), node_remap.end());
  }
  else
  {
    dolfin_error("DofMapBuilder.cpp",
                 "reorder degrees of freedom",
                 "The requested ordering library '%s' is unknown",
                 ordering_library.c_str());
  }

  // Map from old to new index for dofs
  boost::unordered_map<std::size_t, std::size_t> old_to_new_node_index;

  // Renumber owned dofs and buffer nodes that are owned but shared with
  // another process
  std::size_t counter = 0;
  std::vector<std::size_t> send_buffer;
  for (set_iterator owned_node = owned_nodes.begin();
       owned_node != owned_nodes.end(); ++owned_node, counter++)
  {
    // Set new node number
    old_to_new_node_index[*owned_node] = process_offset + node_remap[counter];

    // Update UFC-to-renumbered map for new dof number
    for (std::size_t i = 0; i < block_size; ++i)
    {
      std::size_t ufc_dof_index = *owned_node + i*num_nodes;
      std::size_t new_dof_index = (process_offset
                                   + node_remap[counter])*block_size + i;

      dofmap.ufc_map_to_dofmap[ufc_dof_index] = new_dof_index;
    }

    // If this node is shared and owned, buffer old and new index for
    // sending
    if (shared_owned_nodes.find(*owned_node) != shared_owned_nodes.end())
    {
      send_buffer.push_back(*owned_node);
      send_buffer.push_back(process_offset + node_remap[counter]);
    }
  }

  // FIXME: The below algortihm can be improved (made more scalable)
  //        by distributing (dof, process) pairs to 'owner' range owner,
  //        then letting each process get the sharing process list. This
  //        will avoid interleaving communication and computation.

  // Exchange new node numbers for nodes that are shared
  const std::size_t num_processes = MPI::size(mpi_comm);
  const std::size_t process_number = MPI::rank(mpi_comm);
  std::vector<std::size_t> recv_buffer;
  for (std::size_t k = 1; k < num_processes; ++k)
  {
    const std::size_t src
      = (process_number - k + num_processes) % num_processes;
    const std::size_t dest = (process_number + k) % num_processes;
    MPI::send_recv(mpi_comm, send_buffer, dest, recv_buffer, src);

    // Add dofs renumbered by another process to the old-to-new map
    for (std::size_t i = 0; i < recv_buffer.size(); i += 2)
    {
      const std::size_t received_old_node_index = recv_buffer[i];
      const std::size_t received_new_node_index = recv_buffer[i + 1];

      // Check if this process has shared dof (and is not the owner)
      if (shared_unowned_nodes.find(received_old_node_index)
          != shared_unowned_nodes.end())
      {
        // Add to old-to-new node map
        old_to_new_node_index[received_old_node_index]
          = received_new_node_index;

        // Store map from off-process dof to owner and update
        // UFC-to-renumbered map
        for (std::size_t i = 0; i < block_size; ++i)
        {
          std::size_t ufc_dof_index = received_old_node_index + i*num_nodes;
          std::size_t new_dof_index = received_new_node_index*block_size + i;
          dofmap._off_process_owner[new_dof_index] = src;

          // Update UFC-to-renumbered map
          dofmap.ufc_map_to_dofmap[ufc_dof_index] = new_dof_index;
        }
      }
    }
  }

  // FIXME: Should dofmap._shared_dofs be cleared?

  // Insert the shared-dof-to-process mapping into the dofmap,
  // renumbering as necessary
  for (vec_map::const_iterator it = shared_node_processes.begin();
       it != shared_node_processes.end(); ++it)
  {
    // Check for shared node in old_to_new_node_index map
    boost::unordered_map<std::size_t, std::size_t>::const_iterator
      new_index = old_to_new_node_index.find(it->first);

    if (new_index == old_to_new_node_index.end())
    {
      for (std::size_t i = 0; i < block_size; ++i)
      {
        const std::size_t dof = it->first*block_size + i;
        dofmap._shared_dofs.insert(std::make_pair(dof, it->second));
      }
    }
    else
    {
      for (std::size_t i = 0; i < block_size; ++i)
      {
        const std::size_t dof = (new_index->second)*block_size + i;
        dofmap._shared_dofs.insert(std::make_pair(dof, it->second));
      }
    }
    dofmap._neighbours.insert(it->second.begin(), it->second.end());
  }

  // Build new dofmap
  for (CellIterator cell(mesh); !cell.end(); ++cell)
  {
    // Skip cells not included in restriction
    if (restriction && !restriction->contains(*cell))
      continue;

    // Get cell index and dimension
    const std::size_t cell_index = cell->index();
    const std::size_t cell_dimension = dofmap.cell_dimension(cell_index);

    // Resize cell map and insert dofs
    new_dofmap[cell_index].resize(cell_dimension);
    for (std::size_t i = 0; i < cell_dimension; ++i)
    {
      const std::size_t old_index = old_dofmap[cell_index][i];
      const std::size_t old_node  = old_index % num_nodes;
      const std::size_t new_node  = old_to_new_node_index[old_node];
      new_dofmap[cell_index][i]   = new_node*block_size + old_index/num_nodes;
    }
  }

  // Set new dof map
  dofmap._dofmap = new_dofmap;

  // Set ownership range
  dofmap._ownership_range
    = std::make_pair(block_size*process_offset,
                     block_size*(process_offset + owned_nodes.size()));

  log(TRACE, "Finished renumbering dofs for parallel dof map");
}
Exemplo n.º 5
0
//-----------------------------------------------------------------------------
void DofMapBuilder::compute_node_ownership(boost::array<set, 3>& node_ownership,
                              vec_map& shared_node_processes,
                              DofMap& dofmap,
                              const std::set<std::size_t>& global_dofs,
                              const Mesh& mesh,
                              std::shared_ptr<const Restriction> restriction,
                              const map& restricted_nodes_inverse,
                              std::size_t block_size)
{
  log(TRACE, "Determining dof ownership for parallel dof map");

  const MPI_Comm mpi_comm = mesh.mpi_comm();

  const std::size_t N = dofmap.global_dimension();
  dolfin_assert(N % block_size == 0);
  const std::size_t num_nodes = N/block_size;

  if (restriction)
    dolfin_assert(block_size == 1);

  // References to 'node' ownership sets
  set& owned_nodes          = node_ownership[0];
  set& shared_owned_nodes   = node_ownership[1];
  set& shared_unowned_nodes = node_ownership[2];

  // Clear data structures
  owned_nodes.clear();
  shared_owned_nodes.clear();
  shared_unowned_nodes.clear();

  // Data structures for computing ownership
  boost::unordered_map<std::size_t, std::size_t> node_vote;
  std::vector<std::size_t> facet_dofs(dofmap.num_facet_dofs());

  // Communication buffer
  std::vector<std::size_t> send_buffer;

  // Extract the interior boundary
  BoundaryMesh boundary(mesh, "local");

  // Create a random number generator for ownership 'voting'
  boost::mt19937 engine(MPI::rank(mpi_comm));
  boost::uniform_int<> distribution(0, 100000000);
  boost::variate_generator<boost::mt19937&, boost::uniform_int<> >
    rng(engine, distribution);

  // Build set of dofs on process boundary (first assuming that all
  // are owned by this process)
  const MeshFunction<std::size_t>& cell_map
    = boundary.entity_map(boundary.topology().dim());
  if (!cell_map.empty())
  {
    for (CellIterator _f(boundary); !_f.end(); ++_f)
    {
      // Create a vote per facet
      const std::size_t facet_vote = rng();

      // Get boundary facet
      Facet f(mesh, cell_map[*_f]);

      // Get cell to which facet belongs (pick first)
      Cell c(mesh, f.entities(mesh.topology().dim())[0]);

      // Skip cells not included in restriction
      if (restriction && !restriction->contains(c))
        continue;

      // Tabulate dofs on cell
      const std::vector<dolfin::la_index>& cell_dofs
        = dofmap.cell_dofs(c.index());

      // Tabulate which dofs are on the facet
      dofmap.tabulate_facet_dofs(facet_dofs, c.index(f));

      // Insert shared nodes into set and assign a 'vote'
      dolfin_assert(dofmap.num_facet_dofs() % block_size == 0);
      for (std::size_t i = 0; i < dofmap.num_facet_dofs(); ++i)
      {
        // Get facet node
        size_t facet_node = cell_dofs[facet_dofs[i]] % num_nodes;

        // Map back to original (and common) numbering for restricted
        // space
        if (restriction)
        {
          const map_iterator it = restricted_nodes_inverse.find(facet_node);
          dolfin_assert(it != restricted_nodes_inverse.end());
          facet_node = it->second;
        }

        // Add to list of shared nodes
        if (shared_owned_nodes.find(facet_node) == shared_owned_nodes.end())
        {
          shared_owned_nodes.insert(facet_node);
          node_vote[facet_node] = facet_vote;

          send_buffer.push_back(facet_node);
          send_buffer.push_back(node_vote[facet_node]);
        }
      }
    }
  }

  // FIXME: The below algortihm can be improved (made more scalable)
  //        by distributing (dof, process) pairs to 'owner' range owner,
  //        then letting each process get the sharing process list. This
  //        will avoid interleaving communication and computation.

  // Decide ownership of shared nodes
  const std::size_t num_prococesses = MPI::size(mpi_comm);
  const std::size_t process_number = MPI::rank(mpi_comm);
  std::vector<std::size_t> recv_buffer;
  for (std::size_t k = 1; k < num_prococesses; ++k)
  {
    const std::size_t src
      = (process_number - k + num_prococesses) % num_prococesses;
    const std::size_t dest = (process_number + k) % num_prococesses;
    MPI::send_recv(mpi_comm, send_buffer, dest, recv_buffer, src);
    for (std::size_t i = 0; i < recv_buffer.size(); i += 2)
    {
      const std::size_t received_node = recv_buffer[i];
      const std::size_t received_vote = recv_buffer[i + 1];

      if (shared_owned_nodes.find(received_node) != shared_owned_nodes.end())
      {
        // Move dofs with higher ownership votes from shared to shared
        // but not owned
        if (received_vote < node_vote[received_node])
        {
          shared_unowned_nodes.insert(received_node);
          shared_owned_nodes.erase(received_node);
        }
        else if (received_vote == node_vote[received_node]
                 && process_number > src)
        {
          // If votes are equal, let lower rank process take ownership
          shared_unowned_nodes.insert(received_node);
          shared_owned_nodes.erase(received_node);
        }

        // Remember the sharing of the node
        shared_node_processes[received_node].push_back(src);
      }
      else if (shared_unowned_nodes.find(received_node)
               != shared_unowned_nodes.end())
      {
        // Remember the sharing of the node
        shared_node_processes[received_node].push_back(src);
      }
    }
  }

  // Add/remove global dofs to/from relevant sets (last process owns
  // global dofs)
  if (process_number == num_prococesses - 1)
  {
    shared_owned_nodes.insert(global_dofs.begin(), global_dofs.end());
    for (std::set<std::size_t>::const_iterator dof = global_dofs.begin();
         dof != global_dofs.end(); ++dof)
    {
      set::const_iterator _dof = shared_unowned_nodes.find(*dof);
      if (_dof != shared_unowned_nodes.end())
        shared_unowned_nodes.erase(_dof);
    }
  }
  else
  {
    shared_unowned_nodes.insert(global_dofs.begin(), global_dofs.end());
    for (std::set<std::size_t>::const_iterator dof = global_dofs.begin();
         dof != global_dofs.end(); ++dof)
    {
      set::const_iterator _dof = shared_owned_nodes.find(*dof);
      if (_dof != shared_owned_nodes.end())
        shared_owned_nodes.erase(_dof);
    }
  }

  // Mark all shared-and-owned dofs as owned by the processes
  for (CellIterator cell(mesh); !cell.end(); ++cell)
  {
    const std::vector<dolfin::la_index>& cell_dofs
      = dofmap.cell_dofs(cell->index());
    for (std::size_t i = 0; i < cell_dofs.size(); ++i)
    {
      // Get cell node
      size_t cell_node = cell_dofs[i] % num_nodes;

      // Map back to original (and common) numbering for restricted
      // space
      if (restriction)
      {
        const map_iterator it = restricted_nodes_inverse.find(cell_node);
        dolfin_assert(it != restricted_nodes_inverse.end());
        cell_node = it->second;
      }

      // Mark dof as owned if not in unowned set
      if (shared_unowned_nodes.find(cell_node) == shared_unowned_nodes.end())
        owned_nodes.insert(cell_node);
    }
  }

  // Check or set global dimension
  if (restriction)
  {
    // Global dimension for restricted space needs to be computed here
    // since it is not know by the UFC dof map.
    const std::size_t _owned_dim = owned_nodes.size();
    const std::size_t _global_dimension = block_size*MPI::sum(mpi_comm,
                                                              _owned_dim);
    dofmap._global_dimension = _global_dimension;
  }
  else
  {
    const std::size_t _owned_dim = owned_nodes.size();
    dolfin_assert(block_size*MPI::sum(mpi_comm, _owned_dim)
                  == dofmap.global_dimension());
  }

  log(TRACE, "Finished determining dof ownership for parallel dof map");
}
Exemplo n.º 6
0
//-----------------------------------------------------------------------------
void DofMapBuilder::reorder_local(DofMap& dofmap, const Mesh& mesh,
                                  std::size_t block_size,
				  const std::set<std::size_t>& global_dofs)
{
  // Global dimension
  const std::size_t N = dofmap.global_dimension();

  // Create empty graph
  dolfin_assert(N % block_size == 0);
  const std::size_t num_nodes = N/block_size;
  Graph graph(num_nodes);

  // Build local graph for blocks
  std::vector<dolfin::la_index> dofs_tmp;
  for (CellIterator cell(mesh); !cell.end(); ++cell)
  {
    const std::vector<dolfin::la_index>& dofs0
      = dofmap.cell_dofs(cell->index());
    const std::vector<dolfin::la_index>& dofs1
      = dofmap.cell_dofs(cell->index());

    dolfin_assert(dofs0.size() % block_size == 0);
    const std::size_t nodes_per_cell = dofs0.size()/block_size;

    // Build vector of graph 'edges'
    dofs_tmp.resize(nodes_per_cell);
    std::size_t dof_counter = 0;
    for (std::size_t i = 0; i < nodes_per_cell; ++i)
    {
      if (global_dofs.find(dofs0[i]) == global_dofs.end())
      {
        dofs_tmp[i] = dofs1[i] % num_nodes;
        ++dof_counter;
      }
    }

    // Add graph 'edges'
    for (std::size_t i = 0; i < nodes_per_cell; ++i)
    {
      if (global_dofs.find(dofs1[i]) == global_dofs.end())
      {
        graph[dofs0[i] % num_nodes].insert(dofs_tmp.begin(),
                                           dofs_tmp.begin() + dof_counter);
      }
    }
  }

  // Reorder block graph
  const std::string ordering_library
    = dolfin::parameters["dof_ordering_library"];
  std::vector<std::size_t> block_remap;
  if (ordering_library == "Boost")
    block_remap = BoostGraphOrdering::compute_cuthill_mckee(graph, true);
  else if (ordering_library == "SCOTCH")
    block_remap = SCOTCH::compute_gps(graph);
  else if (ordering_library == "random")
  {
    // NOTE: Randomised dof ordering should only be used for
    // testing/benchmarking
    block_remap.resize(graph.size());
    for (std::size_t i = 0; i < block_remap.size(); ++i)
      block_remap[i] = i;
    std::random_shuffle(block_remap.begin(), block_remap.end());
  }
  else
  {
    dolfin_error("DofMapBuilder.cpp",
                 "reorder degrees of freedom",
                 "The requested ordering library '%s' is unknown", ordering_library.c_str());
  }

  // Re-number dofs for each cell
  std::vector<std::vector<dolfin::la_index> >::iterator cell_map;
  std::vector<dolfin::la_index>::iterator dof;
  for (cell_map = dofmap._dofmap.begin(); cell_map != dofmap._dofmap.end();
       ++cell_map)
  {
    for (dof = cell_map->begin(); dof != cell_map->end(); ++dof)
    {
      const std::size_t old_node = (*dof) % num_nodes;
      const std::size_t new_node = block_remap[old_node];
      *dof = new_node*block_size + (*dof)/num_nodes;
    }
  }

  // Store re-ordering map (from UFC dofmap)
  dolfin_assert(dofmap.ufc_map_to_dofmap.empty());
  for (std::size_t i = 0; i < dofmap.global_dimension(); ++i)
  {
    const std::size_t old_node = i % num_nodes;
    const std::size_t new_node = block_remap[old_node];
    dofmap.ufc_map_to_dofmap[i] = new_node*block_size + i/num_nodes;
  }
}