Example #1
0
void Partitioner::repartition (MeshBase & mesh,
                               const unsigned int n)
{
  // we cannot partition into more pieces than we have
  // active elements!
  const unsigned int n_parts =
    static_cast<unsigned int>
    (std::min(mesh.n_active_elem(), static_cast<dof_id_type>(n)));

  // Set the number of partitions in the mesh
  mesh.set_n_partitions()=n_parts;

  if (n_parts == 1)
    {
      this->single_partition (mesh);
      return;
    }

  // First assign a temporary partitioning to any unpartitioned elements
  Partitioner::partition_unpartitioned_elements(mesh, n_parts);

  // Call the partitioning function
  this->_do_repartition(mesh,n_parts);

  // Set the parent's processor ids
  Partitioner::set_parent_processor_ids(mesh);

  // Set the node's processor ids
  Partitioner::set_node_processor_ids(mesh);
}
Example #2
0
void Partitioner::partition (MeshBase & mesh,
                             const unsigned int n)
{
  libmesh_parallel_only(mesh.comm());

  // BSK - temporary fix while redistribution is integrated 6/26/2008
  // Uncomment this to not repartition in parallel
  //   if (!mesh.is_serial())
  //     return;

  // we cannot partition into more pieces than we have
  // active elements!
  const unsigned int n_parts =
    static_cast<unsigned int>
    (std::min(mesh.n_active_elem(), static_cast<dof_id_type>(n)));

  // Set the number of partitions in the mesh
  mesh.set_n_partitions()=n_parts;

  if (n_parts == 1)
    {
      this->single_partition (mesh);
      return;
    }

  // First assign a temporary partitioning to any unpartitioned elements
  Partitioner::partition_unpartitioned_elements(mesh, n_parts);

  // Call the partitioning function
  this->_do_partition(mesh,n_parts);

  // Set the parent's processor ids
  Partitioner::set_parent_processor_ids(mesh);

  // Redistribute elements if necessary, before setting node processor
  // ids, to make sure those will be set consistently
  mesh.redistribute();

#ifdef DEBUG
  MeshTools::libmesh_assert_valid_remote_elems(mesh);

  // Messed up elem processor_id()s can leave us without the child
  // elements we need to restrict vectors on a distributed mesh
  MeshTools::libmesh_assert_valid_procids<Elem>(mesh);
#endif

  // Set the node's processor ids
  Partitioner::set_node_processor_ids(mesh);

#ifdef DEBUG
  MeshTools::libmesh_assert_valid_procids<Elem>(mesh);
#endif

  // Give derived Mesh classes a chance to update any cached data to
  // reflect the new partitioning
  mesh.update_post_partitioning();
}
Example #3
0
// ------------------------------------------------------------
// LinearPartitioner implementation
void LinearPartitioner::_do_partition (MeshBase& mesh,
                                       const unsigned int n)
{
  libmesh_assert_greater (n, 0);

  // Check for an easy return
  if (n == 1)
    {
      this->single_partition (mesh);
      return;
    }

  // Create a simple linear partitioning
  {
    START_LOG ("partition()", "LinearPartitioner");

    const dof_id_type n_active_elem = mesh.n_active_elem();
    const dof_id_type blksize       = n_active_elem/n;

    dof_id_type e = 0;

    MeshBase::element_iterator       elem_it  = mesh.active_elements_begin();
    const MeshBase::element_iterator elem_end = mesh.active_elements_end();

    for ( ; elem_it != elem_end; ++elem_it)
      {
        if ((e/blksize) < n)
          {
            Elem *elem = *elem_it;
            elem->processor_id() =
              libmesh_cast_int<processor_id_type>(e/blksize);
          }
        else
          {
            Elem *elem = *elem_it;
            elem->processor_id() = 0;
            elem = elem->parent();
          }

        e++;
      }

    STOP_LOG ("partition()", "LinearPartitioner");
  }
}
Example #4
0
// ------------------------------------------------------------
// SFCPartitioner implementation
void SFCPartitioner::_do_partition (MeshBase & mesh,
                                    const unsigned int n)
{

  libmesh_assert_greater (n, 0);

  // Check for an easy return
  if (n == 1)
    {
      this->single_partition (mesh);
      return;
    }

  // What to do if the sfcurves library IS NOT present
#ifndef LIBMESH_HAVE_SFCURVES

  libmesh_here();
  libMesh::err << "ERROR: The library has been built without"    << std::endl
               << "Space Filling Curve support.  Using a linear" << std::endl
               << "partitioner instead!" << std::endl;

  LinearPartitioner lp;

  lp.partition (mesh, n);

  // What to do if the sfcurves library IS present
#else

  LOG_SCOPE("sfc_partition()", "SFCPartitioner");

  const dof_id_type n_active_elem = mesh.n_active_elem();
  const dof_id_type n_elem        = mesh.n_elem();

  // the forward_map maps the active element id
  // into a contiguous block of indices
  std::vector<dof_id_type>
    forward_map (n_elem, DofObject::invalid_id);

  // the reverse_map maps the contiguous ids back
  // to active elements
  std::vector<Elem *> reverse_map (n_active_elem, libmesh_nullptr);

  int size = static_cast<int>(n_active_elem);
  std::vector<double> x      (size);
  std::vector<double> y      (size);
  std::vector<double> z      (size);
  std::vector<int>    table  (size);


  // We need to map the active element ids into a
  // contiguous range.
  {
    MeshBase::element_iterator       elem_it  = mesh.active_elements_begin();
    const MeshBase::element_iterator elem_end = mesh.active_elements_end();

    dof_id_type el_num = 0;

    for (; elem_it != elem_end; ++elem_it)
      {
        libmesh_assert_less ((*elem_it)->id(), forward_map.size());
        libmesh_assert_less (el_num, reverse_map.size());

        forward_map[(*elem_it)->id()] = el_num;
        reverse_map[el_num]           = *elem_it;
        el_num++;
      }
    libmesh_assert_equal_to (el_num, n_active_elem);
  }


  // Get the centroid for each active element
  {
    //     const_active_elem_iterator       elem_it (mesh.const_elements_begin());
    //     const const_active_elem_iterator elem_end(mesh.const_elements_end());

    MeshBase::element_iterator       elem_it  = mesh.active_elements_begin();
    const MeshBase::element_iterator elem_end = mesh.active_elements_end();

    for (; elem_it != elem_end; ++elem_it)
      {
        const Elem * elem = *elem_it;

        libmesh_assert_less (elem->id(), forward_map.size());

        const Point p = elem->centroid();

        x[forward_map[elem->id()]] = p(0);
        y[forward_map[elem->id()]] = p(1);
        z[forward_map[elem->id()]] = p(2);
      }
  }

  // build the space-filling curve
  if (_sfc_type == "Hilbert")
    Sfc::hilbert (&x[0], &y[0], &z[0], &size, &table[0]);

  else if (_sfc_type == "Morton")
    Sfc::morton  (&x[0], &y[0], &z[0], &size, &table[0]);

  else
    {
      libmesh_here();
      libMesh::err << "ERROR: Unknown type: " << _sfc_type << std::endl
                   << " Valid types are"                   << std::endl
                   << "  \"Hilbert\""                      << std::endl
                   << "  \"Morton\""                       << std::endl
                   << " "                                  << std::endl
                   << "Proceeding with a Hilbert curve."   << std::endl;

      Sfc::hilbert (&x[0], &y[0], &z[0], &size, &table[0]);
    }


  // Assign the partitioning to the active elements
  {
    //      {
    //        std::ofstream out ("sfc.dat");
    //        out << "variables=x,y,z" << std::endl;
    //        out << "zone f=point" << std::endl;

    //        for (unsigned int i=0; i<n_active_elem; i++)
    //  out << x[i] << " "
    //      << y[i] << " "
    //      << z[i] << std::endl;
    //      }

    const dof_id_type blksize = (n_active_elem+n-1)/n;

    for (dof_id_type i=0; i<n_active_elem; i++)
      {
        libmesh_assert_less (static_cast<unsigned int>(table[i]-1), reverse_map.size());

        Elem * elem = reverse_map[table[i]-1];

        elem->processor_id() = cast_int<processor_id_type>
          (i/blksize);
      }
  }

#endif

}
Example #5
0
// ------------------------------------------------------------
// MetisPartitioner implementation
void MetisPartitioner::_do_partition (MeshBase & mesh,
                                      const unsigned int n_pieces)
{
  libmesh_assert_greater (n_pieces, 0);
  libmesh_assert (mesh.is_serial());

  // Check for an easy return
  if (n_pieces == 1)
    {
      this->single_partition (mesh);
      return;
    }

  // What to do if the Metis library IS NOT present
#ifndef LIBMESH_HAVE_METIS

  libmesh_here();
  libMesh::err << "ERROR: The library has been built without"    << std::endl
               << "Metis support.  Using a space-filling curve"  << std::endl
               << "partitioner instead!"                         << std::endl;

  SFCPartitioner sfcp;

  sfcp.partition (mesh, n_pieces);

  // What to do if the Metis library IS present
#else

  LOG_SCOPE("partition()", "MetisPartitioner");

  const dof_id_type n_active_elem = mesh.n_active_elem();

  // build the graph
  // std::vector<Metis::idx_t> options(5);
  std::vector<Metis::idx_t> vwgt(n_active_elem);
  std::vector<Metis::idx_t> part(n_active_elem);

  Metis::idx_t
    n = static_cast<Metis::idx_t>(n_active_elem),  // number of "nodes" (elements)
                                                   //   in the graph
    //    wgtflag = 2,                             // weights on vertices only,
    //                                             //   none on edges
    //    numflag = 0,                             // C-style 0-based numbering
    nparts  = static_cast<Metis::idx_t>(n_pieces), // number of subdomains to create
    edgecut = 0;                                   // the numbers of edges cut by the
                                                   //   resulting partition

  // Set the options
  // options[0] = 0; // use default options

  // Metis will only consider the active elements.
  // We need to map the active element ids into a
  // contiguous range.  Further, we want the unique range indexing to be
  // independent of the element ordering, otherwise a circular dependency
  // can result in which the partitioning depends on the ordering which
  // depends on the partitioning...
  vectormap<dof_id_type, dof_id_type> global_index_map;
  global_index_map.reserve (n_active_elem);

  {
    std::vector<dof_id_type> global_index;

    MeshBase::element_iterator       it  = mesh.active_elements_begin();
    const MeshBase::element_iterator end = mesh.active_elements_end();

    MeshCommunication().find_global_indices (mesh.comm(),
                                             MeshTools::bounding_box(mesh),
                                             it, end, global_index);

    libmesh_assert_equal_to (global_index.size(), n_active_elem);

    for (std::size_t cnt=0; it != end; ++it)
      {
        const Elem * elem = *it;

        global_index_map.insert (std::make_pair(elem->id(), global_index[cnt++]));
      }
    libmesh_assert_equal_to (global_index_map.size(), n_active_elem);
  }

  // If we have boundary elements in this mesh, we want to account for
  // the connectivity between them and interior elements.  We can find
  // interior elements from boundary elements, but we need to build up
  // a lookup map to do the reverse.

  typedef LIBMESH_BEST_UNORDERED_MULTIMAP<const Elem *, const Elem *>
    map_type;
  map_type interior_to_boundary_map;

  {
    MeshBase::const_element_iterator       elem_it  = mesh.active_elements_begin();
    const MeshBase::const_element_iterator elem_end = mesh.active_elements_end();

    for (; elem_it != elem_end; ++elem_it)
      {
        const Elem * elem = *elem_it;

        // If we don't have an interior_parent then there's nothing to look us
        // up.
        if ((elem->dim() >= LIBMESH_DIM) ||
            !elem->interior_parent())
          continue;

        // get all relevant interior elements
        std::set<const Elem *> neighbor_set;
        elem->find_interior_neighbors(neighbor_set);

        std::set<const Elem *>::iterator n_it = neighbor_set.begin();
        for (; n_it != neighbor_set.end(); ++n_it)
          {
            // FIXME - non-const versions of the Elem set methods
            // would be nice
            Elem * neighbor = const_cast<Elem *>(*n_it);

#if defined(LIBMESH_HAVE_UNORDERED_MULTIMAP) ||         \
  defined(LIBMESH_HAVE_TR1_UNORDERED_MULTIMAP) ||       \
  defined(LIBMESH_HAVE_HASH_MULTIMAP) ||                \
  defined(LIBMESH_HAVE_EXT_HASH_MULTIMAP)
            interior_to_boundary_map.insert
              (std::make_pair(neighbor, elem));
#else
            interior_to_boundary_map.insert
              (interior_to_boundary_map.begin(),
               std::make_pair(neighbor, elem));
#endif
          }
      }
  }


  // Invoke METIS, but only on processor 0.
  // Then broadcast the resulting decomposition
  if (mesh.processor_id() == 0)
    {
      METIS_CSR_Graph<Metis::idx_t> csr_graph;

      csr_graph.offsets.resize(n_active_elem+1, 0);

      // Local scope for these
      {
        // build the graph in CSR format.  Note that
        // the edges in the graph will correspond to
        // face neighbors

#ifdef LIBMESH_ENABLE_AMR
        std::vector<const Elem *> neighbors_offspring;
#endif

        MeshBase::element_iterator       elem_it  = mesh.active_elements_begin();
        const MeshBase::element_iterator elem_end = mesh.active_elements_end();

#ifndef NDEBUG
        std::size_t graph_size=0;
#endif

        // (1) first pass - get the row sizes for each element by counting the number
        // of face neighbors.  Also populate the vwght array if necessary
        for (; elem_it != elem_end; ++elem_it)
          {
            const Elem * elem = *elem_it;

            const dof_id_type elem_global_index =
              global_index_map[elem->id()];

            libmesh_assert_less (elem_global_index, vwgt.size());

            // maybe there is a better weight?
            // The weight is used to define what a balanced graph is
            if(!_weights)
              vwgt[elem_global_index] = elem->n_nodes();
            else
              vwgt[elem_global_index] = static_cast<Metis::idx_t>((*_weights)[elem->id()]);

            unsigned int num_neighbors = 0;

            // Loop over the element's neighbors.  An element
            // adjacency corresponds to a face neighbor
            for (unsigned int ms=0; ms<elem->n_neighbors(); ms++)
              {
                const Elem * neighbor = elem->neighbor(ms);

                if (neighbor != libmesh_nullptr)
                  {
                    // If the neighbor is active treat it
                    // as a connection
                    if (neighbor->active())
                      num_neighbors++;

#ifdef LIBMESH_ENABLE_AMR

                    // Otherwise we need to find all of the
                    // neighbor's children that are connected to
                    // us and add them
                    else
                      {
                        // The side of the neighbor to which
                        // we are connected
                        const unsigned int ns =
                          neighbor->which_neighbor_am_i (elem);
                        libmesh_assert_less (ns, neighbor->n_neighbors());

                        // Get all the active children (& grandchildren, etc...)
                        // of the neighbor.

                        // FIXME - this is the wrong thing, since we
                        // should be getting the active family tree on
                        // our side only.  But adding too many graph
                        // links may cause hanging nodes to tend to be
                        // on partition interiors, which would reduce
                        // communication overhead for constraint
                        // equations, so we'll leave it.

                        neighbor->active_family_tree (neighbors_offspring);

                        // Get all the neighbor's children that
                        // live on that side and are thus connected
                        // to us
                        for (unsigned int nc=0; nc<neighbors_offspring.size(); nc++)
                          {
                            const Elem * child =
                              neighbors_offspring[nc];

                            // This does not assume a level-1 mesh.
                            // Note that since children have sides numbered
                            // coincident with the parent then this is a sufficient test.
                            if (child->neighbor(ns) == elem)
                              {
                                libmesh_assert (child->active());
                                num_neighbors++;
                              }
                          }
                      }

#endif /* ifdef LIBMESH_ENABLE_AMR */

                  }
              }

            // Check for any interior neighbors
            if ((elem->dim() < LIBMESH_DIM) &&
                elem->interior_parent())
              {
                // get all relevant interior elements
                std::set<const Elem *> neighbor_set;
                elem->find_interior_neighbors(neighbor_set);

                num_neighbors += neighbor_set.size();
              }

            // Check for any boundary neighbors
            typedef map_type::iterator map_it_type;
            std::pair<map_it_type, map_it_type>
              bounds = interior_to_boundary_map.equal_range(elem);
            num_neighbors += std::distance(bounds.first, bounds.second);

            csr_graph.prep_n_nonzeros(elem_global_index, num_neighbors);
#ifndef NDEBUG
            graph_size += num_neighbors;
#endif
          }

        csr_graph.prepare_for_use();

        // (2) second pass - fill the compressed adjacency array
        elem_it  = mesh.active_elements_begin();

        for (; elem_it != elem_end; ++elem_it)
          {
            const Elem * elem = *elem_it;

            const dof_id_type elem_global_index =
              global_index_map[elem->id()];

            unsigned int connection=0;

            // Loop over the element's neighbors.  An element
            // adjacency corresponds to a face neighbor
            for (unsigned int ms=0; ms<elem->n_neighbors(); ms++)
              {
                const Elem * neighbor = elem->neighbor(ms);

                if (neighbor != libmesh_nullptr)
                  {
                    // If the neighbor is active treat it
                    // as a connection
                    if (neighbor->active())
                      csr_graph(elem_global_index, connection++) = global_index_map[neighbor->id()];

#ifdef LIBMESH_ENABLE_AMR

                    // Otherwise we need to find all of the
                    // neighbor's children that are connected to
                    // us and add them
                    else
                      {
                        // The side of the neighbor to which
                        // we are connected
                        const unsigned int ns =
                          neighbor->which_neighbor_am_i (elem);
                        libmesh_assert_less (ns, neighbor->n_neighbors());

                        // Get all the active children (& grandchildren, etc...)
                        // of the neighbor.
                        neighbor->active_family_tree (neighbors_offspring);

                        // Get all the neighbor's children that
                        // live on that side and are thus connected
                        // to us
                        for (unsigned int nc=0; nc<neighbors_offspring.size(); nc++)
                          {
                            const Elem * child =
                              neighbors_offspring[nc];

                            // This does not assume a level-1 mesh.
                            // Note that since children have sides numbered
                            // coincident with the parent then this is a sufficient test.
                            if (child->neighbor(ns) == elem)
                              {
                                libmesh_assert (child->active());

                                csr_graph(elem_global_index, connection++) = global_index_map[child->id()];
                              }
                          }
                      }

#endif /* ifdef LIBMESH_ENABLE_AMR */

                  }
              }

            if ((elem->dim() < LIBMESH_DIM) &&
                elem->interior_parent())
              {
                // get all relevant interior elements
                std::set<const Elem *> neighbor_set;
                elem->find_interior_neighbors(neighbor_set);

                std::set<const Elem *>::iterator n_it = neighbor_set.begin();
                for (; n_it != neighbor_set.end(); ++n_it)
                  {
                    // FIXME - non-const versions of the Elem set methods
                    // would be nice
                    Elem * neighbor = const_cast<Elem *>(*n_it);

                    csr_graph(elem_global_index, connection++) =
                      global_index_map[neighbor->id()];
                  }
              }

            // Check for any boundary neighbors
            typedef map_type::iterator map_it_type;
            std::pair<map_it_type, map_it_type>
              bounds = interior_to_boundary_map.equal_range(elem);

            for (map_it_type it = bounds.first; it != bounds.second; ++it)
              {
                const Elem * neighbor = it->second;
                csr_graph(elem_global_index, connection++) =
                  global_index_map[neighbor->id()];
              }
          }

        // We create a non-empty vals for a disconnected graph, to
        // work around a segfault from METIS.
        libmesh_assert_equal_to (csr_graph.vals.size(),
                                 std::max(graph_size,std::size_t(1)));
      } // done building the graph

      Metis::idx_t ncon = 1;

      // Select which type of partitioning to create

      // Use recursive if the number of partitions is less than or equal to 8
      if (n_pieces <= 8)
        Metis::METIS_PartGraphRecursive(&n, &ncon, &csr_graph.offsets[0], &csr_graph.vals[0], &vwgt[0], libmesh_nullptr,
                                        libmesh_nullptr, &nparts, libmesh_nullptr, libmesh_nullptr, libmesh_nullptr,
                                        &edgecut, &part[0]);

      // Otherwise  use kway
      else
        Metis::METIS_PartGraphKway(&n, &ncon, &csr_graph.offsets[0], &csr_graph.vals[0], &vwgt[0], libmesh_nullptr,
                                   libmesh_nullptr, &nparts, libmesh_nullptr, libmesh_nullptr, libmesh_nullptr,
                                   &edgecut, &part[0]);

    } // end processor 0 part

  // Broadcase the resutling partition
  mesh.comm().broadcast(part);

  // Assign the returned processor ids.  The part array contains
  // the processor id for each active element, but in terms of
  // the contiguous indexing we defined above
  {
    MeshBase::element_iterator       it  = mesh.active_elements_begin();
    const MeshBase::element_iterator end = mesh.active_elements_end();

    for (; it!=end; ++it)
      {
        Elem * elem = *it;

        libmesh_assert (global_index_map.count(elem->id()));

        const dof_id_type elem_global_index =
          global_index_map[elem->id()];

        libmesh_assert_less (elem_global_index, part.size());
        const processor_id_type elem_procid =
          static_cast<processor_id_type>(part[elem_global_index]);

        elem->processor_id() = elem_procid;
      }
  }
#endif
}
Example #6
0
// ------------------------------------------------------------
// MetisPartitioner implementation
void MetisPartitioner::_do_partition (MeshBase& mesh,
				      const unsigned int n_pieces)
{
  libmesh_assert_greater (n_pieces, 0);
  libmesh_assert (mesh.is_serial());

  // Check for an easy return
  if (n_pieces == 1)
    {
      this->single_partition (mesh);
      return;
    }

// What to do if the Metis library IS NOT present
#ifndef LIBMESH_HAVE_METIS

  libmesh_here();
  libMesh::err << "ERROR: The library has been built without"    << std::endl
	        << "Metis support.  Using a space-filling curve"  << std::endl
	        << "partitioner instead!"                         << std::endl;

  SFCPartitioner sfcp;

  sfcp.partition (mesh, n_pieces);

// What to do if the Metis library IS present
#else

  START_LOG("partition()", "MetisPartitioner");

  const dof_id_type n_active_elem = mesh.n_active_elem();

  // build the graph
  // std::vector<int> options(5);
  std::vector<int> vwgt(n_active_elem);
  std::vector<int> part(n_active_elem);

  int
    n = static_cast<int>(n_active_elem),  // number of "nodes" (elements)
                                          //   in the graph
//    wgtflag = 2,                          // weights on vertices only,
//                                          //   none on edges
//    numflag = 0,                          // C-style 0-based numbering
    nparts  = static_cast<int>(n_pieces), // number of subdomains to create
    edgecut = 0;                          // the numbers of edges cut by the
                                          //   resulting partition

  // Set the options
  // options[0] = 0; // use default options

  // Metis will only consider the active elements.
  // We need to map the active element ids into a
  // contiguous range.  Further, we want the unique range indexing to be
  // independednt of the element ordering, otherwise a circular dependency
  // can result in which the partitioning depends on the ordering which
  // depends on the partitioning...
  std::map<const Elem*, dof_id_type> global_index_map;
  {
    std::vector<dof_id_type> global_index;

    MeshBase::element_iterator       it  = mesh.active_elements_begin();
    const MeshBase::element_iterator end = mesh.active_elements_end();

    MeshCommunication().find_global_indices (MeshTools::bounding_box(mesh),
					     it, end, global_index);

    libmesh_assert_equal_to (global_index.size(), n_active_elem);

    for (std::size_t cnt=0; it != end; ++it)
      {
	const Elem *elem = *it;
	libmesh_assert (!global_index_map.count(elem));

	global_index_map[elem]  = global_index[cnt++];
      }
    libmesh_assert_equal_to (global_index_map.size(), n_active_elem);
  }


  // build the graph in CSR format.  Note that
  // the edges in the graph will correspond to
  // face neighbors
  std::vector<int> xadj, adjncy;
  {
    std::vector<const Elem*> neighbors_offspring;

    MeshBase::element_iterator       elem_it  = mesh.active_elements_begin();
    const MeshBase::element_iterator elem_end = mesh.active_elements_end();

    // This will be exact when there is no refinement and all the
    // elements are of the same type.
    std::size_t graph_size=0;
    std::vector<std::vector<dof_id_type> > graph(n_active_elem);

    for (; elem_it != elem_end; ++elem_it)
      {
	const Elem* elem = *elem_it;

	libmesh_assert (global_index_map.count(elem));

	const dof_id_type elem_global_index =
	  global_index_map[elem];

	libmesh_assert_less (elem_global_index, vwgt.size());
	libmesh_assert_less (elem_global_index, graph.size());

	// maybe there is a better weight?
	// The weight is used to define what a balanced graph is
        if(!_weights)
          vwgt[elem_global_index] = elem->n_nodes();
        else
          vwgt[elem_global_index] = static_cast<int>((*_weights)[elem->id()]);

	// Loop over the element's neighbors.  An element
	// adjacency corresponds to a face neighbor
	for (unsigned int ms=0; ms<elem->n_neighbors(); ms++)
	  {
	    const Elem* neighbor = elem->neighbor(ms);

	    if (neighbor != NULL)
	      {
		// If the neighbor is active treat it
		// as a connection
		if (neighbor->active())
		  {
		    libmesh_assert (global_index_map.count(neighbor));

		    const dof_id_type neighbor_global_index =
		      global_index_map[neighbor];

		    graph[elem_global_index].push_back(neighbor_global_index);
		    graph_size++;
		  }

#ifdef LIBMESH_ENABLE_AMR

		// Otherwise we need to find all of the
		// neighbor's children that are connected to
		// us and add them
		else
		  {
		    // The side of the neighbor to which
		    // we are connected
		    const unsigned int ns =
		      neighbor->which_neighbor_am_i (elem);
                    libmesh_assert_less (ns, neighbor->n_neighbors());

		    // Get all the active children (& grandchildren, etc...)
		    // of the neighbor.
		    neighbor->active_family_tree (neighbors_offspring);

		    // Get all the neighbor's children that
		    // live on that side and are thus connected
		    // to us
		    for (unsigned int nc=0; nc<neighbors_offspring.size(); nc++)
		      {
			const Elem* child =
			  neighbors_offspring[nc];

			// This does not assume a level-1 mesh.
			// Note that since children have sides numbered
			// coincident with the parent then this is a sufficient test.
			if (child->neighbor(ns) == elem)
			  {
			    libmesh_assert (child->active());
			    libmesh_assert (global_index_map.count(child));

			    const dof_id_type child_global_index =
			      global_index_map[child];

			    graph[elem_global_index].push_back(child_global_index);
			    graph_size++;
			  }
		      }
		  }

#endif /* ifdef LIBMESH_ENABLE_AMR */

	      }
	  }
      }

    // Convert the graph into the format Metis wants
    xadj.reserve(n_active_elem+1);
    adjncy.reserve(graph_size);

    for (std::size_t r=0; r<graph.size(); r++)
      {
	xadj.push_back(adjncy.size());
	std::vector<dof_id_type> graph_row; // build this emtpy
	graph_row.swap(graph[r]); // this will deallocate at the end of scope
	adjncy.insert(adjncy.end(),
		      graph_row.begin(),
		      graph_row.end());
      }

    // The end of the adjacency array for the last elem
    xadj.push_back(adjncy.size());

    libmesh_assert_equal_to (adjncy.size(), graph_size);
    libmesh_assert_equal_to (xadj.size(), n_active_elem+1);
  } // done building the graph


  if (adjncy.empty())
    adjncy.push_back(0);

  int ncon = 1;

  // Select which type of partitioning to create

  // Use recursive if the number of partitions is less than or equal to 8
  if (n_pieces <= 8)
    Metis::METIS_PartGraphRecursive(&n, &ncon, &xadj[0], &adjncy[0], &vwgt[0], NULL,
				    NULL, &nparts, NULL, NULL, NULL,
				    &edgecut, &part[0]);

  // Otherwise  use kway
  else
    Metis::METIS_PartGraphKway(&n, &ncon, &xadj[0], &adjncy[0], &vwgt[0], NULL,
			       NULL, &nparts, NULL, NULL, NULL,
			       &edgecut, &part[0]);


  // Assign the returned processor ids.  The part array contains
  // the processor id for each active element, but in terms of
  // the contiguous indexing we defined above
  {
    MeshBase::element_iterator       it  = mesh.active_elements_begin();
    const MeshBase::element_iterator end = mesh.active_elements_end();

    for (; it!=end; ++it)
      {
	Elem* elem = *it;

	libmesh_assert (global_index_map.count(elem));

	const dof_id_type elem_global_index =
	  global_index_map[elem];

	libmesh_assert_less (elem_global_index, part.size());
	const processor_id_type elem_procid =
	  static_cast<processor_id_type>(part[elem_global_index]);

        elem->processor_id() = elem_procid;
      }
  }

  STOP_LOG("partition()", "MetisPartitioner");
#endif
}