void LocationMap<Elem>::fill(MeshBase& mesh) { // Populate the elem map MeshBase::element_iterator it = mesh.active_elements_begin(), end = mesh.active_elements_end(); for (; it != end; ++it) this->insert(**it); }
void MetisPartitioner::_do_partition (MeshBase & mesh, const unsigned int n_pieces) { this->partition_range(mesh, mesh.active_elements_begin(), mesh.active_elements_end(), n_pieces); }
void SFCPartitioner::_do_partition (MeshBase & mesh, const unsigned int n) { this->partition_range(mesh, mesh.active_elements_begin(), mesh.active_elements_end(), n); }
unsigned int CheckpointIO::n_active_levels_on_processor(const MeshBase & mesh) const { unsigned int max_level = 0; MeshBase::const_element_iterator el = mesh.active_elements_begin(); const MeshBase::const_element_iterator end_el = mesh.active_elements_end(); for( ; el != end_el; ++el) max_level = std::max((*el)->level(), max_level); return max_level + 1; }
// ------------------------------------------------------------ // LinearPartitioner implementation void LinearPartitioner::_do_partition (MeshBase& mesh, const unsigned int n) { libmesh_assert_greater (n, 0); // Check for an easy return if (n == 1) { this->single_partition (mesh); return; } // Create a simple linear partitioning { START_LOG ("partition()", "LinearPartitioner"); const dof_id_type n_active_elem = mesh.n_active_elem(); const dof_id_type blksize = n_active_elem/n; dof_id_type e = 0; MeshBase::element_iterator elem_it = mesh.active_elements_begin(); const MeshBase::element_iterator elem_end = mesh.active_elements_end(); for ( ; elem_it != elem_end; ++elem_it) { if ((e/blksize) < n) { Elem *elem = *elem_it; elem->processor_id() = libmesh_cast_int<processor_id_type>(e/blksize); } else { Elem *elem = *elem_it; elem->processor_id() = 0; elem = elem->parent(); } e++; } STOP_LOG ("partition()", "LinearPartitioner"); } }
// ------------------------------------------------------------ // SFCPartitioner implementation void SFCPartitioner::_do_partition (MeshBase & mesh, const unsigned int n) { libmesh_assert_greater (n, 0); // Check for an easy return if (n == 1) { this->single_partition (mesh); return; } // What to do if the sfcurves library IS NOT present #ifndef LIBMESH_HAVE_SFCURVES libmesh_here(); libMesh::err << "ERROR: The library has been built without" << std::endl << "Space Filling Curve support. Using a linear" << std::endl << "partitioner instead!" << std::endl; LinearPartitioner lp; lp.partition (mesh, n); // What to do if the sfcurves library IS present #else LOG_SCOPE("sfc_partition()", "SFCPartitioner"); const dof_id_type n_active_elem = mesh.n_active_elem(); const dof_id_type n_elem = mesh.n_elem(); // the forward_map maps the active element id // into a contiguous block of indices std::vector<dof_id_type> forward_map (n_elem, DofObject::invalid_id); // the reverse_map maps the contiguous ids back // to active elements std::vector<Elem *> reverse_map (n_active_elem, libmesh_nullptr); int size = static_cast<int>(n_active_elem); std::vector<double> x (size); std::vector<double> y (size); std::vector<double> z (size); std::vector<int> table (size); // We need to map the active element ids into a // contiguous range. { MeshBase::element_iterator elem_it = mesh.active_elements_begin(); const MeshBase::element_iterator elem_end = mesh.active_elements_end(); dof_id_type el_num = 0; for (; elem_it != elem_end; ++elem_it) { libmesh_assert_less ((*elem_it)->id(), forward_map.size()); libmesh_assert_less (el_num, reverse_map.size()); forward_map[(*elem_it)->id()] = el_num; reverse_map[el_num] = *elem_it; el_num++; } libmesh_assert_equal_to (el_num, n_active_elem); } // Get the centroid for each active element { // const_active_elem_iterator elem_it (mesh.const_elements_begin()); // const const_active_elem_iterator elem_end(mesh.const_elements_end()); MeshBase::element_iterator elem_it = mesh.active_elements_begin(); const MeshBase::element_iterator elem_end = mesh.active_elements_end(); for (; elem_it != elem_end; ++elem_it) { const Elem * elem = *elem_it; libmesh_assert_less (elem->id(), forward_map.size()); const Point p = elem->centroid(); x[forward_map[elem->id()]] = p(0); y[forward_map[elem->id()]] = p(1); z[forward_map[elem->id()]] = p(2); } } // build the space-filling curve if (_sfc_type == "Hilbert") Sfc::hilbert (&x[0], &y[0], &z[0], &size, &table[0]); else if (_sfc_type == "Morton") Sfc::morton (&x[0], &y[0], &z[0], &size, &table[0]); else { libmesh_here(); libMesh::err << "ERROR: Unknown type: " << _sfc_type << std::endl << " Valid types are" << std::endl << " \"Hilbert\"" << std::endl << " \"Morton\"" << std::endl << " " << std::endl << "Proceeding with a Hilbert curve." << std::endl; Sfc::hilbert (&x[0], &y[0], &z[0], &size, &table[0]); } // Assign the partitioning to the active elements { // { // std::ofstream out ("sfc.dat"); // out << "variables=x,y,z" << std::endl; // out << "zone f=point" << std::endl; // for (unsigned int i=0; i<n_active_elem; i++) // out << x[i] << " " // << y[i] << " " // << z[i] << std::endl; // } const dof_id_type blksize = (n_active_elem+n-1)/n; for (dof_id_type i=0; i<n_active_elem; i++) { libmesh_assert_less (static_cast<unsigned int>(table[i]-1), reverse_map.size()); Elem * elem = reverse_map[table[i]-1]; elem->processor_id() = cast_int<processor_id_type> (i/blksize); } } #endif }
void Partitioner::set_node_processor_ids(MeshBase & mesh) { LOG_SCOPE("set_node_processor_ids()","Partitioner"); // This function must be run on all processors at once libmesh_parallel_only(mesh.comm()); // If we have any unpartitioned elements at this // stage there is a problem libmesh_assert (MeshTools::n_elem(mesh.unpartitioned_elements_begin(), mesh.unpartitioned_elements_end()) == 0); // const dof_id_type orig_n_local_nodes = mesh.n_local_nodes(); // libMesh::err << "[" << mesh.processor_id() << "]: orig_n_local_nodes=" // << orig_n_local_nodes << std::endl; // Build up request sets. Each node is currently owned by a processor because // it is connected to an element owned by that processor. However, during the // repartitioning phase that element may have been assigned a new processor id, but // it is still resident on the original processor. We need to know where to look // for new ids before assigning new ids, otherwise we may be asking the wrong processors // for the wrong information. // // The only remaining issue is what to do with unpartitioned nodes. Since they are required // to live on all processors we can simply rely on ourselves to number them properly. std::vector<std::vector<dof_id_type> > requested_node_ids(mesh.n_processors()); // Loop over all the nodes, count the ones on each processor. We can skip ourself std::vector<dof_id_type> ghost_nodes_from_proc(mesh.n_processors(), 0); MeshBase::node_iterator node_it = mesh.nodes_begin(); const MeshBase::node_iterator node_end = mesh.nodes_end(); for (; node_it != node_end; ++node_it) { Node * node = *node_it; libmesh_assert(node); const processor_id_type current_pid = node->processor_id(); if (current_pid != mesh.processor_id() && current_pid != DofObject::invalid_processor_id) { libmesh_assert_less (current_pid, ghost_nodes_from_proc.size()); ghost_nodes_from_proc[current_pid]++; } } // We know how many objects live on each processor, so reserve() // space for each. for (processor_id_type pid=0; pid != mesh.n_processors(); ++pid) requested_node_ids[pid].reserve(ghost_nodes_from_proc[pid]); // We need to get the new pid for each node from the processor // which *currently* owns the node. We can safely skip ourself for (node_it = mesh.nodes_begin(); node_it != node_end; ++node_it) { Node * node = *node_it; libmesh_assert(node); const processor_id_type current_pid = node->processor_id(); if (current_pid != mesh.processor_id() && current_pid != DofObject::invalid_processor_id) { libmesh_assert_less (current_pid, requested_node_ids.size()); libmesh_assert_less (requested_node_ids[current_pid].size(), ghost_nodes_from_proc[current_pid]); requested_node_ids[current_pid].push_back(node->id()); } // Unset any previously-set node processor ids node->invalidate_processor_id(); } // Loop over all the active elements MeshBase::element_iterator elem_it = mesh.active_elements_begin(); const MeshBase::element_iterator elem_end = mesh.active_elements_end(); for ( ; elem_it != elem_end; ++elem_it) { Elem * elem = *elem_it; libmesh_assert(elem); libmesh_assert_not_equal_to (elem->processor_id(), DofObject::invalid_processor_id); // For each node, set the processor ID to the min of // its current value and this Element's processor id. // // TODO: we would probably get better parallel partitioning if // we did something like "min for even numbered nodes, max for // odd numbered". We'd need to be careful about how that would // affect solution ordering for I/O, though. for (unsigned int n=0; n<elem->n_nodes(); ++n) elem->node_ptr(n)->processor_id() = std::min(elem->node_ptr(n)->processor_id(), elem->processor_id()); } // And loop over the subactive elements, but don't reassign // nodes that are already active on another processor. MeshBase::element_iterator sub_it = mesh.subactive_elements_begin(); const MeshBase::element_iterator sub_end = mesh.subactive_elements_end(); for ( ; sub_it != sub_end; ++sub_it) { Elem * elem = *sub_it; libmesh_assert(elem); libmesh_assert_not_equal_to (elem->processor_id(), DofObject::invalid_processor_id); for (unsigned int n=0; n<elem->n_nodes(); ++n) if (elem->node_ptr(n)->processor_id() == DofObject::invalid_processor_id) elem->node_ptr(n)->processor_id() = elem->processor_id(); } // Same for the inactive elements -- we will have already gotten most of these // nodes, *except* for the case of a parent with a subset of children which are // ghost elements. In that case some of the parent nodes will not have been // properly handled yet MeshBase::element_iterator not_it = mesh.not_active_elements_begin(); const MeshBase::element_iterator not_end = mesh.not_active_elements_end(); for ( ; not_it != not_end; ++not_it) { Elem * elem = *not_it; libmesh_assert(elem); libmesh_assert_not_equal_to (elem->processor_id(), DofObject::invalid_processor_id); for (unsigned int n=0; n<elem->n_nodes(); ++n) if (elem->node_ptr(n)->processor_id() == DofObject::invalid_processor_id) elem->node_ptr(n)->processor_id() = elem->processor_id(); } // We can't assert that all nodes are connected to elements, because // a DistributedMesh with NodeConstraints might have pulled in some // remote nodes solely for evaluating those constraints. // MeshTools::libmesh_assert_connected_nodes(mesh); // For such nodes, we'll do a sanity check later when making sure // that we successfully reset their processor ids to something // valid. // Next set node ids from other processors, excluding self for (processor_id_type p=1; p != mesh.n_processors(); ++p) { // Trade my requests with processor procup and procdown processor_id_type procup = cast_int<processor_id_type> ((mesh.processor_id() + p) % mesh.n_processors()); processor_id_type procdown = cast_int<processor_id_type> ((mesh.n_processors() + mesh.processor_id() - p) % mesh.n_processors()); std::vector<dof_id_type> request_to_fill; mesh.comm().send_receive(procup, requested_node_ids[procup], procdown, request_to_fill); // Fill those requests in-place for (std::size_t i=0; i != request_to_fill.size(); ++i) { Node & node = mesh.node_ref(request_to_fill[i]); const processor_id_type new_pid = node.processor_id(); // We may have an invalid processor_id() on nodes that have been // "detatched" from coarsened-away elements but that have not yet // themselves been removed. // libmesh_assert_not_equal_to (new_pid, DofObject::invalid_processor_id); // libmesh_assert_less (new_pid, mesh.n_partitions()); // this is the correct test -- request_to_fill[i] = new_pid; // the number of partitions may } // not equal the number of processors // Trade back the results std::vector<dof_id_type> filled_request; mesh.comm().send_receive(procdown, request_to_fill, procup, filled_request); libmesh_assert_equal_to (filled_request.size(), requested_node_ids[procup].size()); // And copy the id changes we've now been informed of for (std::size_t i=0; i != filled_request.size(); ++i) { Node & node = mesh.node_ref(requested_node_ids[procup][i]); // this is the correct test -- the number of partitions may // not equal the number of processors // But: we may have an invalid processor_id() on nodes that // have been "detatched" from coarsened-away elements but // that have not yet themselves been removed. // libmesh_assert_less (filled_request[i], mesh.n_partitions()); node.processor_id(cast_int<processor_id_type>(filled_request[i])); } } #ifdef DEBUG MeshTools::libmesh_assert_valid_procids<Node>(mesh); #endif }
void Partitioner::set_parent_processor_ids(MeshBase & mesh) { // Ignore the parameter when !LIBMESH_ENABLE_AMR libmesh_ignore(mesh); LOG_SCOPE("set_parent_processor_ids()", "Partitioner"); #ifdef LIBMESH_ENABLE_AMR // If the mesh is serial we have access to all the elements, // in particular all the active ones. We can therefore set // the parent processor ids indirecly through their children, and // set the subactive processor ids while examining their active // ancestors. // By convention a parent is assigned to the minimum processor // of all its children, and a subactive is assigned to the processor // of its active ancestor. if (mesh.is_serial()) { // Loop over all the active elements in the mesh MeshBase::element_iterator it = mesh.active_elements_begin(); const MeshBase::element_iterator end = mesh.active_elements_end(); for ( ; it!=end; ++it) { Elem * child = *it; // First set descendents std::vector<const Elem *> subactive_family; child->total_family_tree(subactive_family); for (unsigned int i = 0; i != subactive_family.size(); ++i) const_cast<Elem *>(subactive_family[i])->processor_id() = child->processor_id(); // Then set ancestors Elem * parent = child->parent(); while (parent) { // invalidate the parent id, otherwise the min below // will not work if the current parent id is less // than all the children! parent->invalidate_processor_id(); for (unsigned int c=0; c<parent->n_children(); c++) { child = parent->child_ptr(c); libmesh_assert(child); libmesh_assert(!child->is_remote()); libmesh_assert_not_equal_to (child->processor_id(), DofObject::invalid_processor_id); parent->processor_id() = std::min(parent->processor_id(), child->processor_id()); } parent = parent->parent(); } } } // When the mesh is parallel we cannot guarantee that parents have access to // all their children. else { // Setting subactive processor ids is easy: we can guarantee // that children have access to all their parents. // Loop over all the active elements in the mesh MeshBase::element_iterator it = mesh.active_elements_begin(); const MeshBase::element_iterator end = mesh.active_elements_end(); for ( ; it!=end; ++it) { Elem * child = *it; std::vector<const Elem *> subactive_family; child->total_family_tree(subactive_family); for (unsigned int i = 0; i != subactive_family.size(); ++i) const_cast<Elem *>(subactive_family[i])->processor_id() = child->processor_id(); } // When the mesh is parallel we cannot guarantee that parents have access to // all their children. // We will use a brute-force approach here. Each processor finds its parent // elements and sets the parent pid to the minimum of its // semilocal descendants. // A global reduction is then performed to make sure the true minimum is found. // As noted, this is required because we cannot guarantee that a parent has // access to all its children on any single processor. libmesh_parallel_only(mesh.comm()); libmesh_assert(MeshTools::n_elem(mesh.unpartitioned_elements_begin(), mesh.unpartitioned_elements_end()) == 0); const dof_id_type max_elem_id = mesh.max_elem_id(); std::vector<processor_id_type> parent_processor_ids (std::min(communication_blocksize, max_elem_id)); for (dof_id_type blk=0, last_elem_id=0; last_elem_id<max_elem_id; blk++) { last_elem_id = std::min(static_cast<dof_id_type>((blk+1)*communication_blocksize), max_elem_id); const dof_id_type first_elem_id = blk*communication_blocksize; std::fill (parent_processor_ids.begin(), parent_processor_ids.end(), DofObject::invalid_processor_id); // first build up local contributions to parent_processor_ids MeshBase::element_iterator not_it = mesh.ancestor_elements_begin(); const MeshBase::element_iterator not_end = mesh.ancestor_elements_end(); bool have_parent_in_block = false; for ( ; not_it != not_end; ++not_it) { Elem * parent = *not_it; const dof_id_type parent_idx = parent->id(); libmesh_assert_less (parent_idx, max_elem_id); if ((parent_idx >= first_elem_id) && (parent_idx < last_elem_id)) { have_parent_in_block = true; processor_id_type parent_pid = DofObject::invalid_processor_id; std::vector<const Elem *> active_family; parent->active_family_tree(active_family); for (unsigned int i = 0; i != active_family.size(); ++i) parent_pid = std::min (parent_pid, active_family[i]->processor_id()); const dof_id_type packed_idx = parent_idx - first_elem_id; libmesh_assert_less (packed_idx, parent_processor_ids.size()); parent_processor_ids[packed_idx] = parent_pid; } } // then find the global minimum mesh.comm().min (parent_processor_ids); // and assign the ids, if we have a parent in this block. if (have_parent_in_block) for (not_it = mesh.ancestor_elements_begin(); not_it != not_end; ++not_it) { Elem * parent = *not_it; const dof_id_type parent_idx = parent->id(); if ((parent_idx >= first_elem_id) && (parent_idx < last_elem_id)) { const dof_id_type packed_idx = parent_idx - first_elem_id; libmesh_assert_less (packed_idx, parent_processor_ids.size()); const processor_id_type parent_pid = parent_processor_ids[packed_idx]; libmesh_assert_not_equal_to (parent_pid, DofObject::invalid_processor_id); parent->processor_id() = parent_pid; } } } } #endif // LIBMESH_ENABLE_AMR }
void LinearElasticityWithContact::move_mesh (MeshBase & input_mesh, const NumericVector<Number> & input_solution) { // Maintain a set of node ids that we've encountered. LIBMESH_BEST_UNORDERED_SET<dof_id_type> encountered_node_ids; // Localize input_solution so that we have the data to move all // elements (not just elements local to this processor). UniquePtr< NumericVector<Number> > localized_input_solution = NumericVector<Number>::build(input_solution.comm()); localized_input_solution->init (input_solution.size(), false, SERIAL); input_solution.localize(*localized_input_solution); MeshBase::const_element_iterator el = input_mesh.active_elements_begin(); const MeshBase::const_element_iterator end_el = input_mesh.active_elements_end(); for ( ; el != end_el; ++el) { Elem * elem = *el; Elem * orig_elem = _sys.get_mesh().elem_ptr(elem->id()); for (unsigned int node_id=0; node_id<elem->n_nodes(); node_id++) { Node & node = elem->node_ref(node_id); if (encountered_node_ids.find(node.id()) != encountered_node_ids.end()) continue; encountered_node_ids.insert(node.id()); std::vector<std::string> uvw_names(3); uvw_names[0] = "u"; uvw_names[1] = "v"; uvw_names[2] = "w"; { const Point master_point = elem->master_point(node_id); Point uvw; for (unsigned int index=0; index<uvw_names.size(); index++) { const unsigned int var = _sys.variable_number(uvw_names[index]); const FEType & fe_type = _sys.get_dof_map().variable_type(var); FEComputeData data (_sys.get_equation_systems(), master_point); FEInterface::compute_data(elem->dim(), fe_type, elem, data); std::vector<dof_id_type> dof_indices_var; _sys.get_dof_map().dof_indices (orig_elem, dof_indices_var, var); for (unsigned int i=0; i<dof_indices_var.size(); i++) { Number value = (*localized_input_solution)(dof_indices_var[i]) * data.shape[i]; #ifdef LIBMESH_USE_COMPLEX_NUMBERS // We explicitly store the real part in uvw uvw(index) += value.real(); #else uvw(index) += value; #endif } } // Update the node's location node += uvw; } } } }
void ParmetisPartitioner::assign_partitioning (MeshBase & mesh) { // This function must be run on all processors at once libmesh_parallel_only(mesh.comm()); const dof_id_type first_local_elem = _pmetis->vtxdist[mesh.processor_id()]; std::vector<std::vector<dof_id_type> > requested_ids(mesh.n_processors()), requests_to_fill(mesh.n_processors()); MeshBase::element_iterator elem_it = mesh.active_elements_begin(); MeshBase::element_iterator elem_end = mesh.active_elements_end(); for (; elem_it != elem_end; ++elem_it) { Elem * elem = *elem_it; // we need to get the index from the owning processor // (note we cannot assign it now -- we are iterating // over elements again and this will be bad!) libmesh_assert_less (elem->processor_id(), requested_ids.size()); requested_ids[elem->processor_id()].push_back(elem->id()); } // Trade with all processors (including self) to get their indices for (processor_id_type pid=0; pid<mesh.n_processors(); pid++) { // Trade my requests with processor procup and procdown const processor_id_type procup = (mesh.processor_id() + pid) % mesh.n_processors(); const processor_id_type procdown = (mesh.n_processors() + mesh.processor_id() - pid) % mesh.n_processors(); mesh.comm().send_receive (procup, requested_ids[procup], procdown, requests_to_fill[procdown]); // we can overwrite these requested ids in-place. for (std::size_t i=0; i<requests_to_fill[procdown].size(); i++) { const dof_id_type requested_elem_index = requests_to_fill[procdown][i]; libmesh_assert(_global_index_by_pid_map.count(requested_elem_index)); const dof_id_type global_index_by_pid = _global_index_by_pid_map[requested_elem_index]; const dof_id_type local_index = global_index_by_pid - first_local_elem; libmesh_assert_less (local_index, _pmetis->part.size()); libmesh_assert_less (local_index, mesh.n_active_local_elem()); const unsigned int elem_procid = static_cast<unsigned int>(_pmetis->part[local_index]); libmesh_assert_less (elem_procid, static_cast<unsigned int>(_pmetis->nparts)); requests_to_fill[procdown][i] = elem_procid; } // Trade back mesh.comm().send_receive (procdown, requests_to_fill[procdown], procup, requested_ids[procup]); } // and finally assign the partitioning. // note we are iterating in exactly the same order // used to build up the request, so we can expect the // required entries to be in the proper sequence. elem_it = mesh.active_elements_begin(); elem_end = mesh.active_elements_end(); for (std::vector<unsigned int> counters(mesh.n_processors(), 0); elem_it != elem_end; ++elem_it) { Elem * elem = *elem_it; const processor_id_type current_pid = elem->processor_id(); libmesh_assert_less (counters[current_pid], requested_ids[current_pid].size()); const processor_id_type elem_procid = requested_ids[current_pid][counters[current_pid]++]; libmesh_assert_less (elem_procid, static_cast<unsigned int>(_pmetis->nparts)); elem->processor_id() = elem_procid; } }
void ParmetisPartitioner::build_graph (const MeshBase & mesh) { // build the graph in distributed CSR format. Note that // the edges in the graph will correspond to // face neighbors const dof_id_type n_active_local_elem = mesh.n_active_local_elem(); // If we have boundary elements in this mesh, we want to account for // the connectivity between them and interior elements. We can find // interior elements from boundary elements, but we need to build up // a lookup map to do the reverse. typedef LIBMESH_BEST_UNORDERED_MULTIMAP<const Elem *, const Elem *> map_type; map_type interior_to_boundary_map; { MeshBase::const_element_iterator elem_it = mesh.active_elements_begin(); const MeshBase::const_element_iterator elem_end = mesh.active_elements_end(); for (; elem_it != elem_end; ++elem_it) { const Elem * elem = *elem_it; // If we don't have an interior_parent then there's nothing to look us // up. if ((elem->dim() >= LIBMESH_DIM) || !elem->interior_parent()) continue; // get all relevant interior elements std::set<const Elem *> neighbor_set; elem->find_interior_neighbors(neighbor_set); std::set<const Elem *>::iterator n_it = neighbor_set.begin(); for (; n_it != neighbor_set.end(); ++n_it) { // FIXME - non-const versions of the Elem set methods // would be nice Elem * neighbor = const_cast<Elem *>(*n_it); #if defined(LIBMESH_HAVE_UNORDERED_MULTIMAP) || \ defined(LIBMESH_HAVE_TR1_UNORDERED_MAP) || \ defined(LIBMESH_HAVE_HASH_MAP) || \ defined(LIBMESH_HAVE_EXT_HASH_MAP) interior_to_boundary_map.insert (std::make_pair(neighbor, elem)); #else interior_to_boundary_map.insert (interior_to_boundary_map.begin(), std::make_pair(neighbor, elem)); #endif } } } #ifdef LIBMESH_ENABLE_AMR std::vector<const Elem *> neighbors_offspring; #endif std::vector<std::vector<dof_id_type> > graph(n_active_local_elem); dof_id_type graph_size=0; const dof_id_type first_local_elem = _pmetis->vtxdist[mesh.processor_id()]; MeshBase::const_element_iterator elem_it = mesh.active_local_elements_begin(); const MeshBase::const_element_iterator elem_end = mesh.active_local_elements_end(); for (; elem_it != elem_end; ++elem_it) { const Elem * elem = *elem_it; libmesh_assert (_global_index_by_pid_map.count(elem->id())); const dof_id_type global_index_by_pid = _global_index_by_pid_map[elem->id()]; const dof_id_type local_index = global_index_by_pid - first_local_elem; libmesh_assert_less (local_index, n_active_local_elem); std::vector<dof_id_type> & graph_row = graph[local_index]; // Loop over the element's neighbors. An element // adjacency corresponds to a face neighbor for (unsigned int ms=0; ms<elem->n_neighbors(); ms++) { const Elem * neighbor = elem->neighbor(ms); if (neighbor != libmesh_nullptr) { // If the neighbor is active treat it // as a connection if (neighbor->active()) { libmesh_assert(_global_index_by_pid_map.count(neighbor->id())); const dof_id_type neighbor_global_index_by_pid = _global_index_by_pid_map[neighbor->id()]; graph_row.push_back(neighbor_global_index_by_pid); graph_size++; } #ifdef LIBMESH_ENABLE_AMR // Otherwise we need to find all of the // neighbor's children that are connected to // us and add them else { // The side of the neighbor to which // we are connected const unsigned int ns = neighbor->which_neighbor_am_i (elem); libmesh_assert_less (ns, neighbor->n_neighbors()); // Get all the active children (& grandchildren, etc...) // of the neighbor // FIXME - this is the wrong thing, since we // should be getting the active family tree on // our side only. But adding too many graph // links may cause hanging nodes to tend to be // on partition interiors, which would reduce // communication overhead for constraint // equations, so we'll leave it. neighbor->active_family_tree (neighbors_offspring); // Get all the neighbor's children that // live on that side and are thus connected // to us for (unsigned int nc=0; nc<neighbors_offspring.size(); nc++) { const Elem * child = neighbors_offspring[nc]; // This does not assume a level-1 mesh. // Note that since children have sides numbered // coincident with the parent then this is a sufficient test. if (child->neighbor(ns) == elem) { libmesh_assert (child->active()); libmesh_assert (_global_index_by_pid_map.count(child->id())); const dof_id_type child_global_index_by_pid = _global_index_by_pid_map[child->id()]; graph_row.push_back(child_global_index_by_pid); graph_size++; } } } #endif /* ifdef LIBMESH_ENABLE_AMR */ } } if ((elem->dim() < LIBMESH_DIM) && elem->interior_parent()) { // get all relevant interior elements std::set<const Elem *> neighbor_set; elem->find_interior_neighbors(neighbor_set); std::set<const Elem *>::iterator n_it = neighbor_set.begin(); for (; n_it != neighbor_set.end(); ++n_it) { // FIXME - non-const versions of the Elem set methods // would be nice Elem * neighbor = const_cast<Elem *>(*n_it); const dof_id_type neighbor_global_index_by_pid = _global_index_by_pid_map[neighbor->id()]; graph_row.push_back(neighbor_global_index_by_pid); graph_size++; } } // Check for any boundary neighbors typedef map_type::iterator map_it_type; std::pair<map_it_type, map_it_type> bounds = interior_to_boundary_map.equal_range(elem); for (map_it_type it = bounds.first; it != bounds.second; ++it) { const Elem * neighbor = it->second; const dof_id_type neighbor_global_index_by_pid = _global_index_by_pid_map[neighbor->id()]; graph_row.push_back(neighbor_global_index_by_pid); graph_size++; } } // Reserve space in the adjacency array _pmetis->xadj.clear(); _pmetis->xadj.reserve (n_active_local_elem + 1); _pmetis->adjncy.clear(); _pmetis->adjncy.reserve (graph_size); for (std::size_t r=0; r<graph.size(); r++) { _pmetis->xadj.push_back(_pmetis->adjncy.size()); std::vector<dof_id_type> graph_row; // build this emtpy graph_row.swap(graph[r]); // this will deallocate at the end of scope _pmetis->adjncy.insert(_pmetis->adjncy.end(), graph_row.begin(), graph_row.end()); } // The end of the adjacency array for the last elem _pmetis->xadj.push_back(_pmetis->adjncy.size()); libmesh_assert_equal_to (_pmetis->xadj.size(), n_active_local_elem+1); libmesh_assert_equal_to (_pmetis->adjncy.size(), graph_size); }
void ParmetisPartitioner::initialize (const MeshBase & mesh, const unsigned int n_sbdmns) { const dof_id_type n_active_local_elem = mesh.n_active_local_elem(); // Set parameters. _pmetis->wgtflag = 2; // weights on vertices only _pmetis->ncon = 1; // one weight per vertex _pmetis->numflag = 0; // C-style 0-based numbering _pmetis->nparts = static_cast<Parmetis::idx_t>(n_sbdmns); // number of subdomains to create _pmetis->edgecut = 0; // the numbers of edges cut by the // partition // Initialize data structures for ParMETIS _pmetis->vtxdist.resize (mesh.n_processors()+1); std::fill (_pmetis->vtxdist.begin(), _pmetis->vtxdist.end(), 0); _pmetis->tpwgts.resize (_pmetis->nparts); std::fill (_pmetis->tpwgts.begin(), _pmetis->tpwgts.end(), 1./_pmetis->nparts); _pmetis->ubvec.resize (_pmetis->ncon); std::fill (_pmetis->ubvec.begin(), _pmetis->ubvec.end(), 1.05); _pmetis->part.resize (n_active_local_elem); std::fill (_pmetis->part.begin(), _pmetis->part.end(), 0); _pmetis->options.resize (5); _pmetis->vwgt.resize (n_active_local_elem); // Set the options _pmetis->options[0] = 1; // don't use default options _pmetis->options[1] = 0; // default (level of timing) _pmetis->options[2] = 15; // random seed (default) _pmetis->options[3] = 2; // processor distribution and subdomain distribution are decoupled // Find the number of active elements on each processor. We cannot use // mesh.n_active_elem_on_proc(pid) since that only returns the number of // elements assigned to pid which are currently stored on the calling // processor. This will not in general be correct for parallel meshes // when (pid!=mesh.processor_id()). _n_active_elem_on_proc.resize(mesh.n_processors()); mesh.comm().allgather(n_active_local_elem, _n_active_elem_on_proc); // count the total number of active elements in the mesh. Note we cannot // use mesh.n_active_elem() in general since this only returns the number // of active elements which are stored on the calling processor. // We should not use n_active_elem for any allocation because that will // be inheritly unscalable, but it can be useful for libmesh_assertions. dof_id_type n_active_elem=0; // Set up the vtxdist array. This will be the same on each processor. // ***** Consult the Parmetis documentation. ***** libmesh_assert_equal_to (_pmetis->vtxdist.size(), cast_int<std::size_t>(mesh.n_processors()+1)); libmesh_assert_equal_to (_pmetis->vtxdist[0], 0); for (processor_id_type pid=0; pid<mesh.n_processors(); pid++) { _pmetis->vtxdist[pid+1] = _pmetis->vtxdist[pid] + _n_active_elem_on_proc[pid]; n_active_elem += _n_active_elem_on_proc[pid]; } libmesh_assert_equal_to (_pmetis->vtxdist.back(), static_cast<Parmetis::idx_t>(n_active_elem)); // ParMetis expects the elements to be numbered in contiguous blocks // by processor, i.e. [0, ne0), [ne0, ne0+ne1), ... // Since we only partition active elements we should have no expectation // that we currently have such a distribution. So we need to create it. // Also, at the same time we are going to map all the active elements into a globally // unique range [0,n_active_elem) which is *independent* of the current partitioning. // This can be fed to ParMetis as the initial partitioning of the subdomains (decoupled // from the partitioning of the objects themselves). This allows us to get the same // resultant partitioning independed of the input partitioning. MeshTools::BoundingBox bbox = MeshTools::bounding_box(mesh); _global_index_by_pid_map.clear(); // Maps active element ids into a contiguous range independent of partitioning. // (only needs local scope) vectormap<dof_id_type, dof_id_type> global_index_map; { std::vector<dof_id_type> global_index; // create the mapping which is contiguous by processor dof_id_type pid_offset=0; for (processor_id_type pid=0; pid<mesh.n_processors(); pid++) { MeshBase::const_element_iterator it = mesh.active_pid_elements_begin(pid); const MeshBase::const_element_iterator end = mesh.active_pid_elements_end(pid); // note that we may not have all (or any!) the active elements which belong on this processor, // but by calling this on all processors a unique range in [0,_n_active_elem_on_proc[pid]) // is constructed. Only the indices for the elements we pass in are returned in the array. MeshCommunication().find_global_indices (mesh.comm(), bbox, it, end, global_index); for (dof_id_type cnt=0; it != end; ++it) { const Elem * elem = *it; libmesh_assert (!_global_index_by_pid_map.count(elem->id())); libmesh_assert_less (cnt, global_index.size()); libmesh_assert_less (global_index[cnt], _n_active_elem_on_proc[pid]); _global_index_by_pid_map.insert(std::make_pair(elem->id(), global_index[cnt++] + pid_offset)); } pid_offset += _n_active_elem_on_proc[pid]; } // create the unique mapping for all active elements independent of partitioning { MeshBase::const_element_iterator it = mesh.active_elements_begin(); const MeshBase::const_element_iterator end = mesh.active_elements_end(); // Calling this on all processors a unique range in [0,n_active_elem) is constructed. // Only the indices for the elements we pass in are returned in the array. MeshCommunication().find_global_indices (mesh.comm(), bbox, it, end, global_index); for (dof_id_type cnt=0; it != end; ++it) { const Elem * elem = *it; libmesh_assert (!global_index_map.count(elem->id())); libmesh_assert_less (cnt, global_index.size()); libmesh_assert_less (global_index[cnt], n_active_elem); global_index_map.insert(std::make_pair(elem->id(), global_index[cnt++])); } } // really, shouldn't be close! libmesh_assert_less_equal (global_index_map.size(), n_active_elem); libmesh_assert_less_equal (_global_index_by_pid_map.size(), n_active_elem); // At this point the two maps should be the same size. If they are not // then the number of active elements is not the same as the sum over all // processors of the number of active elements per processor, which means // there must be some unpartitioned objects out there. if (global_index_map.size() != _global_index_by_pid_map.size()) libmesh_error_msg("ERROR: ParmetisPartitioner cannot handle unpartitioned objects!"); } // Finally, we need to initialize the vertex (partition) weights and the initial subdomain // mapping. The subdomain mapping will be independent of the processor mapping, and is // defined by a simple mapping of the global indices we just found. { std::vector<dof_id_type> subdomain_bounds(mesh.n_processors()); const dof_id_type first_local_elem = _pmetis->vtxdist[mesh.processor_id()]; for (processor_id_type pid=0; pid<mesh.n_processors(); pid++) { dof_id_type tgt_subdomain_size = 0; // watch out for the case that n_subdomains < n_processors if (pid < static_cast<unsigned int>(_pmetis->nparts)) { tgt_subdomain_size = n_active_elem/std::min (cast_int<Parmetis::idx_t>(mesh.n_processors()), _pmetis->nparts); if (pid < n_active_elem%_pmetis->nparts) tgt_subdomain_size++; } if (pid == 0) subdomain_bounds[0] = tgt_subdomain_size; else subdomain_bounds[pid] = subdomain_bounds[pid-1] + tgt_subdomain_size; } libmesh_assert_equal_to (subdomain_bounds.back(), n_active_elem); MeshBase::const_element_iterator elem_it = mesh.active_local_elements_begin(); const MeshBase::const_element_iterator elem_end = mesh.active_local_elements_end(); for (; elem_it != elem_end; ++elem_it) { const Elem * elem = *elem_it; libmesh_assert (_global_index_by_pid_map.count(elem->id())); const dof_id_type global_index_by_pid = _global_index_by_pid_map[elem->id()]; libmesh_assert_less (global_index_by_pid, n_active_elem); const dof_id_type local_index = global_index_by_pid - first_local_elem; libmesh_assert_less (local_index, n_active_local_elem); libmesh_assert_less (local_index, _pmetis->vwgt.size()); // TODO:[BSK] maybe there is a better weight? _pmetis->vwgt[local_index] = elem->n_nodes(); // find the subdomain this element belongs in libmesh_assert (global_index_map.count(elem->id())); const dof_id_type global_index = global_index_map[elem->id()]; libmesh_assert_less (global_index, subdomain_bounds.back()); const unsigned int subdomain_id = std::distance(subdomain_bounds.begin(), std::lower_bound(subdomain_bounds.begin(), subdomain_bounds.end(), global_index)); libmesh_assert_less (subdomain_id, static_cast<unsigned int>(_pmetis->nparts)); libmesh_assert_less (local_index, _pmetis->part.size()); _pmetis->part[local_index] = subdomain_id; } } }
// ------------------------------------------------------------ // MetisPartitioner implementation void MetisPartitioner::_do_partition (MeshBase & mesh, const unsigned int n_pieces) { libmesh_assert_greater (n_pieces, 0); libmesh_assert (mesh.is_serial()); // Check for an easy return if (n_pieces == 1) { this->single_partition (mesh); return; } // What to do if the Metis library IS NOT present #ifndef LIBMESH_HAVE_METIS libmesh_here(); libMesh::err << "ERROR: The library has been built without" << std::endl << "Metis support. Using a space-filling curve" << std::endl << "partitioner instead!" << std::endl; SFCPartitioner sfcp; sfcp.partition (mesh, n_pieces); // What to do if the Metis library IS present #else LOG_SCOPE("partition()", "MetisPartitioner"); const dof_id_type n_active_elem = mesh.n_active_elem(); // build the graph // std::vector<Metis::idx_t> options(5); std::vector<Metis::idx_t> vwgt(n_active_elem); std::vector<Metis::idx_t> part(n_active_elem); Metis::idx_t n = static_cast<Metis::idx_t>(n_active_elem), // number of "nodes" (elements) // in the graph // wgtflag = 2, // weights on vertices only, // // none on edges // numflag = 0, // C-style 0-based numbering nparts = static_cast<Metis::idx_t>(n_pieces), // number of subdomains to create edgecut = 0; // the numbers of edges cut by the // resulting partition // Set the options // options[0] = 0; // use default options // Metis will only consider the active elements. // We need to map the active element ids into a // contiguous range. Further, we want the unique range indexing to be // independent of the element ordering, otherwise a circular dependency // can result in which the partitioning depends on the ordering which // depends on the partitioning... vectormap<dof_id_type, dof_id_type> global_index_map; global_index_map.reserve (n_active_elem); { std::vector<dof_id_type> global_index; MeshBase::element_iterator it = mesh.active_elements_begin(); const MeshBase::element_iterator end = mesh.active_elements_end(); MeshCommunication().find_global_indices (mesh.comm(), MeshTools::bounding_box(mesh), it, end, global_index); libmesh_assert_equal_to (global_index.size(), n_active_elem); for (std::size_t cnt=0; it != end; ++it) { const Elem * elem = *it; global_index_map.insert (std::make_pair(elem->id(), global_index[cnt++])); } libmesh_assert_equal_to (global_index_map.size(), n_active_elem); } // If we have boundary elements in this mesh, we want to account for // the connectivity between them and interior elements. We can find // interior elements from boundary elements, but we need to build up // a lookup map to do the reverse. typedef LIBMESH_BEST_UNORDERED_MULTIMAP<const Elem *, const Elem *> map_type; map_type interior_to_boundary_map; { MeshBase::const_element_iterator elem_it = mesh.active_elements_begin(); const MeshBase::const_element_iterator elem_end = mesh.active_elements_end(); for (; elem_it != elem_end; ++elem_it) { const Elem * elem = *elem_it; // If we don't have an interior_parent then there's nothing to look us // up. if ((elem->dim() >= LIBMESH_DIM) || !elem->interior_parent()) continue; // get all relevant interior elements std::set<const Elem *> neighbor_set; elem->find_interior_neighbors(neighbor_set); std::set<const Elem *>::iterator n_it = neighbor_set.begin(); for (; n_it != neighbor_set.end(); ++n_it) { // FIXME - non-const versions of the Elem set methods // would be nice Elem * neighbor = const_cast<Elem *>(*n_it); #if defined(LIBMESH_HAVE_UNORDERED_MULTIMAP) || \ defined(LIBMESH_HAVE_TR1_UNORDERED_MULTIMAP) || \ defined(LIBMESH_HAVE_HASH_MULTIMAP) || \ defined(LIBMESH_HAVE_EXT_HASH_MULTIMAP) interior_to_boundary_map.insert (std::make_pair(neighbor, elem)); #else interior_to_boundary_map.insert (interior_to_boundary_map.begin(), std::make_pair(neighbor, elem)); #endif } } } // Invoke METIS, but only on processor 0. // Then broadcast the resulting decomposition if (mesh.processor_id() == 0) { METIS_CSR_Graph<Metis::idx_t> csr_graph; csr_graph.offsets.resize(n_active_elem+1, 0); // Local scope for these { // build the graph in CSR format. Note that // the edges in the graph will correspond to // face neighbors #ifdef LIBMESH_ENABLE_AMR std::vector<const Elem *> neighbors_offspring; #endif MeshBase::element_iterator elem_it = mesh.active_elements_begin(); const MeshBase::element_iterator elem_end = mesh.active_elements_end(); #ifndef NDEBUG std::size_t graph_size=0; #endif // (1) first pass - get the row sizes for each element by counting the number // of face neighbors. Also populate the vwght array if necessary for (; elem_it != elem_end; ++elem_it) { const Elem * elem = *elem_it; const dof_id_type elem_global_index = global_index_map[elem->id()]; libmesh_assert_less (elem_global_index, vwgt.size()); // maybe there is a better weight? // The weight is used to define what a balanced graph is if(!_weights) vwgt[elem_global_index] = elem->n_nodes(); else vwgt[elem_global_index] = static_cast<Metis::idx_t>((*_weights)[elem->id()]); unsigned int num_neighbors = 0; // Loop over the element's neighbors. An element // adjacency corresponds to a face neighbor for (unsigned int ms=0; ms<elem->n_neighbors(); ms++) { const Elem * neighbor = elem->neighbor(ms); if (neighbor != libmesh_nullptr) { // If the neighbor is active treat it // as a connection if (neighbor->active()) num_neighbors++; #ifdef LIBMESH_ENABLE_AMR // Otherwise we need to find all of the // neighbor's children that are connected to // us and add them else { // The side of the neighbor to which // we are connected const unsigned int ns = neighbor->which_neighbor_am_i (elem); libmesh_assert_less (ns, neighbor->n_neighbors()); // Get all the active children (& grandchildren, etc...) // of the neighbor. // FIXME - this is the wrong thing, since we // should be getting the active family tree on // our side only. But adding too many graph // links may cause hanging nodes to tend to be // on partition interiors, which would reduce // communication overhead for constraint // equations, so we'll leave it. neighbor->active_family_tree (neighbors_offspring); // Get all the neighbor's children that // live on that side and are thus connected // to us for (unsigned int nc=0; nc<neighbors_offspring.size(); nc++) { const Elem * child = neighbors_offspring[nc]; // This does not assume a level-1 mesh. // Note that since children have sides numbered // coincident with the parent then this is a sufficient test. if (child->neighbor(ns) == elem) { libmesh_assert (child->active()); num_neighbors++; } } } #endif /* ifdef LIBMESH_ENABLE_AMR */ } } // Check for any interior neighbors if ((elem->dim() < LIBMESH_DIM) && elem->interior_parent()) { // get all relevant interior elements std::set<const Elem *> neighbor_set; elem->find_interior_neighbors(neighbor_set); num_neighbors += neighbor_set.size(); } // Check for any boundary neighbors typedef map_type::iterator map_it_type; std::pair<map_it_type, map_it_type> bounds = interior_to_boundary_map.equal_range(elem); num_neighbors += std::distance(bounds.first, bounds.second); csr_graph.prep_n_nonzeros(elem_global_index, num_neighbors); #ifndef NDEBUG graph_size += num_neighbors; #endif } csr_graph.prepare_for_use(); // (2) second pass - fill the compressed adjacency array elem_it = mesh.active_elements_begin(); for (; elem_it != elem_end; ++elem_it) { const Elem * elem = *elem_it; const dof_id_type elem_global_index = global_index_map[elem->id()]; unsigned int connection=0; // Loop over the element's neighbors. An element // adjacency corresponds to a face neighbor for (unsigned int ms=0; ms<elem->n_neighbors(); ms++) { const Elem * neighbor = elem->neighbor(ms); if (neighbor != libmesh_nullptr) { // If the neighbor is active treat it // as a connection if (neighbor->active()) csr_graph(elem_global_index, connection++) = global_index_map[neighbor->id()]; #ifdef LIBMESH_ENABLE_AMR // Otherwise we need to find all of the // neighbor's children that are connected to // us and add them else { // The side of the neighbor to which // we are connected const unsigned int ns = neighbor->which_neighbor_am_i (elem); libmesh_assert_less (ns, neighbor->n_neighbors()); // Get all the active children (& grandchildren, etc...) // of the neighbor. neighbor->active_family_tree (neighbors_offspring); // Get all the neighbor's children that // live on that side and are thus connected // to us for (unsigned int nc=0; nc<neighbors_offspring.size(); nc++) { const Elem * child = neighbors_offspring[nc]; // This does not assume a level-1 mesh. // Note that since children have sides numbered // coincident with the parent then this is a sufficient test. if (child->neighbor(ns) == elem) { libmesh_assert (child->active()); csr_graph(elem_global_index, connection++) = global_index_map[child->id()]; } } } #endif /* ifdef LIBMESH_ENABLE_AMR */ } } if ((elem->dim() < LIBMESH_DIM) && elem->interior_parent()) { // get all relevant interior elements std::set<const Elem *> neighbor_set; elem->find_interior_neighbors(neighbor_set); std::set<const Elem *>::iterator n_it = neighbor_set.begin(); for (; n_it != neighbor_set.end(); ++n_it) { // FIXME - non-const versions of the Elem set methods // would be nice Elem * neighbor = const_cast<Elem *>(*n_it); csr_graph(elem_global_index, connection++) = global_index_map[neighbor->id()]; } } // Check for any boundary neighbors typedef map_type::iterator map_it_type; std::pair<map_it_type, map_it_type> bounds = interior_to_boundary_map.equal_range(elem); for (map_it_type it = bounds.first; it != bounds.second; ++it) { const Elem * neighbor = it->second; csr_graph(elem_global_index, connection++) = global_index_map[neighbor->id()]; } } // We create a non-empty vals for a disconnected graph, to // work around a segfault from METIS. libmesh_assert_equal_to (csr_graph.vals.size(), std::max(graph_size,std::size_t(1))); } // done building the graph Metis::idx_t ncon = 1; // Select which type of partitioning to create // Use recursive if the number of partitions is less than or equal to 8 if (n_pieces <= 8) Metis::METIS_PartGraphRecursive(&n, &ncon, &csr_graph.offsets[0], &csr_graph.vals[0], &vwgt[0], libmesh_nullptr, libmesh_nullptr, &nparts, libmesh_nullptr, libmesh_nullptr, libmesh_nullptr, &edgecut, &part[0]); // Otherwise use kway else Metis::METIS_PartGraphKway(&n, &ncon, &csr_graph.offsets[0], &csr_graph.vals[0], &vwgt[0], libmesh_nullptr, libmesh_nullptr, &nparts, libmesh_nullptr, libmesh_nullptr, libmesh_nullptr, &edgecut, &part[0]); } // end processor 0 part // Broadcase the resutling partition mesh.comm().broadcast(part); // Assign the returned processor ids. The part array contains // the processor id for each active element, but in terms of // the contiguous indexing we defined above { MeshBase::element_iterator it = mesh.active_elements_begin(); const MeshBase::element_iterator end = mesh.active_elements_end(); for (; it!=end; ++it) { Elem * elem = *it; libmesh_assert (global_index_map.count(elem->id())); const dof_id_type elem_global_index = global_index_map[elem->id()]; libmesh_assert_less (elem_global_index, part.size()); const processor_id_type elem_procid = static_cast<processor_id_type>(part[elem_global_index]); elem->processor_id() = elem_procid; } } #endif }
// ------------------------------------------------------------ // MetisPartitioner implementation void MetisPartitioner::_do_partition (MeshBase& mesh, const unsigned int n_pieces) { libmesh_assert_greater (n_pieces, 0); libmesh_assert (mesh.is_serial()); // Check for an easy return if (n_pieces == 1) { this->single_partition (mesh); return; } // What to do if the Metis library IS NOT present #ifndef LIBMESH_HAVE_METIS libmesh_here(); libMesh::err << "ERROR: The library has been built without" << std::endl << "Metis support. Using a space-filling curve" << std::endl << "partitioner instead!" << std::endl; SFCPartitioner sfcp; sfcp.partition (mesh, n_pieces); // What to do if the Metis library IS present #else START_LOG("partition()", "MetisPartitioner"); const dof_id_type n_active_elem = mesh.n_active_elem(); // build the graph // std::vector<int> options(5); std::vector<int> vwgt(n_active_elem); std::vector<int> part(n_active_elem); int n = static_cast<int>(n_active_elem), // number of "nodes" (elements) // in the graph // wgtflag = 2, // weights on vertices only, // // none on edges // numflag = 0, // C-style 0-based numbering nparts = static_cast<int>(n_pieces), // number of subdomains to create edgecut = 0; // the numbers of edges cut by the // resulting partition // Set the options // options[0] = 0; // use default options // Metis will only consider the active elements. // We need to map the active element ids into a // contiguous range. Further, we want the unique range indexing to be // independednt of the element ordering, otherwise a circular dependency // can result in which the partitioning depends on the ordering which // depends on the partitioning... std::map<const Elem*, dof_id_type> global_index_map; { std::vector<dof_id_type> global_index; MeshBase::element_iterator it = mesh.active_elements_begin(); const MeshBase::element_iterator end = mesh.active_elements_end(); MeshCommunication().find_global_indices (MeshTools::bounding_box(mesh), it, end, global_index); libmesh_assert_equal_to (global_index.size(), n_active_elem); for (std::size_t cnt=0; it != end; ++it) { const Elem *elem = *it; libmesh_assert (!global_index_map.count(elem)); global_index_map[elem] = global_index[cnt++]; } libmesh_assert_equal_to (global_index_map.size(), n_active_elem); } // build the graph in CSR format. Note that // the edges in the graph will correspond to // face neighbors std::vector<int> xadj, adjncy; { std::vector<const Elem*> neighbors_offspring; MeshBase::element_iterator elem_it = mesh.active_elements_begin(); const MeshBase::element_iterator elem_end = mesh.active_elements_end(); // This will be exact when there is no refinement and all the // elements are of the same type. std::size_t graph_size=0; std::vector<std::vector<dof_id_type> > graph(n_active_elem); for (; elem_it != elem_end; ++elem_it) { const Elem* elem = *elem_it; libmesh_assert (global_index_map.count(elem)); const dof_id_type elem_global_index = global_index_map[elem]; libmesh_assert_less (elem_global_index, vwgt.size()); libmesh_assert_less (elem_global_index, graph.size()); // maybe there is a better weight? // The weight is used to define what a balanced graph is if(!_weights) vwgt[elem_global_index] = elem->n_nodes(); else vwgt[elem_global_index] = static_cast<int>((*_weights)[elem->id()]); // Loop over the element's neighbors. An element // adjacency corresponds to a face neighbor for (unsigned int ms=0; ms<elem->n_neighbors(); ms++) { const Elem* neighbor = elem->neighbor(ms); if (neighbor != NULL) { // If the neighbor is active treat it // as a connection if (neighbor->active()) { libmesh_assert (global_index_map.count(neighbor)); const dof_id_type neighbor_global_index = global_index_map[neighbor]; graph[elem_global_index].push_back(neighbor_global_index); graph_size++; } #ifdef LIBMESH_ENABLE_AMR // Otherwise we need to find all of the // neighbor's children that are connected to // us and add them else { // The side of the neighbor to which // we are connected const unsigned int ns = neighbor->which_neighbor_am_i (elem); libmesh_assert_less (ns, neighbor->n_neighbors()); // Get all the active children (& grandchildren, etc...) // of the neighbor. neighbor->active_family_tree (neighbors_offspring); // Get all the neighbor's children that // live on that side and are thus connected // to us for (unsigned int nc=0; nc<neighbors_offspring.size(); nc++) { const Elem* child = neighbors_offspring[nc]; // This does not assume a level-1 mesh. // Note that since children have sides numbered // coincident with the parent then this is a sufficient test. if (child->neighbor(ns) == elem) { libmesh_assert (child->active()); libmesh_assert (global_index_map.count(child)); const dof_id_type child_global_index = global_index_map[child]; graph[elem_global_index].push_back(child_global_index); graph_size++; } } } #endif /* ifdef LIBMESH_ENABLE_AMR */ } } } // Convert the graph into the format Metis wants xadj.reserve(n_active_elem+1); adjncy.reserve(graph_size); for (std::size_t r=0; r<graph.size(); r++) { xadj.push_back(adjncy.size()); std::vector<dof_id_type> graph_row; // build this emtpy graph_row.swap(graph[r]); // this will deallocate at the end of scope adjncy.insert(adjncy.end(), graph_row.begin(), graph_row.end()); } // The end of the adjacency array for the last elem xadj.push_back(adjncy.size()); libmesh_assert_equal_to (adjncy.size(), graph_size); libmesh_assert_equal_to (xadj.size(), n_active_elem+1); } // done building the graph if (adjncy.empty()) adjncy.push_back(0); int ncon = 1; // Select which type of partitioning to create // Use recursive if the number of partitions is less than or equal to 8 if (n_pieces <= 8) Metis::METIS_PartGraphRecursive(&n, &ncon, &xadj[0], &adjncy[0], &vwgt[0], NULL, NULL, &nparts, NULL, NULL, NULL, &edgecut, &part[0]); // Otherwise use kway else Metis::METIS_PartGraphKway(&n, &ncon, &xadj[0], &adjncy[0], &vwgt[0], NULL, NULL, &nparts, NULL, NULL, NULL, &edgecut, &part[0]); // Assign the returned processor ids. The part array contains // the processor id for each active element, but in terms of // the contiguous indexing we defined above { MeshBase::element_iterator it = mesh.active_elements_begin(); const MeshBase::element_iterator end = mesh.active_elements_end(); for (; it!=end; ++it) { Elem* elem = *it; libmesh_assert (global_index_map.count(elem)); const dof_id_type elem_global_index = global_index_map[elem]; libmesh_assert_less (elem_global_index, part.size()); const processor_id_type elem_procid = static_cast<processor_id_type>(part[elem_global_index]); elem->processor_id() = elem_procid; } } STOP_LOG("partition()", "MetisPartitioner"); #endif }