//-----------------------------------------------------------------------------
std::vector<std::size_t> GraphOrdering::compute_local_reordering_map()
{
// Initialise Zoltan
float version;
int argc = 0;
char** argv = NULL;
Zoltan_Initialize(argc, argv, &version);
// Create Zoltan object
Zoltan zoltan;
// Set parameters
//zoltan.Set_Param( "ORDER_METHOD", "METIS");
zoltan.Set_Param( "ORDER_METHOD", "SCOTCH");
zoltan.Set_Param( "NUM_GID_ENTRIES", "1"); // global ID is 1 integer
zoltan.Set_Param( "NUM_LID_ENTRIES", "1"); // local ID is 1 integer
zoltan.Set_Param( "OBJ_WEIGHT_DIM", "0"); // omit object weights
// Set call-back functions
zoltan.Set_Num_Obj_Fn(GraphOrdering::get_number_of_objects, this);
zoltan.Set_Obj_List_Fn(GraphOrdering::get_object_list, this);
zoltan.Set_Num_Edges_Multi_Fn(GraphOrdering::get_number_edges, this);
zoltan.Set_Edge_List_Multi_Fn(GraphOrdering::get_all_edges, this);
// Create array for global ids that should be re-ordered
std::vector<ZOLTAN_ID_TYPE> global_ids(num_global_objects());
for (std::size_t i = 0; i < global_ids.size(); ++i)
global_ids[i] = i;
// Create array for re-ordered vertices
std::vector<ZOLTAN_ID_TYPE> new_id(num_global_objects());
// Compute re-ordering
int rc = zoltan.Order(1, num_global_objects(), &global_ids[0], &new_id[0]);
// Check for errors
if (rc != ZOLTAN_OK)
{
dolfin_error("GraphOrdering.cpp",
"compute matrix re-ordering",
"Zoltan partitioning failed");
}
// Copy re-ordering into a vector (in case Zoltan uses something other than std::size_t)
std::vector<std::size_t> map(new_id.begin(), new_id.end());
return map;
}
//-----------------------------------------------------------------------------
void
ZoltanPartition::compute_partition_phg(const MPI_Comm mpi_comm,
std::vector<std::size_t>& cell_partition,
const LocalMeshData& mesh_data)
{
Timer timer0("Partition graph (calling Zoltan PHG)");
// Create data structures to hold graph
std::vector<std::set<std::size_t>> local_graph;
std::set<std::size_t> ghost_vertices;
// Compute local dual graph
GraphBuilder::compute_dual_graph(mpi_comm, mesh_data, local_graph,
ghost_vertices);
// Initialise Zoltan
float version;
int argc = 0;
char** argv = NULL;
Zoltan_Initialize(argc, argv, &version);
// Create Zoltan object
Zoltan zoltan;
// Set Zoltan parameters
zoltan.Set_Param("NUM_GID_ENTRIES", "1");
zoltan.Set_Param("NUM_LID_ENTRIES", "0");
zoltan.Set_Param("NUM_GLOBAL_PARTS",
std::to_string(MPI::size(mpi_comm)));
zoltan.Set_Param("NUM_LOCAL_PARTS", "1");
zoltan.Set_Param("LB_METHOD", "GRAPH");
// Get partition method: 'PARTITION', 'REPARTITION' or 'REFINE'
std::string lb_approach = parameters["partitioning_approach"];
zoltan.Set_Param("LB_APPROACH", lb_approach.c_str());
// Repartitioning weighting
double phg_repart_multiplier = parameters["Zoltan_PHG_REPART_MULTIPLIER"];
zoltan.Set_Param("PHG_REPART_MULTIPLIER",
std::to_string(phg_repart_multiplier));
// Set call-back functions
void *mesh_data_ptr = (void *)&mesh_data;
zoltan.Set_Num_Obj_Fn(get_number_of_objects, mesh_data_ptr);
zoltan.Set_Obj_List_Fn(get_object_list, mesh_data_ptr);
void *graph_data_ptr = (void *)&local_graph;
zoltan.Set_Num_Edges_Multi_Fn(get_number_edges, graph_data_ptr);
zoltan.Set_Edge_List_Multi_Fn(get_all_edges, graph_data_ptr);
// Call Zoltan function to compute partitions
int changes = 0;
int num_gids = 0;
int num_lids = 0;
int num_import, num_export;
ZOLTAN_ID_PTR import_lids;
ZOLTAN_ID_PTR export_lids;
ZOLTAN_ID_PTR import_gids;
ZOLTAN_ID_PTR export_gids;
int* import_procs;
int* export_procs;
int* import_parts;
int* export_parts;
int rc = zoltan.LB_Partition(changes, num_gids, num_lids,
num_import, import_gids, import_lids, import_procs, import_parts,
num_export, export_gids, export_lids, export_procs, export_parts);
dolfin_assert(num_gids == 1);
dolfin_assert(num_lids == 0);
std::size_t proc = MPI::rank(mpi_comm);
if (rc != ZOLTAN_OK)
{
dolfin_error("ZoltanPartition.cpp",
"partition mesh using Zoltan",
"Call to Zoltan failed");
}
cell_partition.assign(local_graph.size(), proc);
std::size_t offset = MPI::global_offset(mpi_comm, local_graph.size(), true);
for(int i = 0; i < num_export; ++i)
{
const std::size_t idx = export_gids[i] - offset;
cell_partition[idx] = (std::size_t)export_procs[i];
}
// Free data structures allocated by Zoltan::LB_Partition
zoltan.LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_parts);
zoltan.LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_parts);
}