//-----------------------------------------------------------------------------
std::vector<std::size_t> GraphOrdering::compute_local_reordering_map()
{
// Initialise Zoltan
float version;
int argc = 0;
char** argv = NULL;
Zoltan_Initialize(argc, argv, &version);
// Create Zoltan object
Zoltan zoltan;
// Set parameters
//zoltan.Set_Param( "ORDER_METHOD", "METIS");
zoltan.Set_Param( "ORDER_METHOD", "SCOTCH");
zoltan.Set_Param( "NUM_GID_ENTRIES", "1"); // global ID is 1 integer
zoltan.Set_Param( "NUM_LID_ENTRIES", "1"); // local ID is 1 integer
zoltan.Set_Param( "OBJ_WEIGHT_DIM", "0"); // omit object weights
// Set call-back functions
zoltan.Set_Num_Obj_Fn(GraphOrdering::get_number_of_objects, this);
zoltan.Set_Obj_List_Fn(GraphOrdering::get_object_list, this);
zoltan.Set_Num_Edges_Multi_Fn(GraphOrdering::get_number_edges, this);
zoltan.Set_Edge_List_Multi_Fn(GraphOrdering::get_all_edges, this);
// Create array for global ids that should be re-ordered
std::vector<ZOLTAN_ID_TYPE> global_ids(num_global_objects());
for (std::size_t i = 0; i < global_ids.size(); ++i)
global_ids[i] = i;
// Create array for re-ordered vertices
std::vector<ZOLTAN_ID_TYPE> new_id(num_global_objects());
// Compute re-ordering
int rc = zoltan.Order(1, num_global_objects(), &global_ids[0], &new_id[0]);
// Check for errors
if (rc != ZOLTAN_OK)
{
dolfin_error("GraphOrdering.cpp",
"compute matrix re-ordering",
"Zoltan partitioning failed");
}
// Copy re-ordering into a vector (in case Zoltan uses something other than std::size_t)
std::vector<std::size_t> map(new_id.begin(), new_id.end());
return map;
}
//-----------------------------------------------------------------------------
void
ZoltanPartition::compute_partition_phg(const MPI_Comm mpi_comm,
std::vector<std::size_t>& cell_partition,
const LocalMeshData& mesh_data)
{
Timer timer0("Partition graph (calling Zoltan PHG)");
// Create data structures to hold graph
std::vector<std::set<std::size_t>> local_graph;
std::set<std::size_t> ghost_vertices;
// Compute local dual graph
GraphBuilder::compute_dual_graph(mpi_comm, mesh_data, local_graph,
ghost_vertices);
// Initialise Zoltan
float version;
int argc = 0;
char** argv = NULL;
Zoltan_Initialize(argc, argv, &version);
// Create Zoltan object
Zoltan zoltan;
// Set Zoltan parameters
zoltan.Set_Param("NUM_GID_ENTRIES", "1");
zoltan.Set_Param("NUM_LID_ENTRIES", "0");
zoltan.Set_Param("NUM_GLOBAL_PARTS",
std::to_string(MPI::size(mpi_comm)));
zoltan.Set_Param("NUM_LOCAL_PARTS", "1");
zoltan.Set_Param("LB_METHOD", "GRAPH");
// Get partition method: 'PARTITION', 'REPARTITION' or 'REFINE'
std::string lb_approach = parameters["partitioning_approach"];
zoltan.Set_Param("LB_APPROACH", lb_approach.c_str());
// Repartitioning weighting
double phg_repart_multiplier = parameters["Zoltan_PHG_REPART_MULTIPLIER"];
zoltan.Set_Param("PHG_REPART_MULTIPLIER",
std::to_string(phg_repart_multiplier));
// Set call-back functions
void *mesh_data_ptr = (void *)&mesh_data;
zoltan.Set_Num_Obj_Fn(get_number_of_objects, mesh_data_ptr);
zoltan.Set_Obj_List_Fn(get_object_list, mesh_data_ptr);
void *graph_data_ptr = (void *)&local_graph;
zoltan.Set_Num_Edges_Multi_Fn(get_number_edges, graph_data_ptr);
zoltan.Set_Edge_List_Multi_Fn(get_all_edges, graph_data_ptr);
// Call Zoltan function to compute partitions
int changes = 0;
int num_gids = 0;
int num_lids = 0;
int num_import, num_export;
ZOLTAN_ID_PTR import_lids;
ZOLTAN_ID_PTR export_lids;
ZOLTAN_ID_PTR import_gids;
ZOLTAN_ID_PTR export_gids;
int* import_procs;
int* export_procs;
int* import_parts;
int* export_parts;
int rc = zoltan.LB_Partition(changes, num_gids, num_lids,
num_import, import_gids, import_lids, import_procs, import_parts,
num_export, export_gids, export_lids, export_procs, export_parts);
dolfin_assert(num_gids == 1);
dolfin_assert(num_lids == 0);
std::size_t proc = MPI::rank(mpi_comm);
if (rc != ZOLTAN_OK)
{
dolfin_error("ZoltanPartition.cpp",
"partition mesh using Zoltan",
"Call to Zoltan failed");
}
cell_partition.assign(local_graph.size(), proc);
std::size_t offset = MPI::global_offset(mpi_comm, local_graph.size(), true);
for(int i = 0; i < num_export; ++i)
{
const std::size_t idx = export_gids[i] - offset;
cell_partition[idx] = (std::size_t)export_procs[i];
}
// Free data structures allocated by Zoltan::LB_Partition
zoltan.LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_parts);
zoltan.LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_parts);
}
//-----------------------------------------------------------------------------
void
ZoltanPartition::compute_partition_rcb(const MPI_Comm mpi_comm,
std::vector<std::size_t>& cell_partition,
const LocalMeshData& mesh_data)
{
Timer timer0("Partition graph (calling Zoltan RCB)");
// Get number of local graph vertices
const std::size_t nlocal = mesh_data.cell_vertices.shape()[0];
// Initialise Zoltan
float version;
int argc = 0;
char** argv = NULL;
Zoltan_Initialize(argc, argv, &version);
// Create Zoltan object
Zoltan zoltan;
// Set Zoltan parameters
zoltan.Set_Param("NUM_GID_ENTRIES", "1");
zoltan.Set_Param("NUM_LID_ENTRIES", "0");
zoltan.Set_Param("NUM_GLOBAL_PARTS",
std::to_string(MPI::size(mpi_comm)));
zoltan.Set_Param("NUM_LOCAL_PARTS", "1");
zoltan.Set_Param("LB_METHOD", "RCB");
// Set call-back functions
void *mesh_data_ptr = (void *)&mesh_data;
zoltan.Set_Num_Obj_Fn(get_number_of_objects, mesh_data_ptr);
zoltan.Set_Obj_List_Fn(get_object_list, mesh_data_ptr);
zoltan.Set_Num_Geom_Fn(get_geom, mesh_data_ptr);
zoltan.Set_Geom_Multi_Fn(get_all_geom, mesh_data_ptr);
// Call Zoltan function to compute partitions
int changes = 0;
int num_gids = 0;
int num_lids = 0;
int num_import, num_export;
ZOLTAN_ID_PTR import_lids;
ZOLTAN_ID_PTR export_lids;
ZOLTAN_ID_PTR import_gids;
ZOLTAN_ID_PTR export_gids;
int* import_procs;
int* export_procs;
int* import_parts;
int* export_parts;
int rc = zoltan.LB_Partition(changes, num_gids, num_lids,
num_import, import_gids, import_lids, import_procs, import_parts,
num_export, export_gids, export_lids, export_procs, export_parts);
dolfin_assert(num_gids == 1);
dolfin_assert(num_lids == 0);
// Get my process rank
const std::size_t my_rank = MPI::rank(mpi_comm);
if (rc != ZOLTAN_OK)
{
dolfin_error("ZoltanPartition.cpp",
"partition mesh using Zoltan",
"Call to Zoltan failed");
}
// Assign all nodes to this processor
cell_partition.assign(nlocal, my_rank);
std::size_t offset = MPI::global_offset(mpi_comm, nlocal, true);
// Change nodes to be exported to the appropriate remote processor
for(int i = 0; i < num_export; ++i)
{
const std::size_t idx = export_gids[i] - offset;
cell_partition[idx] = export_procs[i];
}
// Free data structures allocated by Zoltan::LB_Partition
zoltan.LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_parts);
zoltan.LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_parts);
}
int run(
const RCP<const Comm<int> > &comm,
int numGlobalParts,
int testCnt,
std::string *thisTest
)
{
#ifdef HAVE_ZOLTAN2_MPI
// Zoltan needs an MPI comm
const Teuchos::MpiComm<int> *tmpicomm =
dynamic_cast<const Teuchos::MpiComm<int> *>(comm.getRawPtr());
MPI_Comm mpiComm = *(tmpicomm->getRawMpiComm());
#endif
int me = comm->getRank();
int np = comm->getSize();
double tolerance = 1.05;
//////////////////////////////////////////////
// Read test data from Zoltan's test directory
//////////////////////////////////////////////
UserInputForTests *uinput;
try{
uinput = new UserInputForTests(zoltanTestDirectory,
thisTest[TESTNAMEOFFSET],
comm, true);
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: UserInputForTests "
<< e.what() << endl;
return 1;
}
RCP<tMatrix_t> matrix;
try{
matrix = uinput->getUITpetraCrsMatrix();
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: get matrix "
<< e.what() << endl;
return 1;
}
RCP<const tMatrix_t> matrixConst = rcp_const_cast<const tMatrix_t>(matrix);
RCP<tMVector_t> coords;
try{
coords = uinput->getUICoordinates();
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: get coordinates "
<< e.what() << endl;
return 1;
}
RCP<tMVector_t> weights;
try{
weights = uinput->getUIWeights();
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: get weights "
<< e.what() << endl;
return 1;
}
int nWeights = atoi(thisTest[TESTOBJWGTOFFSET].c_str());
if (me == 0) {
cout << "Test " << testCnt << " filename = "
<< thisTest[TESTNAMEOFFSET] << endl;
cout << "Test " << testCnt << " num processors = "
<< np << endl;
cout << "Test " << testCnt << " zoltan method = "
<< thisTest[TESTMETHODOFFSET] << endl;
cout << "Test " << testCnt << " num_global_parts = "
<< numGlobalParts << endl;
cout << "Test " << testCnt << " imbalance_tolerance = "
<< tolerance << endl;
cout << "Test " << testCnt << " num weights per ID = "
<< nWeights << endl;
}
/////////////////////////////////////////
// PARTITION USING ZOLTAN DIRECTLY
/////////////////////////////////////////
if (me == 0) cout << "Calling Zoltan directly" << endl;
# ifdef HAVE_ZOLTAN2_MPI
Zoltan zz(mpiComm);
# else
Zoltan zz;
# endif
char tmp[56];
zz.Set_Param("LB_METHOD", thisTest[TESTMETHODOFFSET]);
sprintf(tmp, "%d", numGlobalParts);
zz.Set_Param("NUM_GLOBAL_PARTS", tmp);
sprintf(tmp, "%d", nWeights);
zz.Set_Param("OBJ_WEIGHT_DIM", tmp);
sprintf(tmp, "%f", tolerance);
zz.Set_Param("IMBALANCE_TOL", tmp);
zz.Set_Param("RETURN_LISTS", "PART");
zz.Set_Param("FINAL_OUTPUT", "1");
zz.Set_Num_Obj_Fn(znumobj, (void *) coords.getRawPtr());
if (nWeights)
zz.Set_Obj_List_Fn(zobjlist, (void *) weights.getRawPtr());
else
zz.Set_Obj_List_Fn(zobjlist, (void *) coords.getRawPtr());
zz.Set_Num_Geom_Fn(znumgeom, (void *) coords.getRawPtr());
zz.Set_Geom_Multi_Fn(zgeom, (void *) coords.getRawPtr());
int changes, ngid, nlid;
int numd, nump;
ZOLTAN_ID_PTR dgid = NULL, dlid = NULL, pgid = NULL, plid = NULL;
int *dproc = NULL, *dpart = NULL, *pproc = NULL, *ppart = NULL;
int ierr = zz.LB_Partition(changes, ngid, nlid,
numd, dgid, dlid, dproc, dpart,
nump, pgid, plid, pproc, ppart);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
if (me == 0)
cout << "Test " << testCnt << ": FAIL: direct Zoltan call" << endl;
zz.LB_Free_Part(&pgid, &plid, &pproc, &ppart);
return 1;
}
/////////////////////////////////////////
// PARTITION USING ZOLTAN THROUGH ZOLTAN2
/////////////////////////////////////////
if (me == 0) cout << "Calling Zoltan through Zoltan2" << endl;
matrixAdapter_t *ia;
try{
ia = new matrixAdapter_t(matrixConst, nWeights);
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: matrix adapter "
<< e.what() << endl;
return 1;
}
for (int idx=0; idx < nWeights; idx++)
ia->setRowWeights(weights->getData(idx).getRawPtr(), 1, idx);
vectorAdapter_t *ca = NULL;
try{
ca = new vectorAdapter_t(coords);
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: vector adapter "
<< e.what() << endl;
return 1;
}
ia->setCoordinateInput(ca);
Teuchos::ParameterList params;
params.set("timer_output_stream" , "std::cout");
params.set("compute_metrics", "true");
// params.set("debug_level" , "verbose_detailed_status");
params.set("algorithm", "zoltan");
params.set("imbalance_tolerance", tolerance );
params.set("num_global_parts", numGlobalParts);
if (thisTest[TESTMETHODOFFSET] != "default") {
// "default" tests case of no Zoltan parameter sublist
Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
zparams.set("LB_METHOD",thisTest[TESTMETHODOFFSET]);
}
Zoltan2::PartitioningProblem<matrixAdapter_t> *problem;
# ifdef HAVE_ZOLTAN2_MPI
try{
problem = new Zoltan2::PartitioningProblem<matrixAdapter_t>(ia, ¶ms,
mpiComm);
}
# else
try{
problem = new Zoltan2::PartitioningProblem<matrixAdapter_t>(ia, ¶ms);
}
# endif
catch(std::exception &e){
cout << "Test " << testCnt << " FAIL: problem " << e.what() << endl;
return 1;
}
try {
problem->solve();
}
catch(std::exception &e){
cout << "Test " << testCnt << " FAIL: solve " << e.what() << endl;
return 1;
}
if (me == 0){
problem->printMetrics(cout);
}
problem->printTimers();
/////////////////////////////////////////
// COMPARE RESULTS
/////////////////////////////////////////
size_t nObj = coords->getLocalLength();
const int *z2parts = problem->getSolution().getPartListView();
int diffcnt = 0, gdiffcnt = 0;
for (size_t i = 0; i < nObj; i++) {
if (z2parts[plid[i]] != ppart[i]) {
diffcnt++;
cout << me << " DIFF for " << i << " ("
<< coords->getMap()->getGlobalElement(i) << "): "
<< "Z2 = " << z2parts[i] << "; Z1 = " << ppart[plid[i]] << endl;
}
}
/////////////////////////////////////////
// CLEAN UP
/////////////////////////////////////////
zz.LB_Free_Part(&pgid, &plid, &pproc, &ppart);
delete ia;
delete ca;
delete problem;
delete uinput;
Teuchos::reduceAll(*comm, Teuchos::REDUCE_SUM, 1, &diffcnt, &gdiffcnt);
if (gdiffcnt > 0) {
if (me == 0)
cout << "Test " << testCnt << " "
<< thisTest[TESTNAMEOFFSET] << " "
<< thisTest[TESTMETHODOFFSET] << " "
<< thisTest[TESTOBJWGTOFFSET] << " "
<< " FAIL: comparison " << endl;
return 1;
}
return 0;
}