//-----------------------------------------------------------------------------
std::vector<std::size_t> GraphOrdering::compute_local_reordering_map()
{
// Initialise Zoltan
float version;
int argc = 0;
char** argv = NULL;
Zoltan_Initialize(argc, argv, &version);
// Create Zoltan object
Zoltan zoltan;
// Set parameters
//zoltan.Set_Param( "ORDER_METHOD", "METIS");
zoltan.Set_Param( "ORDER_METHOD", "SCOTCH");
zoltan.Set_Param( "NUM_GID_ENTRIES", "1"); // global ID is 1 integer
zoltan.Set_Param( "NUM_LID_ENTRIES", "1"); // local ID is 1 integer
zoltan.Set_Param( "OBJ_WEIGHT_DIM", "0"); // omit object weights
// Set call-back functions
zoltan.Set_Num_Obj_Fn(GraphOrdering::get_number_of_objects, this);
zoltan.Set_Obj_List_Fn(GraphOrdering::get_object_list, this);
zoltan.Set_Num_Edges_Multi_Fn(GraphOrdering::get_number_edges, this);
zoltan.Set_Edge_List_Multi_Fn(GraphOrdering::get_all_edges, this);
// Create array for global ids that should be re-ordered
std::vector<ZOLTAN_ID_TYPE> global_ids(num_global_objects());
for (std::size_t i = 0; i < global_ids.size(); ++i)
global_ids[i] = i;
// Create array for re-ordered vertices
std::vector<ZOLTAN_ID_TYPE> new_id(num_global_objects());
// Compute re-ordering
int rc = zoltan.Order(1, num_global_objects(), &global_ids[0], &new_id[0]);
// Check for errors
if (rc != ZOLTAN_OK)
{
dolfin_error("GraphOrdering.cpp",
"compute matrix re-ordering",
"Zoltan partitioning failed");
}
// Copy re-ordering into a vector (in case Zoltan uses something other than std::size_t)
std::vector<std::size_t> map(new_id.begin(), new_id.end());
return map;
}
//-----------------------------------------------------------------------------
void
ZoltanPartition::compute_partition_phg(const MPI_Comm mpi_comm,
std::vector<std::size_t>& cell_partition,
const LocalMeshData& mesh_data)
{
Timer timer0("Partition graph (calling Zoltan PHG)");
// Create data structures to hold graph
std::vector<std::set<std::size_t>> local_graph;
std::set<std::size_t> ghost_vertices;
// Compute local dual graph
GraphBuilder::compute_dual_graph(mpi_comm, mesh_data, local_graph,
ghost_vertices);
// Initialise Zoltan
float version;
int argc = 0;
char** argv = NULL;
Zoltan_Initialize(argc, argv, &version);
// Create Zoltan object
Zoltan zoltan;
// Set Zoltan parameters
zoltan.Set_Param("NUM_GID_ENTRIES", "1");
zoltan.Set_Param("NUM_LID_ENTRIES", "0");
zoltan.Set_Param("NUM_GLOBAL_PARTS",
std::to_string(MPI::size(mpi_comm)));
zoltan.Set_Param("NUM_LOCAL_PARTS", "1");
zoltan.Set_Param("LB_METHOD", "GRAPH");
// Get partition method: 'PARTITION', 'REPARTITION' or 'REFINE'
std::string lb_approach = parameters["partitioning_approach"];
zoltan.Set_Param("LB_APPROACH", lb_approach.c_str());
// Repartitioning weighting
double phg_repart_multiplier = parameters["Zoltan_PHG_REPART_MULTIPLIER"];
zoltan.Set_Param("PHG_REPART_MULTIPLIER",
std::to_string(phg_repart_multiplier));
// Set call-back functions
void *mesh_data_ptr = (void *)&mesh_data;
zoltan.Set_Num_Obj_Fn(get_number_of_objects, mesh_data_ptr);
zoltan.Set_Obj_List_Fn(get_object_list, mesh_data_ptr);
void *graph_data_ptr = (void *)&local_graph;
zoltan.Set_Num_Edges_Multi_Fn(get_number_edges, graph_data_ptr);
zoltan.Set_Edge_List_Multi_Fn(get_all_edges, graph_data_ptr);
// Call Zoltan function to compute partitions
int changes = 0;
int num_gids = 0;
int num_lids = 0;
int num_import, num_export;
ZOLTAN_ID_PTR import_lids;
ZOLTAN_ID_PTR export_lids;
ZOLTAN_ID_PTR import_gids;
ZOLTAN_ID_PTR export_gids;
int* import_procs;
int* export_procs;
int* import_parts;
int* export_parts;
int rc = zoltan.LB_Partition(changes, num_gids, num_lids,
num_import, import_gids, import_lids, import_procs, import_parts,
num_export, export_gids, export_lids, export_procs, export_parts);
dolfin_assert(num_gids == 1);
dolfin_assert(num_lids == 0);
std::size_t proc = MPI::rank(mpi_comm);
if (rc != ZOLTAN_OK)
{
dolfin_error("ZoltanPartition.cpp",
"partition mesh using Zoltan",
"Call to Zoltan failed");
}
cell_partition.assign(local_graph.size(), proc);
std::size_t offset = MPI::global_offset(mpi_comm, local_graph.size(), true);
for(int i = 0; i < num_export; ++i)
{
const std::size_t idx = export_gids[i] - offset;
cell_partition[idx] = (std::size_t)export_procs[i];
}
// Free data structures allocated by Zoltan::LB_Partition
zoltan.LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_parts);
zoltan.LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_parts);
}
//-----------------------------------------------------------------------------
void
ZoltanPartition::compute_partition_rcb(const MPI_Comm mpi_comm,
std::vector<std::size_t>& cell_partition,
const LocalMeshData& mesh_data)
{
Timer timer0("Partition graph (calling Zoltan RCB)");
// Get number of local graph vertices
const std::size_t nlocal = mesh_data.cell_vertices.shape()[0];
// Initialise Zoltan
float version;
int argc = 0;
char** argv = NULL;
Zoltan_Initialize(argc, argv, &version);
// Create Zoltan object
Zoltan zoltan;
// Set Zoltan parameters
zoltan.Set_Param("NUM_GID_ENTRIES", "1");
zoltan.Set_Param("NUM_LID_ENTRIES", "0");
zoltan.Set_Param("NUM_GLOBAL_PARTS",
std::to_string(MPI::size(mpi_comm)));
zoltan.Set_Param("NUM_LOCAL_PARTS", "1");
zoltan.Set_Param("LB_METHOD", "RCB");
// Set call-back functions
void *mesh_data_ptr = (void *)&mesh_data;
zoltan.Set_Num_Obj_Fn(get_number_of_objects, mesh_data_ptr);
zoltan.Set_Obj_List_Fn(get_object_list, mesh_data_ptr);
zoltan.Set_Num_Geom_Fn(get_geom, mesh_data_ptr);
zoltan.Set_Geom_Multi_Fn(get_all_geom, mesh_data_ptr);
// Call Zoltan function to compute partitions
int changes = 0;
int num_gids = 0;
int num_lids = 0;
int num_import, num_export;
ZOLTAN_ID_PTR import_lids;
ZOLTAN_ID_PTR export_lids;
ZOLTAN_ID_PTR import_gids;
ZOLTAN_ID_PTR export_gids;
int* import_procs;
int* export_procs;
int* import_parts;
int* export_parts;
int rc = zoltan.LB_Partition(changes, num_gids, num_lids,
num_import, import_gids, import_lids, import_procs, import_parts,
num_export, export_gids, export_lids, export_procs, export_parts);
dolfin_assert(num_gids == 1);
dolfin_assert(num_lids == 0);
// Get my process rank
const std::size_t my_rank = MPI::rank(mpi_comm);
if (rc != ZOLTAN_OK)
{
dolfin_error("ZoltanPartition.cpp",
"partition mesh using Zoltan",
"Call to Zoltan failed");
}
// Assign all nodes to this processor
cell_partition.assign(nlocal, my_rank);
std::size_t offset = MPI::global_offset(mpi_comm, nlocal, true);
// Change nodes to be exported to the appropriate remote processor
for(int i = 0; i < num_export; ++i)
{
const std::size_t idx = export_gids[i] - offset;
cell_partition[idx] = export_procs[i];
}
// Free data structures allocated by Zoltan::LB_Partition
zoltan.LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_parts);
zoltan.LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_parts);
}
int main(int argc, char *argv[])
{
/* Local declarations. */
const char *cmd_file;
char cmesg[256]; /* for error messages */
int error, gerror;
int print_output = 1;
/***************************** BEGIN EXECUTION ******************************/
/* Initialize MPI */
// We must use the C bindings to MPI because the C++ bindings are
// are not available or not complete on some of our platforms.
MPI_Init(&argc, &argv);
/* get some machine information */
int Proc = 0, Num_Proc = 0;
MPI_Comm_rank(MPI_COMM_WORLD, &Proc);
MPI_Comm_size(MPI_COMM_WORLD, &Num_Proc);
/* Initialize flags */
Test.DDirectory = 0;
Test.Local_Parts = 0;
Test.Drops = 0;
Test.RCB_Box = 0;
Test.Multi_Callbacks = 0;
Test.Gen_Files = 0;
Test.Fixed_Objects = 0;
Test.Null_Lists = NONE;
Output.Text = 1;
Output.Gnuplot = 0;
Output.Nemesis = 0;
Output.Plot_Partition = 0;
Output.Mesh_Info_File = 0;
/* Interpret the command line */
switch(argc)
{
case 1:
cmd_file = "zdrive.inp";
break;
case 2:
cmd_file = argv[1];
break;
default:
cerr << "MAIN: ERROR in command line " ;
if (Proc == 0)
{
cerr << "usage:" << endl;
cerr << "\t" << DRIVER_NAME << " [command file]";
}
cerr << endl;
exit(1);
break;
}
/* Initialize Zoltan
* Not part of C++ interface at this time. (C++ wishlist)
*/
float version;
if ((error = Zoltan_Initialize(argc, argv, &version)) != ZOLTAN_OK) {
sprintf(cmesg, "fatal: Zoltan_Initialize returned error code, %d", error);
Gen_Error(0, cmesg);
error_report(Proc);
MPI_Finalize();
return 1;
}
/*
* Create a Zoltan structure.
* No exception handling at this time. (C++ wishlist)
* We must dynamically create the object so that we can delete it
* before MPI_Finalize(). (If zz is created on the stack, it will
* be deleted atexit, after MPI_Finalize(). Zoltan_Destroy calls
* MPI functions.)
*/
Zoltan *zz = NULL;
zz = new Zoltan();
/* initialize some variables */
MESH_INFO mesh;
initialize_mesh(&mesh);
PARIO_INFO pio_info;
pio_info.dsk_list_cnt = -1;
pio_info.num_dsk_ctrlrs = -1;
pio_info.pdsk_add_fact = -1;
pio_info.zeros = -1;
pio_info.file_type = -1;
pio_info.init_dist_type = -1;
pio_info.init_size = ZOLTAN_ID_INVALID;
pio_info.init_dim = -1;
pio_info.init_vwgt_dim = -1;
pio_info.init_dist_pins = -1;
pio_info.pdsk_root[0] = '\0';
pio_info.pdsk_subdir[0] = '\0';
pio_info.pexo_fname[0] = '\0';
pio_info.file_comp = STANDARD;
PROB_INFO prob;
prob.method[0] = '\0';
prob.num_params = 0;
prob.params = NULL;
/* Read in the ascii input file */
error = 0;
if (Proc == 0) {
cout << "\n\nReading the command file, " << cmd_file << endl;
if (!read_cmd_file(cmd_file, &prob, &pio_info, NULL)) {
sprintf(cmesg,"fatal: Could not read in the command file"
" \"%s\"!\n", cmd_file);
Gen_Error(0, cmesg);
error_report(Proc);
print_output = 0;
error = 1;
}
if (!check_inp(&prob, &pio_info)) {
Gen_Error(0, "fatal: Error in user specified parameters.\n");
error_report(Proc);
print_output = 0;
error = 1;
}
print_input_info(cout, Num_Proc, &prob, &pio_info, version);
}
MPI_Allreduce(&error, &gerror, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
if (gerror) goto End;
/* broadcast the command info to all of the processor */
brdcst_cmd_info(Proc, &prob, &pio_info, &mesh);
zz->Set_Param("DEBUG_MEMORY", "1");
print_output = Output.Text;
if (!setup_zoltan(*zz, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from setup_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
srand(Proc);
/* Loop over read and balance for a number of iterations */
/* (Useful for testing REUSE parameters in Zoltan.) */
for (int iteration = 1; iteration <= Number_Iterations; iteration++) {
/*
* now read in the mesh and element information.
* This is the only function call to do this. Upon return,
* the mesh struct and the elements array should be filled.
*/
if (iteration == 1) {
if (!read_mesh(Proc, Num_Proc, &prob, &pio_info, &mesh)) {
Gen_Error(0, "fatal: Error returned from read_mesh\n");
error_report(Proc);
print_output = 0;
goto End;
}
/*
* Create a Zoltan DD for tracking elements during repartitioning.
*/
if (mesh.data_type == HYPERGRAPH && !build_elem_dd(&mesh)) {
Gen_Error(0, "fatal: Error returned from build_elem_dd\n");
error_report(Proc);
print_output = 0;
goto End;
}
}
#ifdef KDDKDD_COOL_TEST
/* KDD Cool test of changing number of partitions */
sprintf(cmesg, "%d", Num_Proc * iteration);
zz->Set_Param("NUM_GLOBAL_PARTS", cmesg);
#endif
/*
* Produce files to verify input.
*/
if (iteration == 1)
if (Debug_Driver > 2) {
if (!output_results(cmd_file,"in",Proc,Num_Proc,&prob,&pio_info,&mesh)){
Gen_Error(0, "fatal: Error returned from output_results\n");
error_report(Proc);
}
if (Output.Gnuplot)
if (!output_gnu(cmd_file,"in",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "warning: Error returned from output_gnu\n");
error_report(Proc);
}
}
/*
* now run Zoltan to get a new load balance and perform
* the migration
*/
if (!run_zoltan(*zz, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from run_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
/* Reset the mesh data structure for next iteration. */
if (iteration < Number_Iterations) {
float twiddle = 0.01;
char str[4];
/* Perturb coordinates of mesh */
if (mesh.data_type == ZOLTAN_GRAPH)
for (int i = 0; i < mesh.num_elems; i++) {
for (int j = 0; j < mesh.num_dims; j++) {
/* tmp = ((float) rand())/RAND_MAX; *//* Equiv. to sjplimp's test */
float tmp = (float) (i % 10) / 10.;
mesh.elements[i].coord[0][j] += twiddle * (2.0*tmp-1.0);
mesh.elements[i].avg_coord[j] = mesh.elements[i].coord[0][j];
}
}
/* change the ParMETIS Seed */
sprintf(str, "%d", iteration%10000);
#ifdef ZOLTAN_PARMETIS
zz->Set_Param("PARMETIS_SEED", str);
#endif
}
} /* End of loop over read and balance */
if (Proc == 0) {
cout << "FILE " << cmd_file << ": Total: " ;
cout << Total_Partition_Time << " seconds in Partitioning" << endl;
cout << "FILE " << cmd_file << ": Average: ";
cout << Total_Partition_Time/Number_Iterations << " seconds per Iteration";
cout << endl;
}
End:
if (mesh.data_type == HYPERGRAPH)
{
destroy_elem_dd();
}
delete zz;
Zoltan_Memory_Stats();
/*
* output the results
*/
if (print_output) {
if (!output_results(cmd_file,"out",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "fatal: Error returned from output_results\n");
error_report(Proc);
}
if (Output.Gnuplot) {
if (!output_gnu(cmd_file,"out",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "warning: Error returned from output_gnu\n");
error_report(Proc);
}
}
}
free_mesh_arrays(&mesh);
if (prob.params != NULL) free(prob.params);
MPI_Finalize();
return 0;
}
int run(
const RCP<const Comm<int> > &comm,
int numGlobalParts,
int testCnt,
std::string *thisTest
)
{
#ifdef HAVE_ZOLTAN2_MPI
// Zoltan needs an MPI comm
const Teuchos::MpiComm<int> *tmpicomm =
dynamic_cast<const Teuchos::MpiComm<int> *>(comm.getRawPtr());
MPI_Comm mpiComm = *(tmpicomm->getRawMpiComm());
#endif
int me = comm->getRank();
int np = comm->getSize();
double tolerance = 1.05;
//////////////////////////////////////////////
// Read test data from Zoltan's test directory
//////////////////////////////////////////////
UserInputForTests *uinput;
try{
uinput = new UserInputForTests(zoltanTestDirectory,
thisTest[TESTNAMEOFFSET],
comm, true);
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: UserInputForTests "
<< e.what() << endl;
return 1;
}
RCP<tMatrix_t> matrix;
try{
matrix = uinput->getUITpetraCrsMatrix();
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: get matrix "
<< e.what() << endl;
return 1;
}
RCP<const tMatrix_t> matrixConst = rcp_const_cast<const tMatrix_t>(matrix);
RCP<tMVector_t> coords;
try{
coords = uinput->getUICoordinates();
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: get coordinates "
<< e.what() << endl;
return 1;
}
RCP<tMVector_t> weights;
try{
weights = uinput->getUIWeights();
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: get weights "
<< e.what() << endl;
return 1;
}
int nWeights = atoi(thisTest[TESTOBJWGTOFFSET].c_str());
if (me == 0) {
cout << "Test " << testCnt << " filename = "
<< thisTest[TESTNAMEOFFSET] << endl;
cout << "Test " << testCnt << " num processors = "
<< np << endl;
cout << "Test " << testCnt << " zoltan method = "
<< thisTest[TESTMETHODOFFSET] << endl;
cout << "Test " << testCnt << " num_global_parts = "
<< numGlobalParts << endl;
cout << "Test " << testCnt << " imbalance_tolerance = "
<< tolerance << endl;
cout << "Test " << testCnt << " num weights per ID = "
<< nWeights << endl;
}
/////////////////////////////////////////
// PARTITION USING ZOLTAN DIRECTLY
/////////////////////////////////////////
if (me == 0) cout << "Calling Zoltan directly" << endl;
# ifdef HAVE_ZOLTAN2_MPI
Zoltan zz(mpiComm);
# else
Zoltan zz;
# endif
char tmp[56];
zz.Set_Param("LB_METHOD", thisTest[TESTMETHODOFFSET]);
sprintf(tmp, "%d", numGlobalParts);
zz.Set_Param("NUM_GLOBAL_PARTS", tmp);
sprintf(tmp, "%d", nWeights);
zz.Set_Param("OBJ_WEIGHT_DIM", tmp);
sprintf(tmp, "%f", tolerance);
zz.Set_Param("IMBALANCE_TOL", tmp);
zz.Set_Param("RETURN_LISTS", "PART");
zz.Set_Param("FINAL_OUTPUT", "1");
zz.Set_Num_Obj_Fn(znumobj, (void *) coords.getRawPtr());
if (nWeights)
zz.Set_Obj_List_Fn(zobjlist, (void *) weights.getRawPtr());
else
zz.Set_Obj_List_Fn(zobjlist, (void *) coords.getRawPtr());
zz.Set_Num_Geom_Fn(znumgeom, (void *) coords.getRawPtr());
zz.Set_Geom_Multi_Fn(zgeom, (void *) coords.getRawPtr());
int changes, ngid, nlid;
int numd, nump;
ZOLTAN_ID_PTR dgid = NULL, dlid = NULL, pgid = NULL, plid = NULL;
int *dproc = NULL, *dpart = NULL, *pproc = NULL, *ppart = NULL;
int ierr = zz.LB_Partition(changes, ngid, nlid,
numd, dgid, dlid, dproc, dpart,
nump, pgid, plid, pproc, ppart);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
if (me == 0)
cout << "Test " << testCnt << ": FAIL: direct Zoltan call" << endl;
zz.LB_Free_Part(&pgid, &plid, &pproc, &ppart);
return 1;
}
/////////////////////////////////////////
// PARTITION USING ZOLTAN THROUGH ZOLTAN2
/////////////////////////////////////////
if (me == 0) cout << "Calling Zoltan through Zoltan2" << endl;
matrixAdapter_t *ia;
try{
ia = new matrixAdapter_t(matrixConst, nWeights);
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: matrix adapter "
<< e.what() << endl;
return 1;
}
for (int idx=0; idx < nWeights; idx++)
ia->setRowWeights(weights->getData(idx).getRawPtr(), 1, idx);
vectorAdapter_t *ca = NULL;
try{
ca = new vectorAdapter_t(coords);
}
catch(std::exception &e){
if (me == 0)
cout << "Test " << testCnt << ": FAIL: vector adapter "
<< e.what() << endl;
return 1;
}
ia->setCoordinateInput(ca);
Teuchos::ParameterList params;
params.set("timer_output_stream" , "std::cout");
params.set("compute_metrics", "true");
// params.set("debug_level" , "verbose_detailed_status");
params.set("algorithm", "zoltan");
params.set("imbalance_tolerance", tolerance );
params.set("num_global_parts", numGlobalParts);
if (thisTest[TESTMETHODOFFSET] != "default") {
// "default" tests case of no Zoltan parameter sublist
Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
zparams.set("LB_METHOD",thisTest[TESTMETHODOFFSET]);
}
Zoltan2::PartitioningProblem<matrixAdapter_t> *problem;
# ifdef HAVE_ZOLTAN2_MPI
try{
problem = new Zoltan2::PartitioningProblem<matrixAdapter_t>(ia, ¶ms,
mpiComm);
}
# else
try{
problem = new Zoltan2::PartitioningProblem<matrixAdapter_t>(ia, ¶ms);
}
# endif
catch(std::exception &e){
cout << "Test " << testCnt << " FAIL: problem " << e.what() << endl;
return 1;
}
try {
problem->solve();
}
catch(std::exception &e){
cout << "Test " << testCnt << " FAIL: solve " << e.what() << endl;
return 1;
}
if (me == 0){
problem->printMetrics(cout);
}
problem->printTimers();
/////////////////////////////////////////
// COMPARE RESULTS
/////////////////////////////////////////
size_t nObj = coords->getLocalLength();
const int *z2parts = problem->getSolution().getPartListView();
int diffcnt = 0, gdiffcnt = 0;
for (size_t i = 0; i < nObj; i++) {
if (z2parts[plid[i]] != ppart[i]) {
diffcnt++;
cout << me << " DIFF for " << i << " ("
<< coords->getMap()->getGlobalElement(i) << "): "
<< "Z2 = " << z2parts[i] << "; Z1 = " << ppart[plid[i]] << endl;
}
}
/////////////////////////////////////////
// CLEAN UP
/////////////////////////////////////////
zz.LB_Free_Part(&pgid, &plid, &pproc, &ppart);
delete ia;
delete ca;
delete problem;
delete uinput;
Teuchos::reduceAll(*comm, Teuchos::REDUCE_SUM, 1, &diffcnt, &gdiffcnt);
if (gdiffcnt > 0) {
if (me == 0)
cout << "Test " << testCnt << " "
<< thisTest[TESTNAMEOFFSET] << " "
<< thisTest[TESTMETHODOFFSET] << " "
<< thisTest[TESTOBJWGTOFFSET] << " "
<< " FAIL: comparison " << endl;
return 1;
}
return 0;
}