int Zoltan_Set_Fn(ZZ *zz, ZOLTAN_FN_TYPE fn_type, ZOLTAN_VOID_FN *fn,
void *data)
{
/*
* Function to initialize a given LB interface function.
* Input:
* zz -- Pointer to a Zoltan structure.
* fn_type -- Enum type indicating the function to be set.
* fn -- Pointer to the function to be used in the
* assignment.
* data -- Pointer to data that the LB library will
* pass as an argument to fn(). May be NULL.
* Output:
* zz -- Appropriate field set to value in void *().
*/
char *yo = "Zoltan_Set_Fn";
char msg[256];
int ierr;
switch (fn_type) {
case ZOLTAN_PART_FN_TYPE:
ierr = Zoltan_Set_Part_Fn(zz,
(ZOLTAN_PART_FN *) fn, data);
break;
case ZOLTAN_PART_MULTI_FN_TYPE:
ierr = Zoltan_Set_Part_Multi_Fn(zz,
(ZOLTAN_PART_MULTI_FN *) fn, data);
break;
case ZOLTAN_NUM_EDGES_FN_TYPE:
ierr = Zoltan_Set_Num_Edges_Fn(zz,
(ZOLTAN_NUM_EDGES_FN *) fn, data);
break;
case ZOLTAN_NUM_EDGES_MULTI_FN_TYPE:
ierr = Zoltan_Set_Num_Edges_Multi_Fn(zz,
(ZOLTAN_NUM_EDGES_MULTI_FN *) fn, data);
break;
case ZOLTAN_EDGE_LIST_FN_TYPE:
ierr = Zoltan_Set_Edge_List_Fn(zz,
(ZOLTAN_EDGE_LIST_FN *) fn, data);
break;
case ZOLTAN_EDGE_LIST_MULTI_FN_TYPE:
ierr = Zoltan_Set_Edge_List_Multi_Fn(zz,
(ZOLTAN_EDGE_LIST_MULTI_FN *) fn, data);
break;
case ZOLTAN_NUM_GEOM_FN_TYPE:
ierr = Zoltan_Set_Num_Geom_Fn(zz,
(ZOLTAN_NUM_GEOM_FN *) fn, data);
break;
case ZOLTAN_GEOM_MULTI_FN_TYPE:
ierr = Zoltan_Set_Geom_Multi_Fn(zz,
(ZOLTAN_GEOM_MULTI_FN *) fn, data);
break;
case ZOLTAN_GEOM_FN_TYPE:
ierr = Zoltan_Set_Geom_Fn(zz,
(ZOLTAN_GEOM_FN *) fn, data);
break;
case ZOLTAN_NUM_OBJ_FN_TYPE:
ierr = Zoltan_Set_Num_Obj_Fn(zz,
(ZOLTAN_NUM_OBJ_FN *) fn, data);
break;
case ZOLTAN_OBJ_LIST_FN_TYPE:
ierr = Zoltan_Set_Obj_List_Fn(zz,
(ZOLTAN_OBJ_LIST_FN *) fn, data);
break;
case ZOLTAN_FIRST_OBJ_FN_TYPE:
ierr = Zoltan_Set_First_Obj_Fn(zz,
(ZOLTAN_FIRST_OBJ_FN *) fn, data);
break;
case ZOLTAN_NEXT_OBJ_FN_TYPE:
ierr = Zoltan_Set_Next_Obj_Fn(zz,
(ZOLTAN_NEXT_OBJ_FN *) fn, data);
break;
case ZOLTAN_NUM_BORDER_OBJ_FN_TYPE:
ierr = Zoltan_Set_Num_Border_Obj_Fn(zz,
(ZOLTAN_NUM_BORDER_OBJ_FN *) fn, data);
break;
case ZOLTAN_BORDER_OBJ_LIST_FN_TYPE:
ierr = Zoltan_Set_Border_Obj_List_Fn(zz,
(ZOLTAN_BORDER_OBJ_LIST_FN *) fn, data);
break;
case ZOLTAN_FIRST_BORDER_OBJ_FN_TYPE:
ierr = Zoltan_Set_First_Border_Obj_Fn(zz,
(ZOLTAN_FIRST_BORDER_OBJ_FN *) fn, data);
break;
case ZOLTAN_NEXT_BORDER_OBJ_FN_TYPE:
ierr = Zoltan_Set_Next_Border_Obj_Fn(zz,
(ZOLTAN_NEXT_BORDER_OBJ_FN *) fn, data);
break;
case ZOLTAN_PRE_MIGRATE_PP_FN_TYPE:
ierr = Zoltan_Set_Pre_Migrate_PP_Fn(zz,
(ZOLTAN_PRE_MIGRATE_PP_FN *) fn, data);
break;
case ZOLTAN_MID_MIGRATE_PP_FN_TYPE:
ierr = Zoltan_Set_Mid_Migrate_PP_Fn(zz,
(ZOLTAN_MID_MIGRATE_PP_FN *) fn, data);
break;
case ZOLTAN_POST_MIGRATE_PP_FN_TYPE:
ierr = Zoltan_Set_Post_Migrate_PP_Fn(zz,
(ZOLTAN_POST_MIGRATE_PP_FN *) fn, data);
break;
case ZOLTAN_PRE_MIGRATE_FN_TYPE:
ierr = Zoltan_Set_Pre_Migrate_Fn(zz,
(ZOLTAN_PRE_MIGRATE_FN *) fn, data);
break;
case ZOLTAN_MID_MIGRATE_FN_TYPE:
ierr = Zoltan_Set_Mid_Migrate_Fn(zz,
(ZOLTAN_MID_MIGRATE_FN *) fn, data);
break;
case ZOLTAN_POST_MIGRATE_FN_TYPE:
ierr = Zoltan_Set_Post_Migrate_Fn(zz,
(ZOLTAN_POST_MIGRATE_FN *) fn, data);
break;
case ZOLTAN_OBJ_SIZE_FN_TYPE:
ierr = Zoltan_Set_Obj_Size_Fn(zz,
(ZOLTAN_OBJ_SIZE_FN *) fn, data);
break;
case ZOLTAN_OBJ_SIZE_MULTI_FN_TYPE:
ierr = Zoltan_Set_Obj_Size_Multi_Fn(zz,
(ZOLTAN_OBJ_SIZE_MULTI_FN *) fn, data);
break;
case ZOLTAN_PACK_OBJ_FN_TYPE:
ierr = Zoltan_Set_Pack_Obj_Fn(zz,
(ZOLTAN_PACK_OBJ_FN *) fn, data);
break;
case ZOLTAN_PACK_OBJ_MULTI_FN_TYPE:
ierr = Zoltan_Set_Pack_Obj_Multi_Fn(zz,
(ZOLTAN_PACK_OBJ_MULTI_FN *) fn, data);
break;
case ZOLTAN_UNPACK_OBJ_FN_TYPE:
ierr = Zoltan_Set_Unpack_Obj_Fn(zz,
(ZOLTAN_UNPACK_OBJ_FN *) fn, data);
break;
case ZOLTAN_UNPACK_OBJ_MULTI_FN_TYPE:
ierr = Zoltan_Set_Unpack_Obj_Multi_Fn(zz,
(ZOLTAN_UNPACK_OBJ_MULTI_FN *) fn, data);
break;
case ZOLTAN_NUM_COARSE_OBJ_FN_TYPE:
ierr = Zoltan_Set_Num_Coarse_Obj_Fn(zz,
(ZOLTAN_NUM_COARSE_OBJ_FN *) fn, data);
break;
case ZOLTAN_COARSE_OBJ_LIST_FN_TYPE:
ierr = Zoltan_Set_Coarse_Obj_List_Fn(zz,
(ZOLTAN_COARSE_OBJ_LIST_FN *) fn, data);
break;
case ZOLTAN_FIRST_COARSE_OBJ_FN_TYPE:
ierr = Zoltan_Set_First_Coarse_Obj_Fn(zz,
(ZOLTAN_FIRST_COARSE_OBJ_FN *) fn, data);
break;
case ZOLTAN_NEXT_COARSE_OBJ_FN_TYPE:
ierr = Zoltan_Set_Next_Coarse_Obj_Fn(zz,
(ZOLTAN_NEXT_COARSE_OBJ_FN *) fn, data);
break;
case ZOLTAN_NUM_CHILD_FN_TYPE:
ierr = Zoltan_Set_Num_Child_Fn(zz,
(ZOLTAN_NUM_CHILD_FN *) fn, data);
break;
case ZOLTAN_CHILD_LIST_FN_TYPE:
ierr = Zoltan_Set_Child_List_Fn(zz,
(ZOLTAN_CHILD_LIST_FN *) fn, data);
break;
case ZOLTAN_CHILD_WEIGHT_FN_TYPE:
ierr = Zoltan_Set_Child_Weight_Fn(zz,
(ZOLTAN_CHILD_WEIGHT_FN *) fn, data);
break;
case ZOLTAN_HG_SIZE_CS_FN_TYPE:
ierr = Zoltan_Set_HG_Size_CS_Fn(zz,
(ZOLTAN_HG_SIZE_CS_FN *) fn, data);
break;
case ZOLTAN_HG_CS_FN_TYPE:
ierr = Zoltan_Set_HG_CS_Fn(zz,
(ZOLTAN_HG_CS_FN *) fn, data);
break;
case ZOLTAN_HG_SIZE_EDGE_WTS_FN_TYPE:
ierr = Zoltan_Set_HG_Size_Edge_Wts_Fn(zz,
(ZOLTAN_HG_SIZE_EDGE_WTS_FN *) fn, data);
break;
case ZOLTAN_HG_EDGE_WTS_FN_TYPE:
ierr = Zoltan_Set_HG_Edge_Wts_Fn(zz,
(ZOLTAN_HG_EDGE_WTS_FN *) fn, data);
break;
case ZOLTAN_NUM_FIXED_OBJ_FN_TYPE:
ierr = Zoltan_Set_Num_Fixed_Obj_Fn(zz,
(ZOLTAN_NUM_FIXED_OBJ_FN *) fn, data);
break;
case ZOLTAN_FIXED_OBJ_LIST_FN_TYPE:
ierr = Zoltan_Set_Fixed_Obj_List_Fn(zz,
(ZOLTAN_FIXED_OBJ_LIST_FN *) fn, data);
break;
case ZOLTAN_HIER_NUM_LEVELS_FN_TYPE:
ierr = Zoltan_Set_Hier_Num_Levels_Fn(zz,
(ZOLTAN_HIER_NUM_LEVELS_FN *) fn, data);
break;
case ZOLTAN_HIER_PART_FN_TYPE:
ierr = Zoltan_Set_Hier_Part_Fn(zz,
(ZOLTAN_HIER_PART_FN *) fn, data);
break;
case ZOLTAN_HIER_METHOD_FN_TYPE:
ierr = Zoltan_Set_Hier_Method_Fn(zz,
(ZOLTAN_HIER_METHOD_FN *) fn, data);
break;
default:
sprintf(msg, "ZOLTAN_FN_TYPE %d is invalid.\n"
"Value must be in range 0 to %d.", fn_type, ZOLTAN_MAX_FN_TYPES);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ierr = ZOLTAN_WARN;
}
return (ierr);
}
void lb_zoltan(PSTopology top, LBMethod method, unsigned int dimen, list_type& pl)
{
const par::communicator& comm = par::comm_world();
float ver;
Zoltan_Initialize(0, 0, &ver);
struct Zoltan_Struct *zz;
// Create ZData (moves pl into zd)
ZData zd(par::comm_world(), std::move(pl));
// Allocate the Zoltan data
zz = Zoltan_Create(zd.comm.raw());
// Set some default sane parameters
if (method == LBMethod::RCB)
Zoltan_Set_Param(zz, "LB_METHOD", "RCB");
else
throw std::runtime_error("Unknown load balancing method");
// Zoltan_Set_Param(zz, "KEEP_CUTS", "1");
// Zoltan_Set_Param(zz, "LB_APPROACH", "REPARTITION");
// Zoltan_Set_Param(zz, "MIGRATE_ONLY_PROC_CHANGES", "1");
Zoltan_Set_Param(zz, "AUTO_MIGRATE", "TRUE");
// Set higher for more debugging output
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
// Set partition query methods
Zoltan_Set_Num_Obj_Fn(zz, pl_num_obj,
static_cast<void *>(&zd));
Zoltan_Set_Obj_List_Fn(zz, pl_obj_list,
static_cast<void *>(&zd));
Zoltan_Set_Num_Geom_Fn(zz, pl_num_geom,
static_cast<void *>(&zd));
Zoltan_Set_Geom_Multi_Fn(zz, pl_geom_multi,
static_cast<void *>(&zd));
// Migration query methods
Zoltan_Set_Mid_Migrate_PP_Fn(zz, pl_mid_migrate_pp,
static_cast<void *>(&zd));
Zoltan_Set_Obj_Size_Multi_Fn(zz, pl_obj_size_multi,
static_cast<void *>(&zd));
Zoltan_Set_Pack_Obj_Multi_Fn(zz, pl_pack_obj_multi,
static_cast<void *>(&zd));
Zoltan_Set_Unpack_Obj_Multi_Fn(zz, pl_unpack_obj_multi,
static_cast<void *>(&zd));
int
zerr,
changes,
num_gid_entries, num_lid_entries,
num_import,
*import_procs,
*import_to_part,
num_export,
*export_procs,
*export_to_part;
unsigned int
*import_global_ids,
*import_local_ids,
*export_global_ids,
*export_local_ids;
zerr = Zoltan_LB_Partition(zz,
&changes,
&num_gid_entries,
&num_lid_entries,
&num_import,
&import_global_ids,
&import_local_ids,
&import_procs,
&import_to_part,
&num_export,
&export_global_ids,
&export_local_ids,
&export_procs,
&export_to_part);
if (zerr != ZOLTAN_OK) comm.abort("Zoltan error", 1);
// Move the data back again out of the struct
pl = std::move(zd.list);
Zoltan_Destroy(&zz);
}
int main(int argc, char *argv[])
{
int i, rc;
int myRank, numProcs;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport, start_gid, num_nbors;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int *parts=NULL;
ZOLTAN_ID_PTR lids=NULL;
FILE *fp;
struct Zoltan_DD_Struct *dd;
GRAPH_DATA myGraph;
int gid_length = 1; /* our global IDs consist of 1 integer */
int lid_length = 1; /* our local IDs consist of 1 integer */
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
exit(0);
}
/******************************************************************
** Read graph from input file and distribute it
******************************************************************/
fp = fopen(fname, "r");
if (!fp){
if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
MPI_Finalize();
exit(1);
}
fclose(fp);
read_input_file(myRank, numProcs, fname, &myGraph);
/*fprintf(stderr,"%d have %d objects\n",myRank,myGraph.numMyVertices);*/
/******************************************************************
** Create a distributed data directory which maps vertex
** global IDs to their current partition number. We'll use this
** after migrating vertices, to update the partition in which
** our vertices neighbors are.
**
** Our local IDs (array "lids") are of type ZOLTAN_ID_TYPE because
** we are using Zoltan's distributed data directory. It assumes
** that a global ID is a sequence of "gid_length" ZOLTAN_ID_TYPEs.
** It assumes that a local ID is a sequence of "lid_length"
** ZOLTAN_ID_TYPEs.
******************************************************************/
rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD,
gid_length, /* length of a global ID */
lid_length, /* length of a local ID */
0, /* length of user data */
myGraph.numMyVertices, /* hash table size */
0); /* debug level */
parts = malloc(myGraph.numMyVertices * sizeof(int));
lids = malloc(myGraph.numMyVertices * sizeof(ZOLTAN_ID_TYPE));
for (i=0; i < myGraph.numMyVertices; i++){
parts[i] = myRank; /* part number of this vertex */
lids[i] = (ZOLTAN_ID_TYPE)i; /* local ID on my process for this vertex */
}
rc = Zoltan_DD_Update(dd,
myGraph.vertexGID,
lids,
NULL,
parts,
myGraph.numMyVertices);
myGraph.dd = dd;
/******************************************************************
** Create a Zoltan library structure for this instance of load
** balancing. Set the parameters and query functions that will
** govern the library's calculation. See the Zoltan User's
** Guide for the definition of these and many other parameters.
******************************************************************/
zz = Zoltan_Create(MPI_COMM_WORLD);
/* General parameters */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
/* Graph parameters */
Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */
/* Query functions, defined in this source file */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph);
Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph);
Zoltan_Set_Obj_Size_Multi_Fn(zz, get_message_sizes,&myGraph);
Zoltan_Set_Pack_Obj_Multi_Fn(zz, pack_object_messages,&myGraph);
Zoltan_Set_Unpack_Obj_Multi_Fn(zz, unpack_object_messages,&myGraph);
Zoltan_Set_Mid_Migrate_PP_Fn(zz, mid_migrate,&myGraph);
/******************************************************************
** Visualize the graph partitioning before calling Zoltan.
******************************************************************/
if (myRank== 0){
printf("\nGraph partition before calling Zoltan\n");
}
showGraphPartitions(myRank, myGraph.dd);
/******************************************************************
** Zoltan can now partition the simple graph.
** In this simple example, we assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/*fprintf(stderr,"%d export %d import %d\n",myRank,numExport,numImport);*/
/******************************************************************
** Update the data directory with the new partition numbers
******************************************************************/
for (i=0; i < numExport; i++){
parts[exportLocalGids[i]] = exportToPart[i];
}
rc = Zoltan_DD_Update(dd,
myGraph.vertexGID,
lids,
NULL,
parts,
myGraph.numMyVertices);
/******************************************************************
** Migrate vertices to new partitions
******************************************************************/
rc = Zoltan_Migrate(zz,
numImport, importGlobalGids, importLocalGids,
importProcs, importToPart,
numExport, exportGlobalGids, exportLocalGids,
exportProcs, exportToPart);
/******************************************************************
** Use the data dictionary to find neighbors' partitions
******************************************************************/
start_gid = myGraph.numMyVertices - numImport;
num_nbors = myGraph.nborIndex[myGraph.numMyVertices] - myGraph.nborIndex[start_gid];
rc = Zoltan_DD_Find(dd,
(ZOLTAN_ID_PTR)(myGraph.nborGID + start_gid), NULL, NULL,
myGraph.nborPart + start_gid, num_nbors, NULL);
/******************************************************************
** Visualize the graph partitioning after calling Zoltan.
******************************************************************/
if (myRank == 0){
printf("Graph partition after calling Zoltan\n");
}
showGraphPartitions(myRank, myGraph.dd);
/******************************************************************
** Free the arrays allocated by Zoltan_LB_Partition, and free
** the storage allocated for the Zoltan structure.
******************************************************************/
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
Zoltan_Destroy(&zz);
/**********************
** all done ***********
**********************/
MPI_Finalize();
if (myGraph.vertex_capacity > 0){
free(myGraph.vertexGID);
free(myGraph.nborIndex);
if (myGraph.nbor_capacity > 0){
free(myGraph.nborGID);
free(myGraph.nborPart);
}
}
if (parts) free(parts);
if (lids) free(lids);
return 0;
}
