int main(int argc, char *argv[])
{
int rc, i;
ZOLTAN_GNO_TYPE numGlobalVertices;
float ver;
char dimstring[16];
double min, max, avg, local;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
#ifdef HOST_LINUX
signal(SIGSEGV, meminfo_signal_handler);
signal(SIGINT, meminfo_signal_handler);
signal(SIGTERM, meminfo_signal_handler);
signal(SIGABRT, meminfo_signal_handler);
signal(SIGFPE, meminfo_signal_handler);
#endif
/******************************************************************
** Problem size
******************************************************************/
numGlobalVertices = NUM_GLOBAL_VERTICES;
vertexWeightDim = VERTEX_WEIGHT_DIMENSION;
vertexDim = VERTEX_DIMENSION;
if (argc > 1){
sscanf(argv[1], "%zd", &numGlobalVertices);
if (argc > 2){
vertexWeightDim = atoi(argv[2]);
if (argc > 3){
vertexDim = atoi(argv[3]);
}
}
}
sprintf(dimstring,"%d",vertexWeightDim);
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
exit(1);
}
Zoltan_Memory_Debug(2);
/******************************************************************
** Create vertices
******************************************************************/
rc = create_vertices(numGlobalVertices, vertexDim, vertexWeightDim, numProcs, myRank);
if (rc){
fprintf(stderr,"Process rank %d: insufficient memory\n",myRank);
MPI_Finalize();
exit(1);
}
first_gid = vertex_gid[myRank];
/******************************************************************
** Create a Zoltan library structure for this instance of load
** balancing. Set the parameters and query functions that will
** govern the library's calculation. See the Zoltan User's
** Guide for the definition of these and many other parameters.
******************************************************************/
zz = Zoltan_Create(MPI_COMM_WORLD);
/* General parameters */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", "RIB");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", dimstring);
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
/* RIB parameters */
Zoltan_Set_Param(zz, "RIB_OUTPUT_LEVEL", "0");
/* Query functions, to provide geometry to Zoltan */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, NULL);
Zoltan_Set_Obj_List_Fn(zz, get_object_list, NULL);
Zoltan_Set_Num_Geom_Fn(zz, get_num_geometry, NULL);
Zoltan_Set_Geom_Multi_Fn(zz, get_geometry_list, NULL);
Zoltan_Set_Part_Multi_Fn(zz, get_partition_list, NULL);
/******************************************************************
** Zoltan can now partition the vertices in the simple mesh.
** In this simple example, we assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
if (myRank == 0){
printf("Run Zoltan\n");
}
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK){
if (myRank == 0)printf("sorry...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Check the balance of the partitions before running zoltan.
** The query function get_partition_list() will give the
** partitions of the vertices before we called Zoltan.
******************************************************************/
if (myRank == 0){
printf("\nBALANCE before running Zoltan\n");
}
rc = Zoltan_LB_Eval_Balance(zz, 1, NULL);
if (rc != ZOLTAN_OK){
printf("sorry first LB_Eval_Balance...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Print out the balance of the new partitions.
******************************************************************/
vertex_part = (int *)malloc(sizeof(int) * numLocalVertices);
if (!vertex_part){
printf("sorry memory error...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
for (i=0; i < numLocalVertices; i++){
vertex_part[i] = myRank;
}
if (numExport > 0){
for (i=0; i < numExport; i++){
vertex_part[exportLocalGids[i]] = exportToPart[i];
}
}
if (myRank == 0){
printf("\nBALANCE after running Zoltan\n");
}
rc = Zoltan_LB_Eval_Balance(zz, 1, NULL);
if (rc != ZOLTAN_OK){
printf("sorry second LB_Eval_Balance...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Free the arrays allocated by Zoltan_LB_Partition, and free
** the storage allocated for the Zoltan structure.
******************************************************************/
if (myRank == 0){
printf("Free structures\n");
}
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
Zoltan_Destroy(&zz);
if (vertex_part) free(vertex_part);
if (v_x) free(v_x);
if (v_y) free(v_y);
if (v_z) free(v_z);
if (vertex_weight) free(vertex_weight);
if (vertex_gid) free(vertex_gid);
/**********************
** all done ***********
**********************/
local= (double)Zoltan_Memory_Usage(ZOLTAN_MEM_STAT_MAXIMUM)/(1024.0*1024);
MPI_Reduce(&local, &avg, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
avg /= (double)numProcs;
MPI_Reduce(&local, &max, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&local, &min, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
if (myRank == 0){
printf("Total MBytes in use by test while Zoltan is running: %12.3lf\n",
mbytes/(1024.0*1024));
printf("Min/Avg/Max of maximum MBytes in use by Zoltan: %12.3lf / %12.3lf / %12.3lf\n",
min, avg, max);
}
MPI_Finalize();
return 0;
}
int Zoltan_Set_Fn(ZZ *zz, ZOLTAN_FN_TYPE fn_type, ZOLTAN_VOID_FN *fn,
void *data)
{
/*
* Function to initialize a given LB interface function.
* Input:
* zz -- Pointer to a Zoltan structure.
* fn_type -- Enum type indicating the function to be set.
* fn -- Pointer to the function to be used in the
* assignment.
* data -- Pointer to data that the LB library will
* pass as an argument to fn(). May be NULL.
* Output:
* zz -- Appropriate field set to value in void *().
*/
char *yo = "Zoltan_Set_Fn";
char msg[256];
int ierr;
switch (fn_type) {
case ZOLTAN_PART_FN_TYPE:
ierr = Zoltan_Set_Part_Fn(zz,
(ZOLTAN_PART_FN *) fn, data);
break;
case ZOLTAN_PART_MULTI_FN_TYPE:
ierr = Zoltan_Set_Part_Multi_Fn(zz,
(ZOLTAN_PART_MULTI_FN *) fn, data);
break;
case ZOLTAN_NUM_EDGES_FN_TYPE:
ierr = Zoltan_Set_Num_Edges_Fn(zz,
(ZOLTAN_NUM_EDGES_FN *) fn, data);
break;
case ZOLTAN_NUM_EDGES_MULTI_FN_TYPE:
ierr = Zoltan_Set_Num_Edges_Multi_Fn(zz,
(ZOLTAN_NUM_EDGES_MULTI_FN *) fn, data);
break;
case ZOLTAN_EDGE_LIST_FN_TYPE:
ierr = Zoltan_Set_Edge_List_Fn(zz,
(ZOLTAN_EDGE_LIST_FN *) fn, data);
break;
case ZOLTAN_EDGE_LIST_MULTI_FN_TYPE:
ierr = Zoltan_Set_Edge_List_Multi_Fn(zz,
(ZOLTAN_EDGE_LIST_MULTI_FN *) fn, data);
break;
case ZOLTAN_NUM_GEOM_FN_TYPE:
ierr = Zoltan_Set_Num_Geom_Fn(zz,
(ZOLTAN_NUM_GEOM_FN *) fn, data);
break;
case ZOLTAN_GEOM_MULTI_FN_TYPE:
ierr = Zoltan_Set_Geom_Multi_Fn(zz,
(ZOLTAN_GEOM_MULTI_FN *) fn, data);
break;
case ZOLTAN_GEOM_FN_TYPE:
ierr = Zoltan_Set_Geom_Fn(zz,
(ZOLTAN_GEOM_FN *) fn, data);
break;
case ZOLTAN_NUM_OBJ_FN_TYPE:
ierr = Zoltan_Set_Num_Obj_Fn(zz,
(ZOLTAN_NUM_OBJ_FN *) fn, data);
break;
case ZOLTAN_OBJ_LIST_FN_TYPE:
ierr = Zoltan_Set_Obj_List_Fn(zz,
(ZOLTAN_OBJ_LIST_FN *) fn, data);
break;
case ZOLTAN_FIRST_OBJ_FN_TYPE:
ierr = Zoltan_Set_First_Obj_Fn(zz,
(ZOLTAN_FIRST_OBJ_FN *) fn, data);
break;
case ZOLTAN_NEXT_OBJ_FN_TYPE:
ierr = Zoltan_Set_Next_Obj_Fn(zz,
(ZOLTAN_NEXT_OBJ_FN *) fn, data);
break;
case ZOLTAN_NUM_BORDER_OBJ_FN_TYPE:
ierr = Zoltan_Set_Num_Border_Obj_Fn(zz,
(ZOLTAN_NUM_BORDER_OBJ_FN *) fn, data);
break;
case ZOLTAN_BORDER_OBJ_LIST_FN_TYPE:
ierr = Zoltan_Set_Border_Obj_List_Fn(zz,
(ZOLTAN_BORDER_OBJ_LIST_FN *) fn, data);
break;
case ZOLTAN_FIRST_BORDER_OBJ_FN_TYPE:
ierr = Zoltan_Set_First_Border_Obj_Fn(zz,
(ZOLTAN_FIRST_BORDER_OBJ_FN *) fn, data);
break;
case ZOLTAN_NEXT_BORDER_OBJ_FN_TYPE:
ierr = Zoltan_Set_Next_Border_Obj_Fn(zz,
(ZOLTAN_NEXT_BORDER_OBJ_FN *) fn, data);
break;
case ZOLTAN_PRE_MIGRATE_PP_FN_TYPE:
ierr = Zoltan_Set_Pre_Migrate_PP_Fn(zz,
(ZOLTAN_PRE_MIGRATE_PP_FN *) fn, data);
break;
case ZOLTAN_MID_MIGRATE_PP_FN_TYPE:
ierr = Zoltan_Set_Mid_Migrate_PP_Fn(zz,
(ZOLTAN_MID_MIGRATE_PP_FN *) fn, data);
break;
case ZOLTAN_POST_MIGRATE_PP_FN_TYPE:
ierr = Zoltan_Set_Post_Migrate_PP_Fn(zz,
(ZOLTAN_POST_MIGRATE_PP_FN *) fn, data);
break;
case ZOLTAN_PRE_MIGRATE_FN_TYPE:
ierr = Zoltan_Set_Pre_Migrate_Fn(zz,
(ZOLTAN_PRE_MIGRATE_FN *) fn, data);
break;
case ZOLTAN_MID_MIGRATE_FN_TYPE:
ierr = Zoltan_Set_Mid_Migrate_Fn(zz,
(ZOLTAN_MID_MIGRATE_FN *) fn, data);
break;
case ZOLTAN_POST_MIGRATE_FN_TYPE:
ierr = Zoltan_Set_Post_Migrate_Fn(zz,
(ZOLTAN_POST_MIGRATE_FN *) fn, data);
break;
case ZOLTAN_OBJ_SIZE_FN_TYPE:
ierr = Zoltan_Set_Obj_Size_Fn(zz,
(ZOLTAN_OBJ_SIZE_FN *) fn, data);
break;
case ZOLTAN_OBJ_SIZE_MULTI_FN_TYPE:
ierr = Zoltan_Set_Obj_Size_Multi_Fn(zz,
(ZOLTAN_OBJ_SIZE_MULTI_FN *) fn, data);
break;
case ZOLTAN_PACK_OBJ_FN_TYPE:
ierr = Zoltan_Set_Pack_Obj_Fn(zz,
(ZOLTAN_PACK_OBJ_FN *) fn, data);
break;
case ZOLTAN_PACK_OBJ_MULTI_FN_TYPE:
ierr = Zoltan_Set_Pack_Obj_Multi_Fn(zz,
(ZOLTAN_PACK_OBJ_MULTI_FN *) fn, data);
break;
case ZOLTAN_UNPACK_OBJ_FN_TYPE:
ierr = Zoltan_Set_Unpack_Obj_Fn(zz,
(ZOLTAN_UNPACK_OBJ_FN *) fn, data);
break;
case ZOLTAN_UNPACK_OBJ_MULTI_FN_TYPE:
ierr = Zoltan_Set_Unpack_Obj_Multi_Fn(zz,
(ZOLTAN_UNPACK_OBJ_MULTI_FN *) fn, data);
break;
case ZOLTAN_NUM_COARSE_OBJ_FN_TYPE:
ierr = Zoltan_Set_Num_Coarse_Obj_Fn(zz,
(ZOLTAN_NUM_COARSE_OBJ_FN *) fn, data);
break;
case ZOLTAN_COARSE_OBJ_LIST_FN_TYPE:
ierr = Zoltan_Set_Coarse_Obj_List_Fn(zz,
(ZOLTAN_COARSE_OBJ_LIST_FN *) fn, data);
break;
case ZOLTAN_FIRST_COARSE_OBJ_FN_TYPE:
ierr = Zoltan_Set_First_Coarse_Obj_Fn(zz,
(ZOLTAN_FIRST_COARSE_OBJ_FN *) fn, data);
break;
case ZOLTAN_NEXT_COARSE_OBJ_FN_TYPE:
ierr = Zoltan_Set_Next_Coarse_Obj_Fn(zz,
(ZOLTAN_NEXT_COARSE_OBJ_FN *) fn, data);
break;
case ZOLTAN_NUM_CHILD_FN_TYPE:
ierr = Zoltan_Set_Num_Child_Fn(zz,
(ZOLTAN_NUM_CHILD_FN *) fn, data);
break;
case ZOLTAN_CHILD_LIST_FN_TYPE:
ierr = Zoltan_Set_Child_List_Fn(zz,
(ZOLTAN_CHILD_LIST_FN *) fn, data);
break;
case ZOLTAN_CHILD_WEIGHT_FN_TYPE:
ierr = Zoltan_Set_Child_Weight_Fn(zz,
(ZOLTAN_CHILD_WEIGHT_FN *) fn, data);
break;
case ZOLTAN_HG_SIZE_CS_FN_TYPE:
ierr = Zoltan_Set_HG_Size_CS_Fn(zz,
(ZOLTAN_HG_SIZE_CS_FN *) fn, data);
break;
case ZOLTAN_HG_CS_FN_TYPE:
ierr = Zoltan_Set_HG_CS_Fn(zz,
(ZOLTAN_HG_CS_FN *) fn, data);
break;
case ZOLTAN_HG_SIZE_EDGE_WTS_FN_TYPE:
ierr = Zoltan_Set_HG_Size_Edge_Wts_Fn(zz,
(ZOLTAN_HG_SIZE_EDGE_WTS_FN *) fn, data);
break;
case ZOLTAN_HG_EDGE_WTS_FN_TYPE:
ierr = Zoltan_Set_HG_Edge_Wts_Fn(zz,
(ZOLTAN_HG_EDGE_WTS_FN *) fn, data);
break;
case ZOLTAN_NUM_FIXED_OBJ_FN_TYPE:
ierr = Zoltan_Set_Num_Fixed_Obj_Fn(zz,
(ZOLTAN_NUM_FIXED_OBJ_FN *) fn, data);
break;
case ZOLTAN_FIXED_OBJ_LIST_FN_TYPE:
ierr = Zoltan_Set_Fixed_Obj_List_Fn(zz,
(ZOLTAN_FIXED_OBJ_LIST_FN *) fn, data);
break;
case ZOLTAN_HIER_NUM_LEVELS_FN_TYPE:
ierr = Zoltan_Set_Hier_Num_Levels_Fn(zz,
(ZOLTAN_HIER_NUM_LEVELS_FN *) fn, data);
break;
case ZOLTAN_HIER_PART_FN_TYPE:
ierr = Zoltan_Set_Hier_Part_Fn(zz,
(ZOLTAN_HIER_PART_FN *) fn, data);
break;
case ZOLTAN_HIER_METHOD_FN_TYPE:
ierr = Zoltan_Set_Hier_Method_Fn(zz,
(ZOLTAN_HIER_METHOD_FN *) fn, data);
break;
default:
sprintf(msg, "ZOLTAN_FN_TYPE %d is invalid.\n"
"Value must be in range 0 to %d.", fn_type, ZOLTAN_MAX_FN_TYPES);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ierr = ZOLTAN_WARN;
}
return (ierr);
}
