int main(int argc, char *argv[])
{
FILE *in_file = NULL; /* file with data for problems */
ZOLTAN_COMM_OBJ *plan; /* communication data structure pointer */
struct Params params; /* parameters describing a problem */
struct Data data; /* data describing a problem instance */
struct Data my_send_data; /* data I initially own */
struct Answer true_answer; /* expected outcome of exchange */
int nvals_recv; /* number of vals I own after exchange */
float *recv_data; /* values I own after exchange */
float *reverse_data; /* values I own after reversing communication */
char file_name[100]; /* name of input file */
int nbytes; /* size of objects */
int my_proc; /* my processor ID */
int nprocs; /* total number of processors */
int flag; /* return code from comm operations */
int more_problems; /* are there more problems to do? */
int i, j;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &my_proc);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
Zoltan_Memory_Debug(2);
if (argc > 1) strcpy(file_name, argv[1]);
else strcpy(file_name,"comm_input.dat");
if (my_proc == 0) {
in_file = fopen(file_name, "r");
if (in_file == NULL) {
printf("No input file `%s' found.\n", file_name);
}
}
/* Read some problem descriptors from a file */
more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc);
while (more_problems) {
/* Generate full data at random on each proc */
gen_comm_data(¶ms, &data, nprocs);
if (out_level > 2) if (my_proc == 0) print_data("DATA", &data);
/* Figure out from the data what what to expect to receive */
extract_comm_answer(&data, &true_answer, my_proc, nprocs);
/* Extract what to send */
set_up_comm_from_send(&data, &my_send_data, params.blocked, my_proc, nprocs);
if (out_level > 2) print_data("MY_DATA", &my_send_data);
if (out_level > 1) printf("%d: About to call comm_create\n", my_proc);
/* Call comm routines */
Zoltan_Comm_Create(&plan, my_send_data.nvals, my_send_data.proc_dest,
MPI_COMM_WORLD, 1, &nvals_recv);
if (out_level > 1) printf("%d: About to call comm_info\n", my_proc);
check_comm_info(plan, &my_send_data, nvals_recv, my_proc);
if (out_level > 2) print_plan("BEFORE RESIZE", plan, my_proc);
/* "4" reflects the max_sizes value in gen_comm_data */
recv_data = (float *) ZOLTAN_MALLOC(nvals_recv * 4 * sizeof(float));
reverse_data = (float *) ZOLTAN_MALLOC(my_send_data.nvals * 4 * sizeof(float));
if (my_send_data.sizes != NULL) {
if (out_level > 1) printf("%d: About to call comm_resize\n", my_proc);
Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
Zoltan_Comm_Resize(plan, NULL, 43, NULL);
Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
if (out_level > 2) print_plan("AFTER RESIZE", plan, my_proc);
}
if (my_send_data.sizes == NULL) nbytes = my_send_data.same_size * sizeof(float);
else nbytes = sizeof(float);
if (out_level > 1) printf("%d: About to call comm_do\n", my_proc);
flag = Zoltan_Comm_Do(plan, 2, (char *) my_send_data.vals, nbytes,
(char *) recv_data);
if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer\n", my_proc);
/* Check answers */
check_comm_answer(&true_answer, recv_data, my_proc);
}
else {
printf("%d: Comm_Do returned error code %d\n", my_proc, flag);
}
if (out_level > 1) printf("%d: About to call comm_do_reverse\n", my_proc);
i = (true_answer.sizes != NULL && plan->indices_to != NULL);
MPI_Allreduce(&i, &j, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (j == 0) flag = Zoltan_Comm_Do_Reverse(plan, 2, (char *) recv_data,
nbytes, true_answer.sizes, (char *) reverse_data);
else {
if (my_proc == 0)
printf(">> Non-blocked, variable-sized recvs not supported\n");
flag = ZOLTAN_FATAL;
}
if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer_reverse\n", my_proc);
/* Check answers */
check_comm_answer_reverse(&my_send_data, reverse_data, my_proc);
}
else {
if (out_level > 1) printf("%d: Comm_Do_Reverse returned error code %d\n", my_proc, flag);
}
/* Free up data structures */
ZOLTAN_FREE(&reverse_data);
ZOLTAN_FREE(&recv_data);
free_comm_data(&data, &my_send_data, &true_answer);
Zoltan_Comm_Destroy(&plan);
/* Read some problem descriptors from a file */
more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc);
}
Zoltan_Memory_Stats();
MPI_Finalize();
return(0);
}
int main(int argc, char *argv[])
{
int rc, do_hier, status;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport;
int generate_files = 0;
char *platform=NULL, *topology=NULL;
char *graph_package=NULL;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
struct option opts[10];
double comm_time[10];
float cut_weight[3] = {0., 0., 0.};
long nvert=0;
char *debug_level=NULL;
status = 0;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
Zoltan_Initialize(argc, argv, &ver);
zz = Zoltan_Create(MPI_COMM_WORLD);
/******************************************************************
** Check that this test makes sense.
******************************************************************/
if (sizeof(long) < sizeof(ZOLTAN_ID_TYPE)){
if (myRank == 0){
printf("ERROR: This code assumes that a long is at least %d bytes\n",(int)sizeof(ZOLTAN_ID_TYPE));
}
status = 1;
}
check_error_status(status, "configuration error");
/******************************************************************
** Initialize zoltan
******************************************************************/
/* options */
opts[0].name = "platform";
opts[0].has_arg = 1;
opts[0].flag = NULL;
opts[0].val = 1;
opts[1].name = "topology";
opts[1].has_arg = 1;
opts[1].flag = NULL;
opts[1].val = 2;
opts[2].name = "size";
opts[2].has_arg = 1;
opts[2].flag = NULL;
opts[2].val = 4;
opts[3].name = "verbose";
opts[3].has_arg = 0;
opts[3].flag = NULL;
opts[3].val = 5;
opts[4].name = "help";
opts[4].has_arg = 0;
opts[4].flag = NULL;
opts[4].val = 6;
opts[5].name = "graph_package";
opts[5].has_arg = 1;
opts[5].flag = NULL;
opts[5].val = 7;
opts[6].name = "generate_files";
opts[6].has_arg = 0;
opts[6].flag = NULL;
opts[6].val = 8;
opts[7].name = "debug_level";
opts[7].has_arg = 1;
opts[7].flag = NULL;
opts[7].val = 9;
opts[8].name = 0;
opts[8].has_arg = 0;
opts[8].flag = NULL;
opts[8].val = 0;
status = 0;
while (1){
rc = getopt_long_only(argc, argv, "", opts, NULL);
if (rc == '?'){
MPI_Barrier(MPI_COMM_WORLD);
if (myRank == 0) usage();
MPI_Finalize();
exit(0);
}
else if (rc == 1){
platform = optarg;
if (myRank == 0)
printf( "For platform %s\n",optarg );
}
else if (rc == 2){
topology = optarg;
if (myRank == 0)
printf( "For topology %s\n",optarg);
}
else if (rc == 7){
graph_package = optarg;
if (myRank == 0)
printf( "Zoltan parameter GRAPH_PACKAGE = %s\n",graph_package);
}
else if (rc == 8){
generate_files = 1;
if (myRank == 0)
printf( "Zoltan_Generate_Files will be called for each level.\n");
}
else if (rc == 4){
nvert = atol(optarg);
if (nvert < 1) status = 1;
check_error_status(status, "--size={approximate number of vertices}");
if (myRank == 0){
printf( "Graph will have approximately %ld vertices.\n",nvert);
}
}
else if (rc == 5){
verbose = 1;
}
else if (rc == 6){
if (myRank == 0) usage();
MPI_Finalize();
exit(0);
}
else if (rc == 9){
debug_level = optarg;
}
else if (rc <= 0){
break;
}
}
if ((platform==NULL) && (topology==NULL)){
if (myRank == 0)
fprintf(stdout,"No platform or topology, so we'll skip hierarchical partitioning\n");
do_hier = 0;
}
else if (graph_package == NULL){
if (myRank == 0)
fprintf(stdout,"No graph package, so we'll skip hierarchical partitioning\n");
do_hier = 0;
}
else{
do_hier = 1;
}
/* start */
Zoltan_Memory_Debug(0);
if (nvert > 0)
numGlobalVertices = nvert;
else
numGlobalVertices = NUM_GLOBAL_VERTICES;
status = create_a_graph();
check_error_status(status, "creating the graph");
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "REMAP", "0");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* export AND import lists */
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "1"); /* number of weights per vertex */
Zoltan_Set_Param(zz, "EDGE_WEIGHT_DIM", "1");/* number of weights per hyperedge */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, NULL);
Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, NULL);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, NULL);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, NULL);
/* GRAPH PARTITION */
Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
if (graph_package)
Zoltan_Set_Param(zz, "GRAPH_PACKAGE", graph_package);
if (verbose){
debug(zz, "Initial graph", 0);
}
if (generate_files){
rc = Zoltan_Generate_Files(zz, "flat", myRank, 0, 1, 0);
if (rc != ZOLTAN_OK) status = 1;
check_error_status(status, "Zoltan_Generate_Files");
}
/* Performance before partitioning */
time_communication(comm_time+0);
cut_weight[0] = get_edge_cut_weight(zz);
if (cut_weight[0] < 0.0) status = 1;
check_error_status(status, "First call to get_edge_cut_weight");
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK) status = 1;
check_error_status(status, "First call to LB_Partition");
status = migrate_graph(numExport, numImport, exportLocalGids, importGlobalGids);
check_error_status(status, "migration");
if (verbose){
debug(zz, "After flat partitioning and migration", 0);
}
time_communication(comm_time+1); /* With graph partitioning */
cut_weight[1] = get_edge_cut_weight(zz);
if (cut_weight[1] < 0.0) status = 1;
check_error_status(status, "Second call to get_edge_cut_weight");
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
if (do_hier){
/* HIERARCHICAL PARTITION */
free_graph();
status = create_a_graph();
check_error_status(status, "create graph for hierarchical partitioning");
Zoltan_Set_Param(zz, "LB_METHOD", "HIER");
Zoltan_Set_Param(zz, "HIER_ASSIST", "1");
if (generate_files){
Zoltan_Set_Param(zz, "HIER_GENERATE_FILES", "1");
}
if (debug_level) /* 1, 2 or 3 */
Zoltan_Set_Param(zz, "HIER_DEBUG_LEVEL", debug_level);
else
Zoltan_Set_Param(zz, "HIER_DEBUG_LEVEL", "0");
/* TODO: Suppose graph is not symmetric, and we request SYMMETRIZE. Do we still get
* a "good" answer when each sub-graph in the hierarchy is symmetrized?
*/
if (topology)
Zoltan_Set_Param(zz, "TOPOLOGY", topology);
else if (platform)
Zoltan_Set_Param(zz, "PLATFORM", platform);
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK) status = 1;
check_error_status(status, "Second call to LB_Partition");
status = migrate_graph(numExport, numImport, exportLocalGids, importGlobalGids);
check_error_status(status, "second migration");
if (verbose){
debug(zz, "After hierarchical partitioning and migration", 0);
}
time_communication(comm_time+2); /* With hierarchical graph partitioning */
cut_weight[2] = get_edge_cut_weight(zz);
if (cut_weight[2] < 0.0) status = 1;
check_error_status(status, "Third call to get_edge_cut_weight");
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
}
Zoltan_Destroy(&zz);
free_graph();
if (myRank == 0){
fprintf(stdout,"Graph cut weight before partitioning: %f\n",cut_weight[0]);
fprintf(stdout," after flat partitioning: %f\n",cut_weight[1]);
if (do_hier)
fprintf(stdout," after hierarchical partitioning: %f\n",cut_weight[2]);
fflush(stdout);
}
if (cut_weight[1] >= cut_weight[0]){
status = 1;
if (zz->Proc == 0){
fprintf(stderr,"FAILED: No improvement shown in flat partitioning");
}
}
if (do_hier && (cut_weight[2] > cut_weight[0])){
status = 1;
if (zz->Proc == 0){
fprintf(stderr,"FAILED: No improvement shown in hierarchical partitioning");
}
}
MPI_Finalize();
return status;
}
