int main(int argc, char *argv[])
{
FILE *in_file = NULL; /* file with data for problems */
ZOLTAN_COMM_OBJ *plan; /* communication data structure pointer */
struct Params params; /* parameters describing a problem */
struct Data data; /* data describing a problem instance */
struct Data my_send_data; /* data I initially own */
struct Answer true_answer; /* expected outcome of exchange */
int nvals_recv; /* number of vals I own after exchange */
float *recv_data; /* values I own after exchange */
float *reverse_data; /* values I own after reversing communication */
char file_name[100]; /* name of input file */
int nbytes; /* size of objects */
int my_proc; /* my processor ID */
int nprocs; /* total number of processors */
int flag; /* return code from comm operations */
int more_problems; /* are there more problems to do? */
int i, j;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &my_proc);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
Zoltan_Memory_Debug(2);
if (argc > 1) strcpy(file_name, argv[1]);
else strcpy(file_name,"comm_input.dat");
if (my_proc == 0) {
in_file = fopen(file_name, "r");
if (in_file == NULL) {
printf("No input file `%s' found.\n", file_name);
}
}
/* Read some problem descriptors from a file */
more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc);
while (more_problems) {
/* Generate full data at random on each proc */
gen_comm_data(¶ms, &data, nprocs);
if (out_level > 2) if (my_proc == 0) print_data("DATA", &data);
/* Figure out from the data what what to expect to receive */
extract_comm_answer(&data, &true_answer, my_proc, nprocs);
/* Extract what to send */
set_up_comm_from_send(&data, &my_send_data, params.blocked, my_proc, nprocs);
if (out_level > 2) print_data("MY_DATA", &my_send_data);
if (out_level > 1) printf("%d: About to call comm_create\n", my_proc);
/* Call comm routines */
Zoltan_Comm_Create(&plan, my_send_data.nvals, my_send_data.proc_dest,
MPI_COMM_WORLD, 1, &nvals_recv);
if (out_level > 1) printf("%d: About to call comm_info\n", my_proc);
check_comm_info(plan, &my_send_data, nvals_recv, my_proc);
if (out_level > 2) print_plan("BEFORE RESIZE", plan, my_proc);
/* "4" reflects the max_sizes value in gen_comm_data */
recv_data = (float *) ZOLTAN_MALLOC(nvals_recv * 4 * sizeof(float));
reverse_data = (float *) ZOLTAN_MALLOC(my_send_data.nvals * 4 * sizeof(float));
if (my_send_data.sizes != NULL) {
if (out_level > 1) printf("%d: About to call comm_resize\n", my_proc);
Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
Zoltan_Comm_Resize(plan, NULL, 43, NULL);
Zoltan_Comm_Resize(plan, my_send_data.sizes, 43, NULL);
if (out_level > 2) print_plan("AFTER RESIZE", plan, my_proc);
}
if (my_send_data.sizes == NULL) nbytes = my_send_data.same_size * sizeof(float);
else nbytes = sizeof(float);
if (out_level > 1) printf("%d: About to call comm_do\n", my_proc);
flag = Zoltan_Comm_Do(plan, 2, (char *) my_send_data.vals, nbytes,
(char *) recv_data);
if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer\n", my_proc);
/* Check answers */
check_comm_answer(&true_answer, recv_data, my_proc);
}
else {
printf("%d: Comm_Do returned error code %d\n", my_proc, flag);
}
if (out_level > 1) printf("%d: About to call comm_do_reverse\n", my_proc);
i = (true_answer.sizes != NULL && plan->indices_to != NULL);
MPI_Allreduce(&i, &j, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (j == 0) flag = Zoltan_Comm_Do_Reverse(plan, 2, (char *) recv_data,
nbytes, true_answer.sizes, (char *) reverse_data);
else {
if (my_proc == 0)
printf(">> Non-blocked, variable-sized recvs not supported\n");
flag = ZOLTAN_FATAL;
}
if (flag == ZOLTAN_OK) {
if (out_level > 1) printf("%d: About to call check_answer_reverse\n", my_proc);
/* Check answers */
check_comm_answer_reverse(&my_send_data, reverse_data, my_proc);
}
else {
if (out_level > 1) printf("%d: Comm_Do_Reverse returned error code %d\n", my_proc, flag);
}
/* Free up data structures */
ZOLTAN_FREE(&reverse_data);
ZOLTAN_FREE(&recv_data);
free_comm_data(&data, &my_send_data, &true_answer);
Zoltan_Comm_Destroy(&plan);
/* Read some problem descriptors from a file */
more_problems = read_comm_problem(in_file, ¶ms, &out_level, my_proc);
}
Zoltan_Memory_Stats();
MPI_Finalize();
return(0);
}
int main(int argc, char *argv[])
{
/* Local declarations. */
struct Zoltan_Struct *zz = NULL;
char *cmd_file;
char cmesg[256]; /* for error messages */
float version;
int Proc, Num_Proc;
int iteration;
int error, gerror;
int print_output = 1;
MESH_INFO mesh; /* mesh information struct */
PARIO_INFO pio_info;
PROB_INFO prob;
/***************************** BEGIN EXECUTION ******************************/
/* initialize MPI */
MPI_Init(&argc, &argv);
#ifdef VAMPIR
VT_initialize(&argc, &argv);
#endif
/* get some machine information */
MPI_Comm_rank(MPI_COMM_WORLD, &Proc);
MPI_Comm_size(MPI_COMM_WORLD, &Num_Proc);
my_rank = Proc;
#ifdef HOST_LINUX
signal(SIGSEGV, meminfo_signal_handler);
signal(SIGINT, meminfo_signal_handler);
signal(SIGTERM, meminfo_signal_handler);
signal(SIGABRT, meminfo_signal_handler);
signal(SIGFPE, meminfo_signal_handler);
#endif
#ifdef ZOLTAN_PURIFY
printf("%d of %d ZDRIVE LAUNCH pid = %d file = %s\n",
Proc, Num_Proc, getpid(), argv[1]);
#endif
/* Initialize flags */
Test.DDirectory = 0;
Test.Local_Parts = 0;
Test.Fixed_Objects = 0;
Test.Drops = 0;
Test.RCB_Box = 0;
Test.Multi_Callbacks = 0;
Test.Graph_Callbacks = 1;
Test.Hypergraph_Callbacks = 1;
Test.Gen_Files = 0;
Test.Null_Lists = NO_NULL_LISTS;
Test.Dynamic_Weights = .0;
Test.Dynamic_Graph = .0;
Test.Vtx_Inc = 0;
Output.Text = 1;
Output.Gnuplot = 0;
Output.Nemesis = 0;
Output.Plot_Partition = 0;
Output.Mesh_Info_File = 0;
/* Interpret the command line */
switch(argc)
{
case 1:
cmd_file = "zdrive.inp";
break;
case 2:
cmd_file = argv[1];
break;
default:
fprintf(stderr, "MAIN: ERROR in command line,");
if(Proc == 0)
{
fprintf(stderr, " usage:\n");
fprintf(stderr, "\t%s [command file]", DRIVER_NAME);
}
exit(1);
break;
}
/* initialize Zoltan */
if ((error = Zoltan_Initialize(argc, argv, &version)) != ZOLTAN_OK) {
sprintf(cmesg, "fatal: Zoltan_Initialize returned error code, %d", error);
Gen_Error(0, cmesg);
error_report(Proc);
print_output = 0;
goto End;
}
/* initialize some variables */
initialize_mesh(&mesh, Proc);
pio_info.dsk_list_cnt = -1;
pio_info.file_comp = STANDARD;
pio_info.num_dsk_ctrlrs = -1;
pio_info.pdsk_add_fact = -1;
pio_info.zeros = -1;
pio_info.file_type = -1;
pio_info.chunk_reader = 0;
pio_info.init_dist_type = -1;
pio_info.init_size = ZOLTAN_ID_INVALID;
pio_info.init_dim = -1;
pio_info.init_vwgt_dim = -1;
pio_info.init_dist_pins = -1;
pio_info.pdsk_root[0] = '\0';
pio_info.pdsk_subdir[0] = '\0';
pio_info.pexo_fname[0] = '\0';
prob.method[0] = '\0';
prob.num_params = 0;
prob.params = NULL;
/* Read in the ascii input file */
error = gerror = 0;
if (Proc == 0) {
printf("\n\nReading the command file, %s\n", cmd_file);
if (!read_cmd_file(cmd_file, &prob, &pio_info, NULL)) {
sprintf(cmesg,"fatal: Could not read in the command file"
" \"%s\"!\n", cmd_file);
Gen_Error(0, cmesg);
error_report(Proc);
print_output = 0;
error = 1;
}
if (!check_inp(&prob, &pio_info)) {
Gen_Error(0, "fatal: Error in user specified parameters.\n");
error_report(Proc);
print_output = 0;
error = 1;
}
print_input_info(stdout, Num_Proc, &prob, &pio_info, version);
}
MPI_Allreduce(&error, &gerror, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
if (gerror) goto End;
/* broadcast the command info to all of the processor */
brdcst_cmd_info(Proc, &prob, &pio_info, &mesh);
Zoltan_Set_Param(NULL, "DEBUG_MEMORY", "1");
print_output = Output.Text;
/*
* Create a Zoltan structure.
*/
if ((zz = Zoltan_Create(MPI_COMM_WORLD)) == NULL) {
Gen_Error(0, "fatal: NULL returned from Zoltan_Create()\n");
return 0;
}
if (!setup_zoltan(zz, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from setup_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
/* srand(Proc); Different seeds on different procs. */
srand(1); /* Same seed everywhere. */
if (Test.Dynamic_Weights){
/* Set obj weight dim to 1; can be overridden by user parameter */
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "1");
}
/* Loop over read and balance for a number of iterations */
/* (Useful for testing REUSE parameters in Zoltan.) */
for (iteration = 1; iteration <= Number_Iterations; iteration++) {
if (Proc == 0) {
printf("Starting iteration %d\n", iteration);
fflush(stdout);
}
/*
* now read in the mesh and element information.
* This is the only function call to do this. Upon return,
* the mesh struct and the elements array should be filled.
*/
if (iteration == 1) {
if (!read_mesh(Proc, Num_Proc, &prob, &pio_info, &mesh)) {
Gen_Error(0, "fatal: Error returned from read_mesh\n");
error_report(Proc);
print_output = 0;
goto End;
}
/*
* Create a Zoltan DD for tracking elements during repartitioning.
*/
if (mesh.data_type == ZOLTAN_HYPERGRAPH && !build_elem_dd(&mesh)) {
Gen_Error(0, "fatal: Error returned from build_elem_dd\n");
error_report(Proc);
print_output = 0;
goto End;
}
}
#ifdef KDDKDD_COOL_TEST
/* KDD Cool test of changing number of partitions */
sprintf(cmesg, "%d", Num_Proc * iteration);
Zoltan_Set_Param(zz, "NUM_GLOBAL_PARTS", cmesg);
#endif
/*
* Produce files to verify input.
*/
if (iteration == 1) {
if (Debug_Driver > 2) {
if (!output_results(cmd_file,"in",Proc,Num_Proc,&prob,&pio_info,&mesh)){
Gen_Error(0, "fatal: Error returned from output_results\n");
error_report(Proc);
}
if (Output.Gnuplot)
if (!output_gnu(cmd_file,"in",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "warning: Error returned from output_gnu\n");
error_report(Proc);
}
}
if (Test.Vtx_Inc<0){
/* Read Citeseer data from file */
FILE *fp;
int i=0;
if (Proc==0){
fp = fopen("months.txt", "r");
if (!fp)
printf("ERROR: Couldn't open file months.txt\n");
while (fscanf(fp, "%d", &CITESEER[i])==1){
++i;
}
fclose(fp);
}
MPI_Bcast (CITESEER, 200, MPI_INT, 0, MPI_COMM_WORLD);
}
}
if (Test.Dynamic_Graph > 0.0){
if (mesh.data_type == ZOLTAN_GRAPH) {
remove_random_vertices(&mesh, iteration, Test.Dynamic_Graph);
}
else{
Gen_Error(0, "fatal: \"test dynamic graph\" only works on graphs, not hypergraphs\n");
error_report(Proc);
print_output = 0;
goto End;
}
}
if (Test.Vtx_Inc){
if (mesh.data_type == ZOLTAN_HYPERGRAPH ) {
if (Test.Vtx_Inc>0)
mesh.visible_nvtx += Test.Vtx_Inc; /* Increment uniformly */
else
mesh.visible_nvtx = CITESEER[iteration-1]; /* Citeseer document matrix. */
}
else{
Gen_Error(0, "fatal: \"vertex increment\" only works on hypergraphs\n");
error_report(Proc);
print_output = 0;
goto End;
}
}
/*
* now run Zoltan to get a new load balance and perform
* the migration
*/
#ifdef IGNORE_FIRST_ITERATION_STATS
if (iteration == 1) {
/* Exercise partitioner once on Tbird because first run is slow. */
/* Lee Ann suspects Tbird is loading shared libraries. */
struct Zoltan_Struct *zzcopy;
zzcopy = Zoltan_Copy(zz);
/* Don't do any migration or accumulate any stats. */
if (Proc == 0) printf("%d KDDKDD IGNORING FIRST ITERATION STATS\n", Proc);
Zoltan_Set_Param(zzcopy, "RETURN_LISTS", "NONE");
Zoltan_Set_Param(zzcopy, "FINAL_OUTPUT", "0");
Zoltan_Set_Param(zzcopy, "USE_TIMERS", "0");
if (!run_zoltan(zzcopy, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from run_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
Zoltan_Destroy(&zzcopy);
}
#endif /* IGNORE_FIRST_ITERATION_STATS */
#ifdef RANDOM_DIST
if (iteration % 2 == 0) {
char LB_METHOD[1024];
if (Proc == 0) printf("%d CCCC Randomizing the input\n", Proc);
strcpy(LB_METHOD, prob.method);
strcpy(prob.method, "RANDOM");
Zoltan_Set_Param(zz, "LB_METHOD", "RANDOM");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
if (!run_zoltan(zz, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from run_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
Zoltan_Set_Param(zz, "RETURN_LISTS", "NONE");
Zoltan_Set_Param(zz, "LB_METHOD", LB_METHOD);
strcpy(prob.method, LB_METHOD);
if (Proc == 0) printf("%d CCCC Randomizing the input -- END\n", Proc);
}
#endif /* RANDOM_DIST */
if (!run_zoltan(zz, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from run_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
/* Reset the mesh data structure for next iteration. */
if (iteration < Number_Iterations) {
int i, j;
float tmp;
float twiddle = 0.01;
char str[4];
/* Perturb coordinates of mesh */
if (mesh.data_type == ZOLTAN_GRAPH){
for (i = 0; i < mesh.num_elems; i++) {
for (j = 0; j < mesh.num_dims; j++) {
/* tmp = ((float) rand())/RAND_MAX; *//* Equiv. to sjplimp's test */
tmp = (float) (i % 10) / 10.;
mesh.elements[i].coord[0][j] += twiddle * (2.0*tmp-1.0);
mesh.elements[i].avg_coord[j] = mesh.elements[i].coord[0][j];
}
}
/* Increase weights in some parts */
if (Test.Dynamic_Weights){
/* Randomly pick 10% of parts to "refine" */
/* Note: Assumes at least 10 parts! */
/* Increase vertex weight, and also edge weights? TODO */
j = (int) ((10.0*rand())/RAND_MAX + .5);
for (i = 0; i < mesh.num_elems; i++) {
if ((mesh.elements[i].my_part%10) == j){
mesh.elements[i].cpu_wgt[0] = Test.Dynamic_Weights*(1+rand()%5);
}
}
}
}
/* change the ParMETIS Seed */
sprintf(str, "%d", iteration);
#ifdef ZOLTAN_PARMETIS
Zoltan_Set_Param(zz, "PARMETIS_SEED", str);
#endif
}
} /* End of loop over read and balance */
if (Proc == 0) {
printf("FILE %s: Total: %e seconds in Partitioning\n",
cmd_file, Total_Partition_Time);
printf("FILE %s: Average: %e seconds per Iteration\n",
cmd_file, Total_Partition_Time/Number_Iterations);
}
End:
Zoltan_Destroy(&zz);
if (mesh.dd) Zoltan_DD_Destroy(&(mesh.dd));
Zoltan_Memory_Stats();
/*
* output the results
*/
if (print_output) {
if (!output_results(cmd_file,"out",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "fatal: Error returned from output_results\n");
error_report(Proc);
}
if (Output.Gnuplot) {
if (!output_gnu(cmd_file,"out",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "warning: Error returned from output_gnu\n");
error_report(Proc);
}
}
}
free_mesh_arrays(&mesh);
if (prob.params != NULL) free(prob.params);
MPI_Finalize();
#ifdef VAMPIR
VT_finalize();
#endif
return 0;
}