int main(int argc, char* argv[]) {
int id=0, numprocs;
int color;
MPI_Comm local;
double t1, t2;
VT_initialize(&argc, &argv);
VT_symdef(1, "step", "USR");
VT_symdef(2, "sequential", "USR");
VT_symdef(3, "p2p", "USR");
VT_symdef(4, "parallel", "USR");
VT_symdef(5, "main", "USR");
VT_begin(5);
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &id);
printf("%03d: ctest-vt start\n", id);
color = (id >= numprocs/2);
MPI_Comm_split(MPI_COMM_WORLD, color, id, &local);
t1 = MPI_Wtime();
parallel(MPI_COMM_WORLD);
parallel(local);
parallel(MPI_COMM_WORLD);
t2 = MPI_Wtime();
MPI_Comm_free(&local);
MPI_Finalize();
printf("%03d: ctest-vt end (%12.9f)\n", id, (t2-t1));
VT_end(5);
VT_finalize();
return 0;
}
int main(int argc, char *argv[])
{
/* Local declarations. */
struct Zoltan_Struct *zz = NULL;
char *cmd_file;
char cmesg[256]; /* for error messages */
float version;
int Proc, Num_Proc;
int iteration;
int error, gerror;
int print_output = 1;
MESH_INFO mesh; /* mesh information struct */
PARIO_INFO pio_info;
PROB_INFO prob;
/***************************** BEGIN EXECUTION ******************************/
/* initialize MPI */
MPI_Init(&argc, &argv);
#ifdef VAMPIR
VT_initialize(&argc, &argv);
#endif
/* get some machine information */
MPI_Comm_rank(MPI_COMM_WORLD, &Proc);
MPI_Comm_size(MPI_COMM_WORLD, &Num_Proc);
my_rank = Proc;
#ifdef HOST_LINUX
signal(SIGSEGV, meminfo_signal_handler);
signal(SIGINT, meminfo_signal_handler);
signal(SIGTERM, meminfo_signal_handler);
signal(SIGABRT, meminfo_signal_handler);
signal(SIGFPE, meminfo_signal_handler);
#endif
#ifdef ZOLTAN_PURIFY
printf("%d of %d ZDRIVE LAUNCH pid = %d file = %s\n",
Proc, Num_Proc, getpid(), argv[1]);
#endif
/* Initialize flags */
Test.DDirectory = 0;
Test.Local_Parts = 0;
Test.Fixed_Objects = 0;
Test.Drops = 0;
Test.RCB_Box = 0;
Test.Multi_Callbacks = 0;
Test.Graph_Callbacks = 1;
Test.Hypergraph_Callbacks = 1;
Test.Gen_Files = 0;
Test.Null_Lists = NO_NULL_LISTS;
Test.Dynamic_Weights = .0;
Test.Dynamic_Graph = .0;
Test.Vtx_Inc = 0;
Output.Text = 1;
Output.Gnuplot = 0;
Output.Nemesis = 0;
Output.Plot_Partition = 0;
Output.Mesh_Info_File = 0;
/* Interpret the command line */
switch(argc)
{
case 1:
cmd_file = "zdrive.inp";
break;
case 2:
cmd_file = argv[1];
break;
default:
fprintf(stderr, "MAIN: ERROR in command line,");
if(Proc == 0)
{
fprintf(stderr, " usage:\n");
fprintf(stderr, "\t%s [command file]", DRIVER_NAME);
}
exit(1);
break;
}
/* initialize Zoltan */
if ((error = Zoltan_Initialize(argc, argv, &version)) != ZOLTAN_OK) {
sprintf(cmesg, "fatal: Zoltan_Initialize returned error code, %d", error);
Gen_Error(0, cmesg);
error_report(Proc);
print_output = 0;
goto End;
}
/* initialize some variables */
initialize_mesh(&mesh, Proc);
pio_info.dsk_list_cnt = -1;
pio_info.file_comp = STANDARD;
pio_info.num_dsk_ctrlrs = -1;
pio_info.pdsk_add_fact = -1;
pio_info.zeros = -1;
pio_info.file_type = -1;
pio_info.chunk_reader = 0;
pio_info.init_dist_type = -1;
pio_info.init_size = ZOLTAN_ID_INVALID;
pio_info.init_dim = -1;
pio_info.init_vwgt_dim = -1;
pio_info.init_dist_pins = -1;
pio_info.pdsk_root[0] = '\0';
pio_info.pdsk_subdir[0] = '\0';
pio_info.pexo_fname[0] = '\0';
prob.method[0] = '\0';
prob.num_params = 0;
prob.params = NULL;
/* Read in the ascii input file */
error = gerror = 0;
if (Proc == 0) {
printf("\n\nReading the command file, %s\n", cmd_file);
if (!read_cmd_file(cmd_file, &prob, &pio_info, NULL)) {
sprintf(cmesg,"fatal: Could not read in the command file"
" \"%s\"!\n", cmd_file);
Gen_Error(0, cmesg);
error_report(Proc);
print_output = 0;
error = 1;
}
if (!check_inp(&prob, &pio_info)) {
Gen_Error(0, "fatal: Error in user specified parameters.\n");
error_report(Proc);
print_output = 0;
error = 1;
}
print_input_info(stdout, Num_Proc, &prob, &pio_info, version);
}
MPI_Allreduce(&error, &gerror, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
if (gerror) goto End;
/* broadcast the command info to all of the processor */
brdcst_cmd_info(Proc, &prob, &pio_info, &mesh);
Zoltan_Set_Param(NULL, "DEBUG_MEMORY", "1");
print_output = Output.Text;
/*
* Create a Zoltan structure.
*/
if ((zz = Zoltan_Create(MPI_COMM_WORLD)) == NULL) {
Gen_Error(0, "fatal: NULL returned from Zoltan_Create()\n");
return 0;
}
if (!setup_zoltan(zz, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from setup_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
/* srand(Proc); Different seeds on different procs. */
srand(1); /* Same seed everywhere. */
if (Test.Dynamic_Weights){
/* Set obj weight dim to 1; can be overridden by user parameter */
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "1");
}
/* Loop over read and balance for a number of iterations */
/* (Useful for testing REUSE parameters in Zoltan.) */
for (iteration = 1; iteration <= Number_Iterations; iteration++) {
if (Proc == 0) {
printf("Starting iteration %d\n", iteration);
fflush(stdout);
}
/*
* now read in the mesh and element information.
* This is the only function call to do this. Upon return,
* the mesh struct and the elements array should be filled.
*/
if (iteration == 1) {
if (!read_mesh(Proc, Num_Proc, &prob, &pio_info, &mesh)) {
Gen_Error(0, "fatal: Error returned from read_mesh\n");
error_report(Proc);
print_output = 0;
goto End;
}
/*
* Create a Zoltan DD for tracking elements during repartitioning.
*/
if (mesh.data_type == ZOLTAN_HYPERGRAPH && !build_elem_dd(&mesh)) {
Gen_Error(0, "fatal: Error returned from build_elem_dd\n");
error_report(Proc);
print_output = 0;
goto End;
}
}
#ifdef KDDKDD_COOL_TEST
/* KDD Cool test of changing number of partitions */
sprintf(cmesg, "%d", Num_Proc * iteration);
Zoltan_Set_Param(zz, "NUM_GLOBAL_PARTS", cmesg);
#endif
/*
* Produce files to verify input.
*/
if (iteration == 1) {
if (Debug_Driver > 2) {
if (!output_results(cmd_file,"in",Proc,Num_Proc,&prob,&pio_info,&mesh)){
Gen_Error(0, "fatal: Error returned from output_results\n");
error_report(Proc);
}
if (Output.Gnuplot)
if (!output_gnu(cmd_file,"in",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "warning: Error returned from output_gnu\n");
error_report(Proc);
}
}
if (Test.Vtx_Inc<0){
/* Read Citeseer data from file */
FILE *fp;
int i=0;
if (Proc==0){
fp = fopen("months.txt", "r");
if (!fp)
printf("ERROR: Couldn't open file months.txt\n");
while (fscanf(fp, "%d", &CITESEER[i])==1){
++i;
}
fclose(fp);
}
MPI_Bcast (CITESEER, 200, MPI_INT, 0, MPI_COMM_WORLD);
}
}
if (Test.Dynamic_Graph > 0.0){
if (mesh.data_type == ZOLTAN_GRAPH) {
remove_random_vertices(&mesh, iteration, Test.Dynamic_Graph);
}
else{
Gen_Error(0, "fatal: \"test dynamic graph\" only works on graphs, not hypergraphs\n");
error_report(Proc);
print_output = 0;
goto End;
}
}
if (Test.Vtx_Inc){
if (mesh.data_type == ZOLTAN_HYPERGRAPH ) {
if (Test.Vtx_Inc>0)
mesh.visible_nvtx += Test.Vtx_Inc; /* Increment uniformly */
else
mesh.visible_nvtx = CITESEER[iteration-1]; /* Citeseer document matrix. */
}
else{
Gen_Error(0, "fatal: \"vertex increment\" only works on hypergraphs\n");
error_report(Proc);
print_output = 0;
goto End;
}
}
/*
* now run Zoltan to get a new load balance and perform
* the migration
*/
#ifdef IGNORE_FIRST_ITERATION_STATS
if (iteration == 1) {
/* Exercise partitioner once on Tbird because first run is slow. */
/* Lee Ann suspects Tbird is loading shared libraries. */
struct Zoltan_Struct *zzcopy;
zzcopy = Zoltan_Copy(zz);
/* Don't do any migration or accumulate any stats. */
if (Proc == 0) printf("%d KDDKDD IGNORING FIRST ITERATION STATS\n", Proc);
Zoltan_Set_Param(zzcopy, "RETURN_LISTS", "NONE");
Zoltan_Set_Param(zzcopy, "FINAL_OUTPUT", "0");
Zoltan_Set_Param(zzcopy, "USE_TIMERS", "0");
if (!run_zoltan(zzcopy, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from run_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
Zoltan_Destroy(&zzcopy);
}
#endif /* IGNORE_FIRST_ITERATION_STATS */
#ifdef RANDOM_DIST
if (iteration % 2 == 0) {
char LB_METHOD[1024];
if (Proc == 0) printf("%d CCCC Randomizing the input\n", Proc);
strcpy(LB_METHOD, prob.method);
strcpy(prob.method, "RANDOM");
Zoltan_Set_Param(zz, "LB_METHOD", "RANDOM");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
if (!run_zoltan(zz, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from run_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
Zoltan_Set_Param(zz, "RETURN_LISTS", "NONE");
Zoltan_Set_Param(zz, "LB_METHOD", LB_METHOD);
strcpy(prob.method, LB_METHOD);
if (Proc == 0) printf("%d CCCC Randomizing the input -- END\n", Proc);
}
#endif /* RANDOM_DIST */
if (!run_zoltan(zz, Proc, &prob, &mesh, &pio_info)) {
Gen_Error(0, "fatal: Error returned from run_zoltan\n");
error_report(Proc);
print_output = 0;
goto End;
}
/* Reset the mesh data structure for next iteration. */
if (iteration < Number_Iterations) {
int i, j;
float tmp;
float twiddle = 0.01;
char str[4];
/* Perturb coordinates of mesh */
if (mesh.data_type == ZOLTAN_GRAPH){
for (i = 0; i < mesh.num_elems; i++) {
for (j = 0; j < mesh.num_dims; j++) {
/* tmp = ((float) rand())/RAND_MAX; *//* Equiv. to sjplimp's test */
tmp = (float) (i % 10) / 10.;
mesh.elements[i].coord[0][j] += twiddle * (2.0*tmp-1.0);
mesh.elements[i].avg_coord[j] = mesh.elements[i].coord[0][j];
}
}
/* Increase weights in some parts */
if (Test.Dynamic_Weights){
/* Randomly pick 10% of parts to "refine" */
/* Note: Assumes at least 10 parts! */
/* Increase vertex weight, and also edge weights? TODO */
j = (int) ((10.0*rand())/RAND_MAX + .5);
for (i = 0; i < mesh.num_elems; i++) {
if ((mesh.elements[i].my_part%10) == j){
mesh.elements[i].cpu_wgt[0] = Test.Dynamic_Weights*(1+rand()%5);
}
}
}
}
/* change the ParMETIS Seed */
sprintf(str, "%d", iteration);
#ifdef ZOLTAN_PARMETIS
Zoltan_Set_Param(zz, "PARMETIS_SEED", str);
#endif
}
} /* End of loop over read and balance */
if (Proc == 0) {
printf("FILE %s: Total: %e seconds in Partitioning\n",
cmd_file, Total_Partition_Time);
printf("FILE %s: Average: %e seconds per Iteration\n",
cmd_file, Total_Partition_Time/Number_Iterations);
}
End:
Zoltan_Destroy(&zz);
if (mesh.dd) Zoltan_DD_Destroy(&(mesh.dd));
Zoltan_Memory_Stats();
/*
* output the results
*/
if (print_output) {
if (!output_results(cmd_file,"out",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "fatal: Error returned from output_results\n");
error_report(Proc);
}
if (Output.Gnuplot) {
if (!output_gnu(cmd_file,"out",Proc,Num_Proc,&prob,&pio_info,&mesh)) {
Gen_Error(0, "warning: Error returned from output_gnu\n");
error_report(Proc);
}
}
}
free_mesh_arrays(&mesh);
if (prob.params != NULL) free(prob.params);
MPI_Finalize();
#ifdef VAMPIR
VT_finalize();
#endif
return 0;
}
