int main(int argc, char *argv[])
{
int rc, i, ngids, maxcol, ncolors;
float ver;
struct Zoltan_Struct *zz=NULL;
#ifdef ZOLTANV31
int numGidEntries, numLidEntries;
#else
ZOLTAN_GRAPH_EVAL graph;
#endif
int *color;
ZOLTAN_ID_PTR gid_list;
UZData guz, *uz=&guz;
int msg_tag = 9999;
int nlvtx, next, maxdeg=0;
double times[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
double gtimes[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &uz->myRank);
MPI_Comm_size(MPI_COMM_WORLD, &uz->numProcs);
MPI_Barrier(MPI_COMM_WORLD);
times[0] = u_wseconds();
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
fprintf(stderr, "Sorry Zoltan initialize failed...\n");
goto End;
}
zz = Zoltan_Create(MPI_COMM_WORLD);
if (argc<3 && !uz->myRank) {
fprintf(stderr, "usage: %s [meshR] [meshC] [X-point stencil] [procR] [procC] [ws-beta] [<ZoltanParam>=<Val>] ...\n\n", argv[0]);
fprintf(stderr, "ws-beta: is the probablity of adding an edge to a vertex to generate Watts-Strogatz graphs\n");
fprintf(stderr, "Valid values for Stencil are 5, 7 and 9\n");
fprintf(stderr, "Zoltan Coloring Parameters and values are\n");
fprintf(stderr, "\tDISTANCE : 1 or 2\n");
fprintf(stderr, "\tSUPERSTEP_SIZE : suggested >= 100\n");
fprintf(stderr, "\tCOMM_PATTERN : S or A\n");
fprintf(stderr, "\tCOLOR_ORDER : I, B, U\n");
fprintf(stderr, "\tCOLORING_METHOD : F (for now)\n");
fprintf(stderr, "\n");
}
uz->procR = uz->procC = 0;
uz->meshR = uz->meshC = 1024;
uz->stencil = 9;
if (argc>1)
uz->meshR = atoi(argv[1]);
if (argc>2)
uz->meshC = atoi(argv[2]);
if (argc>3)
uz->stencil = atoi(argv[3]);
if (uz->stencil!=5 && uz->stencil!=7 && uz->stencil!=9) {
fprintf(stderr, "\t invalid stencil value. Valid values are 5, 7 and 9. Assumed 9.\n");
uz->stencil = 9;
}
--uz->stencil;
if (argc>4)
uz->procR = atoi(argv[4]);
if (argc>5)
uz->procC = atoi(argv[5]);
if (uz->procR <= 0 || uz->procC <= 0)
computeProcMesh(uz);
if (uz->procR*uz->procC!=uz->numProcs) {
fprintf(stderr, "#Procs=%d but requested %dx%d Proc Mesh Partitioning...\n", uz->numProcs, uz->procR, uz->procC);
goto End;
}
if (argc>6)
uz->beta = atof(argv[6]);
else
uz->beta = 0.0;
/* compute which part of mesh I will compute */
uz->myR = uz->myRank / uz->procC;
uz->myC = uz->myRank % uz->procC;
uz->_sr = uz->myR * (uz->meshR / uz->procR);
uz->_er = (uz->myR+1) * (uz->meshR / uz->procR);
if (uz->_er>uz->meshR)
uz->_er = uz->meshR;
uz->_sc = uz->myC * (uz->meshC / uz->procC);
uz->_ec = (uz->myC+1) * (uz->meshC / uz->procC);
if (uz->_ec>uz->meshC)
uz->_ec = uz->meshC;
if ( (uz->meshR % uz->procR) !=0 || (uz->meshC % uz->procC)!=0) {
printf("Mesh dimensions are not divisible with proc mesh.\nRequested mesh is %dx%d and proc mesh is %d x %d\n", uz->meshR, uz->meshC, uz->procR, uz->procC);
exit(1);
}
nlvtx= (uz->_er-uz->_sr) * (uz->_ec-uz->_sc);
if (uz->myRank==0)
printf("Running %s on %d x %d processor mesh, generating %d-point %d x %d mesh with beta=%.3lf\n", argv[0], uz->procR, uz->procC, uz->stencil+1, uz->meshR, uz->meshC, uz->beta);
times[1] = u_wseconds();
uz->numredge = 0;
uz->redgeto = NULL;
if (uz->beta>0) { /* create random edges for WS graph */
int ngvtx=uz->meshC*uz->meshR, trsh=(int) (uz->beta*100.0);
int ierr=0;
int *edges=NULL, *redges=NULL, *proclist=NULL, nedge;
ZOLTAN_COMM_OBJ *plan;
uz->redgeto = (int *) malloc(nlvtx*sizeof(int));
for (i=0; i<nlvtx; ++i) {
int rv = Zoltan_Rand_InRange(NULL, 100);
if ( rv < trsh) {
if ((uz->redgeto[i] = Zoltan_Rand_InRange(NULL, ngvtx))==gIDfLID(i)) /* is it a self edge */
uz->redgeto[i] = -1;
else
++uz->numredge;
} else
uz->redgeto[i] = -1;
}
edges = (int *) malloc(sizeof(int)*2*uz->numredge);
proclist = (int *) malloc(sizeof(int)*uz->numredge);
next = 0;
for (i=0; i<nlvtx; ++i)
if (uz->redgeto[i]>0) {
edges[2*next] = uz->redgeto[i];
edges[2*next+1] = gIDfLID(i);
proclist[next] = pIDfGID(uz->redgeto[i]);
++next;
}
ierr = Zoltan_Comm_Create(&plan, uz->numredge, proclist, MPI_COMM_WORLD,
msg_tag, &nedge);
redges = (int *) malloc(sizeof(int)*2*nedge);
--msg_tag;
ierr |= Zoltan_Comm_Do(plan, msg_tag, (char *) edges, 2*sizeof(int),
(char *) redges);
ierr |= Zoltan_Comm_Destroy(&plan);
free(proclist);
free(edges);
if (ierr) {
printf("error while communicating edges!\n");
exit(1);
}
xadj = (int *) calloc(1+nlvtx, sizeof(int));
adj = (int *) malloc(sizeof(int)*nedge);
for (i=0; i<nedge; ++i) {
if (redges[2*i] < gID(uz->_sr, uz->_sc) || redges[2*i] >= gID(uz->_er, uz->_ec)) {
printf("[%d/%d] received gid=%d doesn't blong to processor range [%d, %d) should go to proc %d\n", uz->myRank, uz->numProcs, redges[2*i], gID(uz->_sr, uz->_sc), gID(uz->_er, uz->_ec), pIDfGID(redges[2*i]));
}
++xadj[lIDfGID(redges[2*i])];
}
xadj[nlvtx] = nedge;
maxdeg = xadj[0];
for (i=1; i<nlvtx; ++i) {
maxdeg = xadj[i]>maxdeg ? xadj[i] : maxdeg;
xadj[i] += xadj[i-1];
}
for (i=0; i<nedge; ++i) {
int u = lIDfGID(redges[2*i]);
int v = redges[2*i+1];
adj[--xadj[u]] = v;
}
free(redges);
}
maxdeg += uz->stencil+1;
adjTemp = (int *) malloc(sizeof(int)*2*maxdeg);
times[2] = u_wseconds();
/*
printf("My rank %d/%d at proc-mesh loc (%d, %d) generating [%d, %d) x [%d, %d) + %d random edges TotEdge=%d\n", uz->myRank, uz->numProcs, uz->myR, uz->myC, uz->_sr, uz->_er, uz->_sc, uz->_ec, uz->numredge, xadj[nlvtx]); */
printStats("Number of Vertices ", nlvtx, uz->myRank, uz->numProcs);
if (xadj)
printStats("Number of Rand Edges", xadj[nlvtx], uz->myRank, uz->numProcs);
/* General parameters */
#ifndef ZOLTANV31
Zoltan_Set_Param(zz, "GRAPH_BUILD_TYPE", "FAST_NO_DUP");
#endif
/* General parameters */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "3");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");
/* coloring parameters */
Zoltan_Set_Param(zz, "SUPERSTEP_SIZE", "500"); /* let's make S=500 default */
for (i=7; i<argc; ++i) {
char param[256], *eq;
if (!uz->myRank)
printf("processing argv[%d]='%s'\n", i, argv[i]);
strncpy(param, argv[i], sizeof(param));
eq = strchr(param, '=');
if (!eq) {
fprintf(stderr, "invalid argument '%s', Zoltan Paramters should be in the format <ZoltanParam>=<Val>\n", param);
goto End;
}
*eq = 0;
Zoltan_Set_Param(zz, param, eq+1);
}
#if 0
/* Graph parameters */
Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
#endif
/* set call backs */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, uz);
Zoltan_Set_Obj_List_Fn(zz, get_object_list, uz);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, uz);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, uz);
#if 0
#ifndef ZOLTANV31
Zoltan_LB_Eval_Graph(zz, 0, &graph);
if (!uz->myRank) {
printf("EdgeCut Min=%8.0f Max=%8.0f Sum=%8.0f\n", graph.cuts[EVAL_GLOBAL_MIN], graph.cuts[EVAL_GLOBAL_MAX], graph.cuts[EVAL_GLOBAL_SUM]);
printf("#Vertices Min=%8.0f Max=%8.0f Sum=%8.0f imbal=%.2f\n", graph.nobj[EVAL_GLOBAL_MIN], graph.nobj[EVAL_GLOBAL_MAX], graph.nobj[EVAL_GLOBAL_SUM], graph.obj_imbalance);
}
#endif
#endif
/* now color */
ngids = get_number_of_objects(uz, &rc);
gid_list = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * ngids);
#ifndef ZOLTANV31
next = 0;
for (i=uz->_sr; i<uz->_er; ++i) {
int j;
for (j=uz->_sc; j<uz->_ec; ++j) {
gid_list[next++] = i*uz->meshC + j;
}
}
#endif
color = (int *) malloc(sizeof(int) * ngids);
MPI_Barrier(MPI_COMM_WORLD);
times[3] = u_wseconds();
#ifdef ZOLTANV31
rc = Zoltan_Color(zz, /* input (all remaining fields are output) */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
ngids, /* #objects to color in this proc */
gid_list, /* global ids of colored vertices */
NULL, /* we ignore local ids */
color); /* result color */
#else
rc = Zoltan_Color(zz, /* input (all remaining fields are output) */
1, /* Number of integers used for a global ID */
ngids, /* #objects to color in this proc */
gid_list, /* global ids of colored vertices */
color); /* result color */
#endif
MPI_Barrier(MPI_COMM_WORLD);
times[4] = u_wseconds();
MPI_Reduce(times, gtimes, 5, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (rc != ZOLTAN_OK)
fprintf(stderr, "Zoltan_Color failed with return code %d...\n", rc);
for (maxcol=i=0; i<ngids; ++i)
if (color[i] > maxcol)
maxcol = color[i];
MPI_Reduce(&maxcol, &ncolors, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
if (uz->myRank==0) {
struct rusage usage;
printf("%s setup Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[1]-times[0], gtimes[1]-gtimes[0]);
printf("%s gen rand edges Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[2]-times[1], gtimes[2]-gtimes[1]);
printf("%s set gids Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[3]-times[2], gtimes[3]-gtimes[2]);
printf("%s Zoltan_Color call Proc-0: %8.2lf Max: %8.2lf\n", argv[0], times[4]-times[3], gtimes[4]-gtimes[3]);
printf("%s Coloring Time : %.2lf # Colors used : %d\n", argv[0], gtimes[4]-gtimes[0], ncolors);
getrusage(RUSAGE_SELF, &usage);
printf("%s maxrss=%ld minflt=%ld majflt=%ld nswap=%ld\n", argv[0], usage.ru_maxrss, usage.ru_minflt, usage.ru_majflt, usage.ru_nswap);
}
#ifdef _DEBUG
saveColor(argv[0], uz, (int *) gid_list, color, ngids);
#endif
/******************************************************************
** Clean up
******************************************************************/
if (gid_list)
free(gid_list);
if (color)
free(color);
if (xadj)
free(xadj);
if (adj)
free(adj);
if (adjTemp)
free(adjTemp);
if (uz->redgeto)
free(uz->redgeto);
End:
Zoltan_Destroy(&zz);
MPI_Finalize();
return 0;
}
/*### MAIN ########################################################################*/
int main (int argc, char *argv[]) {
int i, j;
char *readsfile, *readsfile2 = NULL, *indexdir=NULL, *indexprefix=NULL, *mainworkdir, *outdir, workdir[1000];
char command[1000], usage[1000];
int ninpfile = 0;
char inpfile[10][1000], **ofname;
char fname1[500];
int nline_per_read = -1;
MPI_Comm comm=MPI_COMM_WORLD;
int myProc, nProc;
FILE *pfp;
double times[5][2];
//#############################################
//### PARSE INPUT PARAMETERS ##################
sprintf(usage, "Usage: %s <work directory> <output directory> <reads file> [-r <reads file2>] [-l <num.of lines per read>] [-i <index directory> <index prefix>] [-f <inpfile>]\n", argv[0]);
MPI_Init(&argc, &argv);
MPI_Comm_size(comm, &nProc);
MPI_Comm_rank(comm, &myProc);
if (argc < 4) {
if (myProc == 0)
printf("%s", usage);
exit(-1);
}
i=1;
mainworkdir = argv[i++];
outdir = argv[i++];
readsfile = argv[i++];
for ( ; i<argc ; i++) {
if (!strcmp(argv[i], "-r")) {
readsfile2 = argv[++i];
} else if (!strcmp(argv[i], "-i")) {
indexdir = argv[++i];
indexprefix = argv[++i];
} else if (!strcmp(argv[i], "-f")) {
if (ninpfile > 10 && myProc == 0) {
fprintf (stderr, "pmap_dist error: At most 10 inpfiles can be specified\n");
exit(-1);
}
strcpy(inpfile[ninpfile++], argv[++i]);
} else if (!strcmp(argv[i], "-l")) {
nline_per_read = atoi(argv[++i]);
if (nline_per_read <= 0 && myProc == 0) {
fprintf(stderr, "pmap_dist error: Number of lines per read should be greater than 0\n");
exit(-1);
}
} else if (myProc == 0) {
fprintf (stderr, "pmap_dist error: Unknown option %s\n", argv[i]);
exit(-1);
}
}
// Create a workdir, indexdir and outdir
sprintf(workdir, "%s/pmap_node%d", mainworkdir, myProc);
sprintf(command, "mkdir -p %s/index", workdir);
system(command);
if (myProc == 0) {
sprintf(command, "mkdir -p %s", outdir);
system(command);
}
// Check if files/folders exist
if (myProc == 0) {
if (indexdir && !folder_exists(indexdir)) {
if (indexdir[0] != '/')
fprintf(stderr, "pmap_dist error: Cannot find indexdir: %s. Make sure the path is an absolute path.\n", indexdir);
else
fprintf(stderr, "pmap_dist error: Cannot find indexdir: %s\n", indexdir);
exit(-1);
}
if (!file_exists(readsfile)) {
if (readsfile[0] != '/')
fprintf(stderr, "pmap_dist error: Cannot find readsfile: %s. Make sure the path is an absolute path.\n", readsfile);
else
fprintf(stderr, "pmap_dist error: Cannot find readsfile: %s\n", readsfile);
exit(-1);
}
if (readsfile2 && !file_exists(readsfile2)) {
if (readsfile2[0] != '/')
fprintf(stderr, "pmap_dist error: Cannot find readsfile2: %s. Make sure the path is an absolute path.\n", readsfile2);
else
fprintf(stderr, "pmap_dist error: Cannot find readsfile2: %s\n", readsfile2);
exit(-1);
}
for (i=0; i<ninpfile; i++) {
if (!file_exists(inpfile[i])) {
if (inpfile[i][0] != '/')
fprintf(stderr, "pmap_dist error: Cannot find inpfile: %s. Make sure the path is an absolute path.\n", inpfile[i]);
else
fprintf(stderr, "pmap_dist error: Cannot find inpfile: %s\n", inpfile[i]);
exit(-1);
}
}
if (nline_per_read == -1) {
nline_per_read = get_nline_per_read(readsfile);
}
}
// Memory allocation
ofname = (char **) malloc(sizeof(char *) * nProc);
for (i=0; i<nProc; i++)
ofname[i] = malloc(sizeof(char) * 1000);
//#########################################################
//### PARTITION/DISTRIBUTE THE DATA #######################
times[1][0] = u_wseconds();
sprintf(fname1, "%s/pmap.reads", workdir);
Partition_Reads (readsfile, fname1, nline_per_read, nProc, myProc, comm);
if (readsfile2) {
sprintf(fname1, "%s/pmap.reads2", workdir);
Partition_Reads (readsfile2, fname1, nline_per_read, nProc, myProc, comm);
}
times[1][1] = u_wseconds();
times[2][0] = u_wseconds();
if (indexdir) {
sprintf(fname1, "%s/index", workdir);
Broadcast_Folder(indexdir, indexprefix, fname1, "index", nProc, myProc, comm);
}
times[2][1] = u_wseconds();
times[3][0] = u_wseconds();
for (i=0; i<ninpfile; i++) {
char *ptr = strrchr(inpfile[i], '/');
for (j=0; j<nProc; j++)
sprintf(ofname[j], "%s/%s", workdir, ptr+1);
Broadcast_File(inpfile[i], ofname, 0, myProc, comm);
}
times[3][1] = u_wseconds();
//#########################################################
//### PRINT TIMING INFO AND FINALIZE ######################
if (myProc == 0) {
sprintf(fname1, "%s/dist_time.txt", outdir);
pfp = fopen(fname1, "w");
fprintf(pfp, "Partition reads : %8.1lf\n", times[1][1]-times[1][0]);
fprintf(pfp, "Replicate index : %8.1lf\n", times[2][1]-times[2][0]);
fprintf(pfp, "Replicate inpfile(s): %8.1lf\n", times[3][1]-times[3][0]);
fclose(pfp);
}
for (i=0; i<nProc; i++)
free(ofname[i]);
free(ofname);
MPI_Finalize();
return 0;
}
