/***************************************************************************** * This function computes a partitioning using coordinate data. *****************************************************************************/ void ParMETIS_PartGeomKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *ndims, float *xyz, int *nparts, int *options, int *edgecut, idxtype *part, MPI_Comm *comm) { int i; int ncon = 1; float *tpwgts, ubvec[MAXNCON]; int myoptions[10]; tpwgts = fmalloc(*nparts*ncon, "tpwgts"); for (i=0; i<*nparts*ncon; i++) tpwgts[i] = 1.0/(float)(*nparts); for (i=0; i<ncon; i++) ubvec[i] = UNBALANCE_FRACTION; if (options[0] == 0) { myoptions[0] = 0; } else { myoptions[0] = 1; myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL]; myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED; } ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ndims, xyz, &ncon, nparts, tpwgts, ubvec, myoptions, edgecut, part, comm); GKfree((void **)&tpwgts, LTERM); return; }
/************************************************************************* * Let the game begin **************************************************************************/ int main(int argc, char *argv[]) { idx_t i, j, npes, mype, optype, nparts, adptf, options[10]; idx_t *part=NULL, *sizes=NULL; graph_t graph; real_t ipc2redist, *xyz=NULL, *tpwgts=NULL, ubvec[MAXNCON]; MPI_Comm comm; idx_t numflag=0, wgtflag=0, ndims, edgecut; char xyzfilename[8192]; MPI_Init(&argc, &argv); MPI_Comm_dup(MPI_COMM_WORLD, &comm); gkMPI_Comm_size(comm, &npes); gkMPI_Comm_rank(comm, &mype); if (argc != 8) { if (mype == 0) printf("Usage: %s <graph-file> <op-type> <nparts> <adapth-factor> <ipc2redist> <dbglvl> <seed>\n", argv[0]); MPI_Finalize(); exit(0); } optype = atoi(argv[2]); nparts = atoi(argv[3]); adptf = atoi(argv[4]); ipc2redist = atof(argv[5]); options[0] = 1; options[PMV3_OPTION_DBGLVL] = atoi(argv[6]); options[PMV3_OPTION_SEED] = atoi(argv[7]); if (mype == 0) printf("reading file: %s\n", argv[1]); ParallelReadGraph(&graph, argv[1], comm); /* Remove the edges for testing */ /*iset(graph.vtxdist[mype+1]-graph.vtxdist[mype]+1, 0, graph.xadj); */ rset(graph.ncon, 1.05, ubvec); tpwgts = rmalloc(nparts*graph.ncon, "tpwgts"); rset(nparts*graph.ncon, 1.0/(real_t)nparts, tpwgts); /* ChangeToFortranNumbering(graph.vtxdist, graph.xadj, graph.adjncy, mype, npes); numflag = 1; nvtxs = graph.vtxdist[mype+1]-graph.vtxdist[mype]; nedges = graph.xadj[nvtxs]; printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1)); */ if (optype >= 20) { sprintf(xyzfilename, "%s.xyz", argv[1]); xyz = ReadTestCoordinates(&graph, xyzfilename, &ndims, comm); } if (mype == 0) printf("finished reading file: %s\n", argv[1]); part = ismalloc(graph.nvtxs, mype%nparts, "main: part"); sizes = imalloc(2*npes, "main: sizes"); switch (optype) { case 1: wgtflag = 3; ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD); break; case 2: wgtflag = 3; options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED; ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD); break; case 3: options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED; graph.vwgt = ismalloc(graph.nvtxs, 1, "main: vwgt"); if (npes > 1) { AdaptGraph(&graph, adptf, comm); } else { wgtflag = 3; ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); printf("Initial partitioning with edgecut of %"PRIDX"\n", edgecut); for (i=0; i<graph.ncon; i++) { for (j=0; j<graph.nvtxs; j++) { if (part[j] == i) graph.vwgt[j*graph.ncon+i] = adptf; else graph.vwgt[j*graph.ncon+i] = 1; } } } wgtflag = 3; ParMETIS_V3_AdaptiveRepart(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec, &ipc2redist, options, &edgecut, part, &comm); break; case 4: ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options, part, sizes, &comm); /* WriteOVector(argv[1], graph.vtxdist, part, comm); */ break; case 5: ParMETIS_SerialNodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options, part, sizes, &comm); /* WriteOVector(argv[1], graph.vtxdist, part, comm); */ printf("%"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX"\n", sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], sizes[5], sizes[6]); break; case 11: /* TestAdaptiveMETIS(graph.vtxdist, graph.xadj, graph.adjncy, part, options, adptf, comm); */ break; case 20: wgtflag = 3; ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &ndims, xyz, &graph.ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); break; case 21: ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm); break; } /* printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1)); */ gk_free((void **)&part, &sizes, &tpwgts, &graph.vtxdist, &graph.xadj, &graph.adjncy, &graph.vwgt, &graph.adjwgt, &xyz, LTERM); MPI_Comm_free(&comm); MPI_Finalize(); return 0; }
void TestParMetis_GPart(char *filename, char *xyzfile, MPI_Comm comm) { idx_t ncon, nparts, npes, mype, opt2, realcut; graph_t graph, mgraph; idx_t *part, *mpart, *savepart, *order, *sizes; idx_t numflag=0, wgtflag=0, options[10], edgecut, ndims; real_t ipc2redist, *xyz=NULL, *tpwgts = NULL, ubvec[MAXNCON]; gkMPI_Comm_size(comm, &npes); gkMPI_Comm_rank(comm, &mype); ParallelReadGraph(&graph, filename, comm); if (xyzfile) xyz = ReadTestCoordinates(&graph, xyzfile, &ndims, comm); gkMPI_Barrier(comm); part = imalloc(graph.nvtxs, "TestParMetis_V3: part"); tpwgts = rmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts"); rset(MAXNCON, 1.05, ubvec); graph.vwgt = ismalloc(graph.nvtxs*5, 1, "TestParMetis_GPart: vwgt"); /*====================================================================== / ParMETIS_V3_PartKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; wgtflag = 2; numflag = 0; edgecut = 0; for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) { for (ncon=1; ncon<=NCON; ncon++) { if (ncon > 1 && nparts > 1) Mc_AdaptGraph(&graph, part, ncon, nparts, comm); else iset(graph.nvtxs, 1, graph.vwgt); if (mype == 0) printf("\nTesting ParMETIS_V3_PartKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts); rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) { printf("ParMETIS_V3_PartKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } if (mype == 0) printf("\nTesting ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts); options[3] = PARMETIS_PSR_UNCOUPLED; ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) { printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } } } /*====================================================================== / ParMETIS_V3_PartGeomKway /=======================================================================*/ if (xyzfile != NULL) { options[0] = 1; options[1] = 3; options[2] = 1; wgtflag = 2; numflag = 0; for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) { for (ncon=1; ncon<=NCON; ncon++) { if (ncon > 1) Mc_AdaptGraph(&graph, part, ncon, nparts, comm); else iset(graph.nvtxs, 1, graph.vwgt); if (mype == 0) printf("\nTesting ParMETIS_V3_PartGeomKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts); rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) printf("ParMETIS_V3_PartGeomKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } } } /*====================================================================== / ParMETIS_V3_PartGeom /=======================================================================*/ if (xyz != NULL) { wgtflag = 0; numflag = 0; if (mype == 0) printf("\nTesting ParMETIS_V3_PartGeom\n"); ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm); realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) printf("ParMETIS_V3_PartGeom reported a cut of %"PRIDX"\n", realcut); } /*====================================================================== / Coupled ParMETIS_V3_RefineKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; options[3] = PARMETIS_PSR_COUPLED; nparts = npes; wgtflag = 0; numflag = 0; ncon = 1; rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); if (mype == 0) printf("\nTesting coupled ParMETIS_V3_RefineKway with default options (before move)\n"); ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); /* Compute a good partition and move the graph. Do so quietly! */ options[0] = 0; nparts = npes; wgtflag = 0; numflag = 0; ncon = 1; rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut, part, &comm); TestMoveGraph(&graph, &mgraph, part, comm); gk_free((void **)&(graph.vwgt), LTERM); mpart = ismalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart"); savepart = imalloc(mgraph.nvtxs, "TestParMetis_V3: savepart"); /*====================================================================== / Coupled ParMETIS_V3_RefineKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; options[3] = PARMETIS_PSR_COUPLED; nparts = npes; wgtflag = 0; numflag = 0; for (ncon=1; ncon<=NCON; ncon++) { if (mype == 0) printf("\nTesting coupled ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts); rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, mpart, &comm); realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart, filename, comm); if (mype == 0) printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } /*ADAPTIVE:*/ /*====================================================================== / ParMETIS_V3_AdaptiveRepart /=======================================================================*/ mgraph.vwgt = ismalloc(mgraph.nvtxs*NCON, 1, "TestParMetis_V3: mgraph.vwgt"); mgraph.vsize = ismalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize"); AdaptGraph(&mgraph, 4, comm); options[0] = 1; options[1] = 7; options[2] = 1; options[3] = PARMETIS_PSR_COUPLED; wgtflag = 2; numflag = 0; for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) { options[0] = 0; ncon = 1; wgtflag = 0; rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, savepart, &comm); options[0] = 1; wgtflag = 2; for (ncon=1; ncon<=NCON; ncon++) { rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts); if (ncon > 1) Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm); else AdaptGraph(&mgraph, 4, comm); for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) { icopy(mgraph.nvtxs, savepart, mpart); if (mype == 0) printf("\nTesting ParMETIS_V3_AdaptiveRepart with ipc2redist: %.3"PRREAL", ncon: %"PRIDX", nparts: %"PRIDX"\n", ipc2redist, ncon, nparts); ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, mgraph.vwgt, mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, &ipc2redist, options, &edgecut, mpart, &comm); realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart, filename, comm); if (mype == 0) printf("ParMETIS_V3_AdaptiveRepart reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut, (edgecut == realcut ? "OK" : "ERROR"), realcut); } } } gk_free((void **)&tpwgts, &part, &mpart, &savepart, &xyz, &mgraph.vwgt, &mgraph.vsize, LTERM); }
int Zoltan_ParMetis( ZZ *zz, /* Zoltan structure */ float *part_sizes, /* Input: Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ int *num_imp, /* number of objects to be imported */ ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */ ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */ int **imp_procs, /* list of processors to import from */ int **imp_to_part, /* list of partitions to which imported objects are assigned. */ int *num_exp, /* number of objects to be exported */ ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */ ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */ int **exp_procs, /* list of processors to export to */ int **exp_to_part /* list of partitions to which exported objects are assigned. */ ) { char *yo = "Zoltan_ParMetis"; int ierr; ZOLTAN_Third_Graph gr; ZOLTAN_Third_Geom *geo = NULL; ZOLTAN_Third_Vsize vsp; ZOLTAN_Third_Part prt; ZOLTAN_Output_Part part; ZOLTAN_ID_PTR global_ids = NULL; ZOLTAN_ID_PTR local_ids = NULL; int use_timers = 0; int timer_p = -1; int get_times = 0; double times[5]; double pmv3_itr = 0.0; realtype itr = 0.0; indextype options[MAX_PARMETIS_OPTIONS]; char alg[MAX_PARAM_STRING_LEN]; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */ /* change our zz struct. */ #endif indextype i; realtype *imb_tols; indextype ncon; indextype edgecut; indextype wgtflag; indextype numflag = 0; indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */ ZOLTAN_TRACE_ENTER(zz, yo); Zoltan_Third_Init(&gr, &prt, &vsp, &part, imp_gids, imp_lids, imp_procs, imp_to_part, exp_gids, exp_lids, exp_procs, exp_to_part); if (sizeof(realtype) != sizeof(float)) { int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1); prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype)); for (i = 0; i < tmp; i++) prt.input_part_sizes[i] = (realtype) part_sizes[i]; /* KDD 2/2014: removed re-scaling part sizes so they sum to one. * part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes. * plus, the code here was wrong for multiple object weights. * similar scaling code did not exist in the Scotch interface. */ prt.part_sizes = prt.input_part_sizes; } else prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes; ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } gr.graph_type = 0; #ifdef ZOLTAN_PARMETIS SET_GLOBAL_GRAPH(&gr.graph_type); /* Select type of graph, negative because we impose them */ /* TODO: add a parameter to select the type, shared with Scotch */ /* if (strcmp (graph_type, "GLOBAL") != 0) { */ /* gr.graph_type = - LOCAL_GRAPH; */ /* if (zz->Num_Proc > 1) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */ /* gr.graph_type = - GLOBAL_GRAPH; */ /* retval = ZOLTAN_WARN; */ /* } */ /* } */ #else /* graph is local */ SET_LOCAL_GRAPH(&gr.graph_type); #endif /* ZOLTAN_PARMETIS */ /* Some algorithms use geometry data */ if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */ geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom)); memset (geo, 0, sizeof(ZOLTAN_Third_Geom)); /* ParMETIS will crash if geometric method and some procs have no nodes. */ /* Avoid fatal crash by setting scatter to level 2 or higher. */ gr.scatter_min = 2; if (geo == NULL) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory."); return (ZOLTAN_MEMERR); } if (strcmp(alg, "PARTGEOM") == 0) { gr.get_data = 0; } } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } vsp.vsize_malloc = 0; #ifdef PARMETIS31_ALWAYS_FREES_VSIZE if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) { /* ParMETIS will free this memory; use malloc to allocate so ZOLTAN_MALLOC counters don't show an error. */ vsp.vsize_malloc = 1 ; } #endif /* PARMETIS31_ALWAYS_FREES_VSIZE */ ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr, geo, &prt, &vsp); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } /* Get object sizes if requested */ if (options[PMV3_OPT_USE_OBJ_SIZE] && (zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) && (!strcmp(alg, "ADAPTIVEREPART") || gr.final_output)) gr.showMoveVol = 1; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); /* Get ready to call ParMETIS */ edgecut = -1; wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0); numflag = 0; ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1); if (!prt.part_sizes){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL."); } if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) { for (i=0; i<num_part; i++){ indextype j; printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i); for (j=0; j<ncon; j++) printf("%f ", prt.part_sizes[i*ncon+j]); printf("\n"); } } /* if (strcmp(alg, "ADAPTIVEREPART") == 0) */ for (i = 0; i < num_part*ncon; i++) if (prt.part_sizes[i] == 0) ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! " "ParMETIS 3.x will likely fail. Please use a " "different method, or remove the zero-sized " "parts from the problem."); /* Set Imbalance Tolerance for each weight component. */ imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype)); if (!imb_tols){ /* Not enough memory */ ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } for (i=0; i<ncon; i++) imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]); /* Now we can call ParMetis */ /* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */ #ifdef ZOLTAN_PARMETIS if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */ /* First check for ParMetis 3 routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartKway"); ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOMKWAY") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeomKway"); ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts, &wgtflag, &numflag, &ndims, geo->xyz, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOM") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeom"); ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "ADAPTIVEREPART") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_AdaptiveRepart"); ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, &itr, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "REFINEKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_RefineKway"); ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown ParMetis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_METIS /* TODO: I don't know how to set balance ! */ if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Check for Metis routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library "); /* Use default options for METIS */ #if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5 options[0] = 0; METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &num_part, prt.part_sizes, options, &edgecut, prt.part); #else METIS_SetDefaultOptions(options); METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown Metis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (gr.final_output) { /* Do final output now because after the data will not be coherent: unscatter only unscatter part data, not graph */ ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr); } /* Ignore the timings of Final Ouput */ if (get_times) times[3] = Zoltan_Time(zz->Timer); ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr, geo, &prt, &vsp, NULL, &part); Zoltan_Third_Export_User(&part, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part); /* Get a time here */ if (get_times) times[4] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers && timer_p >= 0) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL); if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols); if (geo != NULL) ZOLTAN_FREE(&geo); ZOLTAN_FREE(&global_ids); ZOLTAN_FREE(&local_ids); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int main(int argc, char *argv[]) { PetscErrorCode ierr; PetscBool flg; PetscMPIInt rank, size; int i, status; idx_t ni,isize,*vtxdist, *xadj, *adjncy, *vwgt, *part; idx_t wgtflag=0, numflag=0, ncon=1, ndims=3, edgecut=0; idx_t options[5]; real_t *xyz, *tpwgts, ubvec[1]; MPI_Comm comm; FILE *fp; char fname[PETSC_MAX_PATH_LEN],prefix[PETSC_MAX_PATH_LEN] = ""; PetscInitialize(&argc,&argv,NULL,help); #if defined(PETSC_USE_64BIT_INDICES) ierr = PetscPrintf(PETSC_COMM_WORLD,"This example only works with 32 bit indices\n"); PetscFinalize(); #endif MPI_Comm_rank(PETSC_COMM_WORLD,&rank); MPI_Comm_size(PETSC_COMM_WORLD,&size); ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"Parmetis test options","");CHKERRQ(ierr); ierr = PetscOptionsString("-prefix","Path and prefix of test file","",prefix,prefix,sizeof(prefix),&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"Must specify -prefix");CHKERRQ(ierr); ierr = PetscOptionsEnd();CHKERRQ(ierr); ierr = PetscMalloc1(size+1,&vtxdist);CHKERRQ(ierr); ierr = PetscSNPrintf(fname,sizeof(fname),"%s.%d.graph",prefix,rank);CHKERRQ(ierr); ierr = PetscFOpen(PETSC_COMM_SELF,fname,"r",&fp);CHKERRQ(ierr); fread(vtxdist, sizeof(idx_t), size+1, fp); ni = vtxdist[rank+1]-vtxdist[rank]; ierr = PetscMalloc1(ni+1,&xadj);CHKERRQ(ierr); fread(xadj, sizeof(idx_t), ni+1, fp); ierr = PetscMalloc1(xadj[ni],&adjncy);CHKERRQ(ierr); for (i=0; i<ni; i++) fread(&adjncy[xadj[i]], sizeof(idx_t), xadj[i+1]-xadj[i], fp); ierr = PetscFClose(PETSC_COMM_SELF,fp);CHKERRQ(ierr); ierr = PetscSNPrintf(fname,sizeof(fname),"%s.%d.graph.xyz",prefix,rank);CHKERRQ(ierr); ierr = PetscFOpen(PETSC_COMM_SELF,fname,"r",&fp);CHKERRQ(ierr); ierr = PetscMalloc3(ni*ndims,&xyz,ni,&part,size,&tpwgts);CHKERRQ(ierr); fread(xyz, sizeof(real_t), ndims*ni, fp); ierr = PetscFClose(PETSC_COMM_SELF,fp);CHKERRQ(ierr); vwgt = NULL; for (i = 0; i < size; i++) tpwgts[i] = 1. / size; isize = size; ubvec[0] = 1.05; options[0] = 1; options[1] = 2; options[2] = 15; options[3] = 0; options[4] = 0; ierr = MPI_Comm_dup(MPI_COMM_WORLD, &comm);CHKERRQ(ierr); status = ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, vwgt, NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &isize, tpwgts, ubvec, options, &edgecut, part, &comm);CHKERRQPARMETIS(status); ierr = MPI_Comm_free(&comm);CHKERRQ(ierr); ierr = PetscFree(vtxdist);CHKERRQ(ierr); ierr = PetscFree(xadj);CHKERRQ(ierr); ierr = PetscFree(adjncy);CHKERRQ(ierr); ierr = PetscFree3(xyz,part,tpwgts);CHKERRQ(ierr); PetscFinalize(); return 0; }
/*********************************************************************************** * This function is the testing routine for the adaptive multilevel partitioning code. * It computes a partition from scratch, it then moves the graph and changes some * of the vertex weights and then call the adaptive code. ************************************************************************************/ void TestParMetis_V3(char *filename, MPI_Comm comm) { int ncon, nparts, npes, mype, opt2, realcut; GraphType graph, mgraph; idxtype *part, *mpart, *savepart, *order, *sizes; int numflag=0, wgtflag=0, options[10], edgecut, ndims; float ipc2redist, *xyz, *tpwgts = NULL, ubvec[MAXNCON]; MPI_Comm_size(comm, &npes); MPI_Comm_rank(comm, &mype); ndims = 2; ParallelReadGraph(&graph, filename, comm); xyz = ReadTestCoordinates(&graph, filename, 2, comm); MPI_Barrier(comm); part = idxmalloc(graph.nvtxs, "TestParMetis_V3: part"); tpwgts = fmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts"); sset(MAXNCON, 1.05, ubvec); graph.vwgt = idxsmalloc(graph.nvtxs*5, 1, "TestParMetis_V3: vwgt"); /*====================================================================== / ParMETIS_V3_PartKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; wgtflag = 2; numflag = 0; edgecut = 0; for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) { for (ncon=1; ncon<=5; ncon+=2) { if (ncon > 1 && nparts > 1) Mc_AdaptGraph(&graph, part, ncon, nparts, comm); else idxset(graph.nvtxs, 1, graph.vwgt); for (opt2=1; opt2<=2; opt2++) { options[2] = opt2; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (mype == 0) printf("\nTesting ParMETIS_V3_PartKway with options[1-2] = {%d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], ncon, nparts); ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); if (mype == 0) { printf("ParMETIS_V3_PartKway reported a cut of %d\n", edgecut); } } } } /*====================================================================== / ParMETIS_V3_PartGeomKway /=======================================================================*/ options[0] = 1; options[1] = 3; wgtflag = 2; numflag = 0; for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) { for (ncon=1; ncon<=5; ncon+=2) { if (ncon > 1) Mc_AdaptGraph(&graph, part, ncon, nparts, comm); else idxset(graph.nvtxs, 1, graph.vwgt); for (opt2=1; opt2<=2; opt2++) { options[2] = opt2; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (mype == 0) printf("\nTesting ParMETIS_V3_PartGeomKway with options[1-2] = {%d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], ncon, nparts); ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); if (mype == 0) { printf("ParMETIS_V3_PartGeomKway reported a cut of %d\n", edgecut); } } } } /*====================================================================== / ParMETIS_V3_PartGeom /=======================================================================*/ wgtflag = 0; numflag = 0; if (mype == 0) printf("\nTesting ParMETIS_V3_PartGeom\n"); /* ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm); */ if (mype == 0) printf("ParMETIS_V3_PartGeom partition complete\n"); /* realcut = ComputeRealCut(graph.vtxdist, part, filename, comm); if (mype == 0) printf("ParMETIS_V3_PartGeom reported a cut of %d\n", realcut); */ /*====================================================================== / ParMETIS_V3_RefineKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[2] = 1; options[3] = COUPLED; nparts = npes; wgtflag = 0; numflag = 0; ncon = 1; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (mype == 0) printf("\nTesting ParMETIS_V3_RefineKway with default options (before move)\n"); ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm); MALLOC_CHECK(NULL); if (mype == 0) { printf("ParMETIS_V3_RefineKway reported a cut of %d\n", edgecut); } MALLOC_CHECK(NULL); /* Compute a good partition and move the graph. Do so quietly! */ options[0] = 0; nparts = npes; wgtflag = 0; numflag = 0; ncon = 1; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut, part, &comm); TestMoveGraph(&graph, &mgraph, part, comm); GKfree((void *)&(graph.vwgt), LTERM); mpart = idxsmalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart"); savepart = idxmalloc(mgraph.nvtxs, "TestParMetis_V3: savepart"); MALLOC_CHECK(NULL); /*====================================================================== / ParMETIS_V3_RefineKway /=======================================================================*/ options[0] = 1; options[1] = 3; options[3] = COUPLED; nparts = npes; wgtflag = 0; numflag = 0; for (ncon=1; ncon<=5; ncon+=2) { for (opt2=1; opt2<=2; opt2++) { options[2] = opt2; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (mype == 0) printf("\nTesting ParMETIS_V3_RefineKway with options[1-3] = {%d %d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], options[3], ncon, nparts); ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, mpart, &comm); if (mype == 0) { printf("ParMETIS_V3_RefineKway reported a cut of %d\n", edgecut); } } } /*====================================================================== / ParMETIS_V3_AdaptiveRepart /=======================================================================*/ mgraph.vwgt = idxsmalloc(mgraph.nvtxs*5, 1, "TestParMetis_V3: mgraph.vwgt"); mgraph.vsize = idxsmalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize"); AdaptGraph(&mgraph, 4, comm); options[0] = 1; options[1] = 7; options[3] = COUPLED; wgtflag = 2; numflag = 0; for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) { ncon = 1; wgtflag = 0; options[0] = 0; sset(nparts*ncon, 1.0/(float)nparts, tpwgts); ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, savepart, &comm); options[0] = 1; wgtflag = 2; for (ncon=1; ncon<=3; ncon+=2) { sset(nparts*ncon, 1.0/(float)nparts, tpwgts); if (ncon > 1) Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm); else AdaptGraph(&mgraph, 4, comm); /* idxset(mgraph.nvtxs, 1, mgraph.vwgt); */ for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) { for (opt2=1; opt2<=2; opt2++) { idxcopy(mgraph.nvtxs, savepart, mpart); options[2] = opt2; if (mype == 0) printf("\nTesting ParMETIS_V3_AdaptiveRepart with options[1-3] = {%d %d %d}, ipc2redist: %.3f, Ncon: %d, Nparts: %d\n", options[1], options[2], options[3], ipc2redist, ncon, nparts); ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, mgraph.vwgt, mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, &ipc2redist, options, &edgecut, mpart, &comm); if (mype == 0) { printf("ParMETIS_V3_AdaptiveRepart reported a cut of %d\n", edgecut); } } } } } free(mgraph.vwgt); free(mgraph.vsize); /*====================================================================== / ParMETIS_V3_NodeND /=======================================================================*/ sizes = idxmalloc(2*npes, "TestParMetis_V3: sizes"); order = idxmalloc(graph.nvtxs, "TestParMetis_V3: sizes"); options[0] = 1; options[PMV3_OPTION_DBGLVL] = 3; options[PMV3_OPTION_SEED] = 1; numflag = 0; for (opt2=1; opt2<=2; opt2++) { options[PMV3_OPTION_IPART] = opt2; if (mype == 0) printf("\nTesting ParMETIS_V3_NodeND with options[1-3] = {%d %d %d}\n", options[1], options[2], options[3]); ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options, order, sizes, &comm); } GKfree(&tpwgts, &part, &mpart, &savepart, &order, &sizes, LTERM); }