/*****************************************************************************
* This function computes a partitioning using coordinate data.
*****************************************************************************/
void ParMETIS_PartGeomKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, idxtype *vwgt,
idxtype *adjwgt, int *wgtflag, int *numflag, int *ndims, float *xyz, int *nparts,
int *options, int *edgecut, idxtype *part, MPI_Comm *comm)
{
int i;
int ncon = 1;
float *tpwgts, ubvec[MAXNCON];
int myoptions[10];
tpwgts = fmalloc(*nparts*ncon, "tpwgts");
for (i=0; i<*nparts*ncon; i++)
tpwgts[i] = 1.0/(float)(*nparts);
for (i=0; i<ncon; i++)
ubvec[i] = UNBALANCE_FRACTION;
if (options[0] == 0) {
myoptions[0] = 0;
}
else {
myoptions[0] = 1;
myoptions[PMV3_OPTION_DBGLVL] = options[OPTION_DBGLVL];
myoptions[PMV3_OPTION_SEED] = GLOBAL_SEED;
}
ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ndims, xyz,
&ncon, nparts, tpwgts, ubvec, myoptions, edgecut, part, comm);
GKfree((void **)&tpwgts, LTERM);
return;
}
/*************************************************************************
* Let the game begin
**************************************************************************/
int main(int argc, char *argv[])
{
idx_t i, j, npes, mype, optype, nparts, adptf, options[10];
idx_t *part=NULL, *sizes=NULL;
graph_t graph;
real_t ipc2redist, *xyz=NULL, *tpwgts=NULL, ubvec[MAXNCON];
MPI_Comm comm;
idx_t numflag=0, wgtflag=0, ndims, edgecut;
char xyzfilename[8192];
MPI_Init(&argc, &argv);
MPI_Comm_dup(MPI_COMM_WORLD, &comm);
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
if (argc != 8) {
if (mype == 0)
printf("Usage: %s <graph-file> <op-type> <nparts> <adapth-factor> <ipc2redist> <dbglvl> <seed>\n", argv[0]);
MPI_Finalize();
exit(0);
}
optype = atoi(argv[2]);
nparts = atoi(argv[3]);
adptf = atoi(argv[4]);
ipc2redist = atof(argv[5]);
options[0] = 1;
options[PMV3_OPTION_DBGLVL] = atoi(argv[6]);
options[PMV3_OPTION_SEED] = atoi(argv[7]);
if (mype == 0)
printf("reading file: %s\n", argv[1]);
ParallelReadGraph(&graph, argv[1], comm);
/* Remove the edges for testing */
/*iset(graph.vtxdist[mype+1]-graph.vtxdist[mype]+1, 0, graph.xadj); */
rset(graph.ncon, 1.05, ubvec);
tpwgts = rmalloc(nparts*graph.ncon, "tpwgts");
rset(nparts*graph.ncon, 1.0/(real_t)nparts, tpwgts);
/*
ChangeToFortranNumbering(graph.vtxdist, graph.xadj, graph.adjncy, mype, npes);
numflag = 1;
nvtxs = graph.vtxdist[mype+1]-graph.vtxdist[mype];
nedges = graph.xadj[nvtxs];
printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1));
*/
if (optype >= 20) {
sprintf(xyzfilename, "%s.xyz", argv[1]);
xyz = ReadTestCoordinates(&graph, xyzfilename, &ndims, comm);
}
if (mype == 0)
printf("finished reading file: %s\n", argv[1]);
part = ismalloc(graph.nvtxs, mype%nparts, "main: part");
sizes = imalloc(2*npes, "main: sizes");
switch (optype) {
case 1:
wgtflag = 3;
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD);
break;
case 2:
wgtflag = 3;
options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED;
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD);
break;
case 3:
options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED;
graph.vwgt = ismalloc(graph.nvtxs, 1, "main: vwgt");
if (npes > 1) {
AdaptGraph(&graph, adptf, comm);
}
else {
wgtflag = 3;
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts,
ubvec, options, &edgecut, part, &comm);
printf("Initial partitioning with edgecut of %"PRIDX"\n", edgecut);
for (i=0; i<graph.ncon; i++) {
for (j=0; j<graph.nvtxs; j++) {
if (part[j] == i)
graph.vwgt[j*graph.ncon+i] = adptf;
else
graph.vwgt[j*graph.ncon+i] = 1;
}
}
}
wgtflag = 3;
ParMETIS_V3_AdaptiveRepart(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
&ipc2redist, options, &edgecut, part, &comm);
break;
case 4:
ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options,
part, sizes, &comm);
/* WriteOVector(argv[1], graph.vtxdist, part, comm); */
break;
case 5:
ParMETIS_SerialNodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options,
part, sizes, &comm);
/* WriteOVector(argv[1], graph.vtxdist, part, comm); */
printf("%"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX"\n", sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], sizes[5], sizes[6]);
break;
case 11:
/* TestAdaptiveMETIS(graph.vtxdist, graph.xadj, graph.adjncy, part, options, adptf, comm); */
break;
case 20:
wgtflag = 3;
ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &ndims, xyz, &graph.ncon, &nparts,
tpwgts, ubvec, options, &edgecut, part, &comm);
break;
case 21:
ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm);
break;
}
/* printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1)); */
gk_free((void **)&part, &sizes, &tpwgts, &graph.vtxdist, &graph.xadj, &graph.adjncy,
&graph.vwgt, &graph.adjwgt, &xyz, LTERM);
MPI_Comm_free(&comm);
MPI_Finalize();
return 0;
}
void TestParMetis_GPart(char *filename, char *xyzfile, MPI_Comm comm)
{
idx_t ncon, nparts, npes, mype, opt2, realcut;
graph_t graph, mgraph;
idx_t *part, *mpart, *savepart, *order, *sizes;
idx_t numflag=0, wgtflag=0, options[10], edgecut, ndims;
real_t ipc2redist, *xyz=NULL, *tpwgts = NULL, ubvec[MAXNCON];
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
ParallelReadGraph(&graph, filename, comm);
if (xyzfile)
xyz = ReadTestCoordinates(&graph, xyzfile, &ndims, comm);
gkMPI_Barrier(comm);
part = imalloc(graph.nvtxs, "TestParMetis_V3: part");
tpwgts = rmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts");
rset(MAXNCON, 1.05, ubvec);
graph.vwgt = ismalloc(graph.nvtxs*5, 1, "TestParMetis_GPart: vwgt");
/*======================================================================
/ ParMETIS_V3_PartKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
wgtflag = 2;
numflag = 0;
edgecut = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=NCON; ncon++) {
if (ncon > 1 && nparts > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
iset(graph.nvtxs, 1, graph.vwgt);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options,
&edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0) {
printf("ParMETIS_V3_PartKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
if (mype == 0)
printf("\nTesting ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
options[3] = PARMETIS_PSR_UNCOUPLED;
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options,
&edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0) {
printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeomKway
/=======================================================================*/
if (xyzfile != NULL) {
options[0] = 1;
options[1] = 3;
options[2] = 1;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=NCON; ncon++) {
if (ncon > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
iset(graph.nvtxs, 1, graph.vwgt);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeomKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0)
printf("ParMETIS_V3_PartGeomKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeom
/=======================================================================*/
if (xyz != NULL) {
wgtflag = 0;
numflag = 0;
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeom\n");
ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0)
printf("ParMETIS_V3_PartGeom reported a cut of %"PRIDX"\n", realcut);
}
/*======================================================================
/ Coupled ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
if (mype == 0)
printf("\nTesting coupled ParMETIS_V3_RefineKway with default options (before move)\n");
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
part, &comm);
/* Compute a good partition and move the graph. Do so quietly! */
options[0] = 0;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut,
part, &comm);
TestMoveGraph(&graph, &mgraph, part, comm);
gk_free((void **)&(graph.vwgt), LTERM);
mpart = ismalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart");
savepart = imalloc(mgraph.nvtxs, "TestParMetis_V3: savepart");
/*======================================================================
/ Coupled ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
for (ncon=1; ncon<=NCON; ncon++) {
if (mype == 0)
printf("\nTesting coupled ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
mpart, &comm);
realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart,
filename, comm);
if (mype == 0)
printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
/*ADAPTIVE:*/
/*======================================================================
/ ParMETIS_V3_AdaptiveRepart
/=======================================================================*/
mgraph.vwgt = ismalloc(mgraph.nvtxs*NCON, 1, "TestParMetis_V3: mgraph.vwgt");
mgraph.vsize = ismalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize");
AdaptGraph(&mgraph, 4, comm);
options[0] = 1;
options[1] = 7;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) {
options[0] = 0;
ncon = 1;
wgtflag = 0;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
savepart, &comm);
options[0] = 1;
wgtflag = 2;
for (ncon=1; ncon<=NCON; ncon++) {
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
if (ncon > 1)
Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm);
else
AdaptGraph(&mgraph, 4, comm);
for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) {
icopy(mgraph.nvtxs, savepart, mpart);
if (mype == 0)
printf("\nTesting ParMETIS_V3_AdaptiveRepart with ipc2redist: %.3"PRREAL", ncon: %"PRIDX", nparts: %"PRIDX"\n",
ipc2redist, ncon, nparts);
ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy,
mgraph.vwgt, mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts,
tpwgts, ubvec, &ipc2redist, options, &edgecut, mpart, &comm);
realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart,
filename, comm);
if (mype == 0)
printf("ParMETIS_V3_AdaptiveRepart reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
gk_free((void **)&tpwgts, &part, &mpart, &savepart, &xyz, &mgraph.vwgt, &mgraph.vsize, LTERM);
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
realtype itr = 0.0;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
/* change our zz struct. */
#endif
indextype i;
realtype *imb_tols;
indextype ncon;
indextype edgecut;
indextype wgtflag;
indextype numflag = 0;
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
ZOLTAN_TRACE_ENTER(zz, yo);
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
if (sizeof(realtype) != sizeof(float)) {
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));
for (i = 0; i < tmp; i++)
prt.input_part_sizes[i] = (realtype) part_sizes[i];
/* KDD 2/2014: removed re-scaling part sizes so they sum to one.
* part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
* plus, the code here was wrong for multiple object weights.
* similar scaling code did not exist in the Scotch interface.
*/
prt.part_sizes = prt.input_part_sizes;
}
else
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
indextype j;
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
/* Now we can call ParMetis */
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
/* Use default options for METIS */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0;
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
#else
METIS_SetDefaultOptions(options);
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
prt.part_sizes, imb_tols, options,
&edgecut, prt.part);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) {
/* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
PetscBool flg;
PetscMPIInt rank, size;
int i, status;
idx_t ni,isize,*vtxdist, *xadj, *adjncy, *vwgt, *part;
idx_t wgtflag=0, numflag=0, ncon=1, ndims=3, edgecut=0;
idx_t options[5];
real_t *xyz, *tpwgts, ubvec[1];
MPI_Comm comm;
FILE *fp;
char fname[PETSC_MAX_PATH_LEN],prefix[PETSC_MAX_PATH_LEN] = "";
PetscInitialize(&argc,&argv,NULL,help);
#if defined(PETSC_USE_64BIT_INDICES)
ierr = PetscPrintf(PETSC_COMM_WORLD,"This example only works with 32 bit indices\n");
PetscFinalize();
#endif
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"Parmetis test options","");CHKERRQ(ierr);
ierr = PetscOptionsString("-prefix","Path and prefix of test file","",prefix,prefix,sizeof(prefix),&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"Must specify -prefix");CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscMalloc1(size+1,&vtxdist);CHKERRQ(ierr);
ierr = PetscSNPrintf(fname,sizeof(fname),"%s.%d.graph",prefix,rank);CHKERRQ(ierr);
ierr = PetscFOpen(PETSC_COMM_SELF,fname,"r",&fp);CHKERRQ(ierr);
fread(vtxdist, sizeof(idx_t), size+1, fp);
ni = vtxdist[rank+1]-vtxdist[rank];
ierr = PetscMalloc1(ni+1,&xadj);CHKERRQ(ierr);
fread(xadj, sizeof(idx_t), ni+1, fp);
ierr = PetscMalloc1(xadj[ni],&adjncy);CHKERRQ(ierr);
for (i=0; i<ni; i++) fread(&adjncy[xadj[i]], sizeof(idx_t), xadj[i+1]-xadj[i], fp);
ierr = PetscFClose(PETSC_COMM_SELF,fp);CHKERRQ(ierr);
ierr = PetscSNPrintf(fname,sizeof(fname),"%s.%d.graph.xyz",prefix,rank);CHKERRQ(ierr);
ierr = PetscFOpen(PETSC_COMM_SELF,fname,"r",&fp);CHKERRQ(ierr);
ierr = PetscMalloc3(ni*ndims,&xyz,ni,&part,size,&tpwgts);CHKERRQ(ierr);
fread(xyz, sizeof(real_t), ndims*ni, fp);
ierr = PetscFClose(PETSC_COMM_SELF,fp);CHKERRQ(ierr);
vwgt = NULL;
for (i = 0; i < size; i++) tpwgts[i] = 1. / size;
isize = size;
ubvec[0] = 1.05;
options[0] = 1;
options[1] = 2;
options[2] = 15;
options[3] = 0;
options[4] = 0;
ierr = MPI_Comm_dup(MPI_COMM_WORLD, &comm);CHKERRQ(ierr);
status = ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, vwgt,
NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &isize, tpwgts, ubvec,
options, &edgecut, part, &comm);CHKERRQPARMETIS(status);
ierr = MPI_Comm_free(&comm);CHKERRQ(ierr);
ierr = PetscFree(vtxdist);CHKERRQ(ierr);
ierr = PetscFree(xadj);CHKERRQ(ierr);
ierr = PetscFree(adjncy);CHKERRQ(ierr);
ierr = PetscFree3(xyz,part,tpwgts);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
/***********************************************************************************
* This function is the testing routine for the adaptive multilevel partitioning code.
* It computes a partition from scratch, it then moves the graph and changes some
* of the vertex weights and then call the adaptive code.
************************************************************************************/
void TestParMetis_V3(char *filename, MPI_Comm comm)
{
int ncon, nparts, npes, mype, opt2, realcut;
GraphType graph, mgraph;
idxtype *part, *mpart, *savepart, *order, *sizes;
int numflag=0, wgtflag=0, options[10], edgecut, ndims;
float ipc2redist, *xyz, *tpwgts = NULL, ubvec[MAXNCON];
MPI_Comm_size(comm, &npes);
MPI_Comm_rank(comm, &mype);
ndims = 2;
ParallelReadGraph(&graph, filename, comm);
xyz = ReadTestCoordinates(&graph, filename, 2, comm);
MPI_Barrier(comm);
part = idxmalloc(graph.nvtxs, "TestParMetis_V3: part");
tpwgts = fmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts");
sset(MAXNCON, 1.05, ubvec);
graph.vwgt = idxsmalloc(graph.nvtxs*5, 1, "TestParMetis_V3: vwgt");
/*======================================================================
/ ParMETIS_V3_PartKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
wgtflag = 2;
numflag = 0;
edgecut = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=5; ncon+=2) {
if (ncon > 1 && nparts > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
idxset(graph.nvtxs, 1, graph.vwgt);
for (opt2=1; opt2<=2; opt2++) {
options[2] = opt2;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartKway with options[1-2] = {%d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], ncon, nparts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag,
&numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm);
if (mype == 0) {
printf("ParMETIS_V3_PartKway reported a cut of %d\n", edgecut);
}
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeomKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=5; ncon+=2) {
if (ncon > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
idxset(graph.nvtxs, 1, graph.vwgt);
for (opt2=1; opt2<=2; opt2++) {
options[2] = opt2;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeomKway with options[1-2] = {%d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], ncon, nparts);
ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt, NULL, &wgtflag,
&numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm);
if (mype == 0) {
printf("ParMETIS_V3_PartGeomKway reported a cut of %d\n", edgecut);
}
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeom
/=======================================================================*/
wgtflag = 0;
numflag = 0;
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeom\n");
/* ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm); */
if (mype == 0)
printf("ParMETIS_V3_PartGeom partition complete\n");
/*
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0)
printf("ParMETIS_V3_PartGeom reported a cut of %d\n", realcut);
*/
/*======================================================================
/ ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
options[3] = COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (mype == 0)
printf("\nTesting ParMETIS_V3_RefineKway with default options (before move)\n");
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag,
&numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, part, &comm);
MALLOC_CHECK(NULL);
if (mype == 0) {
printf("ParMETIS_V3_RefineKway reported a cut of %d\n", edgecut);
}
MALLOC_CHECK(NULL);
/* Compute a good partition and move the graph. Do so quietly! */
options[0] = 0;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL, &wgtflag,
&numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut, part, &comm);
TestMoveGraph(&graph, &mgraph, part, comm);
GKfree((void *)&(graph.vwgt), LTERM);
mpart = idxsmalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart");
savepart = idxmalloc(mgraph.nvtxs, "TestParMetis_V3: savepart");
MALLOC_CHECK(NULL);
/*======================================================================
/ ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[3] = COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
for (ncon=1; ncon<=5; ncon+=2) {
for (opt2=1; opt2<=2; opt2++) {
options[2] = opt2;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (mype == 0)
printf("\nTesting ParMETIS_V3_RefineKway with options[1-3] = {%d %d %d}, Ncon: %d, Nparts: %d\n", options[1], options[2], options[3], ncon, nparts);
ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL, &wgtflag,
&numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, mpart, &comm);
if (mype == 0) {
printf("ParMETIS_V3_RefineKway reported a cut of %d\n", edgecut);
}
}
}
/*======================================================================
/ ParMETIS_V3_AdaptiveRepart
/=======================================================================*/
mgraph.vwgt = idxsmalloc(mgraph.nvtxs*5, 1, "TestParMetis_V3: mgraph.vwgt");
mgraph.vsize = idxsmalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize");
AdaptGraph(&mgraph, 4, comm);
options[0] = 1;
options[1] = 7;
options[3] = COUPLED;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) {
ncon = 1;
wgtflag = 0;
options[0] = 0;
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut, savepart, &comm);
options[0] = 1;
wgtflag = 2;
for (ncon=1; ncon<=3; ncon+=2) {
sset(nparts*ncon, 1.0/(float)nparts, tpwgts);
if (ncon > 1)
Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm);
else
AdaptGraph(&mgraph, 4, comm);
/* idxset(mgraph.nvtxs, 1, mgraph.vwgt); */
for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) {
for (opt2=1; opt2<=2; opt2++) {
idxcopy(mgraph.nvtxs, savepart, mpart);
options[2] = opt2;
if (mype == 0)
printf("\nTesting ParMETIS_V3_AdaptiveRepart with options[1-3] = {%d %d %d}, ipc2redist: %.3f, Ncon: %d, Nparts: %d\n", options[1], options[2], options[3], ipc2redist, ncon, nparts);
ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, mgraph.vwgt,
mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, &ipc2redist,
options, &edgecut, mpart, &comm);
if (mype == 0) {
printf("ParMETIS_V3_AdaptiveRepart reported a cut of %d\n", edgecut);
}
}
}
}
}
free(mgraph.vwgt);
free(mgraph.vsize);
/*======================================================================
/ ParMETIS_V3_NodeND
/=======================================================================*/
sizes = idxmalloc(2*npes, "TestParMetis_V3: sizes");
order = idxmalloc(graph.nvtxs, "TestParMetis_V3: sizes");
options[0] = 1;
options[PMV3_OPTION_DBGLVL] = 3;
options[PMV3_OPTION_SEED] = 1;
numflag = 0;
for (opt2=1; opt2<=2; opt2++) {
options[PMV3_OPTION_IPART] = opt2;
if (mype == 0)
printf("\nTesting ParMETIS_V3_NodeND with options[1-3] = {%d %d %d}\n", options[1], options[2], options[3]);
ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options,
order, sizes, &comm);
}
GKfree(&tpwgts, &part, &mpart, &savepart, &order, &sizes, LTERM);
}
