/***********************************************************************************
* This function is the entry point of the parallel kmetis algorithm that uses
* coordinates to compute an initial graph distribution.
************************************************************************************/
void ParMETIS_V3_PartGeomKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *ndims,
float *xyz, int *ncon, int *nparts, float *tpwgts, float *ubvec,
int *options, int *edgecut, idxtype *part, MPI_Comm *comm)
{
int h, i, j;
int nvtxs = -1, npes, mype;
int uwgtflag, cut, gcut, maxnvtxs;
int ltvwgts[MAXNCON];
int moptions[10];
CtrlType ctrl;
idxtype *uvwgt;
WorkSpaceType wspace;
GraphType *graph, *mgraph;
float avg, maximb, balance, *mytpwgts;
int seed, dbglvl = 0;
int iwgtflag, inumflag, incon, inparts, ioptions[10];
float *itpwgts, iubvec[MAXNCON];
MPI_Comm_size(*comm, &npes);
MPI_Comm_rank(*comm, &mype);
/********************************/
/* Try and take care bad inputs */
/********************************/
if (options != NULL && options[0] == 1)
dbglvl = options[PMV3_OPTION_DBGLVL];
CheckInputs(STATIC_PARTITION, npes, dbglvl, wgtflag, &iwgtflag, numflag, &inumflag,
ncon, &incon, nparts, &inparts, tpwgts, &itpwgts, ubvec, iubvec,
NULL, NULL, options, ioptions, part, comm);
/*********************************/
/* Take care the nparts = 1 case */
/*********************************/
if (inparts <= 1) {
idxset(vtxdist[mype+1]-vtxdist[mype], 0, part);
*edgecut = 0;
return;
}
/******************************/
/* Take care of npes = 1 case */
/******************************/
if (npes == 1 && inparts > 1) {
moptions[0] = 0;
nvtxs = vtxdist[1];
if (incon == 1) {
METIS_WPartGraphKway(&nvtxs, xadj, adjncy, vwgt, adjwgt, &iwgtflag, &inumflag,
&inparts, itpwgts, moptions, edgecut, part);
}
else {
/* ADD: this is because METIS does not support tpwgts for all constraints */
mytpwgts = fmalloc(inparts, "mytpwgts");
for (i=0; i<inparts; i++)
mytpwgts[i] = itpwgts[i*incon];
moptions[7] = -1;
METIS_mCPartGraphRecursive2(&nvtxs, &incon, xadj, adjncy, vwgt, adjwgt, &iwgtflag,
&inumflag, &inparts, mytpwgts, moptions, edgecut, part);
free(mytpwgts);
}
return;
}
if (inumflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
/*****************************/
/* Set up control structures */
/*****************************/
if (ioptions[0] == 1) {
dbglvl = ioptions[PMV3_OPTION_DBGLVL];
seed = ioptions[PMV3_OPTION_SEED];
}
else {
dbglvl = GLOBAL_DBGLVL;
seed = GLOBAL_SEED;
}
SetUpCtrl(&ctrl, npes, dbglvl, *comm);
ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*incon*amax(npes, inparts));
ctrl.seed = (seed == 0) ? mype : seed*mype;
ctrl.sync = GlobalSEMax(&ctrl, seed);
ctrl.partType = STATIC_PARTITION;
ctrl.ps_relation = -1;
ctrl.tpwgts = itpwgts;
scopy(incon, iubvec, ctrl.ubvec);
uwgtflag = iwgtflag|2;
uvwgt = idxsmalloc(vtxdist[mype+1]-vtxdist[mype], 1, "uvwgt");
graph = Moc_SetUpGraph(&ctrl, 1, vtxdist, xadj, uvwgt, adjncy, adjwgt, &uwgtflag);
free(graph->nvwgt); graph->nvwgt = NULL;
PreAllocateMemory(&ctrl, graph, &wspace);
/*=================================================================
* Compute the initial npes-way partitioning geometric partitioning
=================================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
Coordinate_Partition(&ctrl, graph, *ndims, xyz, 1, &wspace);
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
/*=================================================================
* Move the graph according to the partitioning
=================================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.MoveTmr));
free(uvwgt);
graph->vwgt = ((iwgtflag&2) != 0) ? vwgt : idxsmalloc(graph->nvtxs*incon, 1, "vwgt");
graph->ncon = incon;
j = ctrl.nparts;
ctrl.nparts = ctrl.npes;
mgraph = Moc_MoveGraph(&ctrl, graph, &wspace);
ctrl.nparts = j;
/**********************************************************/
/* Do the same functionality as Moc_SetUpGraph for mgraph */
/**********************************************************/
/* compute tvwgts */
for (j=0; j<incon; j++)
ltvwgts[j] = 0;
for (i=0; i<graph->nvtxs; i++)
for (j=0; j<incon; j++)
ltvwgts[j] += mgraph->vwgt[i*incon+j];
for (j=0; j<incon; j++)
ctrl.tvwgts[j] = GlobalSESum(&ctrl, ltvwgts[j]);
/* check for zero wgt constraints */
for (i=0; i<incon; i++) {
/* ADD: take care of the case in which tvwgts is zero */
if (ctrl.tvwgts[i] == 0) {
if (ctrl.mype == 0) printf("ERROR: sum weight for constraint %d is zero\n", i);
MPI_Finalize();
exit(-1);
}
}
/* compute nvwgt */
mgraph->nvwgt = fmalloc(mgraph->nvtxs*incon, "mgraph->nvwgt");
for (i=0; i<mgraph->nvtxs; i++)
for (j=0; j<incon; j++)
mgraph->nvwgt[i*incon+j] = (float)(mgraph->vwgt[i*incon+j]) / (float)(ctrl.tvwgts[j]);
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.MoveTmr));
if (ctrl.dbglvl&DBG_INFO) {
cut = 0;
for (i=0; i<graph->nvtxs; i++)
for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++)
if (graph->where[i] != graph->where[graph->adjncy[j]])
cut += graph->adjwgt[j];
gcut = GlobalSESum(&ctrl, cut)/2;
maxnvtxs = GlobalSEMax(&ctrl, mgraph->nvtxs);
balance = (float)(maxnvtxs)/((float)(graph->gnvtxs)/(float)(npes));
rprintf(&ctrl, "XYZ Cut: %6d \tBalance: %6.3f [%d %d %d]\n",
gcut, balance, maxnvtxs, graph->gnvtxs, npes);
}
/*=================================================================
* Set up the newly moved graph
=================================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
ctrl.nparts = inparts;
FreeWSpace(&wspace);
PreAllocateMemory(&ctrl, mgraph, &wspace);
/*=======================================================
* Now compute the partition of the moved graph
=======================================================*/
if (vtxdist[npes] < SMALLGRAPH || vtxdist[npes] < npes*20 || GlobalSESum(&ctrl, mgraph->nedges) == 0) {
IFSET(ctrl.dbglvl, DBG_INFO, rprintf(&ctrl, "Partitioning a graph of size %d serially\n", vtxdist[npes]));
PartitionSmallGraph(&ctrl, mgraph, &wspace);
}
else {
Moc_Global_Partition(&ctrl, mgraph, &wspace);
}
ParallelReMapGraph(&ctrl, mgraph, &wspace);
/* Invert the ordering back to the original graph */
ctrl.nparts = npes;
ProjectInfoBack(&ctrl, graph, part, mgraph->where, &wspace);
*edgecut = mgraph->mincut;
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
/*******************/
/* Print out stats */
/*******************/
IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
if (ctrl.dbglvl&DBG_INFO) {
rprintf(&ctrl, "Final %d-way CUT: %6d \tBalance: ", inparts, mgraph->mincut);
avg = 0.0;
for (h=0; h<incon; h++) {
maximb = 0.0;
for (i=0; i<inparts; i++)
maximb = amax(maximb, mgraph->gnpwgts[i*incon+h]/itpwgts[i*incon+h]);
avg += maximb;
rprintf(&ctrl, "%.3f ", maximb);
}
rprintf(&ctrl, " avg: %.3f\n", avg/(float)incon);
}
GKfree((void **)&itpwgts, LTERM);
FreeGraph(mgraph);
FreeInitialGraphAndRemap(graph, iwgtflag);
FreeWSpace(&wspace);
FreeCtrl(&ctrl);
if (inumflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
}
/***********************************************************************************
* This function is the entry point of the parallel ordering algorithm.
* This function assumes that the graph is already nice partitioned among the
* processors and then proceeds to perform recursive bisection.
************************************************************************************/
void ParMETIS_V3_NodeND(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int *numflag,
int *options, idxtype *order, idxtype *sizes, MPI_Comm *comm)
{
int i, j;
int ltvwgts[MAXNCON];
int nparts, npes, mype, wgtflag = 0, seed = GLOBAL_SEED;
CtrlType ctrl;
WorkSpaceType wspace;
GraphType *graph, *mgraph;
idxtype *morder;
int minnvtxs;
MPI_Comm_size(*comm, &npes);
MPI_Comm_rank(*comm, &mype);
nparts = npes;
if (!ispow2(npes)) {
if (mype == 0)
printf("Error: The number of processors must be a power of 2!\n");
return;
}
if (vtxdist[npes] < (int)((float)(npes*npes)*1.2)) {
if (mype == 0)
printf("Error: Too many processors for this many vertices.\n");
return;
}
minnvtxs = vtxdist[1]-vtxdist[0];
for (i=0; i<npes; i++)
minnvtxs = (minnvtxs < vtxdist[i+1]-vtxdist[i]) ? minnvtxs : vtxdist[i+1]-vtxdist[i];
if (minnvtxs < (int)((float)npes*1.1)) {
if (mype == 0)
printf("Error: vertices are not distributed equally.\n");
return;
}
if (*numflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, order, npes, mype, 1);
SetUpCtrl(&ctrl, nparts, options[PMV3_OPTION_DBGLVL], *comm);
ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*npes);
ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*amax(npes, nparts));
ctrl.seed = mype;
ctrl.sync = seed;
ctrl.partType = STATIC_PARTITION;
ctrl.ps_relation = -1;
ctrl.tpwgts = fsmalloc(nparts, 1.0/(float)(nparts), "tpwgts");
ctrl.ubvec[0] = 1.03;
graph = Moc_SetUpGraph(&ctrl, 1, vtxdist, xadj, NULL, adjncy, NULL, &wgtflag);
PreAllocateMemory(&ctrl, graph, &wspace);
/*=======================================================
* Compute the initial k-way partitioning
=======================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
Moc_Global_Partition(&ctrl, graph, &wspace);
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
/*=======================================================
* Move the graph according to the partitioning
=======================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.MoveTmr));
MALLOC_CHECK(NULL);
graph->ncon = 1;
mgraph = Moc_MoveGraph(&ctrl, graph, &wspace);
MALLOC_CHECK(NULL);
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.MoveTmr));
/*=======================================================
* Now compute an ordering of the moved graph
=======================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
FreeWSpace(&wspace);
PreAllocateMemory(&ctrl, mgraph, &wspace);
ctrl.ipart = ISEP_NODE;
ctrl.CoarsenTo = amin(vtxdist[npes]+1, amax(20*npes, 1000));
/* compute tvwgts */
for (j=0; j<mgraph->ncon; j++)
ltvwgts[j] = 0;
for (i=0; i<mgraph->nvtxs; i++)
for (j=0; j<mgraph->ncon; j++)
ltvwgts[j] += mgraph->vwgt[i*mgraph->ncon+j];
for (j=0; j<mgraph->ncon; j++)
ctrl.tvwgts[j] = GlobalSESum(&ctrl, ltvwgts[j]);
mgraph->nvwgt = fmalloc(mgraph->nvtxs*mgraph->ncon, "mgraph->nvwgt");
for (i=0; i<mgraph->nvtxs; i++)
for (j=0; j<mgraph->ncon; j++)
mgraph->nvwgt[i*mgraph->ncon+j] = (float)(mgraph->vwgt[i*mgraph->ncon+j]) / (float)(ctrl.tvwgts[j]);
morder = idxmalloc(mgraph->nvtxs, "PAROMETIS: morder");
MultilevelOrder(&ctrl, mgraph, morder, sizes, &wspace);
MALLOC_CHECK(NULL);
/* Invert the ordering back to the original graph */
ProjectInfoBack(&ctrl, graph, order, morder, &wspace);
MALLOC_CHECK(NULL);
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
free(ctrl.tpwgts);
free(morder);
FreeGraph(mgraph);
FreeInitialGraphAndRemap(graph, 0);
FreeWSpace(&wspace);
FreeCtrl(&ctrl);
if (*numflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, order, npes, mype, 0);
MALLOC_CHECK(NULL);
}
/*************************************************************************
* This function is the driver to the multi-constraint partitioning algorithm.
**************************************************************************/
void Moc_Global_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
{
int i, ncon, nparts;
floattype ftmp, ubavg, lbavg, lbvec[MAXNCON];
ncon = graph->ncon;
nparts = ctrl->nparts;
ubavg = savg(graph->ncon, ctrl->ubvec);
SetUp(ctrl, graph, wspace);
if (ctrl->dbglvl&DBG_PROGRESS) {
rprintf(ctrl, "[%6d %8d %5d %5d] [%d] [", graph->gnvtxs, GlobalSESum(ctrl, graph->nedges),
GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo);
for (i=0; i<ncon; i++)
rprintf(ctrl, " %.3f", GlobalSEMinFloat(ctrl,graph->nvwgt[samin_strd(graph->nvtxs, graph->nvwgt+i, ncon)*ncon+i]));
rprintf(ctrl, "] [");
for (i=0; i<ncon; i++)
rprintf(ctrl, " %.3f", GlobalSEMaxFloat(ctrl, graph->nvwgt[samax_strd(graph->nvtxs, graph->nvwgt+i, ncon)*ncon+i]));
rprintf(ctrl, "]\n");
}
if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
(graph->finer != NULL &&
graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
/* Done with coarsening. Find a partition */
graph->where = idxmalloc(graph->nvtxs+graph->nrecv, "graph->where");
Moc_InitPartition_RB(ctrl, graph, wspace);
if (ctrl->dbglvl&DBG_PROGRESS) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10d, balance: ", graph->gnvtxs);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3f ", lbvec[i]);
rprintf(ctrl, "\n");
}
/* In case no coarsening took place */
if (graph->finer == NULL) {
Moc_ComputePartitionParams(ctrl, graph, wspace);
Moc_KWayFM(ctrl, graph, wspace, NGR_PASSES);
}
}
else {
Moc_GlobalMatch_Balance(ctrl, graph, wspace);
Moc_Global_Partition(ctrl, graph->coarser, wspace);
Moc_ProjectPartition(ctrl, graph, wspace);
Moc_ComputePartitionParams(ctrl, graph, wspace);
if (graph->ncon > 1 && graph->level < 3) {
for (i=0; i<ncon; i++) {
ftmp = ssum_strd(nparts, graph->gnpwgts+i, ncon);
if (ftmp != 0.0)
lbvec[i] = (floattype)(nparts) *
graph->gnpwgts[samax_strd(nparts, graph->gnpwgts+i, ncon)*ncon+i]/ftmp;
else
lbvec[i] = 1.0;
}
lbavg = savg(graph->ncon, lbvec);
if (lbavg > ubavg + 0.035) {
if (ctrl->dbglvl&DBG_PROGRESS) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3f ", lbvec[i]);
rprintf(ctrl, "\n");
}
Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
}
}
Moc_KWayFM(ctrl, graph, wspace, NGR_PASSES);
if (ctrl->dbglvl&DBG_PROGRESS) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3f ", lbvec[i]);
rprintf(ctrl, "\n");
}
if (graph->level != 0)
GKfree((void **)&graph->lnpwgts, (void **)&graph->gnpwgts, LTERM);
}
return;
}
