/*************************************************************************
* This function computes movement statistics for adaptive refinement
* schemes
**************************************************************************/
void ComputeMoveStatistics(CtrlType *ctrl, GraphType *graph, int *nmoved, int *maxin, int *maxout)
{
int i, j, nvtxs;
idxtype *vwgt, *where;
idxtype *lpvtxs, *gpvtxs;
nvtxs = graph->nvtxs;
vwgt = graph->vwgt;
where = graph->where;
lpvtxs = idxsmalloc(ctrl->nparts, 0, "ComputeMoveStatistics: lpvtxs");
gpvtxs = idxsmalloc(ctrl->nparts, 0, "ComputeMoveStatistics: gpvtxs");
for (j=i=0; i<nvtxs; i++) {
lpvtxs[where[i]]++;
if (where[i] != ctrl->mype)
j++;
}
/* PrintVector(ctrl, ctrl->npes, 0, lpvtxs, "Lpvtxs: "); */
MPI_Allreduce((void *)lpvtxs, (void *)gpvtxs, ctrl->nparts, IDX_DATATYPE, MPI_SUM, ctrl->comm);
*nmoved = GlobalSESum(ctrl, j);
*maxout = GlobalSEMax(ctrl, j);
*maxin = GlobalSEMax(ctrl, gpvtxs[ctrl->mype]-(nvtxs-j));
GKfree((void **)&lpvtxs, (void **)&gpvtxs, LTERM);
}
/*************************************************************************
* This function is the driver for the partition refinement mode of ParMETIS
**************************************************************************/
void Order_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
{
SetUp(ctrl, graph, wspace);
graph->ncon = 1;
IFSET(ctrl->dbglvl, DBG_PROGRESS, rprintf(ctrl, "[%6d %8d %5d %5d][%d][%d]\n",
graph->gnvtxs, GlobalSESum(ctrl, graph->nedges), GlobalSEMin(ctrl, graph->nvtxs),
GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo,
GlobalSEMax(ctrl, graph->vwgt[idxamax(graph->nvtxs, graph->vwgt)])));
if (graph->gnvtxs < 1.3*ctrl->CoarsenTo || (graph->finer != NULL && graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
/* Compute the initial npart-way multisection */
InitMultisection(ctrl, graph, wspace);
if (graph->finer == NULL) { /* Do that only of no-coarsening took place */
ComputeNodePartitionParams(ctrl, graph, wspace);
KWayNodeRefine(ctrl, graph, wspace, 2*NGR_PASSES, ORDER_UNBALANCE_FRACTION);
}
}
else { /* Coarsen it and the partition it */
Mc_LocalMatch_HEM(ctrl, graph, wspace);
Order_Partition(ctrl, graph->coarser, wspace);
Moc_ProjectPartition(ctrl, graph, wspace);
ComputeNodePartitionParams(ctrl, graph, wspace);
KWayNodeRefine(ctrl, graph, wspace, 2*NGR_PASSES, ORDER_UNBALANCE_FRACTION);
}
}
/***********************************************************************************
* This function is the entry point of the parallel kmetis algorithm that uses
* coordinates to compute an initial graph distribution.
************************************************************************************/
void ParMETIS_V3_PartGeomKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *ndims,
float *xyz, int *ncon, int *nparts, float *tpwgts, float *ubvec,
int *options, int *edgecut, idxtype *part, MPI_Comm *comm)
{
int h, i, j;
int nvtxs = -1, npes, mype;
int uwgtflag, cut, gcut, maxnvtxs;
int ltvwgts[MAXNCON];
int moptions[10];
CtrlType ctrl;
idxtype *uvwgt;
WorkSpaceType wspace;
GraphType *graph, *mgraph;
float avg, maximb, balance, *mytpwgts;
int seed, dbglvl = 0;
int iwgtflag, inumflag, incon, inparts, ioptions[10];
float *itpwgts, iubvec[MAXNCON];
MPI_Comm_size(*comm, &npes);
MPI_Comm_rank(*comm, &mype);
/********************************/
/* Try and take care bad inputs */
/********************************/
if (options != NULL && options[0] == 1)
dbglvl = options[PMV3_OPTION_DBGLVL];
CheckInputs(STATIC_PARTITION, npes, dbglvl, wgtflag, &iwgtflag, numflag, &inumflag,
ncon, &incon, nparts, &inparts, tpwgts, &itpwgts, ubvec, iubvec,
NULL, NULL, options, ioptions, part, comm);
/*********************************/
/* Take care the nparts = 1 case */
/*********************************/
if (inparts <= 1) {
idxset(vtxdist[mype+1]-vtxdist[mype], 0, part);
*edgecut = 0;
return;
}
/******************************/
/* Take care of npes = 1 case */
/******************************/
if (npes == 1 && inparts > 1) {
moptions[0] = 0;
nvtxs = vtxdist[1];
if (incon == 1) {
METIS_WPartGraphKway(&nvtxs, xadj, adjncy, vwgt, adjwgt, &iwgtflag, &inumflag,
&inparts, itpwgts, moptions, edgecut, part);
}
else {
/* ADD: this is because METIS does not support tpwgts for all constraints */
mytpwgts = fmalloc(inparts, "mytpwgts");
for (i=0; i<inparts; i++)
mytpwgts[i] = itpwgts[i*incon];
moptions[7] = -1;
METIS_mCPartGraphRecursive2(&nvtxs, &incon, xadj, adjncy, vwgt, adjwgt, &iwgtflag,
&inumflag, &inparts, mytpwgts, moptions, edgecut, part);
free(mytpwgts);
}
return;
}
if (inumflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
/*****************************/
/* Set up control structures */
/*****************************/
if (ioptions[0] == 1) {
dbglvl = ioptions[PMV3_OPTION_DBGLVL];
seed = ioptions[PMV3_OPTION_SEED];
}
else {
dbglvl = GLOBAL_DBGLVL;
seed = GLOBAL_SEED;
}
SetUpCtrl(&ctrl, npes, dbglvl, *comm);
ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*incon*amax(npes, inparts));
ctrl.seed = (seed == 0) ? mype : seed*mype;
ctrl.sync = GlobalSEMax(&ctrl, seed);
ctrl.partType = STATIC_PARTITION;
ctrl.ps_relation = -1;
ctrl.tpwgts = itpwgts;
scopy(incon, iubvec, ctrl.ubvec);
uwgtflag = iwgtflag|2;
uvwgt = idxsmalloc(vtxdist[mype+1]-vtxdist[mype], 1, "uvwgt");
graph = Moc_SetUpGraph(&ctrl, 1, vtxdist, xadj, uvwgt, adjncy, adjwgt, &uwgtflag);
free(graph->nvwgt); graph->nvwgt = NULL;
PreAllocateMemory(&ctrl, graph, &wspace);
/*=================================================================
* Compute the initial npes-way partitioning geometric partitioning
=================================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
Coordinate_Partition(&ctrl, graph, *ndims, xyz, 1, &wspace);
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
/*=================================================================
* Move the graph according to the partitioning
=================================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.MoveTmr));
free(uvwgt);
graph->vwgt = ((iwgtflag&2) != 0) ? vwgt : idxsmalloc(graph->nvtxs*incon, 1, "vwgt");
graph->ncon = incon;
j = ctrl.nparts;
ctrl.nparts = ctrl.npes;
mgraph = Moc_MoveGraph(&ctrl, graph, &wspace);
ctrl.nparts = j;
/**********************************************************/
/* Do the same functionality as Moc_SetUpGraph for mgraph */
/**********************************************************/
/* compute tvwgts */
for (j=0; j<incon; j++)
ltvwgts[j] = 0;
for (i=0; i<graph->nvtxs; i++)
for (j=0; j<incon; j++)
ltvwgts[j] += mgraph->vwgt[i*incon+j];
for (j=0; j<incon; j++)
ctrl.tvwgts[j] = GlobalSESum(&ctrl, ltvwgts[j]);
/* check for zero wgt constraints */
for (i=0; i<incon; i++) {
/* ADD: take care of the case in which tvwgts is zero */
if (ctrl.tvwgts[i] == 0) {
if (ctrl.mype == 0) printf("ERROR: sum weight for constraint %d is zero\n", i);
MPI_Finalize();
exit(-1);
}
}
/* compute nvwgt */
mgraph->nvwgt = fmalloc(mgraph->nvtxs*incon, "mgraph->nvwgt");
for (i=0; i<mgraph->nvtxs; i++)
for (j=0; j<incon; j++)
mgraph->nvwgt[i*incon+j] = (float)(mgraph->vwgt[i*incon+j]) / (float)(ctrl.tvwgts[j]);
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.MoveTmr));
if (ctrl.dbglvl&DBG_INFO) {
cut = 0;
for (i=0; i<graph->nvtxs; i++)
for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++)
if (graph->where[i] != graph->where[graph->adjncy[j]])
cut += graph->adjwgt[j];
gcut = GlobalSESum(&ctrl, cut)/2;
maxnvtxs = GlobalSEMax(&ctrl, mgraph->nvtxs);
balance = (float)(maxnvtxs)/((float)(graph->gnvtxs)/(float)(npes));
rprintf(&ctrl, "XYZ Cut: %6d \tBalance: %6.3f [%d %d %d]\n",
gcut, balance, maxnvtxs, graph->gnvtxs, npes);
}
/*=================================================================
* Set up the newly moved graph
=================================================================*/
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
ctrl.nparts = inparts;
FreeWSpace(&wspace);
PreAllocateMemory(&ctrl, mgraph, &wspace);
/*=======================================================
* Now compute the partition of the moved graph
=======================================================*/
if (vtxdist[npes] < SMALLGRAPH || vtxdist[npes] < npes*20 || GlobalSESum(&ctrl, mgraph->nedges) == 0) {
IFSET(ctrl.dbglvl, DBG_INFO, rprintf(&ctrl, "Partitioning a graph of size %d serially\n", vtxdist[npes]));
PartitionSmallGraph(&ctrl, mgraph, &wspace);
}
else {
Moc_Global_Partition(&ctrl, mgraph, &wspace);
}
ParallelReMapGraph(&ctrl, mgraph, &wspace);
/* Invert the ordering back to the original graph */
ctrl.nparts = npes;
ProjectInfoBack(&ctrl, graph, part, mgraph->where, &wspace);
*edgecut = mgraph->mincut;
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
/*******************/
/* Print out stats */
/*******************/
IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
if (ctrl.dbglvl&DBG_INFO) {
rprintf(&ctrl, "Final %d-way CUT: %6d \tBalance: ", inparts, mgraph->mincut);
avg = 0.0;
for (h=0; h<incon; h++) {
maximb = 0.0;
for (i=0; i<inparts; i++)
maximb = amax(maximb, mgraph->gnpwgts[i*incon+h]/itpwgts[i*incon+h]);
avg += maximb;
rprintf(&ctrl, "%.3f ", maximb);
}
rprintf(&ctrl, " avg: %.3f\n", avg/(float)incon);
}
GKfree((void **)&itpwgts, LTERM);
FreeGraph(mgraph);
FreeInitialGraphAndRemap(graph, iwgtflag);
FreeWSpace(&wspace);
FreeCtrl(&ctrl);
if (inumflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
}
/*************************************************************************
* This function is the driver for the adaptive refinement mode of ParMETIS
**************************************************************************/
void Adaptive_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
{
int i;
int tewgt, tvsize;
floattype gtewgt, gtvsize;
floattype ubavg, lbavg, lbvec[MAXNCON];
/************************************/
/* Set up important data structures */
/************************************/
SetUp(ctrl, graph, wspace);
ubavg = savg(graph->ncon, ctrl->ubvec);
tewgt = idxsum(graph->nedges, graph->adjwgt);
tvsize = idxsum(graph->nvtxs, graph->vsize);
gtewgt = (floattype) GlobalSESum(ctrl, tewgt) + 1.0; /* The +1 were added to remove any FPE */
gtvsize = (floattype) GlobalSESum(ctrl, tvsize) + 1.0;
ctrl->redist_factor = ctrl->redist_base * ((gtewgt/gtvsize)/ ctrl->edge_size_ratio);
IFSET(ctrl->dbglvl, DBG_PROGRESS, rprintf(ctrl, "[%6d %8d %5d %5d][%d]\n",
graph->gnvtxs, GlobalSESum(ctrl, graph->nedges), GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo));
if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
(graph->finer != NULL && graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
/***********************************************/
/* Balance the partition on the coarsest graph */
/***********************************************/
graph->where = idxsmalloc(graph->nvtxs+graph->nrecv, -1, "graph->where");
idxcopy(graph->nvtxs, graph->home, graph->where);
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
lbavg = savg(graph->ncon, lbvec);
if (lbavg > ubavg + 0.035 && ctrl->partType != REFINE_PARTITION)
Balance_Partition(ctrl, graph, wspace);
if (ctrl->dbglvl&DBG_PROGRESS) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10d, balance: ", graph->gnvtxs);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3f ", lbvec[i]);
rprintf(ctrl, "\n");
}
/* check if no coarsening took place */
if (graph->finer == NULL) {
Moc_ComputePartitionParams(ctrl, graph, wspace);
Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
Moc_KWayAdaptiveRefine(ctrl, graph, wspace, NGR_PASSES);
}
}
else {
/*******************************/
/* Coarsen it and partition it */
/*******************************/
switch (ctrl->ps_relation) {
case COUPLED:
Mc_LocalMatch_HEM(ctrl, graph, wspace);
break;
case DISCOUPLED:
default:
Moc_GlobalMatch_Balance(ctrl, graph, wspace);
break;
}
Adaptive_Partition(ctrl, graph->coarser, wspace);
/********************************/
/* project partition and refine */
/********************************/
Moc_ProjectPartition(ctrl, graph, wspace);
Moc_ComputePartitionParams(ctrl, graph, wspace);
if (graph->ncon > 1 && graph->level < 4) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
lbavg = savg(graph->ncon, lbvec);
if (lbavg > ubavg + 0.025) {
Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
}
}
Moc_KWayAdaptiveRefine(ctrl, graph, wspace, NGR_PASSES);
if (ctrl->dbglvl&DBG_PROGRESS) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3f ", lbvec[i]);
rprintf(ctrl, "\n");
}
}
}
/*************************************************************************
* This function tests the repeated shmem_put
**************************************************************************/
void SetUp(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
{
int i, j, k, islocal, penum, gnvtxs, nvtxs, nlocal, firstvtx, lastvtx, nsend, nrecv, nnbrs, nadj;
int npes=ctrl->npes, mype=ctrl->mype;
idxtype *vtxdist, *xadj, *adjncy;
idxtype *peind, *recvptr, *recvind, *sendptr, *sendind;
idxtype *receive, *pemap, *imap, *lperm;
idxtype *pexadj, *peadjncy, *peadjloc, *startsind;
KeyValueType *recvrequests, *sendrequests, *adjpairs;
IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->SetupTmr));
gnvtxs = graph->gnvtxs;
nvtxs = graph->nvtxs;
vtxdist = graph->vtxdist;
xadj = graph->xadj;
adjncy = graph->adjncy;
firstvtx = vtxdist[mype];
lastvtx = vtxdist[mype+1];
pemap = wspace->pv1;
idxset(npes, -1, pemap);
lperm = graph->lperm = idxmalloc(nvtxs, "SetUp: graph->lperm");
for (i=0; i<nvtxs; i++)
lperm[i] = i;
/*************************************************************
* Determine what you need to receive
*************************************************************/
receive = wspace->indices; /* Use the large global received array for now */
adjpairs = wspace->pairs;
for (nlocal = nadj = i = 0; i<nvtxs; i++) {
islocal = 1;
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = adjncy[j];
if (k >= firstvtx && k < lastvtx) {
adjncy[j] = k-firstvtx;
continue; /* local vertex */
}
adjpairs[nadj].key = k;
adjpairs[nadj++].val = j;
islocal = 0;
}
if (islocal) {
lperm[i] = lperm[nlocal];
lperm[nlocal++] = i;
}
}
/* Take care the received part now */
ikeysort(nadj, adjpairs);
adjpairs[nadj].key = gnvtxs+1; /* Boundary condition */
for (nrecv=i=0; i<nadj; i++) {
adjncy[adjpairs[i].val] = nvtxs+nrecv;
if (adjpairs[i].key != adjpairs[i+1].key)
receive[nrecv++] = adjpairs[i].key;
}
/* Allocate space for the setup info attached to this level of the graph */
peind = graph->peind = idxmalloc(npes, "SetUp: peind");
recvptr = graph->recvptr = idxmalloc(npes+1, "SetUp: recvptr");
recvind = graph->recvind = idxmalloc(nrecv, "SetUp: recvind");
/* Take care of the received portion */
idxcopy(nrecv, receive, recvind); /* Copy the vertices to be received into recvind */
i = nnbrs = recvptr[0] = 0;
for (penum=0; penum<npes; penum++) {
for (j=i; j<nrecv; j++) {
if (recvind[j] >= vtxdist[penum+1])
break;
}
if (j > i) {
peind[nnbrs] = penum;
recvptr[++nnbrs] = j;
i = j;
}
}
/*************************************************************
* Determine what you need to send
*************************************************************/
/* Tell the other processors what they need to send you */
recvrequests = wspace->pepairs1;
sendrequests = wspace->pepairs2;
for (i=0; i<npes; i++)
recvrequests[i].key = 0;
for (i=0; i<nnbrs; i++) {
recvrequests[peind[i]].key = recvptr[i+1]-recvptr[i];
recvrequests[peind[i]].val = nvtxs+recvptr[i];
}
MPI_Alltoall((void *)recvrequests, 2, IDX_DATATYPE, (void *)sendrequests, 2, IDX_DATATYPE, ctrl->comm);
sendptr = graph->sendptr = idxmalloc(npes+1, "SetUp: sendptr");
startsind = wspace->pv2;
for (j=i=0; i<npes; i++) {
if (sendrequests[i].key > 0) {
sendptr[j] = sendrequests[i].key;
startsind[j] = sendrequests[i].val;
j++;
}
}
ASSERT(ctrl, nnbrs == j);
MAKECSR(i, j, sendptr);
nsend = sendptr[nnbrs];
sendind = graph->sendind = idxmalloc(nsend, "SetUp: sendind");
/* Issue the receives for sendind */
for (i=0; i<nnbrs; i++) {
MPI_Irecv((void *)(sendind+sendptr[i]), sendptr[i+1]-sendptr[i], IDX_DATATYPE,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends. My recvind[penum] becomes penum's sendind[mype] */
for (i=0; i<nnbrs; i++) {
MPI_Isend((void *)(recvind+recvptr[i]), recvptr[i+1]-recvptr[i], IDX_DATATYPE,
peind[i], 1, ctrl->comm, ctrl->sreq+i);
}
MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/* Create the peadjncy data structure for sparse boundary exchanges */
pexadj = graph->pexadj = idxsmalloc(nvtxs+1, 0, "SetUp: pexadj");
peadjncy = graph->peadjncy = idxmalloc(nsend, "SetUp: peadjncy");
peadjloc = graph->peadjloc = idxmalloc(nsend, "SetUp: peadjloc");
for (i=0; i<nsend; i++) {
ASSERTP(ctrl, sendind[i] >= firstvtx && sendind[i] < lastvtx, (ctrl, "%d %d %d\n", sendind[i], firstvtx, lastvtx));
pexadj[sendind[i]-firstvtx]++;
}
MAKECSR(i, nvtxs, pexadj);
for (i=0; i<nnbrs; i++) {
for (j=sendptr[i]; j<sendptr[i+1]; j++) {
k = pexadj[sendind[j]-firstvtx]++;
peadjncy[k] = i; /* peind[i] is the actual PE number */
peadjloc[k] = startsind[i]++;
}
}
ASSERT(ctrl, pexadj[nvtxs] == nsend);
for (i=nvtxs; i>0; i--)
pexadj[i] = pexadj[i-1];
pexadj[0] = 0;
graph->nnbrs = nnbrs;
graph->nrecv = nrecv;
graph->nsend = nsend;
graph->nlocal = nlocal;
/* Create the inverse map from ladjncy to adjncy */
imap = graph->imap = idxmalloc(nvtxs+nrecv, "SetUp: imap");
for (i=0; i<nvtxs; i++)
imap[i] = firstvtx+i;
for (i=0; i<nrecv; i++)
imap[nvtxs+i] = recvind[i];
/* Check if wspace->nlarge is large enough for nrecv and nsend */
if (wspace->nlarge < nrecv+nsend) {
free(wspace->indices);
free(wspace->pairs);
wspace->nlarge = nrecv+nsend;
wspace->indices = idxmalloc(wspace->nlarge, "SetUp: wspace->indices");
wspace->pairs = (KeyValueType *)GKmalloc(sizeof(KeyValueType)*wspace->nlarge, "SetUp: wspace->pairs");
}
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->SetupTmr));
#ifdef DEBUG_SETUPINFO
rprintf(ctrl, "[%5d %5d] \tl:[%5d %5d] \ts:[%5d, %5d] \tr:[%5d, %5d]\n",
GlobalSEMin(ctrl, nvtxs), GlobalSEMax(ctrl, nvtxs),
GlobalSEMin(ctrl, nlocal), GlobalSEMax(ctrl, nlocal),
GlobalSEMin(ctrl, nsend), GlobalSEMax(ctrl, nsend),
GlobalSEMin(ctrl, nrecv), GlobalSEMax(ctrl, nrecv));
PrintSetUpInfo(ctrl, graph);
#endif
}
/***********************************************************************************
* This function is the entry point of the parallel ordering algorithm.
* This function assumes that the graph is already nice partitioned among the
* processors and then proceeds to perform recursive bisection.
************************************************************************************/
void ParMETIS_V3_NodeND(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy, int *numflag,
int *options, idxtype *order, idxtype *sizes, MPI_Comm *comm)
{
int i, j;
int ltvwgts[MAXNCON];
int nparts, npes, mype, wgtflag = 0, seed;
CtrlType ctrl;
WorkSpaceType wspace;
GraphType *graph, *mgraph;
idxtype *morder;
int minnvtxs;
int dbglvl_original;
MPI_Comm_size(*comm, &npes);
MPI_Comm_rank(*comm, &mype);
nparts = 1*npes;
if (!ispow2(npes)) {
if (mype == 0)
printf("Error: The number of processors must be a power of 2!\n");
return;
}
if (vtxdist[npes] < (int)((float)(npes*npes)*1.2)) {
if (mype == 0)
printf("Error: Too many processors for this many vertices.\n");
return;
}
minnvtxs = vtxdist[1]-vtxdist[0];
for (i=0; i<npes; i++)
minnvtxs = (minnvtxs < vtxdist[i+1]-vtxdist[i]) ? minnvtxs : vtxdist[i+1]-vtxdist[i];
if (minnvtxs < (int)((float)npes*1.1)) {
if (mype == 0)
printf("Error: vertices are not distributed equally.\n");
return;
}
if (*numflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, order, npes, mype, 1);
/*****************************/
/* Set up control structures */
/*****************************/
if (options == NULL && options[0] == 0) {
dbglvl_original = GLOBAL_DBGLVL;
seed = GLOBAL_SEED;
}
else {
dbglvl_original = options[PMV3_OPTION_DBGLVL];
seed = options[PMV3_OPTION_SEED];
}
SetUpCtrl(&ctrl, nparts, 0, *comm);
ctrl.CoarsenTo = amin(vtxdist[npes]+1, 25*amax(npes, nparts));
ctrl.seed = (seed == 0 ? mype : seed*mype);
ctrl.sync = GlobalSEMax(&ctrl, seed);
ctrl.partType = STATIC_PARTITION;
ctrl.ps_relation = -1;
ctrl.tpwgts = fsmalloc(nparts, 1.0/(float)(nparts), "tpwgts");
ctrl.ubvec[0] = 1.03;
graph = Mc_SetUpGraph(&ctrl, 1, vtxdist, xadj, NULL, adjncy, NULL, &wgtflag);
AllocateWSpace(&ctrl, graph, &wspace);
/*=======================================================
* Compute the initial k-way partitioning
=======================================================*/
IFSET(dbglvl_original, DBG_TIME, InitTimers(&ctrl));
IFSET(dbglvl_original, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(dbglvl_original, DBG_TIME, starttimer(ctrl.TotalTmr));
Mc_Global_Partition(&ctrl, graph, &wspace);
/* Collapse the number of partitions to be from 0..npes-1 */
for (i=0; i<graph->nvtxs; i++)
graph->where[i] = graph->where[i]%npes;
ctrl.nparts = nparts = npes;
/*=======================================================
* Move the graph according to the partitioning
=======================================================*/
IFSET(dbglvl_original, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(dbglvl_original, DBG_TIME, starttimer(ctrl.MoveTmr));
MALLOC_CHECK(NULL);
graph->ncon = 1;
mgraph = Mc_MoveGraph(&ctrl, graph, &wspace);
MALLOC_CHECK(NULL);
IFSET(dbglvl_original, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(dbglvl_original, DBG_TIME, stoptimer(ctrl.MoveTmr));
/* restore the user supplied dbglvl */
ctrl.dbglvl = dbglvl_original;
/*=======================================================
* Now compute an ordering of the moved graph
=======================================================*/
AdjustWSpace(&ctrl, mgraph, &wspace);
ctrl.ipart = ISEP_NODE;
ctrl.CoarsenTo = amin(vtxdist[npes]+1, amax(20*npes, 1000));
/* compute tvwgts */
for (j=0; j<mgraph->ncon; j++)
ltvwgts[j] = 0;
for (i=0; i<mgraph->nvtxs; i++)
for (j=0; j<mgraph->ncon; j++)
ltvwgts[j] += mgraph->vwgt[i*mgraph->ncon+j];
for (j=0; j<mgraph->ncon; j++)
ctrl.tvwgts[j] = GlobalSESum(&ctrl, ltvwgts[j]);
mgraph->nvwgt = fmalloc(mgraph->nvtxs*mgraph->ncon, "mgraph->nvwgt");
for (i=0; i<mgraph->nvtxs; i++)
for (j=0; j<mgraph->ncon; j++)
mgraph->nvwgt[i*mgraph->ncon+j] = (float)(mgraph->vwgt[i*mgraph->ncon+j]) / (float)(ctrl.tvwgts[j]);
morder = idxmalloc(mgraph->nvtxs, "PAROMETIS: morder");
MultilevelOrder(&ctrl, mgraph, morder, sizes, &wspace);
MALLOC_CHECK(NULL);
/* Invert the ordering back to the original graph */
ProjectInfoBack(&ctrl, graph, order, morder, &wspace);
MALLOC_CHECK(NULL);
IFSET(dbglvl_original, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(dbglvl_original, DBG_TIME, stoptimer(ctrl.TotalTmr));
IFSET(dbglvl_original, DBG_TIME, PrintTimingInfo(&ctrl));
IFSET(dbglvl_original, DBG_TIME, MPI_Barrier(ctrl.gcomm));
GKfree((void **)&ctrl.tpwgts, &morder, LTERM);
FreeGraph(mgraph);
FreeInitialGraphAndRemap(graph, 0, 1);
FreeWSpace(&wspace);
FreeCtrl(&ctrl);
if (*numflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, order, npes, mype, 0);
MALLOC_CHECK(NULL);
}
/***********************************************************************************
* This function is the entry point of the parallel kmetis algorithm that uses
* coordinates to compute an initial graph distribution.
************************************************************************************/
int ParMETIS_V3_PartGeomKway(idx_t *vtxdist, idx_t *xadj, idx_t *adjncy,
idx_t *vwgt, idx_t *adjwgt, idx_t *wgtflag, idx_t *numflag, idx_t *ndims,
real_t *xyz, idx_t *ncon, idx_t *nparts, real_t *tpwgts, real_t *ubvec,
idx_t *options, idx_t *edgecut, idx_t *part, MPI_Comm *comm)
{
idx_t h, i, j, npes, mype, status, nvtxs, seed, dbglvl;
idx_t cut, gcut, maxnvtxs;
idx_t moptions[METIS_NOPTIONS];
ctrl_t *ctrl;
graph_t *graph, *mgraph;
real_t balance;
size_t curmem;
/* Check the input parameters and return if an error */
status = CheckInputsPartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag,
numflag, ndims, xyz, ncon, nparts, tpwgts, ubvec, options,
edgecut, part, comm);
if (GlobalSEMinComm(*comm, status) == 0)
return METIS_ERROR;
status = METIS_OK;
gk_malloc_init();
curmem = gk_GetCurMemoryUsed();
/* Setup the ctrl */
ctrl = SetupCtrl(PARMETIS_OP_GKMETIS, options, *ncon, *nparts, tpwgts, ubvec, *comm);
npes = ctrl->npes;
mype = ctrl->mype;
/* Take care the nparts == 1 case */
if (*nparts == 1) {
iset(vtxdist[mype+1]-vtxdist[mype], (*numflag == 0 ? 0 : 1), part);
*edgecut = 0;
goto DONE;
}
/* Take care of npes == 1 case */
if (npes == 1) {
nvtxs = vtxdist[1] - vtxdist[0]; /* subtraction is required when numflag==1 */
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_NUMBERING] = *numflag;
status = METIS_PartGraphKway(&nvtxs, ncon, xadj, adjncy, vwgt, NULL, adjwgt,
nparts, tpwgts, ubvec, moptions, edgecut, part);
goto DONE;
}
/* Setup the graph */
if (*numflag > 0)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
graph = SetupGraph(ctrl, *ncon, vtxdist, xadj, vwgt, NULL, adjncy, adjwgt, *wgtflag);
gk_free((void **)&graph->nvwgt, LTERM);
/* Allocate the workspace */
AllocateWSpace(ctrl, 10*graph->nvtxs);
/* Compute the initial npes-way partitioning geometric partitioning */
STARTTIMER(ctrl, ctrl->TotalTmr);
Coordinate_Partition(ctrl, graph, *ndims, xyz, 1);
STOPTIMER(ctrl, ctrl->TotalTmr);
/* Move the graph according to the partitioning */
STARTTIMER(ctrl, ctrl->MoveTmr);
ctrl->nparts = npes;
mgraph = MoveGraph(ctrl, graph);
ctrl->nparts = *nparts;
SetupGraph_nvwgts(ctrl, mgraph); /* compute nvwgts for the moved graph */
if (ctrl->dbglvl&DBG_INFO) {
CommInterfaceData(ctrl, graph, graph->where, graph->where+graph->nvtxs);
for (cut=0, i=0; i<graph->nvtxs; i++) {
for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++) {
if (graph->where[i] != graph->where[graph->adjncy[j]])
cut += graph->adjwgt[j];
}
}
gcut = GlobalSESum(ctrl, cut)/2;
maxnvtxs = GlobalSEMax(ctrl, mgraph->nvtxs);
balance = (real_t)(maxnvtxs)/((real_t)(graph->gnvtxs)/(real_t)(npes));
rprintf(ctrl, "XYZ Cut: %6"PRIDX" \tBalance: %6.3"PRREAL" [%"PRIDX" %"PRIDX" %"PRIDX"]\n",
gcut, balance, maxnvtxs, graph->gnvtxs, npes);
}
STOPTIMER(ctrl, ctrl->MoveTmr);
/* Compute the partition of the moved graph */
STARTTIMER(ctrl, ctrl->TotalTmr);
ctrl->CoarsenTo = gk_min(vtxdist[npes]+1, 25*(*ncon)*gk_max(npes, *nparts));
if (vtxdist[npes] < SMALLGRAPH
|| vtxdist[npes] < npes*20
|| GlobalSESum(ctrl, mgraph->nedges) == 0) { /* serially */
IFSET(ctrl->dbglvl, DBG_INFO,
rprintf(ctrl, "Partitioning a graph of size %"PRIDX" serially\n", vtxdist[npes]));
PartitionSmallGraph(ctrl, mgraph);
}
else { /* in parallel */
Global_Partition(ctrl, mgraph);
}
ParallelReMapGraph(ctrl, mgraph);
/* Invert the ordering back to the original graph */
ctrl->nparts = npes;
ProjectInfoBack(ctrl, graph, part, mgraph->where);
ctrl->nparts = *nparts;
*edgecut = mgraph->mincut;
STOPTIMER(ctrl, ctrl->TotalTmr);
/* Print some stats */
IFSET(ctrl->dbglvl, DBG_TIME, PrintTimingInfo(ctrl));
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->gcomm));
IFSET(ctrl->dbglvl, DBG_INFO, PrintPostPartInfo(ctrl, mgraph, 0));
FreeGraph(mgraph);
FreeInitialGraphAndRemap(graph);
if (*numflag > 0)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
DONE:
FreeCtrl(&ctrl);
if (gk_GetCurMemoryUsed() - curmem > 0) {
printf("ParMETIS appears to have a memory leak of %zdbytes. Report this.\n",
(ssize_t)(gk_GetCurMemoryUsed() - curmem));
}
gk_malloc_cleanup(0);
return (int)status;
}
/*************************************************************************
* This function converts a mesh into a dual graph
**************************************************************************/
void ParMETIS_V3_Mesh2Dual(idxtype *elmdist, idxtype *eptr, idxtype *eind,
int *numflag, int *ncommonnodes, idxtype **xadj,
idxtype **adjncy, MPI_Comm *comm)
{
int i, j, jj, k, kk, m;
int npes, mype, pe, count, mask, pass;
int nelms, lnns, my_nns, node;
int firstelm, firstnode, lnode, nrecv, nsend;
int *scounts, *rcounts, *sdispl, *rdispl;
idxtype *nodedist, *nmap, *auxarray;
idxtype *gnptr, *gnind, *nptr, *nind, *myxadj, *myadjncy = NULL;
idxtype *sbuffer, *rbuffer, *htable;
KeyValueType *nodelist, *recvbuffer;
idxtype ind[200], wgt[200];
int gmaxnode, gminnode;
CtrlType ctrl;
SetUpCtrl(&ctrl, -1, 0, *comm);
npes = ctrl.npes;
mype = ctrl.mype;
nelms = elmdist[mype+1]-elmdist[mype];
if (*numflag == 1)
ChangeNumberingMesh2(elmdist, eptr, eind, NULL, NULL, NULL, npes, mype, 1);
mask = (1<<11)-1;
/*****************************/
/* Determine number of nodes */
/*****************************/
gminnode = GlobalSEMin(&ctrl, eind[idxamin(eptr[nelms], eind)]);
for (i=0; i<eptr[nelms]; i++)
eind[i] -= gminnode;
gmaxnode = GlobalSEMax(&ctrl, eind[idxamax(eptr[nelms], eind)]);
/**************************/
/* Check for input errors */
/**************************/
ASSERTS(nelms > 0);
/* construct node distribution array */
nodedist = idxsmalloc(npes+1, 0, "nodedist");
for (nodedist[0]=0, i=0,j=gmaxnode+1; i<npes; i++) {
k = j/(npes-i);
nodedist[i+1] = nodedist[i]+k;
j -= k;
}
my_nns = nodedist[mype+1]-nodedist[mype];
firstnode = nodedist[mype];
nodelist = (KeyValueType *)GKmalloc(eptr[nelms]*sizeof(KeyValueType), "nodelist");
auxarray = idxmalloc(eptr[nelms], "auxarray");
htable = idxsmalloc(amax(my_nns, mask+1), -1, "htable");
scounts = imalloc(4*npes+2, "scounts");
rcounts = scounts+npes;
sdispl = scounts+2*npes;
rdispl = scounts+3*npes+1;
/*********************************************/
/* first find a local numbering of the nodes */
/*********************************************/
for (i=0; i<nelms; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++) {
nodelist[j].key = eind[j];
nodelist[j].val = j;
auxarray[j] = i; /* remember the local element ID that uses this node */
}
}
ikeysort(eptr[nelms], nodelist);
for (count=1, i=1; i<eptr[nelms]; i++) {
if (nodelist[i].key > nodelist[i-1].key)
count++;
}
lnns = count;
nmap = idxmalloc(lnns, "nmap");
/* renumber the nodes of the elements array */
count = 1;
nmap[0] = nodelist[0].key;
eind[nodelist[0].val] = 0;
nodelist[0].val = auxarray[nodelist[0].val]; /* Store the local element ID */
for (i=1; i<eptr[nelms]; i++) {
if (nodelist[i].key > nodelist[i-1].key) {
nmap[count] = nodelist[i].key;
count++;
}
eind[nodelist[i].val] = count-1;
nodelist[i].val = auxarray[nodelist[i].val]; /* Store the local element ID */
}
MPI_Barrier(*comm);
/**********************************************************/
/* perform comms necessary to construct node-element list */
/**********************************************************/
iset(npes, 0, scounts);
for (pe=i=0; i<eptr[nelms]; i++) {
while (nodelist[i].key >= nodedist[pe+1])
pe++;
scounts[pe] += 2;
}
ASSERTS(pe < npes);
MPI_Alltoall((void *)scounts, 1, MPI_INT, (void *)rcounts, 1, MPI_INT, *comm);
icopy(npes, scounts, sdispl);
MAKECSR(i, npes, sdispl);
icopy(npes, rcounts, rdispl);
MAKECSR(i, npes, rdispl);
ASSERTS(sdispl[npes] == eptr[nelms]*2);
nrecv = rdispl[npes]/2;
recvbuffer = (KeyValueType *)GKmalloc(amax(1, nrecv)*sizeof(KeyValueType), "recvbuffer");
MPI_Alltoallv((void *)nodelist, scounts, sdispl, IDX_DATATYPE, (void *)recvbuffer,
rcounts, rdispl, IDX_DATATYPE, *comm);
/**************************************/
/* construct global node-element list */
/**************************************/
gnptr = idxsmalloc(my_nns+1, 0, "gnptr");
for (i=0; i<npes; i++) {
for (j=rdispl[i]/2; j<rdispl[i+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
ASSERTS(lnode >= 0 && lnode < my_nns)
gnptr[lnode]++;
}
}
MAKECSR(i, my_nns, gnptr);
gnind = idxmalloc(amax(1, gnptr[my_nns]), "gnind");
for (pe=0; pe<npes; pe++) {
firstelm = elmdist[pe];
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
gnind[gnptr[lnode]++] = recvbuffer[j].val+firstelm;
}
}
SHIFTCSR(i, my_nns, gnptr);
/*********************************************************/
/* send the node-element info to the relevant processors */
/*********************************************************/
iset(npes, 0, scounts);
/* use a hash table to ensure that each node is sent to a proc only once */
for (pe=0; pe<npes; pe++) {
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
if (htable[lnode] == -1) {
scounts[pe] += gnptr[lnode+1]-gnptr[lnode];
htable[lnode] = 1;
}
}
/* now reset the hash table */
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
htable[lnode] = -1;
}
}
MPI_Alltoall((void *)scounts, 1, MPI_INT, (void *)rcounts, 1, MPI_INT, *comm);
icopy(npes, scounts, sdispl);
MAKECSR(i, npes, sdispl);
/* create the send buffer */
nsend = sdispl[npes];
sbuffer = (idxtype *)realloc(nodelist, sizeof(idxtype)*amax(1, nsend));
count = 0;
for (pe=0; pe<npes; pe++) {
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
if (htable[lnode] == -1) {
for (k=gnptr[lnode]; k<gnptr[lnode+1]; k++) {
if (k == gnptr[lnode])
sbuffer[count++] = -1*(gnind[k]+1);
else
sbuffer[count++] = gnind[k];
}
htable[lnode] = 1;
}
}
ASSERTS(count == sdispl[pe+1]);
/* now reset the hash table */
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
htable[lnode] = -1;
}
}
icopy(npes, rcounts, rdispl);
MAKECSR(i, npes, rdispl);
nrecv = rdispl[npes];
rbuffer = (idxtype *)realloc(recvbuffer, sizeof(idxtype)*amax(1, nrecv));
MPI_Alltoallv((void *)sbuffer, scounts, sdispl, IDX_DATATYPE, (void *)rbuffer,
rcounts, rdispl, IDX_DATATYPE, *comm);
k = -1;
nptr = idxsmalloc(lnns+1, 0, "nptr");
nind = rbuffer;
for (pe=0; pe<npes; pe++) {
for (j=rdispl[pe]; j<rdispl[pe+1]; j++) {
if (nind[j] < 0) {
k++;
nind[j] = (-1*nind[j])-1;
}
nptr[k]++;
}
}
MAKECSR(i, lnns, nptr);
ASSERTS(k+1 == lnns);
ASSERTS(nptr[lnns] == nrecv)
myxadj = *xadj = idxsmalloc(nelms+1, 0, "xadj");
idxset(mask+1, -1, htable);
firstelm = elmdist[mype];
/* Two passes -- in first pass, simply find out the memory requirements */
for (pass=0; pass<2; pass++) {
for (i=0; i<nelms; i++) {
for (count=0, j=eptr[i]; j<eptr[i+1]; j++) {
node = eind[j];
for (k=nptr[node]; k<nptr[node+1]; k++) {
if ((kk=nind[k]) == firstelm+i)
continue;
m = htable[(kk&mask)];
if (m == -1) {
ind[count] = kk;
wgt[count] = 1;
htable[(kk&mask)] = count++;
}
else {
if (ind[m] == kk) {
wgt[m]++;
}
else {
for (jj=0; jj<count; jj++) {
if (ind[jj] == kk) {
wgt[jj]++;
break;
}
}
if (jj == count) {
ind[count] = kk;
wgt[count++] = 1;
}
}
}
}
}
for (j=0; j<count; j++) {
htable[(ind[j]&mask)] = -1;
if (wgt[j] >= *ncommonnodes) {
if (pass == 0)
myxadj[i]++;
else
myadjncy[myxadj[i]++] = ind[j];
}
}
}
if (pass == 0) {
MAKECSR(i, nelms, myxadj);
myadjncy = *adjncy = idxmalloc(myxadj[nelms], "adjncy");
}
else {
SHIFTCSR(i, nelms, myxadj);
}
}
/*****************************************/
/* correctly renumber the elements array */
/*****************************************/
for (i=0; i<eptr[nelms]; i++)
eind[i] = nmap[eind[i]] + gminnode;
if (*numflag == 1)
ChangeNumberingMesh2(elmdist, eptr, eind, myxadj, myadjncy, NULL, npes, mype, 0);
/* do not free nodelist, recvbuffer, rbuffer */
GKfree((void **)&scounts, (void **)&nodedist, (void **)&nmap, (void **)&sbuffer,
(void **)&htable, (void **)&nptr, (void **)&nind, (void **)&gnptr,
(void **)&gnind, (void **)&auxarray, LTERM);
FreeCtrl(&ctrl);
return;
}
void Adaptive_Partition(ctrl_t *ctrl, graph_t *graph)
{
idx_t i;
idx_t tewgt, tvsize;
real_t gtewgt, gtvsize;
real_t ubavg, lbavg, *lbvec;
WCOREPUSH;
lbvec = rwspacemalloc(ctrl, graph->ncon);
/************************************/
/* Set up important data structures */
/************************************/
CommSetup(ctrl, graph);
ubavg = ravg(graph->ncon, ctrl->ubvec);
tewgt = isum(graph->nedges, graph->adjwgt, 1);
tvsize = isum(graph->nvtxs, graph->vsize, 1);
gtewgt = (real_t) GlobalSESum(ctrl, tewgt) + 1.0/graph->gnvtxs; /* The +1/graph->gnvtxs were added to remove any FPE */
gtvsize = (real_t) GlobalSESum(ctrl, tvsize) + 1.0/graph->gnvtxs;
ctrl->redist_factor = ctrl->redist_base * ((gtewgt/gtvsize)/ ctrl->edge_size_ratio);
IFSET(ctrl->dbglvl, DBG_PROGRESS, rprintf(ctrl, "[%6"PRIDX" %8"PRIDX" %5"PRIDX" %5"PRIDX"][%"PRIDX"]\n",
graph->gnvtxs, GlobalSESum(ctrl, graph->nedges), GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo));
if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
(graph->finer != NULL && graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
AllocateRefinementWorkSpace(ctrl, 2*graph->nedges);
/***********************************************/
/* Balance the partition on the coarsest graph */
/***********************************************/
graph->where = ismalloc(graph->nvtxs+graph->nrecv, -1, "graph->where");
icopy(graph->nvtxs, graph->home, graph->where);
ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
lbavg = ravg(graph->ncon, lbvec);
if (lbavg > ubavg + 0.035 && ctrl->partType != REFINE_PARTITION)
Balance_Partition(ctrl, graph);
if (ctrl->dbglvl&DBG_PROGRESS) {
ComputePartitionParams(ctrl, graph);
ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10"PRIDX", cut: %8"PRIDX", balance: ",
graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3"PRREAL" ", lbvec[i]);
rprintf(ctrl, "\n");
/* free memory allocated by ComputePartitionParams */
gk_free((void **)&graph->ckrinfo, &graph->lnpwgts, &graph->gnpwgts, LTERM);
}
/* check if no coarsening took place */
if (graph->finer == NULL) {
ComputePartitionParams(ctrl, graph);
KWayBalance(ctrl, graph, graph->ncon);
KWayAdaptiveRefine(ctrl, graph, NGR_PASSES);
}
}
else {
/*******************************/
/* Coarsen it and partition it */
/*******************************/
switch (ctrl->ps_relation) {
case PARMETIS_PSR_COUPLED:
Match_Local(ctrl, graph);
break;
case PARMETIS_PSR_UNCOUPLED:
default:
Match_Global(ctrl, graph);
break;
}
Adaptive_Partition(ctrl, graph->coarser);
/********************************/
/* project partition and refine */
/********************************/
ProjectPartition(ctrl, graph);
ComputePartitionParams(ctrl, graph);
if (graph->ncon > 1 && graph->level < 4) {
ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
lbavg = ravg(graph->ncon, lbvec);
if (lbavg > ubavg + 0.025) {
KWayBalance(ctrl, graph, graph->ncon);
}
}
KWayAdaptiveRefine(ctrl, graph, NGR_PASSES);
if (ctrl->dbglvl&DBG_PROGRESS) {
ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10"PRIDX", cut: %8"PRIDX", balance: ",
graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3"PRREAL" ", lbvec[i]);
rprintf(ctrl, "\n");
}
}
WCOREPOP;
}
/*************************************************************************
* This function is the entry point of the initial partitioning algorithm.
* This algorithm assembles the graph to all the processors and preceed
* serially.
**************************************************************************/
idx_t Mc_Diffusion(ctrl_t *ctrl, graph_t *graph, idx_t *vtxdist, idx_t *where,
idx_t *home, idx_t npasses)
{
idx_t h, i, j;
idx_t nvtxs, nedges, ncon, pass, iter, domain, processor;
idx_t nparts, mype, npes, nlinks, me, you, wsize;
idx_t nvisited, nswaps = -1, tnswaps, done, alldone = -1;
idx_t *rowptr, *colind, *diff_where, *sr_where, *ehome, *map, *rmap;
idx_t *pack, *unpack, *match, *proc2sub, *sub2proc;
idx_t *visited, *gvisited;
real_t *transfer, *npwgts, maxdiff, minflow, maxflow;
real_t lbavg, oldlbavg, ubavg, *lbvec;
real_t *diff_flows, *sr_flows;
real_t diff_lbavg, sr_lbavg, diff_cost, sr_cost;
idx_t *rbuffer, *sbuffer;
idx_t *rcount, *rdispl;
real_t *solution, *load, *workspace;
matrix_t matrix;
graph_t *egraph;
if (graph->ncon > 3)
return 0;
WCOREPUSH;
nvtxs = graph->nvtxs;
nedges = graph->nedges;
ncon = graph->ncon;
nparts = ctrl->nparts;
mype = ctrl->mype;
npes = ctrl->npes;
ubavg = ravg(ncon, ctrl->ubvec);
/* initialize variables and allocate memory */
lbvec = rwspacemalloc(ctrl, ncon);
diff_flows = rwspacemalloc(ctrl, ncon);
sr_flows = rwspacemalloc(ctrl, ncon);
load = rwspacemalloc(ctrl, nparts);
solution = rwspacemalloc(ctrl, nparts);
npwgts = graph->gnpwgts = rwspacemalloc(ctrl, ncon*nparts);
matrix.values = rwspacemalloc(ctrl, nedges);
transfer = matrix.transfer = rwspacemalloc(ctrl, ncon*nedges);
proc2sub = iwspacemalloc(ctrl, gk_max(nparts, npes*2));
sub2proc = iwspacemalloc(ctrl, nparts);
match = iwspacemalloc(ctrl, nparts);
rowptr = matrix.rowptr = iwspacemalloc(ctrl, nparts+1);
colind = matrix.colind = iwspacemalloc(ctrl, nedges);
rcount = iwspacemalloc(ctrl, npes);
rdispl = iwspacemalloc(ctrl, npes+1);
pack = iwspacemalloc(ctrl, nvtxs);
unpack = iwspacemalloc(ctrl, nvtxs);
rbuffer = iwspacemalloc(ctrl, nvtxs);
sbuffer = iwspacemalloc(ctrl, nvtxs);
map = iwspacemalloc(ctrl, nvtxs);
rmap = iwspacemalloc(ctrl, nvtxs);
diff_where = iwspacemalloc(ctrl, nvtxs);
ehome = iwspacemalloc(ctrl, nvtxs);
wsize = gk_max(sizeof(real_t)*nparts*6, sizeof(idx_t)*(nvtxs+nparts*2+1));
workspace = (real_t *)gk_malloc(wsize, "Mc_Diffusion: workspace");
graph->ckrinfo = (ckrinfo_t *)gk_malloc(nvtxs*sizeof(ckrinfo_t), "Mc_Diffusion: rinfo");
/* construct subdomain connectivity graph */
matrix.nrows = nparts;
SetUpConnectGraph(graph, &matrix, (idx_t *)workspace);
nlinks = (matrix.nnzs-nparts) / 2;
visited = iwspacemalloc(ctrl, matrix.nnzs);
gvisited = iwspacemalloc(ctrl, matrix.nnzs);
for (pass=0; pass<npasses; pass++) {
rset(matrix.nnzs*ncon, 0.0, transfer);
iset(matrix.nnzs, 0, gvisited);
iset(matrix.nnzs, 0, visited);
iter = nvisited = 0;
/* compute ncon flow solutions */
for (h=0; h<ncon; h++) {
rset(nparts, 0.0, solution);
ComputeLoad(graph, nparts, load, ctrl->tpwgts, h);
lbvec[h] = (rmax(nparts, load)+1.0/nparts) * (real_t)nparts;
ConjGrad2(&matrix, load, solution, 0.001, workspace);
ComputeTransferVector(ncon, &matrix, solution, transfer, h);
}
oldlbavg = ravg(ncon, lbvec);
tnswaps = 0;
maxdiff = 0.0;
for (i=0; i<nparts; i++) {
for (j=rowptr[i]; j<rowptr[i+1]; j++) {
maxflow = rmax(ncon, transfer+j*ncon);
minflow = rmin(ncon, transfer+j*ncon);
maxdiff = (maxflow - minflow > maxdiff) ? maxflow - minflow : maxdiff;
}
}
while (nvisited < nlinks) {
/* compute independent sets of subdomains */
iset(gk_max(nparts, npes*2), UNMATCHED, proc2sub);
CSR_Match_SHEM(&matrix, match, proc2sub, gvisited, ncon);
/* set up the packing arrays */
iset(nparts, UNMATCHED, sub2proc);
for (i=0; i<npes*2; i++) {
if (proc2sub[i] == UNMATCHED)
break;
sub2proc[proc2sub[i]] = i/2;
}
iset(npes, 0, rcount);
for (i=0; i<nvtxs; i++) {
domain = where[i];
processor = sub2proc[domain];
if (processor != UNMATCHED)
rcount[processor]++;
}
rdispl[0] = 0;
for (i=1; i<npes+1; i++)
rdispl[i] = rdispl[i-1] + rcount[i-1];
iset(nvtxs, UNMATCHED, unpack);
for (i=0; i<nvtxs; i++) {
domain = where[i];
processor = sub2proc[domain];
if (processor != UNMATCHED)
unpack[rdispl[processor]++] = i;
}
SHIFTCSR(i, npes, rdispl);
iset(nvtxs, UNMATCHED, pack);
for (i=0; i<rdispl[npes]; i++) {
ASSERT(unpack[i] != UNMATCHED);
domain = where[unpack[i]];
processor = sub2proc[domain];
if (processor != UNMATCHED)
pack[unpack[i]] = i;
}
/* Compute the flows */
if (proc2sub[mype*2] != UNMATCHED) {
me = proc2sub[2*mype];
you = proc2sub[2*mype+1];
ASSERT(me != you);
for (j=rowptr[me]; j<rowptr[me+1]; j++) {
if (colind[j] == you) {
visited[j] = 1;
rcopy(ncon, transfer+j*ncon, diff_flows);
break;
}
}
for (j=rowptr[you]; j<rowptr[you+1]; j++) {
if (colind[j] == me) {
visited[j] = 1;
for (h=0; h<ncon; h++) {
if (transfer[j*ncon+h] > 0.0)
diff_flows[h] = -1.0 * transfer[j*ncon+h];
}
break;
}
}
nswaps = 1;
rcopy(ncon, diff_flows, sr_flows);
iset(nvtxs, 0, sbuffer);
for (i=0; i<nvtxs; i++) {
if (where[i] == me || where[i] == you)
sbuffer[i] = 1;
}
egraph = ExtractGraph(ctrl, graph, sbuffer, map, rmap);
if (egraph != NULL) {
icopy(egraph->nvtxs, egraph->where, diff_where);
for (j=0; j<egraph->nvtxs; j++)
ehome[j] = home[map[j]];
RedoMyLink(ctrl, egraph, ehome, me, you, sr_flows, &sr_cost, &sr_lbavg);
if (ncon <= 4) {
sr_where = egraph->where;
egraph->where = diff_where;
nswaps = BalanceMyLink(ctrl, egraph, ehome, me, you, diff_flows, maxdiff,
&diff_cost, &diff_lbavg, 1.0/(real_t)nvtxs);
if ((sr_lbavg < diff_lbavg &&
(diff_lbavg >= ubavg-1.0 || sr_cost == diff_cost)) ||
(sr_lbavg < ubavg-1.0 && sr_cost < diff_cost)) {
for (i=0; i<egraph->nvtxs; i++)
where[map[i]] = sr_where[i];
}
else {
for (i=0; i<egraph->nvtxs; i++)
where[map[i]] = diff_where[i];
}
}
else {
for (i=0; i<egraph->nvtxs; i++)
where[map[i]] = egraph->where[i];
}
gk_free((void **)&egraph->xadj, &egraph->nvwgt, &egraph->adjncy, &egraph, LTERM);
}
/* Pack the flow data */
iset(nvtxs, UNMATCHED, sbuffer);
for (i=0; i<nvtxs; i++) {
domain = where[i];
if (domain == you || domain == me)
sbuffer[pack[i]] = where[i];
}
}
/* Broadcast the flow data */
gkMPI_Allgatherv((void *)&sbuffer[rdispl[mype]], rcount[mype], IDX_T,
(void *)rbuffer, rcount, rdispl, IDX_T, ctrl->comm);
/* Unpack the flow data */
for (i=0; i<rdispl[npes]; i++) {
if (rbuffer[i] != UNMATCHED)
where[unpack[i]] = rbuffer[i];
}
/* Do other stuff */
gkMPI_Allreduce((void *)visited, (void *)gvisited, matrix.nnzs,
IDX_T, MPI_MAX, ctrl->comm);
nvisited = isum(matrix.nnzs, gvisited, 1)/2;
tnswaps += GlobalSESum(ctrl, nswaps);
if (iter++ == NGD_PASSES)
break;
}
/* perform serial refinement */
Mc_ComputeSerialPartitionParams(ctrl, graph, nparts);
Mc_SerialKWayAdaptRefine(ctrl, graph, nparts, home, ctrl->ubvec, 10);
/* check for early breakout */
for (h=0; h<ncon; h++) {
lbvec[h] = (real_t)(nparts) *
npwgts[rargmax_strd(nparts,npwgts+h,ncon)*ncon+h];
}
lbavg = ravg(ncon, lbvec);
done = 0;
if (tnswaps == 0 || lbavg >= oldlbavg || lbavg <= ubavg + 0.035)
done = 1;
alldone = GlobalSEMax(ctrl, done);
if (alldone == 1)
break;
}
/* ensure that all subdomains have at least one vertex */
/*
iset(nparts, 0, match);
for (i=0; i<nvtxs; i++)
match[where[i]]++;
done = 0;
while (done == 0) {
done = 1;
me = iargmin(nparts, match);
if (match[me] == 0) {
if (ctrl->mype == PE) printf("WARNING: empty subdomain %"PRIDX" in Mc_Diffusion\n", me);
you = iargmax(nparts, match);
for (i=0; i<nvtxs; i++) {
if (where[i] == you) {
where[i] = me;
match[you]--;
match[me]++;
done = 0;
break;
}
}
}
}
*/
/* now free memory and return */
gk_free((void **)&workspace, (void **)&graph->ckrinfo, LTERM);
graph->gnpwgts = NULL;
graph->ckrinfo = NULL;
WCOREPOP;
return 0;
}
/*************************************************************************
* This function is the driver to the multi-constraint partitioning algorithm.
**************************************************************************/
void Moc_Global_Partition(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
{
int i, ncon, nparts;
floattype ftmp, ubavg, lbavg, lbvec[MAXNCON];
ncon = graph->ncon;
nparts = ctrl->nparts;
ubavg = savg(graph->ncon, ctrl->ubvec);
SetUp(ctrl, graph, wspace);
if (ctrl->dbglvl&DBG_PROGRESS) {
rprintf(ctrl, "[%6d %8d %5d %5d] [%d] [", graph->gnvtxs, GlobalSESum(ctrl, graph->nedges),
GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo);
for (i=0; i<ncon; i++)
rprintf(ctrl, " %.3f", GlobalSEMinFloat(ctrl,graph->nvwgt[samin_strd(graph->nvtxs, graph->nvwgt+i, ncon)*ncon+i]));
rprintf(ctrl, "] [");
for (i=0; i<ncon; i++)
rprintf(ctrl, " %.3f", GlobalSEMaxFloat(ctrl, graph->nvwgt[samax_strd(graph->nvtxs, graph->nvwgt+i, ncon)*ncon+i]));
rprintf(ctrl, "]\n");
}
if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
(graph->finer != NULL &&
graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
/* Done with coarsening. Find a partition */
graph->where = idxmalloc(graph->nvtxs+graph->nrecv, "graph->where");
Moc_InitPartition_RB(ctrl, graph, wspace);
if (ctrl->dbglvl&DBG_PROGRESS) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10d, balance: ", graph->gnvtxs);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3f ", lbvec[i]);
rprintf(ctrl, "\n");
}
/* In case no coarsening took place */
if (graph->finer == NULL) {
Moc_ComputePartitionParams(ctrl, graph, wspace);
Moc_KWayFM(ctrl, graph, wspace, NGR_PASSES);
}
}
else {
Moc_GlobalMatch_Balance(ctrl, graph, wspace);
Moc_Global_Partition(ctrl, graph->coarser, wspace);
Moc_ProjectPartition(ctrl, graph, wspace);
Moc_ComputePartitionParams(ctrl, graph, wspace);
if (graph->ncon > 1 && graph->level < 3) {
for (i=0; i<ncon; i++) {
ftmp = ssum_strd(nparts, graph->gnpwgts+i, ncon);
if (ftmp != 0.0)
lbvec[i] = (floattype)(nparts) *
graph->gnpwgts[samax_strd(nparts, graph->gnpwgts+i, ncon)*ncon+i]/ftmp;
else
lbvec[i] = 1.0;
}
lbavg = savg(graph->ncon, lbvec);
if (lbavg > ubavg + 0.035) {
if (ctrl->dbglvl&DBG_PROGRESS) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3f ", lbvec[i]);
rprintf(ctrl, "\n");
}
Moc_KWayBalance(ctrl, graph, wspace, graph->ncon);
}
}
Moc_KWayFM(ctrl, graph, wspace, NGR_PASSES);
if (ctrl->dbglvl&DBG_PROGRESS) {
Moc_ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10d, cut: %8d, balance: ", graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3f ", lbvec[i]);
rprintf(ctrl, "\n");
}
if (graph->level != 0)
GKfree((void **)&graph->lnpwgts, (void **)&graph->gnpwgts, LTERM);
}
return;
}
/***********************************************************************************
* This function is the entry point of the parallel multilevel local diffusion
* algorithm. It uses parallel undirected diffusion followed by adaptive k-way
* refinement. This function utilizes local coarsening.
************************************************************************************/
void ParMETIS_V3_RefineKway(idxtype *vtxdist, idxtype *xadj, idxtype *adjncy,
idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *ncon,
int *nparts, float *tpwgts, float *ubvec, int *options, int *edgecut,
idxtype *part, MPI_Comm *comm)
{
int h, i;
int npes, mype;
CtrlType ctrl;
WorkSpaceType wspace;
GraphType *graph;
int tewgt, tvsize, nmoved, maxin, maxout;
float gtewgt, gtvsize, avg, maximb;
int ps_relation, seed, dbglvl = 0;
int iwgtflag, inumflag, incon, inparts, ioptions[10];
float *itpwgts, iubvec[MAXNCON];
MPI_Comm_size(*comm, &npes);
MPI_Comm_rank(*comm, &mype);
/********************************/
/* Try and take care bad inputs */
/********************************/
if (options != NULL && options[0] == 1)
dbglvl = options[PMV3_OPTION_DBGLVL];
CheckInputs(REFINE_PARTITION, npes, dbglvl, wgtflag, &iwgtflag, numflag, &inumflag,
ncon, &incon, nparts, &inparts, tpwgts, &itpwgts, ubvec, iubvec,
NULL, NULL, options, ioptions, part, comm);
/* ADD: take care of disconnected graph */
/* ADD: take care of highly unbalanced vtxdist */
/*********************************/
/* Take care the nparts = 1 case */
/*********************************/
if (inparts <= 1) {
idxset(vtxdist[mype+1]-vtxdist[mype], 0, part);
*edgecut = 0;
return;
}
/**************************/
/* Set up data structures */
/**************************/
if (inumflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
/*****************************/
/* Set up control structures */
/*****************************/
if (ioptions[0] == 1) {
dbglvl = ioptions[PMV3_OPTION_DBGLVL];
seed = ioptions[PMV3_OPTION_SEED];
ps_relation = (npes == inparts) ? ioptions[PMV3_OPTION_PSR] : DISCOUPLED;
}
else {
dbglvl = GLOBAL_DBGLVL;
seed = GLOBAL_SEED;
ps_relation = (npes == inparts) ? COUPLED : DISCOUPLED;
}
SetUpCtrl(&ctrl, inparts, dbglvl, *comm);
ctrl.CoarsenTo = amin(vtxdist[npes]+1, 50*incon*amax(npes, inparts));
ctrl.ipc_factor = 1000.0;
ctrl.redist_factor = 1.0;
ctrl.redist_base = 1.0;
ctrl.seed = (seed == 0) ? mype : seed*mype;
ctrl.sync = GlobalSEMax(&ctrl, seed);
ctrl.partType = REFINE_PARTITION;
ctrl.ps_relation = ps_relation;
ctrl.tpwgts = itpwgts;
graph = Moc_SetUpGraph(&ctrl, incon, vtxdist, xadj, vwgt, adjncy, adjwgt, &iwgtflag);
graph->vsize = idxsmalloc(graph->nvtxs, 1, "vsize");
graph->home = idxmalloc(graph->nvtxs, "home");
if (ctrl.ps_relation == COUPLED)
idxset(graph->nvtxs, mype, graph->home);
else
idxcopy(graph->nvtxs, part, graph->home);
tewgt = idxsum(graph->nedges, graph->adjwgt);
tvsize = idxsum(graph->nvtxs, graph->vsize);
gtewgt = (float) GlobalSESum(&ctrl, tewgt) + 1.0/graph->gnvtxs;
gtvsize = (float) GlobalSESum(&ctrl, tvsize) + 1.0/graph->gnvtxs;
ctrl.edge_size_ratio = gtewgt/gtvsize;
scopy(incon, iubvec, ctrl.ubvec);
PreAllocateMemory(&ctrl, graph, &wspace);
/***********************/
/* Partition and Remap */
/***********************/
IFSET(ctrl.dbglvl, DBG_TIME, InitTimers(&ctrl));
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, starttimer(ctrl.TotalTmr));
Adaptive_Partition(&ctrl, graph, &wspace);
ParallelReMapGraph(&ctrl, graph, &wspace);
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
IFSET(ctrl.dbglvl, DBG_TIME, stoptimer(ctrl.TotalTmr));
idxcopy(graph->nvtxs, graph->where, part);
if (edgecut != NULL)
*edgecut = graph->mincut;
/***********************/
/* Take care of output */
/***********************/
IFSET(ctrl.dbglvl, DBG_TIME, PrintTimingInfo(&ctrl));
IFSET(ctrl.dbglvl, DBG_TIME, MPI_Barrier(ctrl.gcomm));
if (ctrl.dbglvl&DBG_INFO) {
Mc_ComputeMoveStatistics(&ctrl, graph, &nmoved, &maxin, &maxout);
rprintf(&ctrl, "Final %3d-way Cut: %6d \tBalance: ", inparts, graph->mincut);
avg = 0.0;
for (h=0; h<incon; h++) {
maximb = 0.0;
for (i=0; i<inparts; i++)
maximb = amax(maximb, graph->gnpwgts[i*incon+h]/itpwgts[i*incon+h]);
avg += maximb;
rprintf(&ctrl, "%.3f ", maximb);
}
rprintf(&ctrl, "\nNMoved: %d %d %d %d\n", nmoved, maxin, maxout, maxin+maxout);
}
/*************************************/
/* Free memory, renumber, and return */
/*************************************/
GKfree((void **)&graph->lnpwgts, (void **)&graph->gnpwgts, (void **)&graph->nvwgt, (void **)(&graph->home), (void **)(&graph->vsize), LTERM);
GKfree((void **)&itpwgts, LTERM);
FreeInitialGraphAndRemap(graph, iwgtflag);
FreeWSpace(&wspace);
FreeCtrl(&ctrl);
if (inumflag == 1)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
return;
}
