/*************************************************************************
* This function finds a matching using the HEM heuristic
**************************************************************************/
void MCMatch_SBHEM(CtrlType *ctrl, GraphType *graph, int norm)
{
int i, ii, j, k, nvtxs, cnvtxs, ncon, maxidx, maxwgt, avgdegree;
idxtype *xadj, *adjncy, *adjwgt;
idxtype *match, *cmap, *degrees, *perm, *tperm;
float *nvwgt, vbal;
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MatchTmr));
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
nvwgt = graph->nvwgt;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
cmap = graph->cmap;
match = idxset(nvtxs, UNMATCHED, idxwspacemalloc(ctrl, nvtxs));
perm = idxwspacemalloc(ctrl, nvtxs);
tperm = idxwspacemalloc(ctrl, nvtxs);
degrees = idxwspacemalloc(ctrl, nvtxs);
RandomPermute(nvtxs, tperm, 1);
avgdegree = (int)(0.7*(xadj[nvtxs]/nvtxs));
for (i=0; i<nvtxs; i++)
degrees[i] = (xadj[i+1]-xadj[i] > avgdegree ? avgdegree : xadj[i+1]-xadj[i]);
BucketSortKeysInc(nvtxs, avgdegree, degrees, tperm, perm);
cnvtxs = 0;
/* Take care any islands. Islands are matched with non-islands due to coarsening */
for (ii=0; ii<nvtxs; ii++) {
i = perm[ii];
if (match[i] == UNMATCHED) { /* Unmatched */
if (xadj[i] < xadj[i+1])
break;
maxidx = i;
for (j=nvtxs-1; j>ii; j--) {
k = perm[j];
if (match[k] == UNMATCHED && xadj[k] < xadj[k+1]) {
maxidx = k;
break;
}
}
cmap[i] = cmap[maxidx] = cnvtxs++;
match[i] = maxidx;
match[maxidx] = i;
}
}
/* Continue with normal matching */
for (; ii<nvtxs; ii++) {
i = perm[ii];
if (match[i] == UNMATCHED) { /* Unmatched */
maxidx = i;
maxwgt = -1;
vbal = 0.0;
/* Find a heavy-edge matching, subject to maxvwgt constraints */
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = adjncy[j];
if (match[k] == UNMATCHED && AreAllVwgtsBelowFast(ncon, nvwgt+i*ncon, nvwgt+k*ncon, ctrl->nmaxvwgt)) {
if (maxidx != i)
vbal = BetterVBalance(ncon, norm, nvwgt+i*ncon, nvwgt+maxidx*ncon, nvwgt+k*ncon);
if (vbal > 0 || (vbal > -.01 && maxwgt < adjwgt[j])) {
maxwgt = adjwgt[j];
maxidx = k;
}
}
}
cmap[i] = cmap[maxidx] = cnvtxs++;
match[i] = maxidx;
match[maxidx] = i;
}
}
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MatchTmr));
idxwspacefree(ctrl, nvtxs); /* degrees */
idxwspacefree(ctrl, nvtxs); /* tperm */
CreateCoarseGraph(ctrl, graph, cnvtxs, match, perm);
idxwspacefree(ctrl, nvtxs);
idxwspacefree(ctrl, nvtxs);
}
void Match_Global(ctrl_t *ctrl, graph_t *graph)
{
idx_t h, i, ii, j, k;
idx_t nnbrs, nvtxs, ncon, cnvtxs, firstvtx, lastvtx, maxi, maxidx, nkept;
idx_t otherlastvtx, nrequests, nchanged, pass, nmatched, wside;
idx_t *xadj, *adjncy, *adjwgt, *vtxdist, *home, *myhome;
idx_t *match;
idx_t *peind, *sendptr, *recvptr;
idx_t *perm, *iperm, *nperm, *changed;
real_t *nvwgt, maxnvwgt;
idx_t *nreqs_pe;
ikv_t *match_requests, *match_granted, *pe_requests;
idx_t last_unmatched;
WCOREPUSH;
maxnvwgt = 0.75/((real_t)(ctrl->CoarsenTo));
graph->match_type = PARMETIS_MTYPE_GLOBAL;
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MatchTmr));
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
home = graph->home;
nvwgt = graph->nvwgt;
vtxdist = graph->vtxdist;
firstvtx = vtxdist[ctrl->mype];
lastvtx = vtxdist[ctrl->mype+1];
nnbrs = graph->nnbrs;
peind = graph->peind;
sendptr = graph->sendptr;
recvptr = graph->recvptr;
match = graph->match = ismalloc(nvtxs+graph->nrecv, UNMATCHED, "GlobalMatch: match");
/* wspacemalloc'ed arrays */
myhome = iset(nvtxs+graph->nrecv, UNMATCHED, iwspacemalloc(ctrl, nvtxs+graph->nrecv));
nreqs_pe = iset(nnbrs, 0, iwspacemalloc(ctrl, nnbrs));
perm = iwspacemalloc(ctrl, nvtxs);
iperm = iwspacemalloc(ctrl, nvtxs);
nperm = iwspacemalloc(ctrl, nnbrs);
changed = iwspacemalloc(ctrl, nvtxs);
match_requests = ikvwspacemalloc(ctrl, graph->nsend);
match_granted = ikvwspacemalloc(ctrl, graph->nrecv);
/* create the traversal order */
FastRandomPermute(nvtxs, perm, 1);
for (i=0; i<nvtxs; i++)
iperm[perm[i]] = i;
iincset(nnbrs, 0, nperm);
/* if coasening for adaptive/repartition, exchange home information */
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
PASSERT(ctrl, home != NULL);
icopy(nvtxs, home, myhome);
CommInterfaceData(ctrl, graph, myhome, myhome+nvtxs);
}
/* if coarsening for ordering, replace home with where information */
if (ctrl->partType == ORDER_PARTITION) {
PASSERT(ctrl, graph->where != NULL);
icopy(nvtxs, graph->where, myhome);
CommInterfaceData(ctrl, graph, myhome, myhome+nvtxs);
}
/* mark all heavy vertices as TOO_HEAVY so they will not be matched */
for (nchanged=i=0; i<nvtxs; i++) {
for (h=0; h<ncon; h++) {
if (nvwgt[i*ncon+h] > maxnvwgt) {
match[i] = TOO_HEAVY;
nchanged++;
break;
}
}
}
/* If no heavy vertices, pick one at random and treat it as such so that
at the end of the matching each partition will still have one vertex.
This is to eliminate the cases in which once a matching has been
computed, a processor ends up having no vertices */
if (nchanged == 0)
match[RandomInRange(nvtxs)] = TOO_HEAVY;
CommInterfaceData(ctrl, graph, match, match+nvtxs);
/* set initial value of nkept based on how over/under weight the
partition is to begin with */
nkept = graph->gnvtxs/ctrl->npes - nvtxs;
/* Find a matching by doing multiple iterations */
for (nmatched=pass=0; pass<NMATCH_PASSES; pass++) {
wside = (graph->level+pass)%2;
nchanged = nrequests = 0;
for (last_unmatched=ii=nmatched; ii<nvtxs; ii++) {
i = perm[ii];
if (match[i] == UNMATCHED) { /* Unmatched */
maxidx = i;
maxi = -1;
/* Deal with islands. Find another vertex and match it with */
if (xadj[i] == xadj[i+1]) {
last_unmatched = gk_max(ii, last_unmatched)+1;
for (; last_unmatched<nvtxs; last_unmatched++) {
k = perm[last_unmatched];
if (match[k] == UNMATCHED && myhome[i] == myhome[k]) {
match[i] = firstvtx+k + (i <= k ? KEEP_BIT : 0);
match[k] = firstvtx+i + (i > k ? KEEP_BIT : 0);
changed[nchanged++] = i;
changed[nchanged++] = k;
break;
}
}
continue;
}
/* Find a heavy-edge matching. */
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = adjncy[j];
if (match[k] == UNMATCHED && myhome[k] == myhome[i]) {
if (ncon == 1) {
if (maxi == -1 || adjwgt[maxi] < adjwgt[j] ||
(adjwgt[maxi] == adjwgt[j] && RandomInRange(xadj[i+1]-xadj[i]) == 0)) {
maxi = j;
maxidx = k;
}
}
else {
if (maxi == -1 || adjwgt[maxi] < adjwgt[j] ||
(adjwgt[maxi] == adjwgt[j] && maxidx < nvtxs && k < nvtxs &&
BetterVBalance(ncon,nvwgt+i*ncon,nvwgt+maxidx*ncon,nvwgt+k*ncon) >= 0)) {
maxi = j;
maxidx = k;
}
}
}
}
if (maxi != -1) {
k = adjncy[maxi];
if (k < nvtxs) { /* Take care the local vertices first */
/* Here we give preference the local matching by granting it right away */
match[i] = firstvtx+k + (i <= k ? KEEP_BIT : 0);
match[k] = firstvtx+i + (i > k ? KEEP_BIT : 0);
changed[nchanged++] = i;
changed[nchanged++] = k;
}
else { /* Take care any remote boundary vertices */
match[k] = MAYBE_MATCHED;
/* Alternate among which vertices will issue the requests */
if ((wside == 0 && firstvtx+i < graph->imap[k]) ||
(wside == 1 && firstvtx+i > graph->imap[k])) {
match[i] = MAYBE_MATCHED;
match_requests[nrequests].key = graph->imap[k];
match_requests[nrequests].val = firstvtx+i;
nrequests++;
}
}
}
}
}
#ifdef DEBUG_MATCH
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match1");
myprintf(ctrl, "[c: %2"PRIDX"] Nlocal: %"PRIDX", Nrequests: %"PRIDX"\n", c, nlocal, nrequests);
#endif
/***********************************************************
* Exchange the match_requests, requests for me are stored in
* match_granted
************************************************************/
/* Issue the receives first. Note that from each PE can receive a maximum
of the interface node that it needs to send it in the case of a mat-vec */
for (i=0; i<nnbrs; i++) {
gkMPI_Irecv((void *)(match_granted+recvptr[i]), 2*(recvptr[i+1]-recvptr[i]), IDX_T,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends next. This needs some work */
ikvsorti(nrequests, match_requests);
for (j=i=0; i<nnbrs; i++) {
otherlastvtx = vtxdist[peind[i]+1];
for (k=j; k<nrequests && match_requests[k].key < otherlastvtx; k++);
gkMPI_Isend((void *)(match_requests+j), 2*(k-j), IDX_T, peind[i], 1,
ctrl->comm, ctrl->sreq+i);
j = k;
}
/* OK, now get into the loop waiting for the operations to finish */
gkMPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
for (i=0; i<nnbrs; i++) {
gkMPI_Get_count(ctrl->statuses+i, IDX_T, nreqs_pe+i);
nreqs_pe[i] = nreqs_pe[i]/2; /* Adjust for pairs of IDX_T */
}
gkMPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/***********************************************************
* Now, go and service the requests that you received in
* match_granted
************************************************************/
RandomPermute(nnbrs, nperm, 0);
for (ii=0; ii<nnbrs; ii++) {
i = nperm[ii];
pe_requests = match_granted+recvptr[i];
for (j=0; j<nreqs_pe[i]; j++) {
k = pe_requests[j].key;
PASSERTP(ctrl, k >= firstvtx && k < lastvtx, (ctrl, "%"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX"\n", firstvtx, lastvtx, k, j, peind[i]));
/* myprintf(ctrl, "Requesting a match %"PRIDX" %"PRIDX"\n", pe_requests[j].key, pe_requests[j].val); */
if (match[k-firstvtx] == UNMATCHED) { /* Bingo, lets grant this request */
changed[nchanged++] = k-firstvtx;
if (nkept >= 0) { /* decide who to keep it based on local balance */
match[k-firstvtx] = pe_requests[j].val + KEEP_BIT;
nkept--;
}
else {
match[k-firstvtx] = pe_requests[j].val;
pe_requests[j].key += KEEP_BIT;
nkept++;
}
/* myprintf(ctrl, "Request from pe:%"PRIDX" (%"PRIDX" %"PRIDX") granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */
}
else { /* We are not granting the request */
/* myprintf(ctrl, "Request from pe:%"PRIDX" (%"PRIDX" %"PRIDX") not granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */
pe_requests[j].key = UNMATCHED;
}
}
}
/***********************************************************
* Exchange the match_granted information. It is stored in
* match_requests
************************************************************/
/* Issue the receives first. Note that from each PE can receive a maximum
of the interface node that it needs to send during the case of a mat-vec */
for (i=0; i<nnbrs; i++) {
gkMPI_Irecv((void *)(match_requests+sendptr[i]), 2*(sendptr[i+1]-sendptr[i]), IDX_T,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends next. */
for (i=0; i<nnbrs; i++) {
gkMPI_Isend((void *)(match_granted+recvptr[i]), 2*nreqs_pe[i], IDX_T,
peind[i], 1, ctrl->comm, ctrl->sreq+i);
}
/* OK, now get into the loop waiting for the operations to finish */
gkMPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
for (i=0; i<nnbrs; i++) {
gkMPI_Get_count(ctrl->statuses+i, IDX_T, nreqs_pe+i);
nreqs_pe[i] = nreqs_pe[i]/2; /* Adjust for pairs of IDX_T */
}
gkMPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/***********************************************************
* Now, go and through the match_requests and update local
* match information for the matchings that were granted.
************************************************************/
for (i=0; i<nnbrs; i++) {
pe_requests = match_requests+sendptr[i];
for (j=0; j<nreqs_pe[i]; j++) {
match[pe_requests[j].val-firstvtx] = pe_requests[j].key;
if (pe_requests[j].key != UNMATCHED)
changed[nchanged++] = pe_requests[j].val-firstvtx;
}
}
for (i=0; i<nchanged; i++) {
ii = iperm[changed[i]];
perm[ii] = perm[nmatched];
iperm[perm[nmatched]] = ii;
nmatched++;
}
CommChangedInterfaceData(ctrl, graph, nchanged, changed, match,
match_requests, match_granted);
}
/* Traverse the vertices and those that were unmatched, match them with themselves */
cnvtxs = 0;
for (i=0; i<nvtxs; i++) {
if (match[i] == UNMATCHED || match[i] == TOO_HEAVY) {
match[i] = (firstvtx+i) + KEEP_BIT;
cnvtxs++;
}
else if (match[i] >= KEEP_BIT) { /* A matched vertex which I get to keep */
cnvtxs++;
}
}
if (ctrl->dbglvl&DBG_MATCHINFO) {
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match");
myprintf(ctrl, "Cnvtxs: %"PRIDX"\n", cnvtxs);
rprintf(ctrl, "Done with matching...\n");
}
WCOREPOP;
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MatchTmr));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->ContractTmr));
CreateCoarseGraph_Global(ctrl, graph, cnvtxs);
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->ContractTmr));
}
void Match_Local(ctrl_t *ctrl, graph_t *graph)
{
idx_t h, i, ii, j, k;
idx_t nvtxs, ncon, cnvtxs, firstvtx, maxi, maxidx, edge;
idx_t *xadj, *adjncy, *adjwgt, *vtxdist, *home, *myhome;
idx_t *perm, *match;
real_t maxnvwgt, *nvwgt;
WCOREPUSH;
maxnvwgt = 0.75/((real_t)ctrl->CoarsenTo);
graph->match_type = PARMETIS_MTYPE_LOCAL;
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MatchTmr));
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
nvwgt = graph->nvwgt;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
home = graph->home;
vtxdist = graph->vtxdist;
firstvtx = vtxdist[ctrl->mype];
match = graph->match = imalloc(nvtxs+graph->nrecv, "HEM_Match: match");
/* wspacemalloc'ed arrays */
myhome = iset(nvtxs+graph->nrecv, UNMATCHED, iwspacemalloc(ctrl, nvtxs+graph->nrecv));
perm = iwspacemalloc(ctrl, nvtxs);
/* if coasening for adaptive/repartition, exchange home information */
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
icopy(nvtxs, home, myhome);
CommInterfaceData(ctrl, graph, myhome, myhome+nvtxs);
}
/*************************************************************
* Go now and find a local matching
*************************************************************/
iset(nvtxs, UNMATCHED, match);
iset(graph->nrecv, 0, match+nvtxs); /* Easy way to handle remote vertices */
FastRandomPermute(nvtxs, perm, 1);
for (cnvtxs=0, ii=0; ii<nvtxs; ii++) {
i = perm[ii];
if (match[i] == UNMATCHED) {
maxidx = maxi = -1;
/* Find a heavy-edge matching, if the weight of the vertex is OK */
for (h=0; h<ncon; h++)
if (nvwgt[i*ncon+h] > maxnvwgt)
break;
if (h == ncon && ii<nvtxs) {
for (j=xadj[i]; j<xadj[i+1]; j++) {
edge = adjncy[j];
/* match only with local vertices */
if (myhome[edge] != myhome[i] || edge >= nvtxs)
continue;
for (h=0; h<ncon; h++)
if (nvwgt[edge*ncon+h] > maxnvwgt)
break;
if (h == ncon) {
if (match[edge] == UNMATCHED &&
(maxi == -1 ||
adjwgt[maxi] < adjwgt[j] ||
(adjwgt[maxi] == adjwgt[j] &&
BetterVBalance(ncon,nvwgt+i*ncon,nvwgt+maxidx*ncon,nvwgt+edge*ncon) >= 0))) {
maxi = j;
maxidx = edge;
}
}
}
}
if (maxi != -1) {
k = adjncy[maxi];
match[i] = firstvtx+k + (i <= k ? KEEP_BIT : 0);
match[k] = firstvtx+i + (i > k ? KEEP_BIT : 0);
}
else {
match[i] = firstvtx+i + KEEP_BIT;
}
cnvtxs++;
}
}
CommInterfaceData(ctrl, graph, match, match+nvtxs);
#ifdef DEBUG_MATCH
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match1");
#endif
if (ctrl->dbglvl&DBG_MATCHINFO) {
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match");
myprintf(ctrl, "Cnvtxs: %"PRIDX"\n", cnvtxs);
rprintf(ctrl, "Done with matching...\n");
}
WCOREPOP;
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MatchTmr));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->ContractTmr));
CreateCoarseGraph_Local(ctrl, graph, cnvtxs);
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->ContractTmr));
}
/*************************************************************************
* This function finds a matching
**************************************************************************/
void Moc_GlobalMatch_Balance(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
{
int h, i, ii, j, k;
int nnbrs, nvtxs, ncon, cnvtxs, firstvtx, lastvtx, maxi, maxidx, nkept;
int otherlastvtx, nrequests, nchanged, pass, nmatched, wside;
idxtype *xadj, *ladjncy, *adjwgt, *vtxdist, *home, *myhome, *shome, *rhome;
idxtype *match, *rmatch, *smatch;
idxtype *peind, *sendptr, *recvptr;
idxtype *perm, *iperm, *nperm, *changed;
floattype *nvwgt, maxnvwgt;
int *nreqs_pe;
KeyValueType *match_requests, *match_granted, *pe_requests;
maxnvwgt = 1.0/((floattype)(ctrl->nparts)*MAXNVWGT_FACTOR);
graph->match_type = MATCH_GLOBAL;
IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MatchTmr));
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
ladjncy = graph->adjncy;
adjwgt = graph->adjwgt;
home = graph->home;
nvwgt = graph->nvwgt;
vtxdist = graph->vtxdist;
firstvtx = vtxdist[ctrl->mype];
lastvtx = vtxdist[ctrl->mype+1];
match = graph->match = idxsmalloc(nvtxs+graph->nrecv, UNMATCHED, "HEM_Match: match");
myhome = idxsmalloc(nvtxs+graph->nrecv, UNMATCHED, "HEM_Match: myhome");
/*------------------------------------------------------------
/ Send/Receive the home information of interface vertices
/------------------------------------------------------------*/
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
idxcopy(nvtxs, home, myhome);
shome = wspace->indices;
rhome = myhome + nvtxs;
CommInterfaceData(ctrl, graph, myhome, shome, rhome);
}
nnbrs = graph->nnbrs;
peind = graph->peind;
sendptr = graph->sendptr;
recvptr = graph->recvptr;
/* Use wspace->indices as the tmp space for matching info of the boundary
* vertices that are sent and received */
rmatch = match + nvtxs;
smatch = wspace->indices;
changed = smatch+graph->nsend;
/* Use wspace->indices as the tmp space for match requests of the boundary
* vertices that are sent and received */
match_requests = wspace->pairs;
match_granted = match_requests + graph->nsend;
nreqs_pe = ismalloc(nnbrs, 0, "Match_HEM: nreqs_pe");
nkept = graph->gnvtxs/ctrl->npes - nvtxs;
perm = (idxtype *)wspace->degrees;
iperm = perm + nvtxs;
FastRandomPermute(nvtxs, perm, 1);
for (i=0; i<nvtxs; i++)
iperm[perm[i]] = i;
nperm = iperm + nvtxs;
for (i=0; i<nnbrs; i++)
nperm[i] = i;
/*************************************************************
* Go now and find a matching by doing multiple iterations
*************************************************************/
/* First nullify the heavy vertices */
for (nchanged=i=0; i<nvtxs; i++) {
for (h=0; h<ncon; h++)
if (nvwgt[i*ncon+h] > maxnvwgt) {
break;
}
if (h != ncon) {
match[i] = TOO_HEAVY;
nchanged++;
}
}
if (GlobalSESum(ctrl, nchanged) > 0) {
IFSET(ctrl->dbglvl, DBG_PROGRESS,
rprintf(ctrl, "We found %d heavy vertices!\n", GlobalSESum(ctrl, nchanged)));
CommInterfaceData(ctrl, graph, match, smatch, rmatch);
}
for (nmatched=pass=0; pass<NMATCH_PASSES; pass++) {
wside = (graph->level+pass)%2;
nchanged = nrequests = 0;
for (ii=nmatched; ii<nvtxs; ii++) {
i = perm[ii];
if (match[i] == UNMATCHED) { /* Unmatched */
maxidx = i;
maxi = -1;
/* Find a heavy-edge matching */
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = ladjncy[j];
if (match[k] == UNMATCHED &&
myhome[k] == myhome[i] &&
(maxi == -1 ||
adjwgt[maxi] < adjwgt[j] ||
(maxidx < nvtxs &&
k < nvtxs &&
adjwgt[maxi] == adjwgt[j] &&
BetterVBalance(ncon,nvwgt+i*ncon,nvwgt+maxidx*ncon,nvwgt+k*ncon) >= 0))) {
maxi = j;
maxidx = k;
}
}
if (maxi != -1) {
k = ladjncy[maxi];
if (k < nvtxs) { /* Take care the local vertices first */
/* Here we give preference the local matching by granting it right away */
if (i <= k) {
match[i] = firstvtx+k + KEEP_BIT;
match[k] = firstvtx+i;
}
else {
match[i] = firstvtx+k;
match[k] = firstvtx+i + KEEP_BIT;
}
changed[nchanged++] = i;
changed[nchanged++] = k;
}
else { /* Take care any remote boundary vertices */
match[k] = MAYBE_MATCHED;
/* Alternate among which vertices will issue the requests */
if ((wside ==0 && firstvtx+i < graph->imap[k]) || (wside == 1 && firstvtx+i > graph->imap[k])) {
match[i] = MAYBE_MATCHED;
match_requests[nrequests].key = graph->imap[k];
match_requests[nrequests].val = firstvtx+i;
nrequests++;
}
}
}
}
}
#ifdef DEBUG_MATCH
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match1");
myprintf(ctrl, "[c: %2d] Nlocal: %d, Nrequests: %d\n", c, nlocal, nrequests);
#endif
/***********************************************************
* Exchange the match_requests, requests for me are stored in
* match_granted
************************************************************/
/* Issue the receives first. Note that from each PE can receive a maximum
of the interface node that it needs to send it in the case of a mat-vec */
for (i=0; i<nnbrs; i++) {
MPI_Irecv((void *)(match_granted+recvptr[i]), 2*(recvptr[i+1]-recvptr[i]), IDX_DATATYPE,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends next. This needs some work */
ikeysort(nrequests, match_requests);
for (j=i=0; i<nnbrs; i++) {
otherlastvtx = vtxdist[peind[i]+1];
for (k=j; k<nrequests && match_requests[k].key < otherlastvtx; k++);
MPI_Isend((void *)(match_requests+j), 2*(k-j), IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
j = k;
}
/* OK, now get into the loop waiting for the operations to finish */
MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
for (i=0; i<nnbrs; i++) {
MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nreqs_pe+i);
nreqs_pe[i] = nreqs_pe[i]/2; /* Adjust for pairs of IDX_DATATYPE */
}
MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/***********************************************************
* Now, go and service the requests that you received in
* match_granted
************************************************************/
RandomPermute(nnbrs, nperm, 0);
for (ii=0; ii<nnbrs; ii++) {
i = nperm[ii];
pe_requests = match_granted+recvptr[i];
for (j=0; j<nreqs_pe[i]; j++) {
k = pe_requests[j].key;
ASSERTP(ctrl, k >= firstvtx && k < lastvtx, (ctrl, "%d %d %d %d %d\n", firstvtx, lastvtx, k, j, peind[i]));
/* myprintf(ctrl, "Requesting a match %d %d\n", pe_requests[j].key, pe_requests[j].val); */
if (match[k-firstvtx] == UNMATCHED) { /* Bingo, lets grant this request */
changed[nchanged++] = k-firstvtx;
if (nkept >= 0) { /* Flip a coin for who gets it */
match[k-firstvtx] = pe_requests[j].val + KEEP_BIT;
nkept--;
}
else {
match[k-firstvtx] = pe_requests[j].val;
pe_requests[j].key += KEEP_BIT;
nkept++;
}
/* myprintf(ctrl, "Request from pe:%d (%d %d) granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */
}
else { /* We are not granting the request */
/* myprintf(ctrl, "Request from pe:%d (%d %d) not granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */
pe_requests[j].key = UNMATCHED;
}
}
}
/***********************************************************
* Exchange the match_granted information. It is stored in
* match_requests
************************************************************/
/* Issue the receives first. Note that from each PE can receive a maximum
of the interface node that it needs to send during the case of a mat-vec */
for (i=0; i<nnbrs; i++) {
MPI_Irecv((void *)(match_requests+sendptr[i]), 2*(sendptr[i+1]-sendptr[i]), IDX_DATATYPE,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends next. */
for (i=0; i<nnbrs; i++) {
MPI_Isend((void *)(match_granted+recvptr[i]), 2*nreqs_pe[i], IDX_DATATYPE,
peind[i], 1, ctrl->comm, ctrl->sreq+i);
}
/* OK, now get into the loop waiting for the operations to finish */
MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
for (i=0; i<nnbrs; i++) {
MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nreqs_pe+i);
nreqs_pe[i] = nreqs_pe[i]/2; /* Adjust for pairs of IDX_DATATYPE */
}
MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/***********************************************************
* Now, go and through the match_requests and update local
* match information for the matchings that were granted.
************************************************************/
for (i=0; i<nnbrs; i++) {
pe_requests = match_requests+sendptr[i];
for (j=0; j<nreqs_pe[i]; j++) {
match[pe_requests[j].val-firstvtx] = pe_requests[j].key;
if (pe_requests[j].key != UNMATCHED)
changed[nchanged++] = pe_requests[j].val-firstvtx;
}
}
for (i=0; i<nchanged; i++) {
ii = iperm[changed[i]];
perm[ii] = perm[nmatched];
iperm[perm[nmatched]] = ii;
nmatched++;
}
CommChangedInterfaceData(ctrl, graph, nchanged, changed, match, match_requests, match_granted, wspace->pv4);
}
/* Traverse the vertices and those that were unmatched, match them with themselves */
cnvtxs = 0;
for (i=0; i<nvtxs; i++) {
if (match[i] == UNMATCHED || match[i] == TOO_HEAVY) {
match[i] = (firstvtx+i) + KEEP_BIT;
cnvtxs++;
}
else if (match[i] >= KEEP_BIT) { /* A matched vertex which I get to keep */
cnvtxs++;
}
}
if (ctrl->dbglvl&DBG_MATCHINFO) {
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match");
myprintf(ctrl, "Cnvtxs: %d\n", cnvtxs);
rprintf(ctrl, "Done with matching...\n");
}
GKfree((void **)(&myhome), (void **)(&nreqs_pe), LTERM);
IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MatchTmr));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->ContractTmr));
Moc_Global_CreateCoarseGraph(ctrl, graph, wspace, cnvtxs);
IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->ContractTmr));
}
