/*************************************************************************
* This function implements a simple graph adaption strategy.
**************************************************************************/
void AdaptGraph(graph_t *graph, idx_t afactor, MPI_Comm comm)
{
idx_t i, nvtxs, nadapt, firstvtx, lastvtx;
idx_t npes, mype, mypwgt, max, min, sum;
idx_t *vwgt, *xadj, *adjncy, *adjwgt, *perm;
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
srand(mype*afactor);
nvtxs = graph->nvtxs;
xadj = graph->xadj;
adjncy = graph->adjncy;
if (graph->adjwgt == NULL)
adjwgt = graph->adjwgt = ismalloc(graph->nedges, 1, "AdaptGraph: adjwgt");
else
adjwgt = graph->adjwgt;
vwgt = graph->vwgt;
firstvtx = graph->vtxdist[mype];
lastvtx = graph->vtxdist[mype+1];
perm = imalloc(nvtxs, "AdaptGraph: perm");
FastRandomPermute(nvtxs, perm, 1);
nadapt = RandomInRange(nvtxs);
nadapt = RandomInRange(nvtxs);
nadapt = RandomInRange(nvtxs);
for (i=0; i<nadapt; i++)
vwgt[perm[i]] = afactor*vwgt[perm[i]];
/*
for (i=0; i<nvtxs; i++) {
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = adjncy[j];
if (k >= firstvtx && k < lastvtx) {
adjwgt[j] = (int)pow(1.0*(gk_min(vwgt[i],vwgt[k-firstvtx])), .6667);
if (adjwgt[j] == 0)
adjwgt[j] = 1;
}
}
}
*/
mypwgt = isum(nvtxs, vwgt, 1);
MPI_Allreduce((void *)&mypwgt, (void *)&max, 1, IDX_T, MPI_MAX, comm);
MPI_Allreduce((void *)&mypwgt, (void *)&min, 1, IDX_T, MPI_MIN, comm);
MPI_Allreduce((void *)&mypwgt, (void *)&sum, 1, IDX_T, MPI_SUM, comm);
if (mype == 0)
printf("Initial Load Imbalance: %5.4"PRREAL", [%5"PRIDX" %5"PRIDX" %5"PRIDX"] for afactor: %"PRIDX"\n", (1.0*max*npes)/(1.0*sum), min, max, sum, afactor);
free(perm);
}
idx_t ComputeRealCutFromMoved(idx_t *vtxdist, idx_t *mvtxdist, idx_t *part,
idx_t *mpart, char *filename, MPI_Comm comm)
{
idx_t i, j, nvtxs, mype, npes, cut;
idx_t *xadj, *adjncy, *gpart, *gmpart, *perm, *sizes;
MPI_Status status;
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
if (mype != 0) {
gkMPI_Send((void *)part, vtxdist[mype+1]-vtxdist[mype], IDX_T, 0, 1, comm);
gkMPI_Send((void *)mpart, mvtxdist[mype+1]-mvtxdist[mype], IDX_T, 0, 1, comm);
}
else { /* Processor 0 does all the rest */
gpart = imalloc(vtxdist[npes], "ComputeRealCut: gpart");
icopy(vtxdist[1], part, gpart);
gmpart = imalloc(mvtxdist[npes], "ComputeRealCut: gmpart");
icopy(mvtxdist[1], mpart, gmpart);
for (i=1; i<npes; i++) {
gkMPI_Recv((void *)(gpart+vtxdist[i]), vtxdist[i+1]-vtxdist[i], IDX_T, i, 1, comm, &status);
gkMPI_Recv((void *)(gmpart+mvtxdist[i]), mvtxdist[i+1]-mvtxdist[i], IDX_T, i, 1, comm, &status);
}
/* OK, now go and reconstruct the permutation to go from the graph to mgraph */
perm = imalloc(vtxdist[npes], "ComputeRealCut: perm");
sizes = ismalloc(npes+1, 0, "ComputeRealCut: sizes");
for (i=0; i<vtxdist[npes]; i++)
sizes[gpart[i]]++;
MAKECSR(i, npes, sizes);
for (i=0; i<vtxdist[npes]; i++)
perm[i] = sizes[gpart[i]]++;
/* Ok, now read the graph from the file */
ReadMetisGraph(filename, &nvtxs, &xadj, &adjncy);
/* OK, now compute the cut */
for (cut=0, i=0; i<nvtxs; i++) {
for (j=xadj[i]; j<xadj[i+1]; j++) {
if (gmpart[perm[i]] != gmpart[perm[adjncy[j]]])
cut++;
}
}
cut = cut/2;
gk_free((void **)&gpart, &gmpart, &perm, &sizes, &xadj, &adjncy, LTERM);
return cut;
}
return 0;
}
/*************************************************************************
* This function implements a simple graph adaption strategy.
**************************************************************************/
void AdaptGraph2(graph_t *graph, idx_t afactor, MPI_Comm comm)
{
idx_t i, j, k, nvtxs, firstvtx, lastvtx;
idx_t npes, mype, mypwgt, max, min, sum;
idx_t *vwgt, *xadj, *adjncy, *adjwgt;
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
srand(mype*afactor);
nvtxs = graph->nvtxs;
xadj = graph->xadj;
adjncy = graph->adjncy;
if (graph->adjwgt == NULL)
adjwgt = graph->adjwgt = ismalloc(graph->nedges, 1, "AdaptGraph: adjwgt");
else
adjwgt = graph->adjwgt;
vwgt = graph->vwgt;
firstvtx = graph->vtxdist[mype];
lastvtx = graph->vtxdist[mype+1];
/* if (RandomInRange(npes+1) < .05*npes) { */
if (RandomInRange(npes+1) < 2) {
printf("[%"PRIDX"] is adapting\n", mype);
for (i=0; i<nvtxs; i++)
vwgt[i] = afactor*vwgt[i];
}
for (i=0; i<nvtxs; i++) {
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = adjncy[j];
if (k >= firstvtx && k < lastvtx) {
adjwgt[j] = (int)pow(1.0*(gk_min(vwgt[i],vwgt[k-firstvtx])), .6667);
if (adjwgt[j] == 0)
adjwgt[j] = 1;
}
}
}
mypwgt = isum(nvtxs, vwgt, 1);
gkMPI_Allreduce((void *)&mypwgt, (void *)&max, 1, IDX_T, MPI_MAX, comm);
gkMPI_Allreduce((void *)&mypwgt, (void *)&min, 1, IDX_T, MPI_MIN, comm);
gkMPI_Allreduce((void *)&mypwgt, (void *)&sum, 1, IDX_T, MPI_SUM, comm);
if (mype == 0)
printf("Initial Load Imbalance: %5.4"PRREAL", [%5"PRIDX" %5"PRIDX" %5"PRIDX"]\n", (1.0*max*npes)/(1.0*sum), min, max, sum);
}
/*************************************************************************
* This function computes the balance of the element partitioning
**************************************************************************/
real_t ComputeElementBalance(idx_t ne, idx_t nparts, idx_t *where)
{
idx_t i;
idx_t *kpwgts;
real_t balance;
kpwgts = ismalloc(nparts, 0, "ComputeElementBalance: kpwgts");
for (i=0; i<ne; i++)
kpwgts[where[i]]++;
balance = 1.0*nparts*kpwgts[iargmax(nparts, kpwgts)]/(1.0*isum(nparts, kpwgts, 1));
gk_free((void **)&kpwgts, LTERM);
return balance;
}
idx_t IsConnectedSubdomain(ctrl_t *ctrl, graph_t *graph, idx_t pid, idx_t report) {
idx_t i, j, k, nvtxs, first, last, nleft, ncmps, wgt;
idx_t *xadj, *adjncy, *where, *touched, *queue;
idx_t *cptr;
nvtxs = graph->nvtxs;
xadj = graph->xadj;
adjncy = graph->adjncy;
where = graph->where;
touched = ismalloc(nvtxs, 0, "IsConnected: touched");
queue = imalloc(nvtxs, "IsConnected: queue");
cptr = imalloc(nvtxs + 1, "IsConnected: cptr");
nleft = 0;
for (i = 0; i < nvtxs; i++) {
if (where[i] == pid) {
nleft++;
}
}
for (i = 0; i < nvtxs; i++) {
if (where[i] == pid) {
break;
}
}
touched[i] = 1;
queue[0] = i;
first = 0;
last = 1;
cptr[0] = 0; /* This actually points to queue */
ncmps = 0;
while (first != nleft) {
if (first == last) { /* Find another starting vertex */
cptr[++ncmps] = first;
for (i = 0; i < nvtxs; i++) {
if (where[i] == pid && !touched[i]) {
break;
}
}
queue[last++] = i;
touched[i] = 1;
}
i = queue[first++];
for (j = xadj[i]; j < xadj[i + 1]; j++) {
k = adjncy[j];
if (where[k] == pid && !touched[k]) {
queue[last++] = k;
touched[k] = 1;
}
}
}
cptr[++ncmps] = first;
if (ncmps > 1 && report) {
printf("The graph has %"PRIDX" connected components in partition %"PRIDX":\t", ncmps, pid);
for (i = 0; i < ncmps; i++) {
wgt = 0;
for (j = cptr[i]; j < cptr[i + 1]; j++) {
wgt += graph->vwgt[queue[j]];
}
printf("[%5"PRIDX" %5"PRIDX"] ", cptr[i + 1] - cptr[i], wgt);
/*
if (cptr[i+1]-cptr[i] == 1)
printf("[%"PRIDX" %"PRIDX"] ", queue[cptr[i]], xadj[queue[cptr[i]]+1]-xadj[queue[cptr[i]]]);
*/
}
printf("\n");
}
gk_free((void **) &touched, &queue, &cptr, LTERM);
return (ncmps == 1 ? 1 : 0);
}
idx_t FindSepInducedComponents(ctrl_t *ctrl, graph_t *graph, idx_t *cptr,
idx_t *cind) {
idx_t i, j, k, nvtxs, first, last, nleft, ncmps, wgt;
idx_t *xadj, *adjncy, *where, *touched, *queue;
nvtxs = graph->nvtxs;
xadj = graph->xadj;
adjncy = graph->adjncy;
where = graph->where;
touched = ismalloc(nvtxs, 0, "IsConnected: queue");
for (i = 0; i < graph->nbnd; i++) {
touched[graph->bndind[i]] = 1;
}
queue = cind;
nleft = 0;
for (i = 0; i < nvtxs; i++) {
if (where[i] != 2) {
nleft++;
}
}
for (i = 0; i < nvtxs; i++) {
if (where[i] != 2) {
break;
}
}
touched[i] = 1;
queue[0] = i;
first = 0;
last = 1;
cptr[0] = 0; /* This actually points to queue */
ncmps = 0;
while (first != nleft) {
if (first == last) { /* Find another starting vertex */
cptr[++ncmps] = first;
for (i = 0; i < nvtxs; i++) {
if (!touched[i]) {
break;
}
}
queue[last++] = i;
touched[i] = 1;
}
i = queue[first++];
for (j = xadj[i]; j < xadj[i + 1]; j++) {
k = adjncy[j];
if (!touched[k]) {
queue[last++] = k;
touched[k] = 1;
}
}
}
cptr[++ncmps] = first;
gk_free((void **) &touched, LTERM);
return ncmps;
}
graph_t *PruneGraph(ctrl_t *ctrl, idx_t nvtxs, idx_t *xadj, idx_t *adjncy,
idx_t *vwgt, idx_t *iperm, real_t factor)
{
idx_t i, j, k, l, nlarge, pnvtxs, pnedges;
idx_t *pxadj, *padjncy, *pvwgt;
idx_t *perm;
graph_t *graph=NULL;
perm = imalloc(nvtxs, "PruneGraph: perm");
factor = factor*xadj[nvtxs]/nvtxs;
pnvtxs = pnedges = nlarge = 0;
for (i=0; i<nvtxs; i++) {
if (xadj[i+1]-xadj[i] < factor) {
perm[i] = pnvtxs;
iperm[pnvtxs++] = i;
pnedges += xadj[i+1]-xadj[i];
}
else {
perm[i] = nvtxs - ++nlarge;
iperm[nvtxs-nlarge] = i;
}
}
IFSET(ctrl->dbglvl, METIS_DBG_INFO,
printf(" Pruned %"PRIDX" of %"PRIDX" vertices.\n", nlarge, nvtxs));
if (nlarge > 0 && nlarge < nvtxs) {
/* Prunning is possible, so go ahead and create the prunned graph */
graph = CreateGraph();
/* Allocate memory for the prunned graph*/
pxadj = graph->xadj = imalloc(pnvtxs+1, "PruneGraph: xadj");
pvwgt = graph->vwgt = imalloc(pnvtxs, "PruneGraph: vwgt");
padjncy = graph->adjncy = imalloc(pnedges, "PruneGraph: adjncy");
graph->adjwgt = ismalloc(pnedges, 1, "PruneGraph: adjwgt");
pxadj[0] = pnedges = l = 0;
for (i=0; i<nvtxs; i++) {
if (xadj[i+1]-xadj[i] < factor) {
pvwgt[l] = (vwgt == NULL ? 1 : vwgt[i]);
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = perm[adjncy[j]];
if (k < pnvtxs)
padjncy[pnedges++] = k;
}
pxadj[++l] = pnedges;
}
}
graph->nvtxs = pnvtxs;
graph->nedges = pnedges;
graph->ncon = 1;
SetupGraph_tvwgt(graph);
SetupGraph_label(graph);
}
else if (nlarge > 0 && nlarge == nvtxs) {
IFSET(ctrl->dbglvl, METIS_DBG_INFO,
printf(" Pruning is ignored as it removes all vertices.\n"));
nlarge = 0;
}
gk_free((void **)&perm, LTERM);
return graph;
}
graph_t *CompressGraph(ctrl_t *ctrl, idx_t nvtxs, idx_t *xadj, idx_t *adjncy,
idx_t *vwgt, idx_t *cptr, idx_t *cind)
{
idx_t i, ii, iii, j, jj, k, l, cnvtxs, cnedges;
idx_t *cxadj, *cadjncy, *cvwgt, *mark, *map;
ikv_t *keys;
graph_t *graph=NULL;
mark = ismalloc(nvtxs, -1, "CompressGraph: mark");
map = ismalloc(nvtxs, -1, "CompressGraph: map");
keys = ikvmalloc(nvtxs, "CompressGraph: keys");
/* Compute a key for each adjacency list */
for (i=0; i<nvtxs; i++) {
k = 0;
for (j=xadj[i]; j<xadj[i+1]; j++)
k += adjncy[j];
keys[i].key = k+i; /* Add the diagonal entry as well */
keys[i].val = i;
}
ikvsorti(nvtxs, keys);
l = cptr[0] = 0;
for (cnvtxs=i=0; i<nvtxs; i++) {
ii = keys[i].val;
if (map[ii] == -1) {
mark[ii] = i; /* Add the diagonal entry */
for (j=xadj[ii]; j<xadj[ii+1]; j++)
mark[adjncy[j]] = i;
map[ii] = cnvtxs;
cind[l++] = ii;
for (j=i+1; j<nvtxs; j++) {
iii = keys[j].val;
if (keys[i].key != keys[j].key || xadj[ii+1]-xadj[ii] != xadj[iii+1]-xadj[iii])
break; /* Break if keys or degrees are different */
if (map[iii] == -1) { /* Do a comparison if iii has not been mapped */
for (jj=xadj[iii]; jj<xadj[iii+1]; jj++) {
if (mark[adjncy[jj]] != i)
break;
}
if (jj == xadj[iii+1]) { /* Identical adjacency structure */
map[iii] = cnvtxs;
cind[l++] = iii;
}
}
}
cptr[++cnvtxs] = l;
}
}
IFSET(ctrl->dbglvl, METIS_DBG_INFO,
printf(" Compression: reduction in # of vertices: %"PRIDX".\n", nvtxs-cnvtxs));
if (cnvtxs < COMPRESSION_FRACTION*nvtxs) {
/* Sufficient compression is possible, so go ahead and create the
compressed graph */
graph = CreateGraph();
cnedges = 0;
for (i=0; i<cnvtxs; i++) {
ii = cind[cptr[i]];
cnedges += xadj[ii+1]-xadj[ii];
}
/* Allocate memory for the compressed graph */
cxadj = graph->xadj = imalloc(cnvtxs+1, "CompressGraph: xadj");
cvwgt = graph->vwgt = ismalloc(cnvtxs, 0, "CompressGraph: vwgt");
cadjncy = graph->adjncy = imalloc(cnedges, "CompressGraph: adjncy");
graph->adjwgt = ismalloc(cnedges, 1, "CompressGraph: adjwgt");
/* Now go and compress the graph */
iset(nvtxs, -1, mark);
l = cxadj[0] = 0;
for (i=0; i<cnvtxs; i++) {
mark[i] = i; /* Remove any dioganal entries in the compressed graph */
for (j=cptr[i]; j<cptr[i+1]; j++) {
ii = cind[j];
/* accumulate the vertex weights of the consistuent vertices */
cvwgt[i] += (vwgt == NULL ? 1 : vwgt[ii]);
/* generate the combined adjancency list */
for (jj=xadj[ii]; jj<xadj[ii+1]; jj++) {
k = map[adjncy[jj]];
if (mark[k] != i) {
mark[k] = i;
cadjncy[l++] = k;
}
}
}
cxadj[i+1] = l;
}
graph->nvtxs = cnvtxs;
graph->nedges = l;
graph->ncon = 1;
SetupGraph_tvwgt(graph);
SetupGraph_label(graph);
}
gk_free((void **)&keys, &map, &mark, LTERM);
return graph;
}
int CheckGraph(graph_t *graph, int numflag, int verbose)
{
idx_t i, j, k, l;
idx_t nvtxs, err=0;
idx_t minedge, maxedge, minewgt, maxewgt;
idx_t *xadj, *adjncy, *adjwgt, *htable;
numflag = (numflag == 0 ? 0 : 1); /* make sure that numflag is 0 or 1 */
nvtxs = graph->nvtxs;
xadj = graph->xadj;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
ASSERT(adjwgt != NULL);
htable = ismalloc(nvtxs, 0, "htable");
minedge = maxedge = adjncy[0];
minewgt = maxewgt = adjwgt[0];
for (i=0; i<nvtxs; i++) {
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = adjncy[j];
minedge = (k < minedge) ? k : minedge;
maxedge = (k > maxedge) ? k : maxedge;
minewgt = (adjwgt[j] < minewgt) ? adjwgt[j] : minewgt;
maxewgt = (adjwgt[j] > maxewgt) ? adjwgt[j] : maxewgt;
if (i == k) {
if (verbose)
printf("Vertex %"PRIDX" contains a self-loop "
"(i.e., diagonal entry in the matrix)!\n", i+numflag);
err++;
}
else {
for (l=xadj[k]; l<xadj[k+1]; l++) {
if (adjncy[l] == i) {
if (adjwgt[l] != adjwgt[j]) {
if (verbose)
printf("Edges (u:%"PRIDX" v:%"PRIDX" wgt:%"PRIDX") and "
"(v:%"PRIDX" u:%"PRIDX" wgt:%"PRIDX") "
"do not have the same weight!\n",
i+numflag, k+numflag, adjwgt[j],
k+numflag, i+numflag, adjwgt[l]);
err++;
}
break;
}
}
if (l == xadj[k+1]) {
if (verbose)
printf("Missing edge: (%"PRIDX" %"PRIDX")!\n", k+numflag, i+numflag);
err++;
}
}
if (htable[k] == 0) {
htable[k]++;
}
else {
if (verbose)
printf("Edge %"PRIDX" from vertex %"PRIDX" is repeated %"PRIDX" times\n",
k+numflag, i+numflag, htable[k]++);
err++;
}
}
for (j=xadj[i]; j<xadj[i+1]; j++)
htable[adjncy[j]] = 0;
}
if (err > 0 && verbose) {
printf("A total of %"PRIDX" errors exist in the input file. "
"Correct them, and run again!\n", err);
}
gk_free((void **)&htable, LTERM);
return (err == 0 ? 1 : 0);
}
/*************************************************************************
* This function computes cuts and balance information
**************************************************************************/
void ComputePartitionInfoBipartite(graph_t *graph, idx_t nparts, idx_t *where)
{
idx_t i, j, k, nvtxs, ncon, mustfree=0;
idx_t *xadj, *adjncy, *vwgt, *vsize, *adjwgt, *kpwgts, *tmpptr;
idx_t *padjncy, *padjwgt, *padjcut;
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
adjncy = graph->adjncy;
vwgt = graph->vwgt;
vsize = graph->vsize;
adjwgt = graph->adjwgt;
if (vwgt == NULL) {
vwgt = graph->vwgt = ismalloc(nvtxs, 1, "vwgt");
mustfree = 1;
}
if (adjwgt == NULL) {
adjwgt = graph->adjwgt = ismalloc(xadj[nvtxs], 1, "adjwgt");
mustfree += 2;
}
printf("%"PRIDX"-way Cut: %5"PRIDX", Vol: %5"PRIDX", ", nparts, ComputeCut(graph, where), ComputeVolume(graph, where));
/* Compute balance information */
kpwgts = ismalloc(ncon*nparts, 0, "ComputePartitionInfo: kpwgts");
for (i=0; i<nvtxs; i++) {
for (j=0; j<ncon; j++)
kpwgts[where[i]*ncon+j] += vwgt[i*ncon+j];
}
if (ncon == 1) {
printf("\tBalance: %5.3"PRREAL" out of %5.3"PRREAL"\n",
1.0*nparts*kpwgts[iargmax(nparts, kpwgts)]/(1.0*isum(nparts, kpwgts, 1)),
1.0*nparts*vwgt[iargmax(nvtxs, vwgt)]/(1.0*isum(nparts, kpwgts, 1)));
}
else {
printf("\tBalance:");
for (j=0; j<ncon; j++)
printf(" (%5.3"PRREAL" out of %5.3"PRREAL")",
1.0*nparts*kpwgts[ncon*iargmax_strd(nparts, kpwgts+j, ncon)+j]/(1.0*isum(nparts, kpwgts+j, ncon)),
1.0*nparts*vwgt[ncon*iargmax_strd(nvtxs, vwgt+j, ncon)+j]/(1.0*isum(nparts, kpwgts+j, ncon)));
printf("\n");
}
/* Compute p-adjncy information */
padjncy = ismalloc(nparts*nparts, 0, "ComputePartitionInfo: padjncy");
padjwgt = ismalloc(nparts*nparts, 0, "ComputePartitionInfo: padjwgt");
padjcut = ismalloc(nparts*nparts, 0, "ComputePartitionInfo: padjwgt");
iset(nparts, 0, kpwgts);
for (i=0; i<nvtxs; i++) {
for (j=xadj[i]; j<xadj[i+1]; j++) {
if (where[i] != where[adjncy[j]]) {
padjncy[where[i]*nparts+where[adjncy[j]]] = 1;
padjcut[where[i]*nparts+where[adjncy[j]]] += adjwgt[j];
if (kpwgts[where[adjncy[j]]] == 0) {
padjwgt[where[i]*nparts+where[adjncy[j]]] += vsize[i];
kpwgts[where[adjncy[j]]] = 1;
}
}
}
for (j=xadj[i]; j<xadj[i+1]; j++)
kpwgts[where[adjncy[j]]] = 0;
}
for (i=0; i<nparts; i++)
kpwgts[i] = isum(nparts, padjncy+i*nparts, 1);
printf("Min/Max/Avg/Bal # of adjacent subdomains: %5"PRIDX" %5"PRIDX" %5"PRIDX" %7.3"PRREAL"\n",
kpwgts[iargmin(nparts, kpwgts)], kpwgts[iargmax(nparts, kpwgts)], isum(nparts, kpwgts, 1)/nparts,
1.0*nparts*kpwgts[iargmax(nparts, kpwgts)]/(1.0*isum(nparts, kpwgts, 1)));
for (i=0; i<nparts; i++)
kpwgts[i] = isum(nparts, padjcut+i*nparts, 1);
printf("Min/Max/Avg/Bal # of adjacent subdomain cuts: %5"PRIDX" %5"PRIDX" %5"PRIDX" %7.3"PRREAL"\n",
kpwgts[iargmin(nparts, kpwgts)], kpwgts[iargmax(nparts, kpwgts)], isum(nparts, kpwgts, 1)/nparts,
1.0*nparts*kpwgts[iargmax(nparts, kpwgts)]/(1.0*isum(nparts, kpwgts, 1)));
for (i=0; i<nparts; i++)
kpwgts[i] = isum(nparts, padjwgt+i*nparts, 1);
printf("Min/Max/Avg/Bal/Frac # of interface nodes: %5"PRIDX" %5"PRIDX" %5"PRIDX" %7.3"PRREAL" %7.3"PRREAL"\n",
kpwgts[iargmin(nparts, kpwgts)], kpwgts[iargmax(nparts, kpwgts)], isum(nparts, kpwgts, 1)/nparts,
1.0*nparts*kpwgts[iargmax(nparts, kpwgts)]/(1.0*isum(nparts, kpwgts, 1)), 1.0*isum(nparts, kpwgts, 1)/(1.0*nvtxs));
if (mustfree == 1 || mustfree == 3) {
gk_free((void **)&vwgt, LTERM);
graph->vwgt = NULL;
}
if (mustfree == 2 || mustfree == 3) {
gk_free((void **)&adjwgt, LTERM);
graph->adjwgt = NULL;
}
gk_free((void **)&kpwgts, &padjncy, &padjwgt, &padjcut, LTERM);
}
int local_search(CtrlType *ctrl, GraphType *graph, int nparts, int chain_length, idxtype *w, float *tpwgts, float ubfactor)
//return # of points moved
{
int nvtxs, nedges, nbnd, me, i, j, k, s, ii;
idxtype *sum, *squared_sum, *xadj, *adjncy, *adjwgt, *where, *bndptr, *bndind;
float change, obj, epsilon, **kDist, *accum_change;
int moves, actual_length, *mark, fidx, loopend;
Chains *chain;
nedges = graph->nedges;
nvtxs = graph->nvtxs;
xadj = graph->xadj;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
where = graph->where;
nbnd = graph->nbnd;
bndind = graph->bndind;
bndptr = graph->bndptr;
if(boundary_points == 1)
loopend = nbnd;
else
loopend = nvtxs;
chain = chainmalloc(chain_length, "Local_search: local search chain");
mark = ismalloc(loopend, 0 , "Local_search: mark");
sum = idxsmalloc(nparts,0, "Local_search: weight sum");
squared_sum = idxsmalloc(nparts,0,"Local_search: weight squared sum");
kDist = f2malloc(loopend, nparts, "Local_search: distance matrix");
accum_change = fmalloc(chain_length+1,"Local_search: accumulated change");
//initialization
for (i = 0; i<nparts; i++)
sum[i] = squared_sum[i] = 0;
for (i = 0; i<loopend; i++)
for (j = 0; j<nparts; j++)
kDist[i][j] = 0;
for (i = 0; i<chain_length+1; i++)
accum_change[i] = 0;
obj = 0;
moves = 0;
epsilon =.0001;
actual_length = chain_length;
for (i=0; i<nvtxs; i++)
sum[where[i]] += w[i];
for (i=0; i<nvtxs; i++){
me = where[i];
for (j=xadj[i]; j<xadj[i+1]; j++)
if (where[adjncy[j]] == me)
squared_sum[me] += adjwgt[j];
}
//the diagonal entries won't affect the result so diagonal's assumed zero
//for (i=0; i<nvtxs; i++)
for (ii=0; ii<loopend; ii++){
if (boundary_points == 1)
i = bndind[ii];
else
i = ii;
for (j=xadj[i]; j<xadj[i+1]; j++)
//kDist[i][where[adjncy[j]]] += 1.0*adjwgt[j]/w[i];
kDist[ii][where[adjncy[j]]] += 1.0*adjwgt[j]/w[i];
}
for (k=0; k<nparts; k++)
if (sum[k] >0)
//for (i=0; i<nvtxs; i++)
for (ii=0; ii<loopend; ii++)
//kDist[i][k] = squared_sum[k]/(1.0*sum[k]*sum[k]) - 2*kDist[i][k]/sum[k];
kDist[ii][k] = squared_sum[k]/(1.0*sum[k]*sum[k]) - 2*kDist[ii][k]/sum[k];
for (i=0; i<nparts; i++)
if (sum[i] >0)
obj += squared_sum[i]*1.0/sum[i];
for (i=0; i<chain_length; i++)
{
float tempMinChange, tempchange, temp_q;
int tempid, tempMoveTo, from, to, tempbndind;
tempMinChange = obj;
tempchange =0;
tempid =0;
tempMoveTo = where[tempid];
tempbndind =0;
//for (j=0; j<nvtxs; j++)
for (ii=0; ii<loopend; ii++){
if (boundary_points == 1)
j = bndind[ii];
else
j = ii;
if (mark[ii] ==0){
me = where[j];
if (sum[me] > w[j]) // if this cluster where j belongs is not a singleton
for (k=0; k<nparts; k++)
if (k != me){
//tempchange = -kDist[j][me]*sum[me]*w[j]/(sum[me]-w[j]) + kDist[j][k]*sum[k]*w[j]/(sum[k]+w[j]);
tempchange = -kDist[ii][me]*sum[me]*w[j]/(sum[me]-w[j]) + kDist[ii][k]*sum[k]*w[j]/(sum[k]+w[j]);
if (tempchange < tempMinChange){
tempMinChange = tempchange;
tempid = j;
tempbndind = ii;
tempMoveTo = k;
}
}
}
}
if ((tempMoveTo == where[tempid]) || (mark[tempbndind] >0)){
actual_length = i;
break;
}
else{
// record which point is moved from its original cluster to new cluster
chain[i].point = tempid;
chain[i].from = where[tempid];
chain[i].to = tempMoveTo;
chain[i].change = tempMinChange;
//mark the point moved
mark[tempbndind] = 1;
// update info
accum_change[i+1] = accum_change[i] + tempMinChange;
from = chain[i].from;
to = chain[i].to;
where[tempid] = to;
sum[from] -= w[tempid];
sum[to] += w[tempid];
for (j=xadj[tempid]; j<xadj[tempid+1]; j++)
if (where[adjncy[j]] == from)
squared_sum[from] -= 2*adjwgt[j];
//for (s=0; s<nvtxs; s++){
for (ii=0; ii<loopend; ii++){
//kDist[s][from] = 0;
kDist[ii][from] = 0;
if(boundary_points == 1)
s = bndind[ii];
else
s = ii;
for (j=xadj[s]; j<xadj[s+1]; j++)
if (where[adjncy[j]] == from)
//kDist[s][from] += adjwgt[j]*1.0/w[s];
kDist[ii][from] += adjwgt[j]*1.0/w[s];
}
temp_q = squared_sum[from]/(1.0*sum[from]*sum[from]);
//for (s=0; s<nvtxs; s++)
for (ii=0; ii<loopend; ii++)
kDist[ii][from] = temp_q - 2*kDist[ii][from]/sum[from];
for (j=xadj[tempid]; j<xadj[tempid+1]; j++)
if (where[adjncy[j]] == to)
squared_sum[to] += 2*adjwgt[j];
//for (s=0; s<nvtxs; s++){
for (ii=0; ii<loopend; ii++){
//kDist[s][to] = 0;
kDist[ii][to] = 0;
if(boundary_points == 1)
s = bndind[ii];
else
s = ii;
for (j=xadj[s]; j<xadj[s+1]; j++)
if (where[adjncy[j]] == to)
//kDist[s][to] += adjwgt[j]*1.0/w[s];
kDist[ii][to] += adjwgt[j]*1.0/w[s];
}
temp_q = squared_sum[to]/(1.0*sum[to]*sum[to]);
//for (s=0; s<nvtxs; s++)
for (ii=0; ii<loopend; ii++)
//kDist[s][to] = temp_q - 2*kDist[s][to]/sum[to];
kDist[ii][to] = temp_q - 2*kDist[ii][to]/sum[to];
}
}
fidx = samin(actual_length, accum_change);
if (accum_change[fidx] < -epsilon * obj){
for (i= fidx+1; i<=actual_length; i++)
where[chain[i-1].point] = chain[i-1].from;
moves = fidx;
change = accum_change[fidx];
}
else{
for (i= 0; i<actual_length; i++)
where[chain[i].point] = chain[i].from;
moves = 0;
change = 0;
}
free(sum); free(squared_sum);free(accum_change); free(chain); free(mark);
//for (i= 0; i<nvtxs; i++)
for (i= 0; i<loopend; i++)
free(kDist[i]);
free(kDist);
return moves;
}
/*************************************************************************
* This function returns the min-cover of a bipartite graph.
* The algorithm used is due to Hopcroft and Karp as modified by Duff etal
* adj: the adjacency list of the bipartite graph
* asize: the number of vertices in the first part of the bipartite graph
* bsize-asize: the number of vertices in the second part
* 0..(asize-1) > A vertices
* asize..bsize > B vertices
*
* Returns:
* cover : the actual cover (array)
* csize : the size of the cover
**************************************************************************/
void MinCover(idx_t *xadj, idx_t *adjncy, idx_t asize, idx_t bsize, idx_t *cover, idx_t *csize) {
idx_t i, j;
idx_t *mate, *queue, *flag, *level, *lst;
idx_t fptr, rptr, lstptr;
idx_t row, maxlevel, col;
mate = ismalloc(bsize, -1, "MinCover: mate");
flag = imalloc(bsize, "MinCover: flag");
level = imalloc(bsize, "MinCover: level");
queue = imalloc(bsize, "MinCover: queue");
lst = imalloc(bsize, "MinCover: lst");
/* Get a cheap matching */
for (i = 0; i < asize; i++) {
for (j = xadj[i]; j < xadj[i + 1]; j++) {
if (mate[adjncy[j]] == -1) {
mate[i] = adjncy[j];
mate[adjncy[j]] = i;
break;
}
}
}
/* Get into the main loop */
while (1) {
/* Initialization */
fptr = rptr = 0; /* Empty Queue */
lstptr = 0; /* Empty List */
for (i = 0; i < bsize; i++) {
level[i] = -1;
flag[i] = 0;
}
maxlevel = bsize;
/* Insert free nodes into the queue */
for (i = 0; i < asize; i++) {
if (mate[i] == -1) {
queue[rptr++] = i;
level[i] = 0;
}
}
/* Perform the BFS */
while (fptr != rptr) {
row = queue[fptr++];
if (level[row] < maxlevel) {
flag[row] = 1;
for (j = xadj[row]; j < xadj[row + 1]; j++) {
col = adjncy[j];
if (!flag[col]) { /* If this column has not been accessed yet */
flag[col] = 1;
if (mate[col] == -1) { /* Free column node was found */
maxlevel = level[row];
lst[lstptr++] = col;
} else { /* This column node is matched */
if (flag[mate[col]]) {
printf("\nSomething wrong, flag[%"PRIDX"] is 1", mate[col]);
}
queue[rptr++] = mate[col];
level[mate[col]] = level[row] + 1;
}
}
}
}
}
if (lstptr == 0) {
break;
} /* No free columns can be reached */
/* Perform restricted DFS from the free column nodes */
for (i = 0; i < lstptr; i++)
MinCover_Augment(xadj, adjncy, lst[i], mate, flag, level, maxlevel);
}
MinCover_Decompose(xadj, adjncy, asize, bsize, mate, cover, csize);
gk_free((void **) &mate, &flag, &level, &queue, &lst, LTERM);
}
void CreateCoarseGraph_Local(ctrl_t *ctrl, graph_t *graph, idx_t cnvtxs)
{
idx_t h, i, j, k, l;
idx_t nvtxs, ncon, nedges, firstvtx, cfirstvtx;
idx_t npes=ctrl->npes, mype=ctrl->mype;
idx_t cnedges, v, u;
idx_t *xadj, *vwgt, *vsize, *adjncy, *adjwgt, *vtxdist, *where, *home;
idx_t *match, *cmap;
idx_t *cxadj, *cvwgt, *cvsize = NULL, *cadjncy, *cadjwgt, *cvtxdist,
*chome = NULL, *cwhere = NULL;
real_t *cnvwgt;
graph_t *cgraph;
idx_t htable[LHTSIZE], htableidx[LHTSIZE];
WCOREPUSH;
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
vwgt = graph->vwgt;
home = graph->home;
vsize = graph->vsize;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
where = graph->where;
match = graph->match;
vtxdist = graph->vtxdist;
firstvtx = vtxdist[mype];
cmap = graph->cmap = ismalloc(nvtxs+graph->nrecv, -1, "CreateCoarseGraph: cmap");
/* Initialize the coarser graph */
cgraph = CreateGraph();
cgraph->nvtxs = cnvtxs;
cgraph->level = graph->level+1;
cgraph->ncon = ncon;
cgraph->finer = graph;
graph->coarser = cgraph;
/*************************************************************
* Obtain the vtxdist of the coarser graph
**************************************************************/
cvtxdist = cgraph->vtxdist = imalloc(npes+1, "CreateCoarseGraph: cvtxdist");
cvtxdist[npes] = cnvtxs; /* Use last position in the cvtxdist as a temp buffer */
gkMPI_Allgather((void *)(cvtxdist+npes), 1, IDX_T, (void *)cvtxdist, 1, IDX_T, ctrl->comm);
MAKECSR(i, npes, cvtxdist);
cgraph->gnvtxs = cvtxdist[npes];
#ifdef DEBUG_CONTRACT
PrintVector(ctrl, npes+1, 0, cvtxdist, "cvtxdist");
#endif
/*************************************************************
* Construct the cmap vector
**************************************************************/
cfirstvtx = cvtxdist[mype];
/* Create the cmap of what you know so far locally */
cnvtxs = 0;
for (i=0; i<nvtxs; i++) {
if (match[i] >= KEEP_BIT) {
k = match[i] - KEEP_BIT;
if (k<firstvtx+i)
continue; /* i has been matched via the (k,i) side */
cmap[i] = cfirstvtx + cnvtxs++;
if (k != firstvtx+i) {
cmap[k-firstvtx] = cmap[i];
match[k-firstvtx] += KEEP_BIT; /* Add the KEEP_BIT to simplify coding */
}
}
}
CommInterfaceData(ctrl, graph, cmap, cmap+nvtxs);
#ifdef DEBUG_CONTRACT
PrintVector(ctrl, nvtxs, firstvtx, cmap, "Cmap");
#endif
/*************************************************************
* Finally, create the coarser graph
**************************************************************/
/* Allocate memory for the coarser graph, and fire up coarsening */
cxadj = cgraph->xadj = imalloc(cnvtxs+1, "CreateCoarserGraph: cxadj");
cvwgt = cgraph->vwgt = imalloc(cnvtxs*ncon, "CreateCoarserGraph: cvwgt");
cnvwgt = cgraph->nvwgt = rmalloc(cnvtxs*ncon, "CreateCoarserGraph: cnvwgt");
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION)
chome = cgraph->home = imalloc(cnvtxs, "CreateCoarserGraph: chome");
if (vsize != NULL)
cvsize = cgraph->vsize = imalloc(cnvtxs, "CreateCoarserGraph: cvsize");
if (where != NULL)
cwhere = cgraph->where = imalloc(cnvtxs, "CreateCoarserGraph: cwhere");
cadjncy = iwspacemalloc(ctrl, graph->nedges);
cadjwgt = iwspacemalloc(ctrl, graph->nedges);
iset(LHTSIZE, -1, htable);
cxadj[0] = cnvtxs = cnedges = 0;
for (i=0; i<nvtxs; i++) {
v = firstvtx+i;
u = match[i]-KEEP_BIT;
if (v > u)
continue; /* I have already collapsed it as (u,v) */
/* Collapse the v vertex first, which you know that is local */
for (h=0; h<ncon; h++)
cvwgt[cnvtxs*ncon+h] = vwgt[i*ncon+h];
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION)
chome[cnvtxs] = home[i];
if (vsize != NULL)
cvsize[cnvtxs] = vsize[i];
if (where != NULL)
cwhere[cnvtxs] = where[i];
nedges = 0;
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = cmap[adjncy[j]];
if (k != cfirstvtx+cnvtxs) { /* If this is not an internal edge */
l = k&MASK;
if (htable[l] == -1) { /* Seeing this for first time */
htable[l] = k;
htableidx[l] = cnedges+nedges;
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges++] = adjwgt[j];
}
else if (htable[l] == k) {
cadjwgt[htableidx[l]] += adjwgt[j];
}
else { /* Now you have to go and do a search. Expensive case */
for (l=0; l<nedges; l++) {
if (cadjncy[cnedges+l] == k)
break;
}
if (l < nedges) {
cadjwgt[cnedges+l] += adjwgt[j];
}
else {
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges++] = adjwgt[j];
}
}
}
}
/* Collapse the u vertex next */
if (v != u) {
u -= firstvtx;
for (h=0; h<ncon; h++)
cvwgt[cnvtxs*ncon+h] += vwgt[u*ncon+h];
if (vsize != NULL)
cvsize[cnvtxs] += vsize[u];
if (where != NULL && cwhere[cnvtxs] != where[u])
myprintf(ctrl, "Something went wrong with the where local matching! %"PRIDX" %"PRIDX"\n", cwhere[cnvtxs], where[u]);
for (j=xadj[u]; j<xadj[u+1]; j++) {
k = cmap[adjncy[j]];
if (k != cfirstvtx+cnvtxs) { /* If this is not an internal edge */
l = k&MASK;
if (htable[l] == -1) { /* Seeing this for first time */
htable[l] = k;
htableidx[l] = cnedges+nedges;
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges++] = adjwgt[j];
}
else if (htable[l] == k) {
cadjwgt[htableidx[l]] += adjwgt[j];
}
else { /* Now you have to go and do a search. Expensive case */
for (l=0; l<nedges; l++) {
if (cadjncy[cnedges+l] == k)
break;
}
if (l < nedges) {
cadjwgt[cnedges+l] += adjwgt[j];
}
else {
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges++] = adjwgt[j];
}
}
}
}
}
cnedges += nedges;
for (j=cxadj[cnvtxs]; j<cnedges; j++)
htable[cadjncy[j]&MASK] = -1; /* reset the htable */
cxadj[++cnvtxs] = cnedges;
}
cgraph->nedges = cnedges;
for (j=0; j<cnvtxs; j++) {
for (h=0; h<ncon; h++)
cgraph->nvwgt[j*ncon+h] = ctrl->invtvwgts[h]*cvwgt[j*ncon+h];
}
cgraph->adjncy = imalloc(cnedges, "CreateCoarserGraph: cadjncy");
cgraph->adjwgt = imalloc(cnedges, "CreateCoarserGraph: cadjwgt");
icopy(cnedges, cadjncy, cgraph->adjncy);
icopy(cnedges, cadjwgt, cgraph->adjwgt);
WCOREPOP;
/* Note that graph->where works fine even if it is NULL */
gk_free((void **)&graph->where, LTERM);
}
void TestParMetis_GPart(char *filename, char *xyzfile, MPI_Comm comm)
{
idx_t ncon, nparts, npes, mype, opt2, realcut;
graph_t graph, mgraph;
idx_t *part, *mpart, *savepart, *order, *sizes;
idx_t numflag=0, wgtflag=0, options[10], edgecut, ndims;
real_t ipc2redist, *xyz=NULL, *tpwgts = NULL, ubvec[MAXNCON];
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
ParallelReadGraph(&graph, filename, comm);
if (xyzfile)
xyz = ReadTestCoordinates(&graph, xyzfile, &ndims, comm);
gkMPI_Barrier(comm);
part = imalloc(graph.nvtxs, "TestParMetis_V3: part");
tpwgts = rmalloc(MAXNCON*npes*2, "TestParMetis_V3: tpwgts");
rset(MAXNCON, 1.05, ubvec);
graph.vwgt = ismalloc(graph.nvtxs*5, 1, "TestParMetis_GPart: vwgt");
/*======================================================================
/ ParMETIS_V3_PartKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
wgtflag = 2;
numflag = 0;
edgecut = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=NCON; ncon++) {
if (ncon > 1 && nparts > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
iset(graph.nvtxs, 1, graph.vwgt);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options,
&edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0) {
printf("ParMETIS_V3_PartKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
if (mype == 0)
printf("\nTesting ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
options[3] = PARMETIS_PSR_UNCOUPLED;
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options,
&edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0) {
printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeomKway
/=======================================================================*/
if (xyzfile != NULL) {
options[0] = 1;
options[1] = 3;
options[2] = 1;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2 && nparts > 0; nparts = nparts/2) {
for (ncon=1; ncon<=NCON; ncon++) {
if (ncon > 1)
Mc_AdaptGraph(&graph, part, ncon, nparts, comm);
else
iset(graph.nvtxs, 1, graph.vwgt);
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeomKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, &wgtflag, &numflag, &ndims, xyz, &ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0)
printf("ParMETIS_V3_PartGeomKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
/*======================================================================
/ ParMETIS_V3_PartGeom
/=======================================================================*/
if (xyz != NULL) {
wgtflag = 0;
numflag = 0;
if (mype == 0)
printf("\nTesting ParMETIS_V3_PartGeom\n");
ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm);
realcut = ComputeRealCut(graph.vtxdist, part, filename, comm);
if (mype == 0)
printf("ParMETIS_V3_PartGeom reported a cut of %"PRIDX"\n", realcut);
}
/*======================================================================
/ Coupled ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
if (mype == 0)
printf("\nTesting coupled ParMETIS_V3_RefineKway with default options (before move)\n");
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
part, &comm);
/* Compute a good partition and move the graph. Do so quietly! */
options[0] = 0;
nparts = npes;
wgtflag = 0;
numflag = 0;
ncon = 1;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &npes, tpwgts, ubvec, options, &edgecut,
part, &comm);
TestMoveGraph(&graph, &mgraph, part, comm);
gk_free((void **)&(graph.vwgt), LTERM);
mpart = ismalloc(mgraph.nvtxs, mype, "TestParMetis_V3: mpart");
savepart = imalloc(mgraph.nvtxs, "TestParMetis_V3: savepart");
/*======================================================================
/ Coupled ParMETIS_V3_RefineKway
/=======================================================================*/
options[0] = 1;
options[1] = 3;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
nparts = npes;
wgtflag = 0;
numflag = 0;
for (ncon=1; ncon<=NCON; ncon++) {
if (mype == 0)
printf("\nTesting coupled ParMETIS_V3_RefineKway with ncon: %"PRIDX", nparts: %"PRIDX"\n", ncon, nparts);
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_RefineKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
mpart, &comm);
realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart,
filename, comm);
if (mype == 0)
printf("ParMETIS_V3_RefineKway reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
/*ADAPTIVE:*/
/*======================================================================
/ ParMETIS_V3_AdaptiveRepart
/=======================================================================*/
mgraph.vwgt = ismalloc(mgraph.nvtxs*NCON, 1, "TestParMetis_V3: mgraph.vwgt");
mgraph.vsize = ismalloc(mgraph.nvtxs, 1, "TestParMetis_V3: mgraph.vsize");
AdaptGraph(&mgraph, 4, comm);
options[0] = 1;
options[1] = 7;
options[2] = 1;
options[3] = PARMETIS_PSR_COUPLED;
wgtflag = 2;
numflag = 0;
for (nparts=2*npes; nparts>=npes/2; nparts = nparts/2) {
options[0] = 0;
ncon = 1;
wgtflag = 0;
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
ParMETIS_V3_PartKway(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy, NULL, NULL,
&wgtflag, &numflag, &ncon, &nparts, tpwgts, ubvec, options, &edgecut,
savepart, &comm);
options[0] = 1;
wgtflag = 2;
for (ncon=1; ncon<=NCON; ncon++) {
rset(nparts*ncon, 1.0/(real_t)nparts, tpwgts);
if (ncon > 1)
Mc_AdaptGraph(&mgraph, savepart, ncon, nparts, comm);
else
AdaptGraph(&mgraph, 4, comm);
for (ipc2redist=1000.0; ipc2redist>=0.001; ipc2redist/=1000.0) {
icopy(mgraph.nvtxs, savepart, mpart);
if (mype == 0)
printf("\nTesting ParMETIS_V3_AdaptiveRepart with ipc2redist: %.3"PRREAL", ncon: %"PRIDX", nparts: %"PRIDX"\n",
ipc2redist, ncon, nparts);
ParMETIS_V3_AdaptiveRepart(mgraph.vtxdist, mgraph.xadj, mgraph.adjncy,
mgraph.vwgt, mgraph.vsize, NULL, &wgtflag, &numflag, &ncon, &nparts,
tpwgts, ubvec, &ipc2redist, options, &edgecut, mpart, &comm);
realcut = ComputeRealCutFromMoved(graph.vtxdist, mgraph.vtxdist, part, mpart,
filename, comm);
if (mype == 0)
printf("ParMETIS_V3_AdaptiveRepart reported a cut of %"PRIDX" [%s:%"PRIDX"]\n", edgecut,
(edgecut == realcut ? "OK" : "ERROR"), realcut);
}
}
}
gk_free((void **)&tpwgts, &part, &mpart, &savepart, &xyz, &mgraph.vwgt, &mgraph.vsize, LTERM);
}
void CreateCoarseGraph_Global(ctrl_t *ctrl, graph_t *graph, idx_t cnvtxs)
{
idx_t h, i, j, k, l, ii, jj, ll, nnbrs, nvtxs, nedges, ncon;
idx_t firstvtx, lastvtx, cfirstvtx, clastvtx, otherlastvtx;
idx_t npes=ctrl->npes, mype=ctrl->mype;
idx_t cnedges, nsend, nrecv, nkeepsize, nrecvsize, nsendsize, v, u;
idx_t *xadj, *adjncy, *adjwgt, *vwgt, *vsize, *vtxdist, *home, *where;
idx_t *match, *cmap;
idx_t *cxadj, *cadjncy, *cadjwgt, *cvwgt, *cvsize = NULL, *chome = NULL,
*cwhere = NULL, *cvtxdist;
idx_t *rsizes, *ssizes, *rlens, *slens, *rgraph, *sgraph, *perm;
idx_t *peind, *recvptr, *recvind;
real_t *nvwgt, *cnvwgt;
graph_t *cgraph;
ikv_t *scand, *rcand;
idx_t htable[LHTSIZE], htableidx[LHTSIZE];
WCOREPUSH;
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
vwgt = graph->vwgt;
vsize = graph->vsize;
nvwgt = graph->nvwgt;
home = graph->home;
where = graph->where;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
match = graph->match;
vtxdist = graph->vtxdist;
firstvtx = vtxdist[mype];
lastvtx = vtxdist[mype+1];
cmap = graph->cmap = ismalloc(nvtxs+graph->nrecv, -1, "Global_CreateCoarseGraph: cmap");
nnbrs = graph->nnbrs;
peind = graph->peind;
recvind = graph->recvind;
recvptr = graph->recvptr;
/* Initialize the coarser graph */
cgraph = CreateGraph();
cgraph->nvtxs = cnvtxs;
cgraph->ncon = ncon;
cgraph->level = graph->level+1;
cgraph->finer = graph;
graph->coarser = cgraph;
/*************************************************************
* Obtain the vtxdist of the coarser graph
**************************************************************/
cvtxdist = cgraph->vtxdist = imalloc(npes+1, "Global_CreateCoarseGraph: cvtxdist");
cvtxdist[npes] = cnvtxs; /* Use last position in the cvtxdist as a temp buffer */
gkMPI_Allgather((void *)(cvtxdist+npes), 1, IDX_T, (void *)cvtxdist, 1,
IDX_T, ctrl->comm);
MAKECSR(i, npes, cvtxdist);
cgraph->gnvtxs = cvtxdist[npes];
#ifdef DEBUG_CONTRACT
PrintVector(ctrl, npes+1, 0, cvtxdist, "cvtxdist");
#endif
/*************************************************************
* Construct the cmap vector
**************************************************************/
cfirstvtx = cvtxdist[mype];
clastvtx = cvtxdist[mype+1];
/* Create the cmap of what you know so far locally. */
for (cnvtxs=0, i=0; i<nvtxs; i++) {
if (match[i] >= KEEP_BIT) {
k = match[i] - KEEP_BIT;
if (k>=firstvtx && k<firstvtx+i)
continue; /* Both (i,k) are local and i has been matched via the (k,i) side */
cmap[i] = cfirstvtx + cnvtxs++;
if (k != firstvtx+i && (k>=firstvtx && k<lastvtx)) { /* I'm matched locally */
cmap[k-firstvtx] = cmap[i];
match[k-firstvtx] += KEEP_BIT; /* Add the KEEP_BIT to simplify coding */
}
}
}
PASSERT(ctrl, cnvtxs == clastvtx-cfirstvtx);
CommInterfaceData(ctrl, graph, cmap, cmap+nvtxs);
/* Update the cmap of the locally stored vertices that will go away.
* The remote processor assigned cmap for them */
for (i=0; i<nvtxs; i++) {
if (match[i] < KEEP_BIT) { /* Only vertices that go away satisfy this*/
cmap[i] = cmap[nvtxs+BSearch(graph->nrecv, recvind, match[i])];
}
}
CommInterfaceData(ctrl, graph, cmap, cmap+nvtxs);
#ifndef NDEBUG
for (i=0; i<nvtxs+graph->nrecv; i++) {
if (cmap[i] == -1)
errexit("cmap[%"PRIDX"] == -1\n", i);
}
#endif
#ifdef DEBUG_CONTRACT
PrintVector(ctrl, nvtxs, firstvtx, cmap, "Cmap");
#endif
/*************************************************************
* Determine how many adjcency lists you need to send/receive.
**************************************************************/
/* first pass: determine sizes */
for (nsend=0, nrecv=0, i=0; i<nvtxs; i++) {
if (match[i] < KEEP_BIT) /* This is going away */
nsend++;
else {
k = match[i]-KEEP_BIT;
if (k<firstvtx || k>=lastvtx) /* This is comming from afar */
nrecv++;
}
}
scand = ikvwspacemalloc(ctrl, nsend);
rcand = graph->rcand = ikvmalloc(nrecv, "CreateCoarseGraph: rcand");
/* second pass: place them in the appropriate arrays */
nkeepsize = nsend = nrecv = 0;
for (i=0; i<nvtxs; i++) {
if (match[i] < KEEP_BIT) { /* This is going away */
scand[nsend].key = match[i];
scand[nsend].val = i;
nsend++;
}
else {
nkeepsize += (xadj[i+1]-xadj[i]);
k = match[i]-KEEP_BIT;
if (k<firstvtx || k>=lastvtx) { /* This is comming from afar */
rcand[nrecv].key = k;
rcand[nrecv].val = cmap[i] - cfirstvtx; /* Set it for use during the partition projection */
PASSERT(ctrl, rcand[nrecv].val>=0 && rcand[nrecv].val<cnvtxs);
nrecv++;
}
}
}
#ifdef DEBUG_CONTRACT
PrintPairs(ctrl, nsend, scand, "scand");
PrintPairs(ctrl, nrecv, rcand, "rcand");
#endif
/***************************************************************
* Determine how many lists and their sizes you will send and
* received for each of the neighboring PEs
****************************************************************/
rlens = graph->rlens = imalloc(nnbrs+1, "CreateCoarseGraph: graph->rlens");
slens = graph->slens = imalloc(nnbrs+1, "CreateCoarseGraph: graph->slens");
rsizes = iset(nnbrs, 0, iwspacemalloc(ctrl, nnbrs));
ssizes = iset(nnbrs, 0, iwspacemalloc(ctrl, nnbrs));
/* Take care the sending data first */
ikvsortii(nsend, scand);
slens[0] = 0;
for (k=i=0; i<nnbrs; i++) {
otherlastvtx = vtxdist[peind[i]+1];
for (; k<nsend && scand[k].key < otherlastvtx; k++)
ssizes[i] += (xadj[scand[k].val+1]-xadj[scand[k].val]);
slens[i+1] = k;
}
/* Take care the receiving data next. You cannot yet determine the rsizes[] */
ikvsortii(nrecv, rcand);
rlens[0] = 0;
for (k=i=0; i<nnbrs; i++) {
otherlastvtx = vtxdist[peind[i]+1];
for (; k<nrecv && rcand[k].key < otherlastvtx; k++);
rlens[i+1] = k;
}
#ifdef DEBUG_CONTRACT
PrintVector(ctrl, nnbrs+1, 0, slens, "slens");
PrintVector(ctrl, nnbrs+1, 0, rlens, "rlens");
#endif
/***************************************************************
* Exchange size information
****************************************************************/
/* Issue the receives first. */
for (i=0; i<nnbrs; i++) {
if (rlens[i+1]-rlens[i] > 0) /* Issue a receive only if you are getting something */
gkMPI_Irecv((void *)(rsizes+i), 1, IDX_T, peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Take care the sending data next */
for (i=0; i<nnbrs; i++) {
if (slens[i+1]-slens[i] > 0) /* Issue a send only if you are sending something */
gkMPI_Isend((void *)(ssizes+i), 1, IDX_T, peind[i], 1, ctrl->comm, ctrl->sreq+i);
}
/* OK, now get into the loop waiting for the operations to finish */
for (i=0; i<nnbrs; i++) {
if (rlens[i+1]-rlens[i] > 0)
gkMPI_Wait(ctrl->rreq+i, &ctrl->status);
}
for (i=0; i<nnbrs; i++) {
if (slens[i+1]-slens[i] > 0)
gkMPI_Wait(ctrl->sreq+i, &ctrl->status);
}
#ifdef DEBUG_CONTRACT
PrintVector(ctrl, nnbrs, 0, rsizes, "rsizes");
PrintVector(ctrl, nnbrs, 0, ssizes, "ssizes");
#endif
/*************************************************************
* Allocate memory for received/sent graphs and start sending
* and receiving data.
* rgraph and sgraph is a different data structure than CSR
* to facilitate single message exchange.
**************************************************************/
nrecvsize = isum(nnbrs, rsizes, 1);
nsendsize = isum(nnbrs, ssizes, 1);
rgraph = iwspacemalloc(ctrl, (4+ncon)*nrecv+2*nrecvsize);
WCOREPUSH; /* for freeing sgraph right away */
sgraph = iwspacemalloc(ctrl, (4+ncon)*nsend+2*nsendsize);
/* Deal with the received portion first */
for (l=i=0; i<nnbrs; i++) {
/* Issue a receive only if you are getting something */
if (rlens[i+1]-rlens[i] > 0) {
gkMPI_Irecv((void *)(rgraph+l), (4+ncon)*(rlens[i+1]-rlens[i])+2*rsizes[i],
IDX_T, peind[i], 1, ctrl->comm, ctrl->rreq+i);
l += (4+ncon)*(rlens[i+1]-rlens[i])+2*rsizes[i];
}
}
/* Deal with the sent portion now */
for (ll=l=i=0; i<nnbrs; i++) {
if (slens[i+1]-slens[i] > 0) { /* Issue a send only if you are sending something */
for (k=slens[i]; k<slens[i+1]; k++) {
ii = scand[k].val;
sgraph[ll++] = firstvtx+ii;
sgraph[ll++] = xadj[ii+1]-xadj[ii];
for (h=0; h<ncon; h++)
sgraph[ll++] = vwgt[ii*ncon+h];
sgraph[ll++] = (ctrl->partType == STATIC_PARTITION || ctrl->partType == ORDER_PARTITION
? -1 : vsize[ii]);
sgraph[ll++] = (ctrl->partType == STATIC_PARTITION || ctrl->partType == ORDER_PARTITION
? -1 : home[ii]);
for (jj=xadj[ii]; jj<xadj[ii+1]; jj++) {
sgraph[ll++] = cmap[adjncy[jj]];
sgraph[ll++] = adjwgt[jj];
}
}
PASSERT(ctrl, ll-l == (4+ncon)*(slens[i+1]-slens[i])+2*ssizes[i]);
/*myprintf(ctrl, "Sending to pe:%"PRIDX", %"PRIDX" lists of size %"PRIDX"\n", peind[i], slens[i+1]-slens[i], ssizes[i]); */
gkMPI_Isend((void *)(sgraph+l), ll-l, IDX_T, peind[i], 1, ctrl->comm, ctrl->sreq+i);
l = ll;
}
}
/* OK, now get into the loop waiting for the operations to finish */
for (i=0; i<nnbrs; i++) {
if (rlens[i+1]-rlens[i] > 0)
gkMPI_Wait(ctrl->rreq+i, &ctrl->status);
}
for (i=0; i<nnbrs; i++) {
if (slens[i+1]-slens[i] > 0)
gkMPI_Wait(ctrl->sreq+i, &ctrl->status);
}
#ifdef DEBUG_CONTRACT
rprintf(ctrl, "Graphs were sent!\n");
PrintTransferedGraphs(ctrl, nnbrs, peind, slens, rlens, sgraph, rgraph);
#endif
WCOREPOP; /* free sgraph */
/*************************************************************
* Setup the mapping from indices returned by BSearch to
* those that are actually stored
**************************************************************/
perm = iset(graph->nrecv, -1, iwspacemalloc(ctrl, graph->nrecv));
for (j=i=0; i<nrecv; i++) {
perm[BSearch(graph->nrecv, recvind, rgraph[j])] = j+1;
j += (4+ncon)+2*rgraph[j+1];
}
/*************************************************************
* Finally, create the coarser graph
**************************************************************/
/* Allocate memory for the coarser graph, and fire up coarsening */
cxadj = cgraph->xadj = imalloc(cnvtxs+1, "CreateCoarserGraph: cxadj");
cvwgt = cgraph->vwgt = imalloc(cnvtxs*ncon, "CreateCoarserGraph: cvwgt");
cnvwgt = cgraph->nvwgt = rmalloc(cnvtxs*ncon, "CreateCoarserGraph: cnvwgt");
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
cvsize = cgraph->vsize = imalloc(cnvtxs, "CreateCoarserGraph: cvsize");
chome = cgraph->home = imalloc(cnvtxs, "CreateCoarserGraph: chome");
}
if (where != NULL)
cwhere = cgraph->where = imalloc(cnvtxs, "CreateCoarserGraph: cwhere");
/* these are just upper bound estimates for now */
cadjncy = iwspacemalloc(ctrl, nkeepsize+nrecvsize);
cadjwgt = iwspacemalloc(ctrl, nkeepsize+nrecvsize);
iset(LHTSIZE, -1, htable);
cxadj[0] = cnvtxs = cnedges = 0;
for (i=0; i<nvtxs; i++) {
if (match[i] >= KEEP_BIT) {
v = firstvtx+i;
u = match[i]-KEEP_BIT;
if (u>=firstvtx && u<lastvtx && v > u)
continue; /* I have already collapsed it as (u,v) */
/* Collapse the v vertex first, which you know is local */
for (h=0; h<ncon; h++)
cvwgt[cnvtxs*ncon+h] = vwgt[i*ncon+h];
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
cvsize[cnvtxs] = vsize[i];
chome[cnvtxs] = home[i];
}
if (where != NULL)
cwhere[cnvtxs] = where[i];
nedges = 0;
/* Collapse the v (i) vertex first */
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = cmap[adjncy[j]];
if (k < 0)
printf("k=%d\n", (int)k);
if (k != cfirstvtx+cnvtxs) { /* If this is not an internal edge */
l = k&MASK;
if (htable[l] == -1) { /* Seeing this for first time */
htable[l] = k;
htableidx[l] = cnedges+nedges;
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges++] = adjwgt[j];
}
else if (htable[l] == k) {
cadjwgt[htableidx[l]] += adjwgt[j];
}
else { /* Now you have to go and do a search. Expensive case */
for (l=0; l<nedges; l++) {
if (cadjncy[cnedges+l] == k)
break;
}
if (l < nedges) {
cadjwgt[cnedges+l] += adjwgt[j];
}
else {
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges++] = adjwgt[j];
}
}
}
}
/* Collapse the u vertex next */
if (v != u) {
if (u>=firstvtx && u<lastvtx) { /* Local vertex */
u -= firstvtx;
for (h=0; h<ncon; h++)
cvwgt[cnvtxs*ncon+h] += vwgt[u*ncon+h];
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION)
cvsize[cnvtxs] += vsize[u];
for (j=xadj[u]; j<xadj[u+1]; j++) {
k = cmap[adjncy[j]];
if (k != cfirstvtx+cnvtxs) { /* If this is not an internal edge */
l = k&MASK;
if (htable[l] == -1) { /* Seeing this for first time */
htable[l] = k;
htableidx[l] = cnedges+nedges;
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges] = adjwgt[j];
nedges++;
}
else if (htable[l] == k) {
cadjwgt[htableidx[l]] += adjwgt[j];
}
else { /* Now you have to go and do a search. Expensive case */
for (l=0; l<nedges; l++) {
if (cadjncy[cnedges+l] == k)
break;
}
if (l < nedges) {
cadjwgt[cnedges+l] += adjwgt[j];
}
else {
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges] = adjwgt[j];
nedges++;
}
}
}
}
}
else { /* Remote vertex */
u = perm[BSearch(graph->nrecv, recvind, u)];
for (h=0; h<ncon; h++)
/* Remember that the +1 stores the vertex weight */
cvwgt[cnvtxs*ncon+h] += rgraph[(u+1)+h];
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
cvsize[cnvtxs] += rgraph[u+1+ncon];
chome[cnvtxs] = rgraph[u+2+ncon];
}
for (j=0; j<rgraph[u]; j++) {
k = rgraph[u+3+ncon+2*j];
if (k != cfirstvtx+cnvtxs) { /* If this is not an internal edge */
l = k&MASK;
if (htable[l] == -1) { /* Seeing this for first time */
htable[l] = k;
htableidx[l] = cnedges+nedges;
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges] = rgraph[u+3+ncon+2*j+1];
nedges++;
}
else if (htable[l] == k) {
cadjwgt[htableidx[l]] += rgraph[u+3+ncon+2*j+1];
}
else { /* Now you have to go and do a search. Expensive case */
for (l=0; l<nedges; l++) {
if (cadjncy[cnedges+l] == k)
break;
}
if (l < nedges) {
cadjwgt[cnedges+l] += rgraph[u+3+ncon+2*j+1];
}
else {
cadjncy[cnedges+nedges] = k;
cadjwgt[cnedges+nedges] = rgraph[u+3+ncon+2*j+1];
nedges++;
}
}
}
}
}
}
cnedges += nedges;
for (j=cxadj[cnvtxs]; j<cnedges; j++)
htable[cadjncy[j]&MASK] = -1; /* reset the htable */
cxadj[++cnvtxs] = cnedges;
}
}
cgraph->nedges = cnedges;
/* ADD: In order to keep from having to change this too much */
/* ADD: I kept vwgt array and recomputed nvwgt for each coarser graph */
for (j=0; j<cnvtxs; j++) {
for (h=0; h<ncon; h++)
cgraph->nvwgt[j*ncon+h] = ctrl->invtvwgts[h]*cvwgt[j*ncon+h];
}
cgraph->adjncy = imalloc(cnedges, "CreateCoarserGraph: cadjncy");
cgraph->adjwgt = imalloc(cnedges, "CreateCoarserGraph: cadjwgt");
icopy(cnedges, cadjncy, cgraph->adjncy);
icopy(cnedges, cadjwgt, cgraph->adjwgt);
WCOREPOP;
/* Note that graph->where works fine even if it is NULL */
gk_free((void **)&graph->where, LTERM);
}
void Match_Global(ctrl_t *ctrl, graph_t *graph)
{
idx_t h, i, ii, j, k;
idx_t nnbrs, nvtxs, ncon, cnvtxs, firstvtx, lastvtx, maxi, maxidx, nkept;
idx_t otherlastvtx, nrequests, nchanged, pass, nmatched, wside;
idx_t *xadj, *adjncy, *adjwgt, *vtxdist, *home, *myhome;
idx_t *match;
idx_t *peind, *sendptr, *recvptr;
idx_t *perm, *iperm, *nperm, *changed;
real_t *nvwgt, maxnvwgt;
idx_t *nreqs_pe;
ikv_t *match_requests, *match_granted, *pe_requests;
idx_t last_unmatched;
WCOREPUSH;
maxnvwgt = 0.75/((real_t)(ctrl->CoarsenTo));
graph->match_type = PARMETIS_MTYPE_GLOBAL;
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MatchTmr));
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
home = graph->home;
nvwgt = graph->nvwgt;
vtxdist = graph->vtxdist;
firstvtx = vtxdist[ctrl->mype];
lastvtx = vtxdist[ctrl->mype+1];
nnbrs = graph->nnbrs;
peind = graph->peind;
sendptr = graph->sendptr;
recvptr = graph->recvptr;
match = graph->match = ismalloc(nvtxs+graph->nrecv, UNMATCHED, "GlobalMatch: match");
/* wspacemalloc'ed arrays */
myhome = iset(nvtxs+graph->nrecv, UNMATCHED, iwspacemalloc(ctrl, nvtxs+graph->nrecv));
nreqs_pe = iset(nnbrs, 0, iwspacemalloc(ctrl, nnbrs));
perm = iwspacemalloc(ctrl, nvtxs);
iperm = iwspacemalloc(ctrl, nvtxs);
nperm = iwspacemalloc(ctrl, nnbrs);
changed = iwspacemalloc(ctrl, nvtxs);
match_requests = ikvwspacemalloc(ctrl, graph->nsend);
match_granted = ikvwspacemalloc(ctrl, graph->nrecv);
/* create the traversal order */
FastRandomPermute(nvtxs, perm, 1);
for (i=0; i<nvtxs; i++)
iperm[perm[i]] = i;
iincset(nnbrs, 0, nperm);
/* if coasening for adaptive/repartition, exchange home information */
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
PASSERT(ctrl, home != NULL);
icopy(nvtxs, home, myhome);
CommInterfaceData(ctrl, graph, myhome, myhome+nvtxs);
}
/* if coarsening for ordering, replace home with where information */
if (ctrl->partType == ORDER_PARTITION) {
PASSERT(ctrl, graph->where != NULL);
icopy(nvtxs, graph->where, myhome);
CommInterfaceData(ctrl, graph, myhome, myhome+nvtxs);
}
/* mark all heavy vertices as TOO_HEAVY so they will not be matched */
for (nchanged=i=0; i<nvtxs; i++) {
for (h=0; h<ncon; h++) {
if (nvwgt[i*ncon+h] > maxnvwgt) {
match[i] = TOO_HEAVY;
nchanged++;
break;
}
}
}
/* If no heavy vertices, pick one at random and treat it as such so that
at the end of the matching each partition will still have one vertex.
This is to eliminate the cases in which once a matching has been
computed, a processor ends up having no vertices */
if (nchanged == 0)
match[RandomInRange(nvtxs)] = TOO_HEAVY;
CommInterfaceData(ctrl, graph, match, match+nvtxs);
/* set initial value of nkept based on how over/under weight the
partition is to begin with */
nkept = graph->gnvtxs/ctrl->npes - nvtxs;
/* Find a matching by doing multiple iterations */
for (nmatched=pass=0; pass<NMATCH_PASSES; pass++) {
wside = (graph->level+pass)%2;
nchanged = nrequests = 0;
for (last_unmatched=ii=nmatched; ii<nvtxs; ii++) {
i = perm[ii];
if (match[i] == UNMATCHED) { /* Unmatched */
maxidx = i;
maxi = -1;
/* Deal with islands. Find another vertex and match it with */
if (xadj[i] == xadj[i+1]) {
last_unmatched = gk_max(ii, last_unmatched)+1;
for (; last_unmatched<nvtxs; last_unmatched++) {
k = perm[last_unmatched];
if (match[k] == UNMATCHED && myhome[i] == myhome[k]) {
match[i] = firstvtx+k + (i <= k ? KEEP_BIT : 0);
match[k] = firstvtx+i + (i > k ? KEEP_BIT : 0);
changed[nchanged++] = i;
changed[nchanged++] = k;
break;
}
}
continue;
}
/* Find a heavy-edge matching. */
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = adjncy[j];
if (match[k] == UNMATCHED && myhome[k] == myhome[i]) {
if (ncon == 1) {
if (maxi == -1 || adjwgt[maxi] < adjwgt[j] ||
(adjwgt[maxi] == adjwgt[j] && RandomInRange(xadj[i+1]-xadj[i]) == 0)) {
maxi = j;
maxidx = k;
}
}
else {
if (maxi == -1 || adjwgt[maxi] < adjwgt[j] ||
(adjwgt[maxi] == adjwgt[j] && maxidx < nvtxs && k < nvtxs &&
BetterVBalance(ncon,nvwgt+i*ncon,nvwgt+maxidx*ncon,nvwgt+k*ncon) >= 0)) {
maxi = j;
maxidx = k;
}
}
}
}
if (maxi != -1) {
k = adjncy[maxi];
if (k < nvtxs) { /* Take care the local vertices first */
/* Here we give preference the local matching by granting it right away */
match[i] = firstvtx+k + (i <= k ? KEEP_BIT : 0);
match[k] = firstvtx+i + (i > k ? KEEP_BIT : 0);
changed[nchanged++] = i;
changed[nchanged++] = k;
}
else { /* Take care any remote boundary vertices */
match[k] = MAYBE_MATCHED;
/* Alternate among which vertices will issue the requests */
if ((wside == 0 && firstvtx+i < graph->imap[k]) ||
(wside == 1 && firstvtx+i > graph->imap[k])) {
match[i] = MAYBE_MATCHED;
match_requests[nrequests].key = graph->imap[k];
match_requests[nrequests].val = firstvtx+i;
nrequests++;
}
}
}
}
}
#ifdef DEBUG_MATCH
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match1");
myprintf(ctrl, "[c: %2"PRIDX"] Nlocal: %"PRIDX", Nrequests: %"PRIDX"\n", c, nlocal, nrequests);
#endif
/***********************************************************
* Exchange the match_requests, requests for me are stored in
* match_granted
************************************************************/
/* Issue the receives first. Note that from each PE can receive a maximum
of the interface node that it needs to send it in the case of a mat-vec */
for (i=0; i<nnbrs; i++) {
gkMPI_Irecv((void *)(match_granted+recvptr[i]), 2*(recvptr[i+1]-recvptr[i]), IDX_T,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends next. This needs some work */
ikvsorti(nrequests, match_requests);
for (j=i=0; i<nnbrs; i++) {
otherlastvtx = vtxdist[peind[i]+1];
for (k=j; k<nrequests && match_requests[k].key < otherlastvtx; k++);
gkMPI_Isend((void *)(match_requests+j), 2*(k-j), IDX_T, peind[i], 1,
ctrl->comm, ctrl->sreq+i);
j = k;
}
/* OK, now get into the loop waiting for the operations to finish */
gkMPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
for (i=0; i<nnbrs; i++) {
gkMPI_Get_count(ctrl->statuses+i, IDX_T, nreqs_pe+i);
nreqs_pe[i] = nreqs_pe[i]/2; /* Adjust for pairs of IDX_T */
}
gkMPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/***********************************************************
* Now, go and service the requests that you received in
* match_granted
************************************************************/
RandomPermute(nnbrs, nperm, 0);
for (ii=0; ii<nnbrs; ii++) {
i = nperm[ii];
pe_requests = match_granted+recvptr[i];
for (j=0; j<nreqs_pe[i]; j++) {
k = pe_requests[j].key;
PASSERTP(ctrl, k >= firstvtx && k < lastvtx, (ctrl, "%"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX"\n", firstvtx, lastvtx, k, j, peind[i]));
/* myprintf(ctrl, "Requesting a match %"PRIDX" %"PRIDX"\n", pe_requests[j].key, pe_requests[j].val); */
if (match[k-firstvtx] == UNMATCHED) { /* Bingo, lets grant this request */
changed[nchanged++] = k-firstvtx;
if (nkept >= 0) { /* decide who to keep it based on local balance */
match[k-firstvtx] = pe_requests[j].val + KEEP_BIT;
nkept--;
}
else {
match[k-firstvtx] = pe_requests[j].val;
pe_requests[j].key += KEEP_BIT;
nkept++;
}
/* myprintf(ctrl, "Request from pe:%"PRIDX" (%"PRIDX" %"PRIDX") granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */
}
else { /* We are not granting the request */
/* myprintf(ctrl, "Request from pe:%"PRIDX" (%"PRIDX" %"PRIDX") not granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */
pe_requests[j].key = UNMATCHED;
}
}
}
/***********************************************************
* Exchange the match_granted information. It is stored in
* match_requests
************************************************************/
/* Issue the receives first. Note that from each PE can receive a maximum
of the interface node that it needs to send during the case of a mat-vec */
for (i=0; i<nnbrs; i++) {
gkMPI_Irecv((void *)(match_requests+sendptr[i]), 2*(sendptr[i+1]-sendptr[i]), IDX_T,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends next. */
for (i=0; i<nnbrs; i++) {
gkMPI_Isend((void *)(match_granted+recvptr[i]), 2*nreqs_pe[i], IDX_T,
peind[i], 1, ctrl->comm, ctrl->sreq+i);
}
/* OK, now get into the loop waiting for the operations to finish */
gkMPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
for (i=0; i<nnbrs; i++) {
gkMPI_Get_count(ctrl->statuses+i, IDX_T, nreqs_pe+i);
nreqs_pe[i] = nreqs_pe[i]/2; /* Adjust for pairs of IDX_T */
}
gkMPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/***********************************************************
* Now, go and through the match_requests and update local
* match information for the matchings that were granted.
************************************************************/
for (i=0; i<nnbrs; i++) {
pe_requests = match_requests+sendptr[i];
for (j=0; j<nreqs_pe[i]; j++) {
match[pe_requests[j].val-firstvtx] = pe_requests[j].key;
if (pe_requests[j].key != UNMATCHED)
changed[nchanged++] = pe_requests[j].val-firstvtx;
}
}
for (i=0; i<nchanged; i++) {
ii = iperm[changed[i]];
perm[ii] = perm[nmatched];
iperm[perm[nmatched]] = ii;
nmatched++;
}
CommChangedInterfaceData(ctrl, graph, nchanged, changed, match,
match_requests, match_granted);
}
/* Traverse the vertices and those that were unmatched, match them with themselves */
cnvtxs = 0;
for (i=0; i<nvtxs; i++) {
if (match[i] == UNMATCHED || match[i] == TOO_HEAVY) {
match[i] = (firstvtx+i) + KEEP_BIT;
cnvtxs++;
}
else if (match[i] >= KEEP_BIT) { /* A matched vertex which I get to keep */
cnvtxs++;
}
}
if (ctrl->dbglvl&DBG_MATCHINFO) {
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match");
myprintf(ctrl, "Cnvtxs: %"PRIDX"\n", cnvtxs);
rprintf(ctrl, "Done with matching...\n");
}
WCOREPOP;
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MatchTmr));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->ContractTmr));
CreateCoarseGraph_Global(ctrl, graph, cnvtxs);
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->ContractTmr));
}
/*************************************************************************
* This function converts a mesh into a dual graph
**************************************************************************/
int ParMETIS_V3_Mesh2Dual(idx_t *elmdist, idx_t *eptr, idx_t *eind,
idx_t *numflag, idx_t *ncommon, idx_t **r_xadj,
idx_t **r_adjncy, MPI_Comm *comm)
{
idx_t i, j, jj, k, kk, m;
idx_t npes, mype, pe, count, mask, pass;
idx_t nelms, lnns, my_nns, node;
idx_t firstelm, firstnode, lnode, nrecv, nsend;
idx_t *scounts, *rcounts, *sdispl, *rdispl;
idx_t *nodedist, *nmap, *auxarray;
idx_t *gnptr, *gnind, *nptr, *nind, *myxadj=NULL, *myadjncy = NULL;
idx_t *sbuffer, *rbuffer, *htable;
ikv_t *nodelist, *recvbuffer;
idx_t maxcount, *ind, *wgt;
idx_t gmaxnode, gminnode;
size_t curmem;
gk_malloc_init();
curmem = gk_GetCurMemoryUsed();
/* Get basic comm info */
gkMPI_Comm_size(*comm, &npes);
gkMPI_Comm_rank(*comm, &mype);
nelms = elmdist[mype+1]-elmdist[mype];
if (*numflag > 0)
ChangeNumberingMesh(elmdist, eptr, eind, NULL, NULL, NULL, npes, mype, 1);
mask = (1<<11)-1;
/*****************************/
/* Determine number of nodes */
/*****************************/
gminnode = GlobalSEMinComm(*comm, imin(eptr[nelms], eind));
for (i=0; i<eptr[nelms]; i++)
eind[i] -= gminnode;
gmaxnode = GlobalSEMaxComm(*comm, imax(eptr[nelms], eind));
/**************************/
/* Check for input errors */
/**************************/
ASSERT(nelms > 0);
/* construct node distribution array */
nodedist = ismalloc(npes+1, 0, "nodedist");
for (nodedist[0]=0, i=0,j=gmaxnode+1; i<npes; i++) {
k = j/(npes-i);
nodedist[i+1] = nodedist[i]+k;
j -= k;
}
my_nns = nodedist[mype+1]-nodedist[mype];
firstnode = nodedist[mype];
nodelist = ikvmalloc(eptr[nelms], "nodelist");
auxarray = imalloc(eptr[nelms], "auxarray");
htable = ismalloc(gk_max(my_nns, mask+1), -1, "htable");
scounts = imalloc(npes, "scounts");
rcounts = imalloc(npes, "rcounts");
sdispl = imalloc(npes+1, "sdispl");
rdispl = imalloc(npes+1, "rdispl");
/*********************************************/
/* first find a local numbering of the nodes */
/*********************************************/
for (i=0; i<nelms; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++) {
nodelist[j].key = eind[j];
nodelist[j].val = j;
auxarray[j] = i; /* remember the local element ID that uses this node */
}
}
ikvsorti(eptr[nelms], nodelist);
for (count=1, i=1; i<eptr[nelms]; i++) {
if (nodelist[i].key > nodelist[i-1].key)
count++;
}
lnns = count;
nmap = imalloc(lnns, "nmap");
/* renumber the nodes of the elements array */
count = 1;
nmap[0] = nodelist[0].key;
eind[nodelist[0].val] = 0;
nodelist[0].val = auxarray[nodelist[0].val]; /* Store the local element ID */
for (i=1; i<eptr[nelms]; i++) {
if (nodelist[i].key > nodelist[i-1].key) {
nmap[count] = nodelist[i].key;
count++;
}
eind[nodelist[i].val] = count-1;
nodelist[i].val = auxarray[nodelist[i].val]; /* Store the local element ID */
}
gkMPI_Barrier(*comm);
/**********************************************************/
/* perform comms necessary to construct node-element list */
/**********************************************************/
iset(npes, 0, scounts);
for (pe=i=0; i<eptr[nelms]; i++) {
while (nodelist[i].key >= nodedist[pe+1])
pe++;
scounts[pe] += 2;
}
ASSERT(pe < npes);
gkMPI_Alltoall((void *)scounts, 1, IDX_T, (void *)rcounts, 1, IDX_T, *comm);
icopy(npes, scounts, sdispl);
MAKECSR(i, npes, sdispl);
icopy(npes, rcounts, rdispl);
MAKECSR(i, npes, rdispl);
ASSERT(sdispl[npes] == eptr[nelms]*2);
nrecv = rdispl[npes]/2;
recvbuffer = ikvmalloc(gk_max(1, nrecv), "recvbuffer");
gkMPI_Alltoallv((void *)nodelist, scounts, sdispl, IDX_T, (void *)recvbuffer,
rcounts, rdispl, IDX_T, *comm);
/**************************************/
/* construct global node-element list */
/**************************************/
gnptr = ismalloc(my_nns+1, 0, "gnptr");
for (i=0; i<npes; i++) {
for (j=rdispl[i]/2; j<rdispl[i+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
ASSERT(lnode >= 0 && lnode < my_nns)
gnptr[lnode]++;
}
}
MAKECSR(i, my_nns, gnptr);
gnind = imalloc(gk_max(1, gnptr[my_nns]), "gnind");
for (pe=0; pe<npes; pe++) {
firstelm = elmdist[pe];
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
gnind[gnptr[lnode]++] = recvbuffer[j].val+firstelm;
}
}
SHIFTCSR(i, my_nns, gnptr);
/*********************************************************/
/* send the node-element info to the relevant processors */
/*********************************************************/
iset(npes, 0, scounts);
/* use a hash table to ensure that each node is sent to a proc only once */
for (pe=0; pe<npes; pe++) {
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
if (htable[lnode] == -1) {
scounts[pe] += gnptr[lnode+1]-gnptr[lnode];
htable[lnode] = 1;
}
}
/* now reset the hash table */
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
htable[lnode] = -1;
}
}
gkMPI_Alltoall((void *)scounts, 1, IDX_T, (void *)rcounts, 1, IDX_T, *comm);
icopy(npes, scounts, sdispl);
MAKECSR(i, npes, sdispl);
/* create the send buffer */
nsend = sdispl[npes];
sbuffer = imalloc(gk_max(1, nsend), "sbuffer");
count = 0;
for (pe=0; pe<npes; pe++) {
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
if (htable[lnode] == -1) {
for (k=gnptr[lnode]; k<gnptr[lnode+1]; k++) {
if (k == gnptr[lnode])
sbuffer[count++] = -1*(gnind[k]+1);
else
sbuffer[count++] = gnind[k];
}
htable[lnode] = 1;
}
}
ASSERT(count == sdispl[pe+1]);
/* now reset the hash table */
for (j=rdispl[pe]/2; j<rdispl[pe+1]/2; j++) {
lnode = recvbuffer[j].key-firstnode;
htable[lnode] = -1;
}
}
icopy(npes, rcounts, rdispl);
MAKECSR(i, npes, rdispl);
nrecv = rdispl[npes];
rbuffer = imalloc(gk_max(1, nrecv), "rbuffer");
gkMPI_Alltoallv((void *)sbuffer, scounts, sdispl, IDX_T, (void *)rbuffer,
rcounts, rdispl, IDX_T, *comm);
k = -1;
nptr = ismalloc(lnns+1, 0, "nptr");
nind = rbuffer;
for (pe=0; pe<npes; pe++) {
for (j=rdispl[pe]; j<rdispl[pe+1]; j++) {
if (nind[j] < 0) {
k++;
nind[j] = (-1*nind[j])-1;
}
nptr[k]++;
}
}
MAKECSR(i, lnns, nptr);
ASSERT(k+1 == lnns);
ASSERT(nptr[lnns] == nrecv)
myxadj = *r_xadj = (idx_t *)malloc(sizeof(idx_t)*(nelms+1));
if (myxadj == NULL)
gk_errexit(SIGMEM, "Failed to allocate memory for the dual graph's xadj array.\n");
iset(nelms+1, 0, myxadj);
iset(mask+1, -1, htable);
firstelm = elmdist[mype];
/* Two passes -- in first pass, simply find out the memory requirements */
maxcount = 200;
ind = imalloc(maxcount, "ParMETIS_V3_Mesh2Dual: ind");
wgt = imalloc(maxcount, "ParMETIS_V3_Mesh2Dual: wgt");
for (pass=0; pass<2; pass++) {
for (i=0; i<nelms; i++) {
for (count=0, j=eptr[i]; j<eptr[i+1]; j++) {
node = eind[j];
for (k=nptr[node]; k<nptr[node+1]; k++) {
if ((kk=nind[k]) == firstelm+i)
continue;
m = htable[(kk&mask)];
if (m == -1) {
ind[count] = kk;
wgt[count] = 1;
htable[(kk&mask)] = count++;
}
else {
if (ind[m] == kk) {
wgt[m]++;
}
else {
for (jj=0; jj<count; jj++) {
if (ind[jj] == kk) {
wgt[jj]++;
break;
}
}
if (jj == count) {
ind[count] = kk;
wgt[count++] = 1;
}
}
}
/* Adjust the memory.
This will be replaced by a idxrealloc() when GKlib will be incorporated */
if (count == maxcount-1) {
maxcount *= 2;
ind = irealloc(ind, maxcount, "ParMETIS_V3_Mesh2Dual: ind");
wgt = irealloc(wgt, maxcount, "ParMETIS_V3_Mesh2Dual: wgt");
}
}
}
for (j=0; j<count; j++) {
htable[(ind[j]&mask)] = -1;
if (wgt[j] >= *ncommon) {
if (pass == 0)
myxadj[i]++;
else
myadjncy[myxadj[i]++] = ind[j];
}
}
}
if (pass == 0) {
MAKECSR(i, nelms, myxadj);
myadjncy = *r_adjncy = (idx_t *)malloc(sizeof(idx_t)*myxadj[nelms]);
if (myadjncy == NULL)
gk_errexit(SIGMEM, "Failed to allocate memory for dual graph's adjncy array.\n");
}
else {
SHIFTCSR(i, nelms, myxadj);
}
}
/*****************************************/
/* correctly renumber the elements array */
/*****************************************/
for (i=0; i<eptr[nelms]; i++)
eind[i] = nmap[eind[i]] + gminnode;
if (*numflag == 1)
ChangeNumberingMesh(elmdist, eptr, eind, myxadj, myadjncy, NULL, npes, mype, 0);
/* do not free nodelist, recvbuffer, rbuffer */
gk_free((void **)&nodedist, &nodelist, &auxarray, &htable, &scounts, &rcounts,
&sdispl, &rdispl, &nmap, &recvbuffer, &gnptr, &gnind, &sbuffer, &rbuffer,
&nptr, &ind, &wgt, LTERM);
if (gk_GetCurMemoryUsed() - curmem > 0) {
printf("ParMETIS appears to have a memory leak of %zdbytes. Report this.\n",
(ssize_t)(gk_GetCurMemoryUsed() - curmem));
}
gk_malloc_cleanup(0);
return METIS_OK;
}
idx_t FindPartitionInducedComponents(graph_t *graph, idx_t *where,
idx_t *cptr, idx_t *cind) {
idx_t i, ii, j, jj, k, me = 0, nvtxs, first, last, nleft, ncmps;
idx_t *xadj, *adjncy;
idx_t *touched, *perm, *todo;
idx_t mustfree_ccsr = 0, mustfree_where = 0;
nvtxs = graph->nvtxs;
xadj = graph->xadj;
adjncy = graph->adjncy;
/* Deal with NULL supplied cptr/cind vectors */
if (cptr == NULL) {
cptr = imalloc(nvtxs + 1, "FindPartitionInducedComponents: cptr");
cind = imalloc(nvtxs, "FindPartitionInducedComponents: cind");
mustfree_ccsr = 1;
}
/* Deal with NULL supplied where vector */
if (where == NULL) {
where = ismalloc(nvtxs, 0, "FindPartitionInducedComponents: where");
mustfree_where = 1;
}
/* Allocate memory required for the BFS traversal */
perm = iincset(nvtxs, 0, imalloc(nvtxs, "FindPartitionInducedComponents: perm"));
todo = iincset(nvtxs, 0, imalloc(nvtxs, "FindPartitionInducedComponents: todo"));
touched = ismalloc(nvtxs, 0, "FindPartitionInducedComponents: touched");
/* Find the connected componends induced by the partition */
ncmps = -1;
first = last = 0;
nleft = nvtxs;
while (nleft > 0) {
if (first == last) { /* Find another starting vertex */
cptr[++ncmps] = first;
ASSERT(touched[todo[0]] == 0);
i = todo[0];
cind[last++] = i;
touched[i] = 1;
me = where[i];
}
i = cind[first++];
k = perm[i];
j = todo[k] = todo[--nleft];
perm[j] = k;
for (j = xadj[i]; j < xadj[i + 1]; j++) {
k = adjncy[j];
if (where[k] == me && !touched[k]) {
cind[last++] = k;
touched[k] = 1;
}
}
}
cptr[++ncmps] = first;
if (mustfree_ccsr) {
gk_free((void **) &cptr, &cind, LTERM);
}
if (mustfree_where) {
gk_free((void **) &where, LTERM);
}
gk_free((void **) &perm, &todo, &touched, LTERM);
return ncmps;
}
void InduceRowPartFromColumnPart(idx_t nrows, idx_t *rowptr, idx_t *rowind,
idx_t *rpart, idx_t *cpart, idx_t nparts, real_t *tpwgts)
{
idx_t i, j, k, me;
idx_t nnbrs, *pwgts, *nbrdom, *nbrwgt, *nbrmrk;
idx_t *itpwgts;
pwgts = ismalloc(nparts, 0, "InduceRowPartFromColumnPart: pwgts");
nbrdom = ismalloc(nparts, 0, "InduceRowPartFromColumnPart: nbrdom");
nbrwgt = ismalloc(nparts, 0, "InduceRowPartFromColumnPart: nbrwgt");
nbrmrk = ismalloc(nparts, -1, "InduceRowPartFromColumnPart: nbrmrk");
iset(nrows, -1, rpart);
/* setup the integer target partition weights */
itpwgts = imalloc(nparts, "InduceRowPartFromColumnPart: itpwgts");
if (tpwgts == NULL) {
iset(nparts, 1+nrows/nparts, itpwgts);
}
else {
for (i=0; i<nparts; i++)
itpwgts[i] = 1+nrows*tpwgts[i];
}
/* first assign the rows consisting only of columns that belong to
a single partition. Assign rows that are empty to -2 (un-assigned) */
for (i=0; i<nrows; i++) {
if (rowptr[i+1]-rowptr[i] == 0) {
rpart[i] = -2;
continue;
}
me = cpart[rowind[rowptr[i]]];
for (j=rowptr[i]+1; j<rowptr[i+1]; j++) {
if (cpart[rowind[j]] != me)
break;
}
if (j == rowptr[i+1]) {
rpart[i] = me;
pwgts[me]++;
}
}
/* next assign the rows consisting of columns belonging to multiple
partitions in a balanced way */
for (i=0; i<nrows; i++) {
if (rpart[i] == -1) {
for (nnbrs=0, j=rowptr[i]; j<rowptr[i+1]; j++) {
me = cpart[rowind[j]];
if (nbrmrk[me] == -1) {
nbrdom[nnbrs] = me;
nbrwgt[nnbrs] = 1;
nbrmrk[me] = nnbrs++;
}
else {
nbrwgt[nbrmrk[me]]++;
}
}
ASSERT(nnbrs > 0);
/* assign it first to the domain with most things in common */
rpart[i] = nbrdom[iargmax(nnbrs, nbrwgt)];
/* if overweight, assign it to the light domain */
if (pwgts[rpart[i]] > itpwgts[rpart[i]]) {
for (j=0; j<nnbrs; j++) {
if (pwgts[nbrdom[j]] < itpwgts[nbrdom[j]] ||
pwgts[nbrdom[j]]-itpwgts[nbrdom[j]] < pwgts[rpart[i]]-itpwgts[rpart[i]]) {
rpart[i] = nbrdom[j];
break;
}
}
}
pwgts[rpart[i]]++;
/* reset nbrmrk array */
for (j=0; j<nnbrs; j++)
nbrmrk[nbrdom[j]] = -1;
}
}
gk_free((void **)&pwgts, &nbrdom, &nbrwgt, &nbrmrk, &itpwgts, LTERM);
}
idx_t smbfct(idx_t neqns, idx_t *xadj, idx_t *adjncy, idx_t *perm, idx_t *invp,
idx_t *xlnz, idx_t *maxlnz, idx_t *xnzsub, idx_t *nzsub,
idx_t *maxsub)
{
/* Local variables */
idx_t node, rchm, mrgk, lmax, i, j, k, m, nabor, nzbeg, nzend;
idx_t kxsub, jstop, jstrt, mrkflg, inz, knz, flag;
idx_t *mrglnk, *marker, *rchlnk;
rchlnk = ismalloc(neqns+1, 0, "smbfct: rchlnk");
marker = ismalloc(neqns+1, 0, "smbfct: marker");
mrglnk = ismalloc(neqns+1, 0, "smbfct: mgrlnk");
/* Parameter adjustments */
--marker;
--mrglnk;
--rchlnk;
--nzsub;
--xnzsub;
--xlnz;
--invp;
--perm;
--adjncy;
--xadj;
/* Function Body */
flag = 0;
nzbeg = 1;
nzend = 0;
xlnz[1] = 1;
/* FOR EACH COLUMN KNZ COUNTS THE NUMBER OF NONZEROS IN COLUMN K ACCUMULATED IN RCHLNK. */
for (k=1; k<=neqns; k++) {
xnzsub[k] = nzend;
node = perm[k];
knz = 0;
mrgk = mrglnk[k];
mrkflg = 0;
marker[k] = k;
if (mrgk != 0) {
assert(mrgk > 0 && mrgk <= neqns);
marker[k] = marker[mrgk];
}
if (xadj[node] >= xadj[node+1]) {
xlnz[k+1] = xlnz[k];
continue;
}
/* USE RCHLNK TO LINK THROUGH THE STRUCTURE OF A(*,K) BELOW DIAGONAL */
assert(k <= neqns && k > 0);
rchlnk[k] = neqns+1;
for (j=xadj[node]; j<xadj[node+1]; j++) {
nabor = invp[adjncy[j]];
if (nabor <= k)
continue;
rchm = k;
do {
m = rchm;
assert(m > 0 && m <= neqns);
rchm = rchlnk[m];
} while (rchm <= nabor);
knz++;
assert(m > 0 && m <= neqns);
rchlnk[m] = nabor;
assert(nabor > 0 && nabor <= neqns);
rchlnk[nabor] = rchm;
assert(k > 0 && k <= neqns);
if (marker[nabor] != marker[k])
mrkflg = 1;
}
/* TEST FOR MASS SYMBOLIC ELIMINATION */
lmax = 0;
assert(mrgk >= 0 && mrgk <= neqns);
if (mrkflg != 0 || mrgk == 0 || mrglnk[mrgk] != 0)
goto L350;
xnzsub[k] = xnzsub[mrgk] + 1;
knz = xlnz[mrgk + 1] - (xlnz[mrgk] + 1);
goto L1400;
L350:
/* LINK THROUGH EACH COLUMN I THAT AFFECTS L(*,K) */
i = k;
assert(i > 0 && i <= neqns);
while ((i = mrglnk[i]) != 0) {
assert(i > 0 && i <= neqns);
inz = xlnz[i+1] - (xlnz[i]+1);
jstrt = xnzsub[i] + 1;
jstop = xnzsub[i] + inz;
if (inz > lmax) {
lmax = inz;
xnzsub[k] = jstrt;
}
/* MERGE STRUCTURE OF L(*,I) IN NZSUB INTO RCHLNK. */
rchm = k;
for (j=jstrt; j<=jstop; j++) {
nabor = nzsub[j];
do {
m = rchm;
assert(m > 0 && m <= neqns);
rchm = rchlnk[m];
} while (rchm < nabor);
if (rchm != nabor) {
knz++;
assert(m > 0 && m <= neqns);
rchlnk[m] = nabor;
assert(nabor > 0 && nabor <= neqns);
rchlnk[nabor] = rchm;
rchm = nabor;
}
}
}
/* CHECK IF SUBSCRIPTS DUPLICATE THOSE OF ANOTHER COLUMN */
if (knz == lmax)
goto L1400;
/* OR IF TAIL OF K-1ST COLUMN MATCHES HEAD OF KTH */
if (nzbeg > nzend)
goto L1200;
assert(k > 0 && k <= neqns);
i = rchlnk[k];
for (jstrt = nzbeg; jstrt <= nzend; ++jstrt) {
if (nzsub[jstrt] < i)
continue;
if (nzsub[jstrt] == i)
goto L1000;
else
goto L1200;
}
goto L1200;
L1000:
xnzsub[k] = jstrt;
for (j = jstrt; j <= nzend; ++j) {
if (nzsub[j] != i)
goto L1200;
assert(i > 0 && i <= neqns);
i = rchlnk[i];
if (i > neqns)
goto L1400;
}
nzend = jstrt - 1;
/* COPY THE STRUCTURE OF L(*,K) FROM RCHLNK TO THE DATA STRUCTURE (XNZSUB, NZSUB) */
L1200:
nzbeg = nzend + 1;
nzend += knz;
if (nzend >= *maxsub) {
flag = 1; /* Out of memory */
break;
}
i = k;
for (j=nzbeg; j<=nzend; j++) {
assert(i > 0 && i <= neqns);
i = rchlnk[i];
nzsub[j] = i;
assert(i > 0 && i <= neqns);
marker[i] = k;
}
xnzsub[k] = nzbeg;
assert(k > 0 && k <= neqns);
marker[k] = k;
/*
* UPDATE THE VECTOR MRGLNK. NOTE COLUMN L(*,K) JUST FOUND
* IS REQUIRED TO DETERMINE COLUMN L(*,J), WHERE
* L(J,K) IS THE FIRST NONZERO IN L(*,K) BELOW DIAGONAL.
*/
L1400:
if (knz > 1) {
kxsub = xnzsub[k];
i = nzsub[kxsub];
assert(i > 0 && i <= neqns);
assert(k > 0 && k <= neqns);
mrglnk[k] = mrglnk[i];
mrglnk[i] = k;
}
xlnz[k + 1] = xlnz[k] + knz;
}
if (flag == 0) {
*maxlnz = xlnz[neqns] - 1;
*maxsub = xnzsub[neqns];
xnzsub[neqns + 1] = xnzsub[neqns];
}
marker++;
mrglnk++;
rchlnk++;
nzsub++;
xnzsub++;
xlnz++;
invp++;
perm++;
adjncy++;
xadj++;
gk_free((void **)&rchlnk, &mrglnk, &marker, LTERM);
return flag;
}
/*************************************************************************
* This function finds a matching
**************************************************************************/
void Moc_GlobalMatch_Balance(CtrlType *ctrl, GraphType *graph, WorkSpaceType *wspace)
{
int h, i, ii, j, k;
int nnbrs, nvtxs, ncon, cnvtxs, firstvtx, lastvtx, maxi, maxidx, nkept;
int otherlastvtx, nrequests, nchanged, pass, nmatched, wside;
idxtype *xadj, *ladjncy, *adjwgt, *vtxdist, *home, *myhome, *shome, *rhome;
idxtype *match, *rmatch, *smatch;
idxtype *peind, *sendptr, *recvptr;
idxtype *perm, *iperm, *nperm, *changed;
floattype *nvwgt, maxnvwgt;
int *nreqs_pe;
KeyValueType *match_requests, *match_granted, *pe_requests;
maxnvwgt = 1.0/((floattype)(ctrl->nparts)*MAXNVWGT_FACTOR);
graph->match_type = MATCH_GLOBAL;
IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->MatchTmr));
nvtxs = graph->nvtxs;
ncon = graph->ncon;
xadj = graph->xadj;
ladjncy = graph->adjncy;
adjwgt = graph->adjwgt;
home = graph->home;
nvwgt = graph->nvwgt;
vtxdist = graph->vtxdist;
firstvtx = vtxdist[ctrl->mype];
lastvtx = vtxdist[ctrl->mype+1];
match = graph->match = idxsmalloc(nvtxs+graph->nrecv, UNMATCHED, "HEM_Match: match");
myhome = idxsmalloc(nvtxs+graph->nrecv, UNMATCHED, "HEM_Match: myhome");
/*------------------------------------------------------------
/ Send/Receive the home information of interface vertices
/------------------------------------------------------------*/
if (ctrl->partType == ADAPTIVE_PARTITION || ctrl->partType == REFINE_PARTITION) {
idxcopy(nvtxs, home, myhome);
shome = wspace->indices;
rhome = myhome + nvtxs;
CommInterfaceData(ctrl, graph, myhome, shome, rhome);
}
nnbrs = graph->nnbrs;
peind = graph->peind;
sendptr = graph->sendptr;
recvptr = graph->recvptr;
/* Use wspace->indices as the tmp space for matching info of the boundary
* vertices that are sent and received */
rmatch = match + nvtxs;
smatch = wspace->indices;
changed = smatch+graph->nsend;
/* Use wspace->indices as the tmp space for match requests of the boundary
* vertices that are sent and received */
match_requests = wspace->pairs;
match_granted = match_requests + graph->nsend;
nreqs_pe = ismalloc(nnbrs, 0, "Match_HEM: nreqs_pe");
nkept = graph->gnvtxs/ctrl->npes - nvtxs;
perm = (idxtype *)wspace->degrees;
iperm = perm + nvtxs;
FastRandomPermute(nvtxs, perm, 1);
for (i=0; i<nvtxs; i++)
iperm[perm[i]] = i;
nperm = iperm + nvtxs;
for (i=0; i<nnbrs; i++)
nperm[i] = i;
/*************************************************************
* Go now and find a matching by doing multiple iterations
*************************************************************/
/* First nullify the heavy vertices */
for (nchanged=i=0; i<nvtxs; i++) {
for (h=0; h<ncon; h++)
if (nvwgt[i*ncon+h] > maxnvwgt) {
break;
}
if (h != ncon) {
match[i] = TOO_HEAVY;
nchanged++;
}
}
if (GlobalSESum(ctrl, nchanged) > 0) {
IFSET(ctrl->dbglvl, DBG_PROGRESS,
rprintf(ctrl, "We found %d heavy vertices!\n", GlobalSESum(ctrl, nchanged)));
CommInterfaceData(ctrl, graph, match, smatch, rmatch);
}
for (nmatched=pass=0; pass<NMATCH_PASSES; pass++) {
wside = (graph->level+pass)%2;
nchanged = nrequests = 0;
for (ii=nmatched; ii<nvtxs; ii++) {
i = perm[ii];
if (match[i] == UNMATCHED) { /* Unmatched */
maxidx = i;
maxi = -1;
/* Find a heavy-edge matching */
for (j=xadj[i]; j<xadj[i+1]; j++) {
k = ladjncy[j];
if (match[k] == UNMATCHED &&
myhome[k] == myhome[i] &&
(maxi == -1 ||
adjwgt[maxi] < adjwgt[j] ||
(maxidx < nvtxs &&
k < nvtxs &&
adjwgt[maxi] == adjwgt[j] &&
BetterVBalance(ncon,nvwgt+i*ncon,nvwgt+maxidx*ncon,nvwgt+k*ncon) >= 0))) {
maxi = j;
maxidx = k;
}
}
if (maxi != -1) {
k = ladjncy[maxi];
if (k < nvtxs) { /* Take care the local vertices first */
/* Here we give preference the local matching by granting it right away */
if (i <= k) {
match[i] = firstvtx+k + KEEP_BIT;
match[k] = firstvtx+i;
}
else {
match[i] = firstvtx+k;
match[k] = firstvtx+i + KEEP_BIT;
}
changed[nchanged++] = i;
changed[nchanged++] = k;
}
else { /* Take care any remote boundary vertices */
match[k] = MAYBE_MATCHED;
/* Alternate among which vertices will issue the requests */
if ((wside ==0 && firstvtx+i < graph->imap[k]) || (wside == 1 && firstvtx+i > graph->imap[k])) {
match[i] = MAYBE_MATCHED;
match_requests[nrequests].key = graph->imap[k];
match_requests[nrequests].val = firstvtx+i;
nrequests++;
}
}
}
}
}
#ifdef DEBUG_MATCH
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match1");
myprintf(ctrl, "[c: %2d] Nlocal: %d, Nrequests: %d\n", c, nlocal, nrequests);
#endif
/***********************************************************
* Exchange the match_requests, requests for me are stored in
* match_granted
************************************************************/
/* Issue the receives first. Note that from each PE can receive a maximum
of the interface node that it needs to send it in the case of a mat-vec */
for (i=0; i<nnbrs; i++) {
MPI_Irecv((void *)(match_granted+recvptr[i]), 2*(recvptr[i+1]-recvptr[i]), IDX_DATATYPE,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends next. This needs some work */
ikeysort(nrequests, match_requests);
for (j=i=0; i<nnbrs; i++) {
otherlastvtx = vtxdist[peind[i]+1];
for (k=j; k<nrequests && match_requests[k].key < otherlastvtx; k++);
MPI_Isend((void *)(match_requests+j), 2*(k-j), IDX_DATATYPE, peind[i], 1, ctrl->comm, ctrl->sreq+i);
j = k;
}
/* OK, now get into the loop waiting for the operations to finish */
MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
for (i=0; i<nnbrs; i++) {
MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nreqs_pe+i);
nreqs_pe[i] = nreqs_pe[i]/2; /* Adjust for pairs of IDX_DATATYPE */
}
MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/***********************************************************
* Now, go and service the requests that you received in
* match_granted
************************************************************/
RandomPermute(nnbrs, nperm, 0);
for (ii=0; ii<nnbrs; ii++) {
i = nperm[ii];
pe_requests = match_granted+recvptr[i];
for (j=0; j<nreqs_pe[i]; j++) {
k = pe_requests[j].key;
ASSERTP(ctrl, k >= firstvtx && k < lastvtx, (ctrl, "%d %d %d %d %d\n", firstvtx, lastvtx, k, j, peind[i]));
/* myprintf(ctrl, "Requesting a match %d %d\n", pe_requests[j].key, pe_requests[j].val); */
if (match[k-firstvtx] == UNMATCHED) { /* Bingo, lets grant this request */
changed[nchanged++] = k-firstvtx;
if (nkept >= 0) { /* Flip a coin for who gets it */
match[k-firstvtx] = pe_requests[j].val + KEEP_BIT;
nkept--;
}
else {
match[k-firstvtx] = pe_requests[j].val;
pe_requests[j].key += KEEP_BIT;
nkept++;
}
/* myprintf(ctrl, "Request from pe:%d (%d %d) granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */
}
else { /* We are not granting the request */
/* myprintf(ctrl, "Request from pe:%d (%d %d) not granted!\n", peind[i], pe_requests[j].val, pe_requests[j].key); */
pe_requests[j].key = UNMATCHED;
}
}
}
/***********************************************************
* Exchange the match_granted information. It is stored in
* match_requests
************************************************************/
/* Issue the receives first. Note that from each PE can receive a maximum
of the interface node that it needs to send during the case of a mat-vec */
for (i=0; i<nnbrs; i++) {
MPI_Irecv((void *)(match_requests+sendptr[i]), 2*(sendptr[i+1]-sendptr[i]), IDX_DATATYPE,
peind[i], 1, ctrl->comm, ctrl->rreq+i);
}
/* Issue the sends next. */
for (i=0; i<nnbrs; i++) {
MPI_Isend((void *)(match_granted+recvptr[i]), 2*nreqs_pe[i], IDX_DATATYPE,
peind[i], 1, ctrl->comm, ctrl->sreq+i);
}
/* OK, now get into the loop waiting for the operations to finish */
MPI_Waitall(nnbrs, ctrl->rreq, ctrl->statuses);
for (i=0; i<nnbrs; i++) {
MPI_Get_count(ctrl->statuses+i, IDX_DATATYPE, nreqs_pe+i);
nreqs_pe[i] = nreqs_pe[i]/2; /* Adjust for pairs of IDX_DATATYPE */
}
MPI_Waitall(nnbrs, ctrl->sreq, ctrl->statuses);
/***********************************************************
* Now, go and through the match_requests and update local
* match information for the matchings that were granted.
************************************************************/
for (i=0; i<nnbrs; i++) {
pe_requests = match_requests+sendptr[i];
for (j=0; j<nreqs_pe[i]; j++) {
match[pe_requests[j].val-firstvtx] = pe_requests[j].key;
if (pe_requests[j].key != UNMATCHED)
changed[nchanged++] = pe_requests[j].val-firstvtx;
}
}
for (i=0; i<nchanged; i++) {
ii = iperm[changed[i]];
perm[ii] = perm[nmatched];
iperm[perm[nmatched]] = ii;
nmatched++;
}
CommChangedInterfaceData(ctrl, graph, nchanged, changed, match, match_requests, match_granted, wspace->pv4);
}
/* Traverse the vertices and those that were unmatched, match them with themselves */
cnvtxs = 0;
for (i=0; i<nvtxs; i++) {
if (match[i] == UNMATCHED || match[i] == TOO_HEAVY) {
match[i] = (firstvtx+i) + KEEP_BIT;
cnvtxs++;
}
else if (match[i] >= KEEP_BIT) { /* A matched vertex which I get to keep */
cnvtxs++;
}
}
if (ctrl->dbglvl&DBG_MATCHINFO) {
PrintVector2(ctrl, nvtxs, firstvtx, match, "Match");
myprintf(ctrl, "Cnvtxs: %d\n", cnvtxs);
rprintf(ctrl, "Done with matching...\n");
}
GKfree((void **)(&myhome), (void **)(&nreqs_pe), LTERM);
IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->MatchTmr));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->ContractTmr));
Moc_Global_CreateCoarseGraph(ctrl, graph, wspace, cnvtxs);
IFSET(ctrl->dbglvl, DBG_TIME, MPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->ContractTmr));
}
ctrl_t *SetupCtrl(moptype_et optype, idx_t *options, idx_t ncon, idx_t nparts,
real_t *tpwgts, real_t *ubvec)
{
idx_t i, j;
ctrl_t *ctrl;
ctrl = (ctrl_t *)gk_malloc(sizeof(ctrl_t), "SetupCtrl: ctrl");
memset((void *)ctrl, 0, sizeof(ctrl_t));
switch (optype) {
case METIS_OP_PMETIS:
ctrl->objtype = GETOPTION(options, METIS_OPTION_OBJTYPE, METIS_OBJTYPE_CUT);
ctrl->ctype = GETOPTION(options, METIS_OPTION_CTYPE, METIS_CTYPE_SHEM);
ctrl->rtype = METIS_RTYPE_FM;
ctrl->ncuts = GETOPTION(options, METIS_OPTION_NCUTS, 1);
ctrl->niter = GETOPTION(options, METIS_OPTION_NITER, 10);
ctrl->seed = GETOPTION(options, METIS_OPTION_SEED, -1);
ctrl->dbglvl = GETOPTION(options, METIS_OPTION_DBGLVL, 0);
if (ncon == 1) {
ctrl->iptype = GETOPTION(options, METIS_OPTION_IPTYPE, METIS_IPTYPE_GROW);
ctrl->ufactor = GETOPTION(options, METIS_OPTION_UFACTOR, PMETIS_DEFAULT_UFACTOR);
ctrl->CoarsenTo = 20;
}
else {
ctrl->iptype = GETOPTION(options, METIS_OPTION_IPTYPE, METIS_IPTYPE_RANDOM);
ctrl->ufactor = GETOPTION(options, METIS_OPTION_UFACTOR, MCPMETIS_DEFAULT_UFACTOR);
ctrl->CoarsenTo = 100;
}
break;
case METIS_OP_KMETIS:
ctrl->objtype = GETOPTION(options, METIS_OPTION_OBJTYPE, METIS_OBJTYPE_CUT);
ctrl->ctype = GETOPTION(options, METIS_OPTION_CTYPE, METIS_CTYPE_SHEM);
ctrl->iptype = METIS_IPTYPE_METISRB;
ctrl->rtype = METIS_RTYPE_GREEDY;
ctrl->ncuts = GETOPTION(options, METIS_OPTION_NCUTS, 1);
ctrl->niter = GETOPTION(options, METIS_OPTION_NITER, 10);
ctrl->ufactor = GETOPTION(options, METIS_OPTION_UFACTOR, KMETIS_DEFAULT_UFACTOR);
ctrl->minconn = GETOPTION(options, METIS_OPTION_MINCONN, 0);
ctrl->contig = GETOPTION(options, METIS_OPTION_CONTIG, 0);
ctrl->seed = GETOPTION(options, METIS_OPTION_SEED, -1);
ctrl->dbglvl = GETOPTION(options, METIS_OPTION_DBGLVL, 0);
break;
case METIS_OP_OMETIS:
ctrl->objtype = GETOPTION(options, METIS_OPTION_OBJTYPE, METIS_OBJTYPE_NODE);
ctrl->ctype = GETOPTION(options, METIS_OPTION_CTYPE, METIS_CTYPE_SHEM);
ctrl->rtype = GETOPTION(options, METIS_OPTION_RTYPE, METIS_RTYPE_SEP1SIDED);
ctrl->iptype = GETOPTION(options, METIS_OPTION_IPTYPE, METIS_IPTYPE_EDGE);
ctrl->nseps = GETOPTION(options, METIS_OPTION_NSEPS, 1);
ctrl->niter = GETOPTION(options, METIS_OPTION_NITER, 10);
ctrl->ufactor = GETOPTION(options, METIS_OPTION_UFACTOR, OMETIS_DEFAULT_UFACTOR);
ctrl->compress = GETOPTION(options, METIS_OPTION_COMPRESS, 1);
ctrl->ccorder = GETOPTION(options, METIS_OPTION_CCORDER, 0);
ctrl->seed = GETOPTION(options, METIS_OPTION_SEED, -1);
ctrl->dbglvl = GETOPTION(options, METIS_OPTION_DBGLVL, 0);
ctrl->pfactor = 0.1*GETOPTION(options, METIS_OPTION_PFACTOR, 0);
ctrl->CoarsenTo = 100;
break;
default:
gk_errexit(SIGERR, "Unknown optype of %d\n", optype);
}
ctrl->numflag = GETOPTION(options, METIS_OPTION_NUMBERING, 0);
ctrl->optype = optype;
ctrl->ncon = ncon;
ctrl->nparts = nparts;
ctrl->maxvwgt = ismalloc(ncon, 0, "SetupCtrl: maxvwgt");
/* setup the target partition weights */
if (ctrl->optype != METIS_OP_OMETIS) {
ctrl->tpwgts = rmalloc(nparts*ncon, "SetupCtrl: ctrl->tpwgts");
if (tpwgts) {
rcopy(nparts*ncon, tpwgts, ctrl->tpwgts);
}
else {
for (i=0; i<nparts; i++) {
for (j=0; j<ncon; j++)
ctrl->tpwgts[i*ncon+j] = 1.0/nparts;
}
}
}
else { /* METIS_OP_OMETIS */
/* this is required to allow the pijbm to be defined properly for
the edge-based refinement during initial partitioning */
ctrl->tpwgts = rsmalloc(2, .5, "SetupCtrl: ctrl->tpwgts");
}
/* setup the ubfactors */
ctrl->ubfactors = rsmalloc(ctrl->ncon, I2RUBFACTOR(ctrl->ufactor), "SetupCtrl: ubfactors");
if (ubvec)
rcopy(ctrl->ncon, ubvec, ctrl->ubfactors);
for (i=0; i<ctrl->ncon; i++)
ctrl->ubfactors[i] += 0.0000499;
/* Allocate memory for balance multipliers.
Note that for PMETIS/OMETIS routines the memory allocated is more
than required as balance multipliers for 2 parts is sufficient. */
ctrl->pijbm = rmalloc(nparts*ncon, "SetupCtrl: ctrl->pijbm");
InitRandom(ctrl->seed);
IFSET(ctrl->dbglvl, METIS_DBG_INFO, PrintCtrl(ctrl));
if (!CheckParams(ctrl)) {
FreeCtrl(&ctrl);
return NULL;
}
else {
return ctrl;
}
}
void Adaptive_Partition(ctrl_t *ctrl, graph_t *graph)
{
idx_t i;
idx_t tewgt, tvsize;
real_t gtewgt, gtvsize;
real_t ubavg, lbavg, *lbvec;
WCOREPUSH;
lbvec = rwspacemalloc(ctrl, graph->ncon);
/************************************/
/* Set up important data structures */
/************************************/
CommSetup(ctrl, graph);
ubavg = ravg(graph->ncon, ctrl->ubvec);
tewgt = isum(graph->nedges, graph->adjwgt, 1);
tvsize = isum(graph->nvtxs, graph->vsize, 1);
gtewgt = (real_t) GlobalSESum(ctrl, tewgt) + 1.0/graph->gnvtxs; /* The +1/graph->gnvtxs were added to remove any FPE */
gtvsize = (real_t) GlobalSESum(ctrl, tvsize) + 1.0/graph->gnvtxs;
ctrl->redist_factor = ctrl->redist_base * ((gtewgt/gtvsize)/ ctrl->edge_size_ratio);
IFSET(ctrl->dbglvl, DBG_PROGRESS, rprintf(ctrl, "[%6"PRIDX" %8"PRIDX" %5"PRIDX" %5"PRIDX"][%"PRIDX"]\n",
graph->gnvtxs, GlobalSESum(ctrl, graph->nedges), GlobalSEMin(ctrl, graph->nvtxs), GlobalSEMax(ctrl, graph->nvtxs), ctrl->CoarsenTo));
if (graph->gnvtxs < 1.3*ctrl->CoarsenTo ||
(graph->finer != NULL && graph->gnvtxs > graph->finer->gnvtxs*COARSEN_FRACTION)) {
AllocateRefinementWorkSpace(ctrl, 2*graph->nedges);
/***********************************************/
/* Balance the partition on the coarsest graph */
/***********************************************/
graph->where = ismalloc(graph->nvtxs+graph->nrecv, -1, "graph->where");
icopy(graph->nvtxs, graph->home, graph->where);
ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
lbavg = ravg(graph->ncon, lbvec);
if (lbavg > ubavg + 0.035 && ctrl->partType != REFINE_PARTITION)
Balance_Partition(ctrl, graph);
if (ctrl->dbglvl&DBG_PROGRESS) {
ComputePartitionParams(ctrl, graph);
ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10"PRIDX", cut: %8"PRIDX", balance: ",
graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3"PRREAL" ", lbvec[i]);
rprintf(ctrl, "\n");
/* free memory allocated by ComputePartitionParams */
gk_free((void **)&graph->ckrinfo, &graph->lnpwgts, &graph->gnpwgts, LTERM);
}
/* check if no coarsening took place */
if (graph->finer == NULL) {
ComputePartitionParams(ctrl, graph);
KWayBalance(ctrl, graph, graph->ncon);
KWayAdaptiveRefine(ctrl, graph, NGR_PASSES);
}
}
else {
/*******************************/
/* Coarsen it and partition it */
/*******************************/
switch (ctrl->ps_relation) {
case PARMETIS_PSR_COUPLED:
Match_Local(ctrl, graph);
break;
case PARMETIS_PSR_UNCOUPLED:
default:
Match_Global(ctrl, graph);
break;
}
Adaptive_Partition(ctrl, graph->coarser);
/********************************/
/* project partition and refine */
/********************************/
ProjectPartition(ctrl, graph);
ComputePartitionParams(ctrl, graph);
if (graph->ncon > 1 && graph->level < 4) {
ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
lbavg = ravg(graph->ncon, lbvec);
if (lbavg > ubavg + 0.025) {
KWayBalance(ctrl, graph, graph->ncon);
}
}
KWayAdaptiveRefine(ctrl, graph, NGR_PASSES);
if (ctrl->dbglvl&DBG_PROGRESS) {
ComputeParallelBalance(ctrl, graph, graph->where, lbvec);
rprintf(ctrl, "nvtxs: %10"PRIDX", cut: %8"PRIDX", balance: ",
graph->gnvtxs, graph->mincut);
for (i=0; i<graph->ncon; i++)
rprintf(ctrl, "%.3"PRREAL" ", lbvec[i]);
rprintf(ctrl, "\n");
}
}
WCOREPOP;
}
/*************************************************************************
* Let the game begin
**************************************************************************/
int main(int argc, char *argv[])
{
idx_t i, j, npes, mype, optype, nparts, adptf, options[10];
idx_t *part=NULL, *sizes=NULL;
graph_t graph;
real_t ipc2redist, *xyz=NULL, *tpwgts=NULL, ubvec[MAXNCON];
MPI_Comm comm;
idx_t numflag=0, wgtflag=0, ndims, edgecut;
char xyzfilename[8192];
MPI_Init(&argc, &argv);
MPI_Comm_dup(MPI_COMM_WORLD, &comm);
gkMPI_Comm_size(comm, &npes);
gkMPI_Comm_rank(comm, &mype);
if (argc != 8) {
if (mype == 0)
printf("Usage: %s <graph-file> <op-type> <nparts> <adapth-factor> <ipc2redist> <dbglvl> <seed>\n", argv[0]);
MPI_Finalize();
exit(0);
}
optype = atoi(argv[2]);
nparts = atoi(argv[3]);
adptf = atoi(argv[4]);
ipc2redist = atof(argv[5]);
options[0] = 1;
options[PMV3_OPTION_DBGLVL] = atoi(argv[6]);
options[PMV3_OPTION_SEED] = atoi(argv[7]);
if (mype == 0)
printf("reading file: %s\n", argv[1]);
ParallelReadGraph(&graph, argv[1], comm);
/* Remove the edges for testing */
/*iset(graph.vtxdist[mype+1]-graph.vtxdist[mype]+1, 0, graph.xadj); */
rset(graph.ncon, 1.05, ubvec);
tpwgts = rmalloc(nparts*graph.ncon, "tpwgts");
rset(nparts*graph.ncon, 1.0/(real_t)nparts, tpwgts);
/*
ChangeToFortranNumbering(graph.vtxdist, graph.xadj, graph.adjncy, mype, npes);
numflag = 1;
nvtxs = graph.vtxdist[mype+1]-graph.vtxdist[mype];
nedges = graph.xadj[nvtxs];
printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1));
*/
if (optype >= 20) {
sprintf(xyzfilename, "%s.xyz", argv[1]);
xyz = ReadTestCoordinates(&graph, xyzfilename, &ndims, comm);
}
if (mype == 0)
printf("finished reading file: %s\n", argv[1]);
part = ismalloc(graph.nvtxs, mype%nparts, "main: part");
sizes = imalloc(2*npes, "main: sizes");
switch (optype) {
case 1:
wgtflag = 3;
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD);
break;
case 2:
wgtflag = 3;
options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED;
ParMETIS_V3_RefineKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
options, &edgecut, part, &comm);
WritePVector(argv[1], graph.vtxdist, part, MPI_COMM_WORLD);
break;
case 3:
options[PMV3_OPTION_PSR] = PARMETIS_PSR_COUPLED;
graph.vwgt = ismalloc(graph.nvtxs, 1, "main: vwgt");
if (npes > 1) {
AdaptGraph(&graph, adptf, comm);
}
else {
wgtflag = 3;
ParMETIS_V3_PartKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts,
ubvec, options, &edgecut, part, &comm);
printf("Initial partitioning with edgecut of %"PRIDX"\n", edgecut);
for (i=0; i<graph.ncon; i++) {
for (j=0; j<graph.nvtxs; j++) {
if (part[j] == i)
graph.vwgt[j*graph.ncon+i] = adptf;
else
graph.vwgt[j*graph.ncon+i] = 1;
}
}
}
wgtflag = 3;
ParMETIS_V3_AdaptiveRepart(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
NULL, graph.adjwgt, &wgtflag, &numflag, &graph.ncon, &nparts, tpwgts, ubvec,
&ipc2redist, options, &edgecut, part, &comm);
break;
case 4:
ParMETIS_V3_NodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options,
part, sizes, &comm);
/* WriteOVector(argv[1], graph.vtxdist, part, comm); */
break;
case 5:
ParMETIS_SerialNodeND(graph.vtxdist, graph.xadj, graph.adjncy, &numflag, options,
part, sizes, &comm);
/* WriteOVector(argv[1], graph.vtxdist, part, comm); */
printf("%"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX" %"PRIDX"\n", sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], sizes[5], sizes[6]);
break;
case 11:
/* TestAdaptiveMETIS(graph.vtxdist, graph.xadj, graph.adjncy, part, options, adptf, comm); */
break;
case 20:
wgtflag = 3;
ParMETIS_V3_PartGeomKway(graph.vtxdist, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &ndims, xyz, &graph.ncon, &nparts,
tpwgts, ubvec, options, &edgecut, part, &comm);
break;
case 21:
ParMETIS_V3_PartGeom(graph.vtxdist, &ndims, xyz, part, &comm);
break;
}
/* printf("%"PRIDX" %"PRIDX"\n", isum(nvtxs, graph.xadj, 1), isum(nedges, graph.adjncy, 1)); */
gk_free((void **)&part, &sizes, &tpwgts, &graph.vtxdist, &graph.xadj, &graph.adjncy,
&graph.vwgt, &graph.adjwgt, &xyz, LTERM);
MPI_Comm_free(&comm);
MPI_Finalize();
return 0;
}
void CreateGraphNodal(idx_t ne, idx_t nn, idx_t *eptr, idx_t *eind,
idx_t **r_xadj, idx_t **r_adjncy)
{
idx_t i, j, nnbrs;
idx_t *nptr, *nind;
idx_t *xadj, *adjncy;
idx_t *marker, *nbrs;
/* construct the node-element list first */
nptr = ismalloc(nn+1, 0, "CreateGraphNodal: nptr");
nind = imalloc(eptr[ne], "CreateGraphNodal: nind");
for (i=0; i<ne; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++)
nptr[eind[j]]++;
}
MAKECSR(i, nn, nptr);
for (i=0; i<ne; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++)
nind[nptr[eind[j]]++] = i;
}
SHIFTCSR(i, nn, nptr);
/* Allocate memory for xadj, since you know its size.
These are done using standard malloc as they are returned
to the calling function */
if ((xadj = (idx_t *)malloc((nn+1)*sizeof(idx_t))) == NULL)
gk_errexit(SIGMEM, "***Failed to allocate memory for xadj.\n");
*r_xadj = xadj;
iset(nn+1, 0, xadj);
/* allocate memory for working arrays used by FindCommonElements */
marker = ismalloc(nn, 0, "CreateGraphNodal: marker");
nbrs = imalloc(nn, "CreateGraphNodal: nbrs");
for (i=0; i<nn; i++) {
xadj[i] = FindCommonNodes(i, nptr[i+1]-nptr[i], nind+nptr[i], eptr,
eind, marker, nbrs);
}
MAKECSR(i, nn, xadj);
/* Allocate memory for adjncy, since you now know its size.
These are done using standard malloc as they are returned
to the calling function */
if ((adjncy = (idx_t *)malloc(xadj[nn]*sizeof(idx_t))) == NULL) {
free(xadj);
*r_xadj = NULL;
gk_errexit(SIGMEM, "***Failed to allocate memory for adjncy.\n");
}
*r_adjncy = adjncy;
for (i=0; i<nn; i++) {
nnbrs = FindCommonNodes(i, nptr[i+1]-nptr[i], nind+nptr[i], eptr,
eind, marker, nbrs);
for (j=0; j<nnbrs; j++)
adjncy[xadj[i]++] = nbrs[j];
}
SHIFTCSR(i, nn, xadj);
gk_free((void **)&nptr, &nind, &marker, &nbrs, LTERM);
}
/*************************************************************************
* This function partitions a finite element mesh by partitioning its dual
* graph using KMETIS and then assigning nodes in a load balanced fashion.
**************************************************************************/
int METIS_PartMeshDual(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind,
idx_t *vwgt, idx_t *vsize, idx_t *ncommon, idx_t *nparts,
real_t *tpwgts, idx_t *options, idx_t *objval, idx_t *epart,
idx_t *npart)
{
int sigrval=0, renumber=0, ptype;
idx_t i, j;
idx_t *xadj=NULL, *adjncy=NULL, *nptr=NULL, *nind=NULL;
idx_t ncon=1, pnumflag=0;
int rstatus = METIS_OK;
/* set up malloc cleaning code and signal catchers */
if (!gk_malloc_init())
return METIS_ERROR_MEMORY;
gk_sigtrap();
if ((sigrval = gk_sigcatch()) != 0)
goto SIGTHROW;
renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0);
ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY);
/* renumber the mesh */
if (renumber) {
ChangeMesh2CNumbering(*ne, eptr, eind);
options[METIS_OPTION_NUMBERING] = 0;
}
/* get the dual graph */
rstatus = METIS_MeshToDual(ne, nn, eptr, eind, ncommon, &pnumflag, &xadj, &adjncy);
if (rstatus != METIS_OK)
raise(SIGERR);
/* partition the graph */
if (ptype == METIS_PTYPE_KWAY)
rstatus = METIS_PartGraphKway(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, epart);
else
rstatus = METIS_PartGraphRecursive(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, epart);
if (rstatus != METIS_OK)
raise(SIGERR);
/* construct the node-element list */
nptr = ismalloc(*nn+1, 0, "METIS_PartMeshDual: nptr");
nind = imalloc(eptr[*ne], "METIS_PartMeshDual: nind");
for (i=0; i<*ne; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++)
nptr[eind[j]]++;
}
MAKECSR(i, *nn, nptr);
for (i=0; i<*ne; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++)
nind[nptr[eind[j]]++] = i;
}
SHIFTCSR(i, *nn, nptr);
/* partition the other side of the mesh */
InduceRowPartFromColumnPart(*nn, nptr, nind, npart, epart, *nparts, tpwgts);
gk_free((void **)&nptr, &nind, LTERM);
SIGTHROW:
if (renumber) {
ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart);
options[METIS_OPTION_NUMBERING] = 1;
}
METIS_Free(xadj);
METIS_Free(adjncy);
gk_siguntrap();
gk_malloc_cleanup(0);
return metis_rcode(sigrval);
}
graph_t *FixGraph(graph_t *graph)
{
idx_t i, j, k, l, nvtxs, nedges;
idx_t *xadj, *adjncy, *adjwgt;
idx_t *nxadj, *nadjncy, *nadjwgt;
graph_t *ngraph;
uvw_t *edges;
nvtxs = graph->nvtxs;
xadj = graph->xadj;
adjncy = graph->adjncy;
adjwgt = graph->adjwgt;
ASSERT(adjwgt != NULL);
ngraph = CreateGraph();
ngraph->nvtxs = nvtxs;
/* deal with vertex weights/sizes */
ngraph->ncon = graph->ncon;
ngraph->vwgt = icopy(nvtxs*graph->ncon, graph->vwgt,
imalloc(nvtxs*graph->ncon, "FixGraph: vwgt"));
ngraph->vsize = ismalloc(nvtxs, 1, "FixGraph: vsize");
if (graph->vsize)
icopy(nvtxs, graph->vsize, ngraph->vsize);
/* fix graph by sorting the "superset" of edges */
edges = (uvw_t *)gk_malloc(sizeof(uvw_t)*2*xadj[nvtxs], "FixGraph: edges");
for (nedges=0, i=0; i<nvtxs; i++) {
for (j=xadj[i]; j<xadj[i+1]; j++) {
/* keep only the upper-trianglular part of the adjacency matrix */
if (i < adjncy[j]) {
edges[nedges].u = i;
edges[nedges].v = adjncy[j];
edges[nedges].w = adjwgt[j];
nedges++;
}
else if (i > adjncy[j]) {
edges[nedges].u = adjncy[j];
edges[nedges].v = i;
edges[nedges].w = adjwgt[j];
nedges++;
}
}
}
uvwsorti(nedges, edges);
/* keep the unique subset */
for (k=0, i=1; i<nedges; i++) {
if (edges[k].v != edges[i].v || edges[k].u != edges[i].u) {
edges[++k] = edges[i];
}
}
nedges = k+1;
/* allocate memory for the fixed graph */
nxadj = ngraph->xadj = ismalloc(nvtxs+1, 0, "FixGraph: nxadj");
nadjncy = ngraph->adjncy = imalloc(2*nedges, "FixGraph: nadjncy");
nadjwgt = ngraph->adjwgt = imalloc(2*nedges, "FixGraph: nadjwgt");
/* create the adjacency list of the fixed graph from the upper-triangular
part of the adjacency matrix */
for (k=0; k<nedges; k++) {
nxadj[edges[k].u]++;
nxadj[edges[k].v]++;
}
MAKECSR(i, nvtxs, nxadj);
for (k=0; k<nedges; k++) {
nadjncy[nxadj[edges[k].u]] = edges[k].v;
nadjncy[nxadj[edges[k].v]] = edges[k].u;
nadjwgt[nxadj[edges[k].u]] = edges[k].w;
nadjwgt[nxadj[edges[k].v]] = edges[k].w;
nxadj[edges[k].u]++;
nxadj[edges[k].v]++;
}
SHIFTCSR(i, nvtxs, nxadj);
gk_free((void **)&edges, LTERM);
return ngraph;
}
