int SCOTCH_graphMapCompute ( SCOTCH_Graph * const grafptr, /*+ Graph to order +*/ SCOTCH_Mapping * const mappptr, /*+ Mapping to compute +*/ SCOTCH_Strat * const stratptr) /*+ Mapping strategy +*/ { Kgraph mapgrafdat; /* Effective mapping graph */ const Strat * mapstratptr; /* Pointer to mapping strategy */ LibMapping * restrict lmapptr; int o; #ifdef SCOTCH_DEBUG_GRAPH2 if (graphCheck ((Graph *) grafptr) != 0) { errorPrint ("SCOTCH_graphMapCompute: invalid input graph"); return (1); } #endif /* SCOTCH_DEBUG_GRAPH2 */ lmapptr = (LibMapping *) mappptr; if (*((Strat **) stratptr) == NULL) { /* Set default mapping strategy if necessary */ ArchDom archdomnorg; archDomFrst (&lmapptr->m.archdat, &archdomnorg); if (archVar (&lmapptr->m.archdat)) SCOTCH_stratGraphClusterBuild (stratptr, 0, 1, 0.0, 0.05); else SCOTCH_stratGraphMapBuild (stratptr, 0, archDomSize (&lmapptr->m.archdat, &archdomnorg), 0.05); } mapstratptr = *((Strat **) stratptr); if (mapstratptr->tabl != &kgraphmapststratab) { errorPrint ("SCOTCH_graphMapCompute: not a graph mapping strategy"); return (1); } if (kgraphInit (&mapgrafdat, (Graph *) grafptr, &lmapptr->m) != 0) return (1); o = kgraphMapSt (&mapgrafdat, mapstratptr); /* Perform mapping */ lmapptr->m.domnmax = mapgrafdat.m.domnmax; /* Do not free the mapping, as it has been cloned */ lmapptr->m.domnnbr = mapgrafdat.m.domnnbr; lmapptr->m.domntab = mapgrafdat.m.domntab; /* Update pointer to domntab in case it has changed */ mapgrafdat.m.parttax = NULL; /* Prevent mapping arrays from being freed by graph */ mapgrafdat.m.domntab = NULL; kgraphExit (&mapgrafdat); if (lmapptr->parttax != NULL) { /* Propagate mapping data to user partition array */ Gnum vertnum; Gnum vertnnd; for (vertnum = lmapptr->m.baseval, vertnnd = vertnum + lmapptr->m.vertnbr; vertnum < vertnnd; vertnum ++) lmapptr->parttax[vertnum] = archDomNum (&lmapptr->m.archdat, &lmapptr->m.domntab[lmapptr->m.parttax[vertnum]]); } return (o); }
int SCOTCH_graphMapView ( const SCOTCH_Graph * const libgrafptr, const SCOTCH_Mapping * const libmappptr, FILE * const stream) { const Graph * restrict grafptr; const Mapping * restrict mappptr; Anum * restrict parttax; /* Part array */ MappingSort * restrict domntab; /* Pointer to domain sort array */ ArchDom domnfrst; /* Largest domain in architecture */ Anum tgtnbr; /* Number of processors in target topology */ Anum mapnbr; /* Number of processors effectively used */ Anum mapnum; double mapavg; /* Average mapping weight */ Gnum mapmin; Gnum mapmax; Gnum mapsum; /* (Partial) sum of vertex loads */ double mapdlt; double mapmmy; /* Maximum / average ratio */ Anum * restrict nghbtab; /* Table storing neighbors of current subdomain */ Anum nghbnbr; Anum nghbmin; Anum nghbmax; Anum nghbsum; Gnum vertnum; Gnum veloval; Gnum edloval; Gnum commdist[256]; /* Array of load distribution */ Gnum commload; /* Total edge load (edge sum) */ Gnum commdilat; /* Total edge dilation */ Gnum commexpan; /* Total edge expansion */ Anum distmax; Anum distval; Gnum diammin; Gnum diammax; Gnum diamsum; const Gnum * restrict const verttax = ((Graph *) libgrafptr)->verttax; const Gnum * restrict const vendtax = ((Graph *) libgrafptr)->vendtax; const Gnum * restrict const velotax = ((Graph *) libgrafptr)->velotax; const Gnum * restrict const edgetax = ((Graph *) libgrafptr)->edgetax; const Gnum * restrict const edlotax = ((Graph *) libgrafptr)->edlotax; grafptr = (Graph *) libgrafptr; mappptr = &((LibMapping *) libmappptr)->m; if ((grafptr->vertnbr == 0) || /* Return if nothing to do */ (grafptr->edgenbr == 0)) return (0); if (memAllocGroup ((void **) (void *) &domntab, (size_t) ((grafptr->vertnbr + 1) * sizeof (MappingSort)), &parttax, (size_t) (grafptr->vertnbr * sizeof (Anum)), &nghbtab, (size_t) ((grafptr->vertnbr + 2) * sizeof (Anum)), NULL) == NULL) { errorPrint ("SCOTCH_graphMapView: out of memory"); return (1); } memSet (parttax, ~0, grafptr->vertnbr * sizeof (Anum)); for (vertnum = 0; vertnum < grafptr->vertnbr; vertnum ++) { parttax[vertnum] = domntab[vertnum].labl = archDomNum (&mappptr->archdat, mapDomain (mappptr, vertnum + grafptr->baseval)); domntab[vertnum].peri = vertnum + grafptr->baseval; /* Build inverse permutation */ } domntab[grafptr->vertnbr].labl = ARCHDOMNOTTERM; /* TRICK: avoid testing (i+1) */ domntab[grafptr->vertnbr].peri = ~0; /* Prevent Valgrind from yelling */ parttax -= grafptr->baseval; /* From now on, base part array */ intSort2asc2 (domntab, grafptr->vertnbr); /* Sort domain label array by increasing target labels */ archDomFrst (&mappptr->archdat, &domnfrst); /* Get architecture domain */ tgtnbr = archDomSize (&mappptr->archdat, &domnfrst); /* Get architecture size */ mapsum = 0; mapnbr = 0; veloval = 1; /* Assume unweighted vertices */ for (vertnum = 0; domntab[vertnum].labl != ARCHDOMNOTTERM; vertnum ++) { parttax[domntab[vertnum].peri] = mapnbr; /* Build map of partition parts starting from 0 */ if (domntab[vertnum].labl != domntab[vertnum + 1].labl) /* TRICK: if new (or end) domain label */ mapnbr ++; if (velotax != NULL) veloval = velotax[domntab[vertnum].peri]; mapsum += veloval; } mapavg = (mapnbr == 0) ? 0.0L : (double) mapsum / (double) mapnbr; mapsum = 0; mapmin = GNUMMAX; mapmax = 0; mapdlt = 0.0L; for (vertnum = 0; domntab[vertnum].labl != ARCHDOMNOTTERM; vertnum ++) { if (velotax != NULL) veloval = velotax[domntab[vertnum].peri]; mapsum += veloval; if (domntab[vertnum].labl != domntab[vertnum + 1].labl) { /* TRICK: if new (or end) domain label */ if (mapsum < mapmin) mapmin = mapsum; if (mapsum > mapmax) mapmax = mapsum; mapdlt += fabs ((double) mapsum - mapavg); mapsum = 0; /* Reset domain load sum */ } } mapdlt = (mapnbr != 0) ? mapdlt / ((double) mapnbr * mapavg) : 0.0L; mapmmy = (mapnbr != 0) ? (double) mapmax / (double) mapavg : 0.0L; if (mapnbr > tgtnbr) { /* If more subdomains than architecture size */ #ifdef SCOTCH_DEBUG_MAP2 if (! archVar (&mappptr->archdat)) { /* If not a variable-sized architecture */ errorPrint ("SCOTCH_graphMapView: invalid mapping"); memFree (domntab); /* Free group leader */ return (1); } #endif /* SCOTCH_DEBUG_MAP2 */ tgtnbr = mapnbr; /* Assume it is a variable-sized architecture */ } fprintf (stream, "M\tProcessors " GNUMSTRING "/" GNUMSTRING " (%g)\n", (Gnum) mapnbr, (Gnum) tgtnbr, (double) mapnbr / (double) tgtnbr); fprintf (stream, "M\tTarget min=" GNUMSTRING "\tmax=" GNUMSTRING "\tavg=%g\tdlt=%g\tmaxavg=%g\n", (Gnum) mapmin, (Gnum) mapmax, mapavg, mapdlt, mapmmy); nghbnbr = 0; nghbmin = ANUMMAX; nghbmax = 0; nghbsum = 0; nghbnbr = 0; nghbtab[0] = -2; for (vertnum = 0; domntab[vertnum].labl != ARCHDOMNOTTERM; vertnum ++) { Gnum edgenum; Gnum edgennd; Anum partnum; partnum = parttax[domntab[vertnum].peri]; for (edgenum = verttax[domntab[vertnum].peri], edgennd = vendtax[domntab[vertnum].peri]; edgenum < edgennd; edgenum ++) { Anum partend; partend = parttax[edgetax[edgenum]]; if ((partend != partnum) && /* If edge is not internal */ (partend != nghbtab[nghbnbr])) { /* And neighbor is not sole neighbor or has not just been found */ Anum partmin; Anum partmax; partmin = 0; partmax = nghbnbr; while ((partmax - partmin) > 1) { Anum partmed; partmed = (partmax + partmin) >> 1; if (nghbtab[partmed] > partend) partmax = partmed; else partmin = partmed; } if (nghbtab[partmin] == partend) /* If neighboring part found, skip to next neighbor */ continue; #ifdef SCOTCH_DEBUG_MAP2 if (nghbnbr >= (grafptr->vertnbr + 1)) { errorPrint ("SCOTCH_graphMapView: internal error"); return (1); } #endif /* SCOTCH_DEBUG_MAP2 */ nghbnbr ++; for (partmax = nghbnbr; partmax > (partmin + 1); partmax --) nghbtab[partmax] = nghbtab[partmax - 1]; nghbtab[partmin + 1] = partend; /* Add new neighbor part in the right place */ } } if (domntab[vertnum].labl != domntab[vertnum + 1].labl) { /* TRICK: if new (or end) domain label */ if (nghbnbr < nghbmin) nghbmin = nghbnbr; if (nghbnbr > nghbmax) nghbmax = nghbnbr; nghbsum += nghbnbr; nghbnbr = 0; } }
int SCOTCH_dgraphMapView ( SCOTCH_Dgraph * const libgrafptr, const SCOTCH_Dmapping * const libmappptr, FILE * const stream) { Dgraph * restrict grafptr; const LibDmapping * restrict mappptr; ArchDom domnfrst; /* Largest domain in architecture */ unsigned int * restrict nmskloctab; /* Local neighbor bitfield */ unsigned int * restrict nmskglbtab; /* Local neighbor bitfield */ int nmskidxnbr; /* Size of bitfield; int since sent by MPI */ Gnum * restrict tgloloctab; /* Local array of terminal domain loads */ Gnum * restrict tgloglbtab; /* Global array of terminal domain loads */ Gnum * restrict termgsttax; /* Terminal domain ghost mapping array */ Anum tgtnbr; /* Number of processors in target topology */ Anum tgtnum; Anum mapnbr; /* Number of processors effectively used */ double mapavg; /* Average mapping weight */ Gnum mapmin; Gnum mapmax; Gnum mapsum; /* (Partial) sum of vertex loads */ double mapdlt; double mapmmy; /* Maximum / average ratio */ Anum ngbsum; Anum ngbmin; Anum ngbmax; Gnum vertlocnum; Gnum veloval; Gnum edloval; Gnum commlocdist[256 + 3]; /* Array of local load distribution */ Gnum commglbdist[256 + 3]; Gnum commlocload; /* Total local edge load (edge sum) */ Gnum commlocdilat; /* Total edge dilation */ Gnum commlocexpan; /* Total edge expansion */ Anum distmax; Anum distval; int cheklocval; int chekglbval; DgraphHaloRequest requdat; grafptr = (Dgraph *) libgrafptr; mappptr = (LibDmapping *) libmappptr; if ((grafptr->vertglbnbr == 0) || /* Return if nothing to do */ (grafptr->edgeglbnbr == 0)) return (0); archDomFrst (&mappptr->m.archdat, &domnfrst); /* Get architecture domain */ tgtnbr = archDomSize (&mappptr->m.archdat, &domnfrst); /* Get architecture size */ if (archVar (&mappptr->m.archdat)) { errorPrint ("SCOTCH_dgraphMapView: not implemented"); return (1); } if (dgraphGhst (grafptr) != 0) { /* Compute ghost edge array if not already present */ errorPrint ("SCOTCH_dgraphMapView: cannot compute ghost edge array"); return (1); } nmskidxnbr = (tgtnbr + 1 + ((sizeof (int) << 3) - 1)) / (sizeof (int) << 3); /* Size of neighbor subdomain bitfield; TRICK: "+1" to have a "-1" cell for unmapped vertices */ cheklocval = 0; if (memAllocGroup ((void **) (void *) &nmskloctab, (size_t) (nmskidxnbr * sizeof (unsigned int)), &nmskglbtab, (size_t) (nmskidxnbr * sizeof (unsigned int)), &tgloloctab, (size_t) ((tgtnbr + 1) * sizeof (Gnum)), /* TRICK: "+1" to have a "-1" cell for unmapped vertices */ &tgloglbtab, (size_t) (tgtnbr * sizeof (Gnum)), &termgsttax, (size_t) (grafptr->vertgstnbr * sizeof (Gnum)), NULL) == NULL) { cheklocval = 1; } if (MPI_Allreduce (&cheklocval, &chekglbval, 1, MPI_INT, MPI_MAX, grafptr->proccomm) != MPI_SUCCESS) { errorPrint ("SCOTCH_dgraphMapView: communication error (1)"); return (1); } if (chekglbval != 0) { if (nmskloctab != NULL) memFree (nmskloctab); errorPrint ("SCOTCH_dgraphMapView: out of memory"); return (1); } if (dmapTerm (&mappptr->m, grafptr, termgsttax) != 0) { errorPrint ("SCOTCH_dgraphMapView: cannot build local terminal array"); memFree (nmskloctab); return (1); } dgraphHaloAsync (grafptr, termgsttax, GNUM_MPI, &requdat); termgsttax -= grafptr->baseval; memSet (tgloloctab, 0, (tgtnbr + 1) * sizeof (Gnum)); tgloloctab ++; /* TRICK: trim array for "-1" cell */ veloval = 1; for (vertlocnum = grafptr->baseval; vertlocnum < grafptr->vertlocnnd; vertlocnum ++) { #ifdef SCOTCH_DEBUG_DMAP2 if ((termgsttax[vertlocnum] < -1) || (termgsttax[vertlocnum] >= tgtnbr)) { errorPrint ("SCOTCH_dgraphMapView: invalid local terminal array"); memFree (nmskloctab); /* Free group leader */ return (1); } #endif /* SCOTCH_DEBUG_DMAP2 */ if (grafptr->veloloctax != NULL) veloval = grafptr->veloloctax[vertlocnum]; tgloloctab[termgsttax[vertlocnum]] += veloval; /* One more vertex of given weight assigned to this target */ } if (MPI_Allreduce (tgloloctab, tgloglbtab, tgtnbr, GNUM_MPI, MPI_SUM, grafptr->proccomm) != MPI_SUCCESS) { errorPrint ("SCOTCH_dgraphMapView: communication error (2)"); memFree (nmskloctab); /* Free group leader */ return (1); } mapmin = GNUMMAX; mapmax = 0; mapsum = 0; mapnbr = 0; for (tgtnum = 0; tgtnum < tgtnbr; tgtnum ++) { Gnum tgtsum; tgtsum = tgloglbtab[tgtnum]; if (tgtsum != 0) { mapnbr ++; mapsum += tgtsum; if (tgtsum < mapmin) mapmin = tgtsum; if (tgtsum > mapmax) mapmax = tgtsum; } } mapavg = (mapnbr == 0) ? 0.0L : ((double) mapsum / (double) mapnbr); mapdlt = 0.0L; for (tgtnum = 0; tgtnum < tgtnbr; tgtnum ++) mapdlt += fabs ((double) tgloglbtab[tgtnum] - mapavg); mapdlt = (mapnbr != 0) ? mapdlt / ((double) mapnbr * mapavg) : 0.0L; mapmmy = (mapnbr != 0) ? (double) mapmax / (double) mapavg : 0.0L; if (stream != NULL) { fprintf (stream, "M\tProcessors " GNUMSTRING "/" GNUMSTRING "(%g)\n", (Gnum) mapnbr, (Gnum) tgtnbr, (double) mapnbr / (double) tgtnbr); fprintf (stream, "M\tTarget min=" GNUMSTRING "\tmax=" GNUMSTRING "\tavg=%g\tdlt=%g\tmaxavg=%g\n", (Gnum) mapmin, (Gnum) mapmax, mapavg, mapdlt, mapmmy); } if (dgraphHaloWait (&requdat) != 0) { /* Wait for ghost terminal data to be exchanged */ errorPrint ("SCOTCH_dgraphMapView: cannot complete asynchronous halo exchange"); memFree (nmskloctab); /* Free group leader */ return (1); } ngbmin = ANUMMAX; ngbmax = 0; ngbsum = 0; for (tgtnum = 0; tgtnum < tgtnbr; tgtnum ++) { /* For all subdomain indices */ int nmskidxnum; Gnum vertlocnum; Anum ngbnbr; if (tgloglbtab[tgtnum] <= 0) /* If empty subdomain, skip it */ continue; memSet (nmskloctab, 0, nmskidxnbr * sizeof (int)); /* Reset neighbor bit mask */ for (vertlocnum = grafptr->baseval; vertlocnum < grafptr->vertlocnnd; vertlocnum ++) { /* For all local vertices */ Gnum termnum; Gnum edgelocnum; Gnum edgelocnnd; termnum = termgsttax[vertlocnum]; if (termnum != tgtnum) /* If vertex does not belong to current part or is not mapped, skip it */ continue; for (edgelocnum = grafptr->vertloctax[vertlocnum], edgelocnnd = grafptr->vendloctax[vertlocnum]; edgelocnum < edgelocnnd; edgelocnum ++) { Gnum termend; termend = termgsttax[grafptr->edgegsttax[edgelocnum]]; if (termend != tgtnum) { /* If edge is not internal */ termend ++; /* TRICK: turn unmapped to 0 and so on */ nmskloctab[termend / (sizeof (int) << 3)] |= 1 << (termend & ((sizeof (int) << 3) - 1)); /* Flag neighbor in bit array */ } } } nmskloctab[0] &= ~1; /* Do not account for unmapped vertices (terminal domain 0 because of "+1") */ if (MPI_Allreduce (nmskloctab, nmskglbtab, nmskidxnbr, MPI_INT, MPI_BOR, grafptr->proccomm) != MPI_SUCCESS) { errorPrint ("SCOTCH_dgraphMapView: communication error (3)"); memFree (nmskloctab); /* Free group leader */ return (1); } for (nmskidxnum = 0, ngbnbr = 0; nmskidxnum < nmskidxnbr; nmskidxnum ++) { unsigned int nmskbitval; for (nmskbitval = nmskglbtab[nmskidxnum]; nmskbitval != 0; nmskbitval >>= 1) ngbnbr += nmskbitval & 1; } ngbsum += ngbnbr; if (ngbnbr < ngbmin) ngbmin = ngbnbr; if (ngbnbr > ngbmax) ngbmax = ngbnbr; } if (stream != NULL) { fprintf (stream, "M\tNeighbors min=" GNUMSTRING "\tmax=" GNUMSTRING "\tsum=" GNUMSTRING "\n", (Gnum) ngbmin, (Gnum) ngbmax, (Gnum) ngbsum); } memSet (commlocdist, 0, 256 * sizeof (Gnum)); /* Initialize the data */ commlocload = commlocdilat = commlocexpan = 0; edloval = 1; for (vertlocnum = grafptr->baseval; vertlocnum < grafptr->vertlocnnd; vertlocnum ++) { /* For all local vertices */ Gnum termlocnum; ArchDom termdomdat; Gnum edgelocnum; Gnum edgelocnnd; termlocnum = termgsttax[vertlocnum]; if (termlocnum == ~0) /* Skip unmapped vertices */ continue; archDomTerm (&mappptr->m.archdat, &termdomdat, termlocnum); for (edgelocnum = grafptr->vertloctax[vertlocnum], edgelocnnd = grafptr->vendloctax[vertlocnum]; edgelocnum < edgelocnnd; edgelocnum ++) { ArchDom termdomend; Gnum termgstend; Anum distval; termgstend = termgsttax[grafptr->edgegsttax[edgelocnum]]; if (termgstend == ~0) /* Skip unmapped end vertices */ continue; distval = 0; if (grafptr->edloloctax != NULL) /* Get edge weight if any */ edloval = grafptr->edloloctax[edgelocnum]; if (termgstend != termlocnum) { /* If not same domain, compute distance */ archDomTerm (&mappptr->m.archdat, &termdomend, termgstend); distval = archDomDist (&mappptr->m.archdat, &termdomdat, &termdomend); } commlocdist[(distval > 255) ? 255 : distval] += edloval; commlocload += edloval; commlocdilat += distval; commlocexpan += distval * edloval; } } commlocdist[256] = commlocload; commlocdist[256 + 1] = commlocdilat; commlocdist[256 + 2] = commlocexpan; if (MPI_Allreduce (commlocdist, commglbdist, 256 + 3, GNUM_MPI, MPI_SUM, grafptr->proccomm) != MPI_SUCCESS) { errorPrint ("SCOTCH_dgraphMapView: communication error (4)"); memFree (nmskloctab); /* Free group leader */ return (1); } if (stream != NULL) { Gnum commglbload; commglbload = commglbdist[256]; fprintf (stream, "M\tCommDilat=%f\t(" GNUMSTRING ")\n", /* Print expansion parameters */ (double) commglbdist[256 + 1] / grafptr->edgeglbnbr, (Gnum) (commglbdist[256 + 1] / 2)); fprintf (stream, "M\tCommExpan=%f\t(" GNUMSTRING ")\n", ((commglbload == 0) ? (double) 0.0L : (double) commglbdist[256 + 2] / (double) commglbload), (Gnum) (commglbdist[256 + 2] / 2)); fprintf (stream, "M\tCommCutSz=%f\t(" GNUMSTRING ")\n", ((commglbload == 0) ? (double) 0.0L : (double) (commglbload - commglbdist[0]) / (double) commglbload), (Gnum) ((commglbload - commglbdist[0]) / 2)); fprintf (stream, "M\tCommDelta=%f\n", (((double) commglbload * (double) commglbdist[256 + 1]) == 0.0L) ? (double) 0.0L : ((double) commglbdist[256 + 2] * (double) grafptr->edgeglbnbr) / ((double) commglbload * (double) commglbdist[256 + 2])); for (distmax = 255; distmax != -1; distmax --) /* Find longest distance */ if (commglbdist[distmax] != 0) break; for (distval = 0; distval <= distmax; distval ++) /* Print distance histogram */ fprintf (stream, "M\tCommLoad[" ANUMSTRING "]=%f\n", (Anum) distval, (double) commglbdist[distval] / (double) commglbload); } memFree (nmskloctab); /* Free group leader */ return (0); }
** It returns: ** - 0 : if the coarse graph has been built. ** - 1 : if threshold achieved or on error. */ static int kgraphMapMlCoarsen ( const Kgraph * restrict const finegrafptr, /*+ Finer graph +*/ Kgraph * restrict const coargrafptr, /*+ Coarser graph to build +*/ GraphCoarsenMulti * restrict * const coarmultptr, /*+ Pointer to multinode table to build +*/ const KgraphMapMlParam * const paraptr) /*+ Method parameters +*/ { Gnum vertmax; vertmax = (archVar (&finegrafptr->m.archdat) ? 1 : finegrafptr->m.domnnbr) * paraptr->coarnbr; if (graphCoarsen (&finegrafptr->s, &coargrafptr->s, coarmultptr, vertmax, paraptr->coarrat, paraptr->coartype) != 0) return (1); /* Return if coarsening failed */ coargrafptr->m.baseval = finegrafptr->m.baseval; coargrafptr->m.vertnbr = coargrafptr->s.vertnbr; coargrafptr->m.parttax = NULL; /* Do not allocate part array yet */ coargrafptr->m.domntab = finegrafptr->m.domntab; /* Re-use domain array */ coargrafptr->m.domnnbr = finegrafptr->m.domnnbr; coargrafptr->m.domnmax = finegrafptr->m.domnmax; coargrafptr->m.archdat = finegrafptr->m.archdat; coargrafptr->m.domnorg = finegrafptr->m.domnorg; coargrafptr->frontab = finegrafptr->frontab; /* Re-use frontier array if it exists */ coargrafptr->comploadavg = finegrafptr->comploadavg; /* Re-use existing load arrays */