void AlgPTScotch<Adapter>::partition(
const RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
SCOTCH_Num partnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(partnbr, numGlobalParts, env);
#ifdef HAVE_ZOLTAN2_MPI
int ierr = 0;
int me = problemComm->getRank();
const SCOTCH_Num baseval = 0; // Base value for array indexing.
// GraphModel returns GNOs from base 0.
SCOTCH_Strat stratstr; // Strategy string
// TODO: Set from parameters
SCOTCH_stratInit(&stratstr);
// Allocate and initialize PTScotch Graph data structure.
SCOTCH_Dgraph *gr = SCOTCH_dgraphAlloc(); // Scotch distributed graph
ierr = SCOTCH_dgraphInit(gr, mpicomm);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphInit",
!ierr, BASIC_ASSERTION, problemComm);
// Get vertex info
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
size_t nVtx = model->getVertexList(vtxID, xyz, vwgts);
SCOTCH_Num vertlocnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(vertlocnbr, nVtx, env);
SCOTCH_Num vertlocmax = vertlocnbr; // Assumes no holes in global nums.
// Get edge info
ArrayView<const gno_t> edgeIds;
ArrayView<const int> procIds;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
size_t nEdge = model->getEdgeList(edgeIds, procIds, offsets, ewgts);
SCOTCH_Num edgelocnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(edgelocnbr, nEdge, env);
const SCOTCH_Num edgelocsize = edgelocnbr; // Assumes adj array is compact.
SCOTCH_Num *vertloctab; // starting adj/vtx
TPL_Traits<SCOTCH_Num, lno_t>::ASSIGN_TPL_T_ARRAY(&vertloctab, offsets, env);
SCOTCH_Num *edgeloctab; // adjacencies
TPL_Traits<SCOTCH_Num, gno_t>::ASSIGN_TPL_T_ARRAY(&edgeloctab, edgeIds, env);
// We don't use these arrays, but we need them as arguments to Scotch.
SCOTCH_Num *vendloctab = NULL; // Assume consecutive storage for adj
SCOTCH_Num *vlblloctab = NULL; // Vertex label array
SCOTCH_Num *edgegsttab = NULL; // Array for ghost vertices
// Get weight info.
SCOTCH_Num *velotab = NULL; // Vertex weights
SCOTCH_Num *edlotab = NULL; // Edge weights
int nVwgts = model->getNumWeightsPerVertex();
int nEwgts = model->getNumWeightsPerEdge();
if (nVwgts > 1 && me == 0) {
std::cerr << "Warning: NumWeightsPerVertex is " << nVwgts
<< " but Scotch allows only one weight. "
<< " Zoltan2 will use only the first weight per vertex."
<< std::endl;
}
if (nEwgts > 1 && me == 0) {
std::cerr << "Warning: NumWeightsPerEdge is " << nEwgts
<< " but Scotch allows only one weight. "
<< " Zoltan2 will use only the first weight per edge."
<< std::endl;
}
if (nVwgts) {
velotab = new SCOTCH_Num[nVtx+1]; // +1 since Scotch wants all procs
// to have non-NULL arrays
scale_weights(nVtx, vwgts[0], velotab);
}
if (nEwgts) {
edlotab = new SCOTCH_Num[nEdge+1]; // +1 since Scotch wants all procs
// to have non-NULL arrays
scale_weights(nEdge, ewgts[0], edlotab);
}
// Build PTScotch distributed data structure
ierr = SCOTCH_dgraphBuild(gr, baseval, vertlocnbr, vertlocmax,
vertloctab, vendloctab, velotab, vlblloctab,
edgelocnbr, edgelocsize,
edgeloctab, edgegsttab, edlotab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphBuild",
!ierr, BASIC_ASSERTION, problemComm);
// Create array for Scotch to return results in.
ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx,true);
SCOTCH_Num *partloctab = NULL;
if (nVtx && (sizeof(SCOTCH_Num) == sizeof(part_t))) {
// Can write directly into the solution's memory
partloctab = (SCOTCH_Num *) partList.getRawPtr();
}
else {
// Can't use solution memory directly; will have to copy later.
// Note: Scotch does not like NULL arrays, so add 1 to always have non-null.
// ParMETIS has this same "feature." See Zoltan bug 4299.
partloctab = new SCOTCH_Num[nVtx+1];
}
// Get target part sizes
float *partsizes = new float[numGlobalParts];
if (!solution->criteriaHasUniformPartSizes(0))
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = solution->getCriteriaPartSize(0, i);
else
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = 1.0 / float(numGlobalParts);
// Allocate and initialize PTScotch target architecture data structure
SCOTCH_Arch archdat;
SCOTCH_archInit(&archdat);
SCOTCH_Num velosum = 0;
SCOTCH_dgraphSize (gr, &velosum, NULL, NULL, NULL);
SCOTCH_Num *goalsizes = new SCOTCH_Num[partnbr];
// TODO: The goalsizes are set as in Zoltan; not sure it is correct there
// or here.
// It appears velosum is global NUMBER of vertices, not global total
// vertex weight. I think we should use the latter.
// Fix this when we add vertex weights.
for (SCOTCH_Num i = 0; i < partnbr; i++)
goalsizes[i] = SCOTCH_Num(ceil(velosum * partsizes[i]));
delete [] partsizes;
SCOTCH_archCmpltw(&archdat, partnbr, goalsizes);
// Call partitioning; result returned in partloctab.
ierr = SCOTCH_dgraphMap(gr, &archdat, &stratstr, partloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphMap",
!ierr, BASIC_ASSERTION, problemComm);
SCOTCH_archExit(&archdat);
delete [] goalsizes;
// TODO - metrics
#ifdef SHOW_ZOLTAN2_SCOTCH_MEMORY
int me = env->comm_->getRank();
#endif
#ifdef HAVE_SCOTCH_ZOLTAN2_GETMEMORYMAX
if (me == 0){
size_t scotchBytes = SCOTCH_getMemoryMax();
std::cout << "Rank " << me << ": Maximum bytes used by Scotch: ";
std::cout << scotchBytes << std::endl;
}
#endif
// Clean up PTScotch
SCOTCH_dgraphExit(gr);
free(gr);
SCOTCH_stratExit(&stratstr);
// Load answer into the solution.
if ((sizeof(SCOTCH_Num) != sizeof(part_t)) || (nVtx == 0)) {
for (size_t i = 0; i < nVtx; i++) partList[i] = partloctab[i];
delete [] partloctab;
}
solution->setParts(partList);
env->memory("Zoltan2-Scotch: After creating solution");
// Clean up copies made due to differing data sizes.
TPL_Traits<SCOTCH_Num, lno_t>::DELETE_TPL_T_ARRAY(&vertloctab);
TPL_Traits<SCOTCH_Num, gno_t>::DELETE_TPL_T_ARRAY(&edgeloctab);
if (nVwgts) delete [] velotab;
if (nEwgts) delete [] edlotab;
#else // DO NOT HAVE_MPI
// TODO: Handle serial case with calls to Scotch.
// TODO: For now, assign everything to rank 0 and assume only one part.
// TODO: Can probably use the code above for loading solution,
// TODO: instead of duplicating it here.
// TODO
// TODO: Actual logic should call Scotch when number of processes == 1.
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
size_t nVtx = model->getVertexList(vtxID, xyz, vwgts);
ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx, true);
for (size_t i = 0; i < nVtx; i++) partList[i] = 0;
solution->setParts(partList);
#endif // DO NOT HAVE_MPI
}
int
main (
int argc,
char * argv[])
{
SCOTCH_Dgraph grafdat;
int procglbnbr;
int proclocnum;
int protglbnum; /* Root process */
SCOTCH_Num vertnbr;
SCOTCH_Num velomin;
SCOTCH_Num velomax;
SCOTCH_Num velosum;
double veloavg;
double velodlt;
SCOTCH_Num degrmin;
SCOTCH_Num degrmax;
double degravg;
double degrdlt;
SCOTCH_Num edgenbr;
SCOTCH_Num edlomin;
SCOTCH_Num edlomax;
SCOTCH_Num edlosum;
double edloavg;
double edlodlt;
int flagval;
int i;
#ifdef SCOTCH_PTHREAD
int thrdlvlreqval;
int thrdlvlproval;
#endif /* SCOTCH_PTHREAD */
errorProg ("dgtst");
#ifdef SCOTCH_PTHREAD
thrdlvlreqval = MPI_THREAD_MULTIPLE;
if (MPI_Init_thread (&argc, &argv, thrdlvlreqval, &thrdlvlproval) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (1)");
if (thrdlvlreqval > thrdlvlproval)
errorPrint ("main: MPI implementation is not thread-safe: recompile without SCOTCH_PTHREAD");
#else /* SCOTCH_PTHREAD */
if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (2)");
#endif /* SCOTCH_PTHREAD */
MPI_Comm_size (MPI_COMM_WORLD, &procglbnbr); /* Get communicator data */
MPI_Comm_rank (MPI_COMM_WORLD, &proclocnum);
protglbnum = 0; /* Assume root process is process 0 */
if ((argc >= 2) && (argv[1][0] == '?')) { /* If need for help */
usagePrint (stdout, C_usageList);
return (0);
}
flagval = C_FLAGNONE;
for (i = 0; i < C_FILENBR; i ++) /* Set default stream pointers */
C_fileTab[i].pntr = (C_fileTab[i].mode[0] == 'r') ? stdin : stdout;
for (i = 1; i < argc; i ++) { /* Loop for all option codes */
if ((argv[i][0] != '+') && /* If found a file name */
((argv[i][0] != '-') || (argv[i][1] == '\0'))) {
if (C_fileNum < C_FILEARGNBR) /* A file name has been given */
C_fileTab[C_fileNum ++].name = argv[i];
else
errorPrint ("main: too many file names given");
}
else { /* If found an option name */
switch (argv[i][1]) {
#ifdef SCOTCH_DEBUG_ALL
case 'D' :
case 'd' :
flagval |= C_FLAGDEBUG;
break;
#endif /* SCOTCH_DEBUG_ALL */
case 'H' : /* Give the usage message */
case 'h' :
usagePrint (stdout, C_usageList);
return (0);
case 'R' : /* Root process (if necessary) */
case 'r' :
protglbnum = atoi (&argv[i][2]);
if ((protglbnum < 0) ||
(protglbnum >= procglbnbr) ||
((protglbnum == 0) && (argv[i][2] != '0')))
errorPrint ("main: invalid root process number");
break;
case 'V' :
case 'v' :
fprintf (stderr, "dgtst, version " SCOTCH_VERSION_STRING "\n");
fprintf (stderr, "Copyright 2007,2008,2010,2011 ENSEIRB, INRIA & CNRS, France\n");
fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
return (0);
default :
errorPrint ("main: unprocessed option '%s'", argv[i]);
}
}
}
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0) {
fprintf (stderr, "Proc %4d of %d, pid %d\n", proclocnum, procglbnbr, getpid ());
if (proclocnum == protglbnum) { /* Synchronize on keybord input */
char c;
printf ("Waiting for key press...\n");
scanf ("%c", &c);
}
MPI_Barrier (MPI_COMM_WORLD);
}
#endif /* SCOTCH_DEBUG_ALL */
fileBlockOpenDist (C_fileTab, C_FILENBR, procglbnbr, proclocnum, protglbnum); /* Open all files */
SCOTCH_dgraphInit (&grafdat, MPI_COMM_WORLD);
SCOTCH_dgraphLoad (&grafdat, C_filepntrsrcinp, -1, 0);
SCOTCH_dgraphCheck (&grafdat);
SCOTCH_dgraphSize (&grafdat, &vertnbr, NULL, &edgenbr, NULL);
SCOTCH_dgraphStat (&grafdat, &velomin, &velomax, &velosum, &veloavg, &velodlt,
°rmin, °rmax, °ravg, °rdlt,
&edlomin, &edlomax, &edlosum, &edloavg, &edlodlt);
if (C_filepntrdatout != NULL) {
fprintf (C_filepntrdatout, "S\tVertex\tnbr=" SCOTCH_NUMSTRING "\n",
(SCOTCH_Num) vertnbr);
fprintf (C_filepntrdatout, "S\tVertex load\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) velomin, (SCOTCH_Num) velomax, (SCOTCH_Num) velosum, veloavg, velodlt);
fprintf (C_filepntrdatout, "S\tVertex degree\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) degrmin, (SCOTCH_Num) degrmax, (SCOTCH_Num) edgenbr, degravg, degrdlt);
fprintf (C_filepntrdatout, "S\tEdge\tnbr=" SCOTCH_NUMSTRING "\n",
(SCOTCH_Num) (edgenbr / 2));
fprintf (C_filepntrdatout, "S\tEdge load\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) edlomin, (SCOTCH_Num) edlomax, (SCOTCH_Num) edlosum, edloavg, edlodlt);
}
fileBlockClose (C_fileTab, C_FILENBR); /* Always close explicitely to end eventual (un)compression tasks */
SCOTCH_dgraphExit (&grafdat);
MPI_Finalize ();
#ifdef COMMON_PTHREAD
pthread_exit ((void *) 0); /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
return (0);
}
