static PetscErrorCode MatPartitioningApply_PTScotch_Private(MatPartitioning part, PetscBool useND, IS *partitioning)
{
MPI_Comm pcomm,comm;
MatPartitioning_PTScotch *scotch = (MatPartitioning_PTScotch*)part->data;
PetscErrorCode ierr;
PetscMPIInt rank;
Mat mat = part->adj;
Mat_MPIAdj *adj = (Mat_MPIAdj*)mat->data;
PetscBool flg,distributed;
PetscBool proc_weight_flg;
PetscInt i,j,p,bs=1,nold;
PetscInt *NDorder = NULL;
PetscReal *vwgttab,deltval;
SCOTCH_Num *locals,*velotab,*veloloctab,*edloloctab,vertlocnbr,edgelocnbr,nparts=part->n;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)part,&pcomm);CHKERRQ(ierr);
/* Duplicate the communicator to be sure that PTSCOTCH attribute caching does not interfere with PETSc. */
ierr = MPI_Comm_dup(pcomm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr);
if (!flg) {
/* bs indicates if the converted matrix is "reduced" from the original and hence the
resulting partition results need to be stretched to match the original matrix */
nold = mat->rmap->n;
ierr = MatConvert(mat,MATMPIADJ,MAT_INITIAL_MATRIX,&mat);CHKERRQ(ierr);
if (mat->rmap->n > 0) bs = nold/mat->rmap->n;
adj = (Mat_MPIAdj*)mat->data;
}
proc_weight_flg = PETSC_TRUE;
ierr = PetscOptionsGetBool(NULL, NULL, "-mat_partitioning_ptscotch_proc_weight", &proc_weight_flg, NULL);CHKERRQ(ierr);
ierr = PetscMalloc1(mat->rmap->n+1,&locals);CHKERRQ(ierr);
if (useND) {
#if defined(PETSC_HAVE_SCOTCH_PARMETIS_V3_NODEND)
PetscInt *sizes, *seps, log2size, subd, *level, base = 0;
PetscMPIInt size;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
log2size = PetscLog2Real(size);
subd = PetscPowInt(2,log2size);
if (subd != size) SETERRQ(comm,PETSC_ERR_SUP,"Only power of 2 communicator sizes");
ierr = PetscMalloc1(mat->rmap->n,&NDorder);CHKERRQ(ierr);
ierr = PetscMalloc3(2*size,&sizes,4*size,&seps,size,&level);CHKERRQ(ierr);
SCOTCH_ParMETIS_V3_NodeND(mat->rmap->range,adj->i,adj->j,&base,NULL,NDorder,sizes,&comm);
ierr = MatPartitioningSizesToSep_Private(subd,sizes,seps,level);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
PetscInt loc;
ierr = PetscFindInt(NDorder[i],2*subd,seps,&loc);CHKERRQ(ierr);
if (loc < 0) {
loc = -(loc+1);
if (loc%2) { /* part of subdomain */
locals[i] = loc/2;
} else {
ierr = PetscFindInt(NDorder[i],2*(subd-1),seps+2*subd,&loc);CHKERRQ(ierr);
loc = loc < 0 ? -(loc+1)/2 : loc/2;
locals[i] = level[loc];
}
} else locals[i] = loc/2;
}
ierr = PetscFree3(sizes,seps,level);CHKERRQ(ierr);
#else
SETERRQ(pcomm,PETSC_ERR_SUP,"Need libptscotchparmetis.a compiled with -DSCOTCH_METIS_PREFIX");
#endif
} else {
velotab = NULL;
if (proc_weight_flg) {
ierr = PetscMalloc1(nparts,&vwgttab);CHKERRQ(ierr);
ierr = PetscMalloc1(nparts,&velotab);CHKERRQ(ierr);
for (j=0; j<nparts; j++) {
if (part->part_weights) vwgttab[j] = part->part_weights[j]*nparts;
else vwgttab[j] = 1.0;
}
for (i=0; i<nparts; i++) {
deltval = PetscAbsReal(vwgttab[i]-PetscFloorReal(vwgttab[i]+0.5));
if (deltval>0.01) {
for (j=0; j<nparts; j++) vwgttab[j] /= deltval;
}
}
for (i=0; i<nparts; i++) velotab[i] = (SCOTCH_Num)(vwgttab[i] + 0.5);
ierr = PetscFree(vwgttab);CHKERRQ(ierr);
}
vertlocnbr = mat->rmap->range[rank+1] - mat->rmap->range[rank];
edgelocnbr = adj->i[vertlocnbr];
veloloctab = part->vertex_weights;
edloloctab = adj->values;
/* detect whether all vertices are located at the same process in original graph */
for (p = 0; !mat->rmap->range[p+1] && p < nparts; ++p);
distributed = (mat->rmap->range[p+1] == mat->rmap->N) ? PETSC_FALSE : PETSC_TRUE;
if (distributed) {
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat;
SCOTCH_Dmapping mappdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_dgraphInit(&grafdat,comm);CHKERRQ(ierr);
ierr = SCOTCH_dgraphBuild(&grafdat,0,vertlocnbr,vertlocnbr,adj->i,adj->i+1,veloloctab,
NULL,edgelocnbr,edgelocnbr,adj->j,NULL,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_dgraphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_archInit(&archdat);CHKERRQ(ierr);
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratDgraphMapBuild(&stradat,scotch->strategy,nparts,nparts,scotch->imbalance);CHKERRQ(ierr);
if (velotab) {
ierr = SCOTCH_archCmpltw(&archdat,nparts,velotab);CHKERRQ(ierr);
} else {
ierr = SCOTCH_archCmplt( &archdat,nparts);CHKERRQ(ierr);
}
ierr = SCOTCH_dgraphMapInit(&grafdat,&mappdat,&archdat,locals);CHKERRQ(ierr);
ierr = SCOTCH_dgraphMapCompute(&grafdat,&mappdat,&stradat);CHKERRQ(ierr);
SCOTCH_dgraphMapExit(&grafdat,&mappdat);
SCOTCH_archExit(&archdat);
SCOTCH_stratExit(&stradat);
SCOTCH_dgraphExit(&grafdat);
} else if (rank == p) {
SCOTCH_Graph grafdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_graphInit(&grafdat);CHKERRQ(ierr);
ierr = SCOTCH_graphBuild(&grafdat,0,vertlocnbr,adj->i,adj->i+1,veloloctab,NULL,edgelocnbr,adj->j,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_graphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratGraphMapBuild(&stradat,scotch->strategy,nparts,scotch->imbalance);CHKERRQ(ierr);
ierr = SCOTCH_graphPart(&grafdat,nparts,&stradat,locals);CHKERRQ(ierr);
SCOTCH_stratExit(&stradat);
SCOTCH_graphExit(&grafdat);
}
ierr = PetscFree(velotab);CHKERRQ(ierr);
}
ierr = MPI_Comm_free(&comm);CHKERRQ(ierr);
if (bs > 1) {
PetscInt *newlocals;
ierr = PetscMalloc1(bs*mat->rmap->n,&newlocals);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
for (j=0;j<bs;j++) {
newlocals[bs*i+j] = locals[i];
}
}
ierr = PetscFree(locals);CHKERRQ(ierr);
ierr = ISCreateGeneral(pcomm,bs*mat->rmap->n,newlocals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,locals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
}
if (useND) {
IS ndis;
if (bs > 1) {
ierr = ISCreateBlock(pcomm,bs,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
}
ierr = ISSetPermutation(ndis);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject)(*partitioning),"_petsc_matpartitioning_ndorder",(PetscObject)ndis);CHKERRQ(ierr);
ierr = ISDestroy(&ndis);CHKERRQ(ierr);
}
if (!flg) {
ierr = MatDestroy(&mat);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
void
METISNAMEU(ParMETIS_V3_PartKway) (
const SCOTCH_Num * const vtxdist,
SCOTCH_Num * const xadj,
SCOTCH_Num * const adjncy,
SCOTCH_Num * const vwgt,
SCOTCH_Num * const adjwgt,
const SCOTCH_Num * const wgtflag,
const SCOTCH_Num * const numflag,
const SCOTCH_Num * const ncon, /* Not used */
const SCOTCH_Num * const nparts,
const float * const tpwgts,
const float * const ubvec, /* Not used */
const SCOTCH_Num * const options, /* Not used */
SCOTCH_Num * const edgecut,
SCOTCH_Num * const part,
MPI_Comm * comm)
{
MPI_Comm proccomm;
int procglbnbr;
int proclocnum;
SCOTCH_Num baseval;
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat; /* Scotch distributed graph object to interface with libScotch */
SCOTCH_Dmapping mappdat; /* Scotch distributed mapping object to interface with libScotch */
SCOTCH_Strat stradat;
SCOTCH_Num vertlocnbr;
SCOTCH_Num * veloloctab;
SCOTCH_Num edgelocnbr;
SCOTCH_Num * edloloctab;
SCOTCH_Num * velotab;
double * vwgttab;
SCOTCH_Num i;
if ((vwgttab = malloc (*nparts * sizeof (double))) == NULL)
return;
if ((velotab = malloc (*nparts * sizeof (SCOTCH_Num))) == NULL) {
free (vwgttab);
return;
}
for (i = 0; i < *nparts; i ++)
vwgttab[i] = (double) tpwgts[i] * (double) (*nparts);
for (i = 0; i < *nparts; i ++) {
double deltval;
deltval = fabs (vwgttab[i] - floor (vwgttab[i] + 0.5));
if (deltval > 0.01) {
SCOTCH_Num j;
deltval = 1.0 / deltval;
for (j = 0; j < *nparts; j ++)
vwgttab[j] *= deltval;
}
}
for (i = 0; i < *nparts; i ++)
velotab[i] = (SCOTCH_Num) (vwgttab[i] + 0.5);
proccomm = *comm;
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0)
return;
MPI_Comm_size (proccomm, &procglbnbr);
MPI_Comm_rank (proccomm, &proclocnum);
baseval = *numflag;
vertlocnbr = vtxdist[proclocnum + 1] - vtxdist[proclocnum];
edgelocnbr = xadj[vertlocnbr] - baseval;
veloloctab = ((vwgt != NULL) && ((*wgtflag & 2) != 0)) ? vwgt : NULL;
edloloctab = ((adjwgt != NULL) && ((*wgtflag & 1) != 0)) ? adjwgt : NULL;
if (SCOTCH_dgraphBuild (&grafdat, baseval,
vertlocnbr, vertlocnbr, xadj, xadj + 1, veloloctab, NULL,
edgelocnbr, edgelocnbr, adjncy, NULL, edloloctab) == 0) {
SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_dgraphCheck (&grafdat) == 0) /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
{
SCOTCH_archInit (&archdat);
if ((SCOTCH_archCmpltw (&archdat, *nparts, velotab) == 0) &&
(SCOTCH_dgraphMapInit (&grafdat, &mappdat, &archdat, part) == 0)) {
SCOTCH_dgraphMapCompute (&grafdat, &mappdat, &stradat);
SCOTCH_dgraphMapExit (&grafdat, &mappdat);
}
SCOTCH_archExit (&archdat);
}
SCOTCH_stratExit (&stradat);
}
SCOTCH_dgraphExit (&grafdat);
*edgecut = 0; /* TODO : compute real edge cut for people who might want it */
free (vwgttab);
free (velotab);
if (baseval != 0) { /* MeTiS part array is based, Scotch is not */
SCOTCH_Num vertlocnum;
for (vertlocnum = 0; vertlocnum < vertlocnbr; vertlocnum ++)
part[vertlocnum] += baseval;
}
}
// Call scotch with options from dictionary.
Foam::label Foam::ptscotchDecomp::decompose
(
const fileName& meshPath,
const List<int>& adjncy,
const List<int>& xadj,
const scalarField& cWeights,
List<int>& finalDecomp
) const
{
if (debug)
{
Pout<< "ptscotchDecomp : entering with xadj:" << xadj.size() << endl;
}
// Dump graph
if (decompositionDict_.found("ptscotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("ptscotchCoeffs");
if (scotchCoeffs.lookupOrDefault("writeGraph", false))
{
OFstream str
(
meshPath + "_" + Foam::name(Pstream::myProcNo()) + ".dgr"
);
Pout<< "Dumping Scotch graph file to " << str.name() << endl
<< "Use this in combination with dgpart." << endl;
globalIndex globalCells(xadj.size()-1);
// Distributed graph file (.grf)
label version = 2;
str << version << nl;
// Number of files (procglbnbr)
str << Pstream::nProcs();
// My file number (procloc)
str << ' ' << Pstream::myProcNo() << nl;
// Total number of vertices (vertglbnbr)
str << globalCells.size();
// Total number of connections (edgeglbnbr)
str << ' ' << returnReduce(xadj[xadj.size()-1], sumOp<label>())
<< nl;
// Local number of vertices (vertlocnbr)
str << xadj.size()-1;
// Local number of connections (edgelocnbr)
str << ' ' << xadj[xadj.size()-1] << nl;
// Numbering starts from 0
label baseval = 0;
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs
str << baseval << ' ' << "000" << nl;
for (label cellI = 0; cellI < xadj.size()-1; cellI++)
{
label start = xadj[cellI];
label end = xadj[cellI+1];
str << end-start;
for (label i = start; i < end; i++)
{
str << ' ' << adjncy[i];
}
str << nl;
}
}
}
// Strategy
// ~~~~~~~~
// Default.
SCOTCH_Strat stradat;
check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
string strategy;
if (scotchCoeffs.readIfPresent("strategy", strategy))
{
if (debug)
{
Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
}
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
//fprintf(stdout, "\n");
}
}
// Graph
// ~~~~~
List<int> velotab;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningIn
(
"ptscotchDecomp::decompose(..)"
) << "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != xadj.size()-1)
{
FatalErrorIn
(
"ptscotchDecomp::decompose(..)"
) << "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << xadj.size()-1
<< exit(FatalError);
}
// Convert to integers.
velotab.setSize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int(cWeights[i]/minWeights);
}
}
if (debug)
{
Pout<< "SCOTCH_dgraphInit" << endl;
}
SCOTCH_Dgraph grafdat;
check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
if (debug)
{
Pout<< "SCOTCH_dgraphBuild with:" << nl
<< "xadj.size()-1 : " << xadj.size()-1 << nl
<< "xadj : " << long(xadj.begin()) << nl
<< "velotab : " << long(velotab.begin()) << nl
<< "adjncy.size() : " << adjncy.size() << nl
<< "adjncy : " << long(adjncy.begin()) << nl
<< endl;
}
check
(
SCOTCH_dgraphBuild
(
&grafdat, // grafdat
0, // baseval, c-style numbering
xadj.size()-1, // vertlocnbr, nCells
xadj.size()-1, // vertlocmax
const_cast<SCOTCH_Num*>(xadj.begin()),
// vertloctab, start index per cell into
// adjncy
const_cast<SCOTCH_Num*>(&xadj[1]),// vendloctab, end index ,,
const_cast<SCOTCH_Num*>(velotab.begin()),// veloloctab, vtx weights
NULL, // vlblloctab
adjncy.size(), // edgelocnbr, number of arcs
adjncy.size(), // edgelocsiz
const_cast<SCOTCH_Num*>(adjncy.begin()), // edgeloctab
NULL, // edgegsttab
NULL // edlotab, edge weights
),
"SCOTCH_dgraphBuild"
);
if (debug)
{
Pout<< "SCOTCH_dgraphCheck" << endl;
}
check(SCOTCH_dgraphCheck(&grafdat), "SCOTCH_dgraphCheck");
// Architecture
// ~~~~~~~~~~~~
// (fully connected network topology since using switch)
if (debug)
{
Pout<< "SCOTCH_archInit" << endl;
}
SCOTCH_Arch archdat;
check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
List<label> processorWeights;
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
scotchCoeffs.readIfPresent("processorWeights", processorWeights);
}
if (processorWeights.size())
{
if (debug)
{
Info<< "ptscotchDecomp : Using procesor weights "
<< processorWeights
<< endl;
}
check
(
SCOTCH_archCmpltw(&archdat, nProcessors_, processorWeights.begin()),
"SCOTCH_archCmpltw"
);
}
else
{
if (debug)
{
Pout<< "SCOTCH_archCmplt" << endl;
}
check
(
SCOTCH_archCmplt(&archdat, nProcessors_),
"SCOTCH_archCmplt"
);
}
//SCOTCH_Mapping mapdat;
//SCOTCH_dgraphMapInit(&grafdat, &mapdat, &archdat, NULL);
//SCOTCH_dgraphMapCompute(&grafdat, &mapdat, &stradat); /*Perform mapping*/
//SCOTCHdgraphMapExit(&grafdat, &mapdat);
// Hack:switch off fpu error trapping
# ifdef LINUX_GNUC
int oldExcepts = fedisableexcept
(
FE_DIVBYZERO
| FE_INVALID
| FE_OVERFLOW
);
# endif
if (debug)
{
Pout<< "SCOTCH_dgraphMap" << endl;
}
finalDecomp.setSize(xadj.size()-1);
finalDecomp = 0;
check
(
SCOTCH_dgraphMap
(
&grafdat,
&archdat,
&stradat, // const SCOTCH_Strat *
finalDecomp.begin() // parttab
),
"SCOTCH_graphMap"
);
# ifdef LINUX_GNUC
feenableexcept(oldExcepts);
# endif
//finalDecomp.setSize(xadj.size()-1);
//check
//(
// SCOTCH_dgraphPart
// (
// &grafdat,
// nProcessors_, // partnbr
// &stradat, // const SCOTCH_Strat *
// finalDecomp.begin() // parttab
// ),
// "SCOTCH_graphPart"
//);
if (debug)
{
Pout<< "SCOTCH_dgraphExit" << endl;
}
// Release storage for graph
SCOTCH_dgraphExit(&grafdat);
// Release storage for strategy
SCOTCH_stratExit(&stradat);
// Release storage for network topology
SCOTCH_archExit(&archdat);
return 0;
}
int
main (
int argc,
char * argv[])
{
SCOTCH_Dgraph grafdat;
int procglbnbr;
int proclocnum;
int protglbnum; /* Root process */
SCOTCH_Num vertnbr;
SCOTCH_Num velomin;
SCOTCH_Num velomax;
SCOTCH_Num velosum;
double veloavg;
double velodlt;
SCOTCH_Num degrmin;
SCOTCH_Num degrmax;
double degravg;
double degrdlt;
SCOTCH_Num edgenbr;
SCOTCH_Num edlomin;
SCOTCH_Num edlomax;
SCOTCH_Num edlosum;
double edloavg;
double edlodlt;
int flagval;
int i;
#ifdef SCOTCH_PTHREAD
int thrdlvlreqval;
int thrdlvlproval;
#endif /* SCOTCH_PTHREAD */
errorProg ("dgtst");
#ifdef SCOTCH_PTHREAD
thrdlvlreqval = MPI_THREAD_MULTIPLE;
if (MPI_Init_thread (&argc, &argv, thrdlvlreqval, &thrdlvlproval) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (1)");
if (thrdlvlreqval > thrdlvlproval)
errorPrint ("main: MPI implementation is not thread-safe: recompile without SCOTCH_PTHREAD");
#else /* SCOTCH_PTHREAD */
if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (2)");
#endif /* SCOTCH_PTHREAD */
MPI_Comm_size (MPI_COMM_WORLD, &procglbnbr); /* Get communicator data */
MPI_Comm_rank (MPI_COMM_WORLD, &proclocnum);
protglbnum = 0; /* Assume root process is process 0 */
if ((argc >= 2) && (argv[1][0] == '?')) { /* If need for help */
usagePrint (stdout, C_usageList);
return (0);
}
flagval = C_FLAGNONE;
for (i = 0; i < C_FILENBR; i ++) /* Set default stream pointers */
C_fileTab[i].pntr = (C_fileTab[i].mode[0] == 'r') ? stdin : stdout;
for (i = 1; i < argc; i ++) { /* Loop for all option codes */
if ((argv[i][0] != '+') && /* If found a file name */
((argv[i][0] != '-') || (argv[i][1] == '\0'))) {
if (C_fileNum < C_FILEARGNBR) /* A file name has been given */
C_fileTab[C_fileNum ++].name = argv[i];
else
errorPrint ("main: too many file names given");
}
else { /* If found an option name */
switch (argv[i][1]) {
#ifdef SCOTCH_DEBUG_ALL
case 'D' :
case 'd' :
flagval |= C_FLAGDEBUG;
break;
#endif /* SCOTCH_DEBUG_ALL */
case 'H' : /* Give the usage message */
case 'h' :
usagePrint (stdout, C_usageList);
return (0);
case 'R' : /* Root process (if necessary) */
case 'r' :
protglbnum = atoi (&argv[i][2]);
if ((protglbnum < 0) ||
(protglbnum >= procglbnbr) ||
((protglbnum == 0) && (argv[i][2] != '0')))
errorPrint ("main: invalid root process number");
break;
case 'V' :
case 'v' :
fprintf (stderr, "dgtst, version " SCOTCH_VERSION_STRING "\n");
fprintf (stderr, "Copyright 2007,2008,2010,2011 ENSEIRB, INRIA & CNRS, France\n");
fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
return (0);
default :
errorPrint ("main: unprocessed option '%s'", argv[i]);
}
}
}
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0) {
fprintf (stderr, "Proc %4d of %d, pid %d\n", proclocnum, procglbnbr, getpid ());
if (proclocnum == protglbnum) { /* Synchronize on keybord input */
char c;
printf ("Waiting for key press...\n");
scanf ("%c", &c);
}
MPI_Barrier (MPI_COMM_WORLD);
}
#endif /* SCOTCH_DEBUG_ALL */
fileBlockOpenDist (C_fileTab, C_FILENBR, procglbnbr, proclocnum, protglbnum); /* Open all files */
SCOTCH_dgraphInit (&grafdat, MPI_COMM_WORLD);
SCOTCH_dgraphLoad (&grafdat, C_filepntrsrcinp, -1, 0);
SCOTCH_dgraphCheck (&grafdat);
SCOTCH_dgraphSize (&grafdat, &vertnbr, NULL, &edgenbr, NULL);
SCOTCH_dgraphStat (&grafdat, &velomin, &velomax, &velosum, &veloavg, &velodlt,
°rmin, °rmax, °ravg, °rdlt,
&edlomin, &edlomax, &edlosum, &edloavg, &edlodlt);
if (C_filepntrdatout != NULL) {
fprintf (C_filepntrdatout, "S\tVertex\tnbr=" SCOTCH_NUMSTRING "\n",
(SCOTCH_Num) vertnbr);
fprintf (C_filepntrdatout, "S\tVertex load\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) velomin, (SCOTCH_Num) velomax, (SCOTCH_Num) velosum, veloavg, velodlt);
fprintf (C_filepntrdatout, "S\tVertex degree\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) degrmin, (SCOTCH_Num) degrmax, (SCOTCH_Num) edgenbr, degravg, degrdlt);
fprintf (C_filepntrdatout, "S\tEdge\tnbr=" SCOTCH_NUMSTRING "\n",
(SCOTCH_Num) (edgenbr / 2));
fprintf (C_filepntrdatout, "S\tEdge load\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) edlomin, (SCOTCH_Num) edlomax, (SCOTCH_Num) edlosum, edloavg, edlodlt);
}
fileBlockClose (C_fileTab, C_FILENBR); /* Always close explicitely to end eventual (un)compression tasks */
SCOTCH_dgraphExit (&grafdat);
MPI_Finalize ();
#ifdef COMMON_PTHREAD
pthread_exit ((void *) 0); /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
return (0);
}
int
main (
int argc,
char * argv[])
{
SCOTCH_Graph * cgrfptr;
SCOTCH_Graph cgrfdat;
SCOTCH_Dgraph dgrfdat;
int procglbnbr;
int proclocnum;
int protglbnum; /* Root process */
int flagval;
int i;
int reduloctab[2];
int reduglbtab[2];
#ifdef SCOTCH_PTHREAD
int thrdlvlreqval;
int thrdlvlproval;
#endif /* SCOTCH_PTHREAD */
errorProg ("dggath");
#ifdef SCOTCH_PTHREAD
thrdlvlreqval = MPI_THREAD_MULTIPLE;
if (MPI_Init_thread (&argc, &argv, thrdlvlreqval, &thrdlvlproval) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (1)");
if (thrdlvlreqval > thrdlvlproval)
errorPrint ("main: MPI implementation is not thread-safe: recompile without SCOTCH_PTHREAD");
#else /* SCOTCH_PTHREAD */
if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (2)");
#endif /* SCOTCH_PTHREAD */
MPI_Comm_size (MPI_COMM_WORLD, &procglbnbr); /* Get communicator data */
MPI_Comm_rank (MPI_COMM_WORLD, &proclocnum);
protglbnum = 0; /* Assume root process is process 0 */
if ((argc >= 2) && (argv[1][0] == '?')) { /* If need for help */
usagePrint (stdout, C_usageList);
return (0);
}
flagval = C_FLAGNONE;
for (i = 0; i < C_FILENBR; i ++) /* Set default stream pointers */
C_fileTab[i].pntr = (C_fileTab[i].mode[0] == 'r') ? stdin : stdout;
for (i = 1; i < argc; i ++) { /* Loop for all option codes */
if ((argv[i][0] != '-') || (argv[i][1] == '\0') || (argv[i][1] == '.')) { /* If found a file name */
if (C_fileNum < C_FILEARGNBR) /* File name has been given */
C_fileTab[C_fileNum ++].name = argv[i];
else
errorPrint ("main: too many file names given");
}
else { /* If found an option name */
switch (argv[i][1]) {
case 'C' :
case 'c' :
flagval |= C_FLAGCHECK;
break;
#ifdef SCOTCH_DEBUG_ALL
case 'D' :
case 'd' :
flagval |= C_FLAGDEBUG;
break;
#endif /* SCOTCH_DEBUG_ALL */
case 'H' : /* Give the usage message */
case 'h' :
usagePrint (stdout, C_usageList);
return (0);
case 'R' : /* Root process (if necessary) */
case 'r' :
protglbnum = atoi (&argv[i][2]);
if ((protglbnum < 0) ||
(protglbnum >= procglbnbr) ||
((protglbnum == 0) && (argv[i][2] != '0'))) {
errorPrint ("main: invalid root process number");
}
break;
case 'V' :
case 'v' :
fprintf (stderr, "dggath, version " SCOTCH_VERSION_STRING "\n");
fprintf (stderr, "Copyright 2008,2010 ENSEIRB, INRIA & CNRS, France\n");
fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
return (0);
default :
errorPrint ("main: unprocessed option (\"%s\")", argv[i]);
}
}
}
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0) {
fprintf (stderr, "Proc %4d of %d, pid %d\n", proclocnum, procglbnbr, getpid ());
if (proclocnum == protglbnum) { /* Synchronize on keybord input */
char c;
printf ("Waiting for key press...\n");
scanf ("%c", &c);
}
MPI_Barrier (MPI_COMM_WORLD);
}
#endif /* SCOTCH_DEBUG_ALL */
fileBlockOpenDist (C_fileTab, C_FILENBR, procglbnbr, proclocnum, protglbnum); /* Open all files */
if (C_filepntrsrcout == NULL) {
cgrfptr = NULL;
reduloctab[0] =
reduloctab[1] = 0;
}
else {
cgrfptr = &cgrfdat;
reduloctab[0] = 1;
reduloctab[1] = proclocnum;
}
if (MPI_Allreduce (reduloctab, reduglbtab, 2, MPI_INT, MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS)
errorPrint ("main: communication error");
if (reduglbtab[0] != 1)
errorPrint ("main: should have only one root");
if (reduglbtab[1] != protglbnum)
errorPrint ("main: root process mismatch");
SCOTCH_dgraphInit (&dgrfdat, MPI_COMM_WORLD);
SCOTCH_dgraphLoad (&dgrfdat, C_filepntrsrcinp, -1, 0);
if ((flagval & C_FLAGCHECK) != 0)
SCOTCH_dgraphCheck (&dgrfdat);
SCOTCH_graphInit (&cgrfdat);
SCOTCH_dgraphGather (&dgrfdat, cgrfptr);
if (cgrfptr != NULL)
SCOTCH_graphSave (cgrfptr, C_filepntrsrcout);
fileBlockClose (C_fileTab, C_FILENBR); /* Always close explicitely to end eventual (un)compression tasks */
SCOTCH_graphExit (&cgrfdat);
SCOTCH_dgraphExit (&dgrfdat);
MPI_Finalize ();
#ifdef COMMON_PTHREAD
pthread_exit ((void *) 0); /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
return (0);
}
void
METISNAMEU(ParMETIS_V3_NodeND) (
const int * const vtxdist,
int * const xadj,
int * const adjncy,
const int * const numflag,
const int * const options, /* Not used */
int * const order,
int * const sizes, /* Of size twice the number of processors ; not used */
MPI_Comm * comm)
{
MPI_Comm proccomm;
int procglbnbr;
int proclocnum;
SCOTCH_Num baseval;
SCOTCH_Dgraph grafdat; /* Scotch distributed graph object to interface with libScotch */
SCOTCH_Dordering ordedat; /* Scotch distributed ordering object to interface with libScotch */
SCOTCH_Strat stradat;
SCOTCH_Num vertlocnbr;
SCOTCH_Num edgelocnbr;
if (sizeof (SCOTCH_Num) != sizeof (int)) {
SCOTCH_errorPrint ("ParMETIS_V3_NodeND (as of SCOTCH): SCOTCH_Num type should equate to int");
return;
}
proccomm = *comm;
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0)
return;
MPI_Comm_size (proccomm, &procglbnbr);
MPI_Comm_rank (proccomm, &proclocnum);
baseval = *numflag;
vertlocnbr = vtxdist[proclocnum + 1] - vtxdist[proclocnum];
edgelocnbr = xadj[vertlocnbr] - baseval;
if (sizes != NULL)
memSet (sizes, ~0, (2 * procglbnbr - 1) * sizeof (int)); /* Array not used if procglbnbr is not a power of 2 or if error */
if (SCOTCH_dgraphBuild (&grafdat, baseval,
vertlocnbr, vertlocnbr, xadj, xadj + 1, NULL, NULL,
edgelocnbr, edgelocnbr, adjncy, NULL, NULL) == 0) {
SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_dgraphCheck (&grafdat) == 0) /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
{
if (SCOTCH_dgraphOrderInit (&grafdat, &ordedat) == 0) {
int levlmax;
int bitsnbr;
SCOTCH_Num proctmp;
SCOTCH_dgraphOrderCompute (&grafdat, &ordedat, &stradat);
SCOTCH_dgraphOrderPerm (&grafdat, &ordedat, order);
for (levlmax = -1, bitsnbr = 0, proctmp = procglbnbr; /* Count number of bits set to 1 in procglbnbr */
proctmp != 0; levlmax ++, proctmp >>= 1)
bitsnbr += proctmp & 1;
if (bitsnbr == 1) {
SCOTCH_Num cblkglbnbr;
if ((cblkglbnbr = SCOTCH_dgraphOrderCblkDist (&grafdat, &ordedat)) >= 0) {
SCOTCH_Num * treeglbtab;
SCOTCH_Num * sizeglbtab;
SCOTCH_Num * sepaglbtab;
if (memAllocGroup ((void **) (void *)
&treeglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num)),
&sizeglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num)),
&sepaglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num) * 3), NULL) != NULL) {
if (SCOTCH_dgraphOrderTreeDist (&grafdat, &ordedat, treeglbtab, sizeglbtab) == 0) {
SCOTCH_Num rootnum;
SCOTCH_Num cblknum;
memSet (sepaglbtab, ~0, cblkglbnbr * sizeof (SCOTCH_Num) * 3);
for (rootnum = -1, cblknum = 0; cblknum < cblkglbnbr; cblknum ++) {
SCOTCH_Num fathnum;
fathnum = treeglbtab[cblknum] - baseval; /* Use un-based indices */
if (fathnum < 0) { /* If father index indicates root */
if (rootnum != -1) { /* If another root already found */
rootnum = -1; /* Indicate an error */
break;
}
rootnum = cblknum; /* Record index of root node */
}
else {
int i;
for (i = 0; i < 3; i ++) {
int j;
j = 3 * fathnum + i; /* Slot number of prospective son */
if (sepaglbtab[j] < 0) { /* If potentially empty slot found */
if (sepaglbtab[j] == -1) /* If we don't have too many sons */
sepaglbtab[j] = cblknum; /* Add link to son in slot */
break;
}
}
if (i == 3) { /* If no empty slot found */
sepaglbtab[3 * fathnum] = -2; /* Indicate there are too many sons */
break;
}
}
}
if ((rootnum >= 0) && (sizes != NULL)) { /* If no error above, go on processing separator tree */
memSet (sizes, 0, (2 * procglbnbr - 1) * sizeof (int)); /* Set array of sizes to 0 by default */
_SCOTCH_ParMETIS_V3_NodeNDTree (sizes + (2 * procglbnbr - 1), sizeglbtab, sepaglbtab, levlmax, 0, rootnum, 1);
}
}
memFree (treeglbtab); /* Free group leader */
}
}
}
SCOTCH_dgraphOrderExit (&grafdat, &ordedat);
}
}
