static PetscErrorCode MatPartitioningApply_PTScotch_Private(MatPartitioning part, PetscBool useND, IS *partitioning)
{
MPI_Comm pcomm,comm;
MatPartitioning_PTScotch *scotch = (MatPartitioning_PTScotch*)part->data;
PetscErrorCode ierr;
PetscMPIInt rank;
Mat mat = part->adj;
Mat_MPIAdj *adj = (Mat_MPIAdj*)mat->data;
PetscBool flg,distributed;
PetscBool proc_weight_flg;
PetscInt i,j,p,bs=1,nold;
PetscInt *NDorder = NULL;
PetscReal *vwgttab,deltval;
SCOTCH_Num *locals,*velotab,*veloloctab,*edloloctab,vertlocnbr,edgelocnbr,nparts=part->n;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)part,&pcomm);CHKERRQ(ierr);
/* Duplicate the communicator to be sure that PTSCOTCH attribute caching does not interfere with PETSc. */
ierr = MPI_Comm_dup(pcomm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr);
if (!flg) {
/* bs indicates if the converted matrix is "reduced" from the original and hence the
resulting partition results need to be stretched to match the original matrix */
nold = mat->rmap->n;
ierr = MatConvert(mat,MATMPIADJ,MAT_INITIAL_MATRIX,&mat);CHKERRQ(ierr);
if (mat->rmap->n > 0) bs = nold/mat->rmap->n;
adj = (Mat_MPIAdj*)mat->data;
}
proc_weight_flg = PETSC_TRUE;
ierr = PetscOptionsGetBool(NULL, NULL, "-mat_partitioning_ptscotch_proc_weight", &proc_weight_flg, NULL);CHKERRQ(ierr);
ierr = PetscMalloc1(mat->rmap->n+1,&locals);CHKERRQ(ierr);
if (useND) {
#if defined(PETSC_HAVE_SCOTCH_PARMETIS_V3_NODEND)
PetscInt *sizes, *seps, log2size, subd, *level, base = 0;
PetscMPIInt size;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
log2size = PetscLog2Real(size);
subd = PetscPowInt(2,log2size);
if (subd != size) SETERRQ(comm,PETSC_ERR_SUP,"Only power of 2 communicator sizes");
ierr = PetscMalloc1(mat->rmap->n,&NDorder);CHKERRQ(ierr);
ierr = PetscMalloc3(2*size,&sizes,4*size,&seps,size,&level);CHKERRQ(ierr);
SCOTCH_ParMETIS_V3_NodeND(mat->rmap->range,adj->i,adj->j,&base,NULL,NDorder,sizes,&comm);
ierr = MatPartitioningSizesToSep_Private(subd,sizes,seps,level);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
PetscInt loc;
ierr = PetscFindInt(NDorder[i],2*subd,seps,&loc);CHKERRQ(ierr);
if (loc < 0) {
loc = -(loc+1);
if (loc%2) { /* part of subdomain */
locals[i] = loc/2;
} else {
ierr = PetscFindInt(NDorder[i],2*(subd-1),seps+2*subd,&loc);CHKERRQ(ierr);
loc = loc < 0 ? -(loc+1)/2 : loc/2;
locals[i] = level[loc];
}
} else locals[i] = loc/2;
}
ierr = PetscFree3(sizes,seps,level);CHKERRQ(ierr);
#else
SETERRQ(pcomm,PETSC_ERR_SUP,"Need libptscotchparmetis.a compiled with -DSCOTCH_METIS_PREFIX");
#endif
} else {
velotab = NULL;
if (proc_weight_flg) {
ierr = PetscMalloc1(nparts,&vwgttab);CHKERRQ(ierr);
ierr = PetscMalloc1(nparts,&velotab);CHKERRQ(ierr);
for (j=0; j<nparts; j++) {
if (part->part_weights) vwgttab[j] = part->part_weights[j]*nparts;
else vwgttab[j] = 1.0;
}
for (i=0; i<nparts; i++) {
deltval = PetscAbsReal(vwgttab[i]-PetscFloorReal(vwgttab[i]+0.5));
if (deltval>0.01) {
for (j=0; j<nparts; j++) vwgttab[j] /= deltval;
}
}
for (i=0; i<nparts; i++) velotab[i] = (SCOTCH_Num)(vwgttab[i] + 0.5);
ierr = PetscFree(vwgttab);CHKERRQ(ierr);
}
vertlocnbr = mat->rmap->range[rank+1] - mat->rmap->range[rank];
edgelocnbr = adj->i[vertlocnbr];
veloloctab = part->vertex_weights;
edloloctab = adj->values;
/* detect whether all vertices are located at the same process in original graph */
for (p = 0; !mat->rmap->range[p+1] && p < nparts; ++p);
distributed = (mat->rmap->range[p+1] == mat->rmap->N) ? PETSC_FALSE : PETSC_TRUE;
if (distributed) {
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat;
SCOTCH_Dmapping mappdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_dgraphInit(&grafdat,comm);CHKERRQ(ierr);
ierr = SCOTCH_dgraphBuild(&grafdat,0,vertlocnbr,vertlocnbr,adj->i,adj->i+1,veloloctab,
NULL,edgelocnbr,edgelocnbr,adj->j,NULL,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_dgraphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_archInit(&archdat);CHKERRQ(ierr);
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratDgraphMapBuild(&stradat,scotch->strategy,nparts,nparts,scotch->imbalance);CHKERRQ(ierr);
if (velotab) {
ierr = SCOTCH_archCmpltw(&archdat,nparts,velotab);CHKERRQ(ierr);
} else {
ierr = SCOTCH_archCmplt( &archdat,nparts);CHKERRQ(ierr);
}
ierr = SCOTCH_dgraphMapInit(&grafdat,&mappdat,&archdat,locals);CHKERRQ(ierr);
ierr = SCOTCH_dgraphMapCompute(&grafdat,&mappdat,&stradat);CHKERRQ(ierr);
SCOTCH_dgraphMapExit(&grafdat,&mappdat);
SCOTCH_archExit(&archdat);
SCOTCH_stratExit(&stradat);
SCOTCH_dgraphExit(&grafdat);
} else if (rank == p) {
SCOTCH_Graph grafdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_graphInit(&grafdat);CHKERRQ(ierr);
ierr = SCOTCH_graphBuild(&grafdat,0,vertlocnbr,adj->i,adj->i+1,veloloctab,NULL,edgelocnbr,adj->j,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_graphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratGraphMapBuild(&stradat,scotch->strategy,nparts,scotch->imbalance);CHKERRQ(ierr);
ierr = SCOTCH_graphPart(&grafdat,nparts,&stradat,locals);CHKERRQ(ierr);
SCOTCH_stratExit(&stradat);
SCOTCH_graphExit(&grafdat);
}
ierr = PetscFree(velotab);CHKERRQ(ierr);
}
ierr = MPI_Comm_free(&comm);CHKERRQ(ierr);
if (bs > 1) {
PetscInt *newlocals;
ierr = PetscMalloc1(bs*mat->rmap->n,&newlocals);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
for (j=0;j<bs;j++) {
newlocals[bs*i+j] = locals[i];
}
}
ierr = PetscFree(locals);CHKERRQ(ierr);
ierr = ISCreateGeneral(pcomm,bs*mat->rmap->n,newlocals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,locals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
}
if (useND) {
IS ndis;
if (bs > 1) {
ierr = ISCreateBlock(pcomm,bs,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
}
ierr = ISSetPermutation(ndis);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject)(*partitioning),"_petsc_matpartitioning_ndorder",(PetscObject)ndis);CHKERRQ(ierr);
ierr = ISDestroy(&ndis);CHKERRQ(ierr);
}
if (!flg) {
ierr = MatDestroy(&mat);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
void AlgPTScotch<Adapter>::partition(
const RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
SCOTCH_Num partnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(partnbr, numGlobalParts, env);
#ifdef HAVE_ZOLTAN2_MPI
int ierr = 0;
int me = problemComm->getRank();
const SCOTCH_Num baseval = 0; // Base value for array indexing.
// GraphModel returns GNOs from base 0.
SCOTCH_Strat stratstr; // Strategy string
// TODO: Set from parameters
SCOTCH_stratInit(&stratstr);
// Allocate and initialize PTScotch Graph data structure.
SCOTCH_Dgraph *gr = SCOTCH_dgraphAlloc(); // Scotch distributed graph
ierr = SCOTCH_dgraphInit(gr, mpicomm);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphInit",
!ierr, BASIC_ASSERTION, problemComm);
// Get vertex info
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
size_t nVtx = model->getVertexList(vtxID, xyz, vwgts);
SCOTCH_Num vertlocnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(vertlocnbr, nVtx, env);
SCOTCH_Num vertlocmax = vertlocnbr; // Assumes no holes in global nums.
// Get edge info
ArrayView<const gno_t> edgeIds;
ArrayView<const int> procIds;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
size_t nEdge = model->getEdgeList(edgeIds, procIds, offsets, ewgts);
SCOTCH_Num edgelocnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(edgelocnbr, nEdge, env);
const SCOTCH_Num edgelocsize = edgelocnbr; // Assumes adj array is compact.
SCOTCH_Num *vertloctab; // starting adj/vtx
TPL_Traits<SCOTCH_Num, lno_t>::ASSIGN_TPL_T_ARRAY(&vertloctab, offsets, env);
SCOTCH_Num *edgeloctab; // adjacencies
TPL_Traits<SCOTCH_Num, gno_t>::ASSIGN_TPL_T_ARRAY(&edgeloctab, edgeIds, env);
// We don't use these arrays, but we need them as arguments to Scotch.
SCOTCH_Num *vendloctab = NULL; // Assume consecutive storage for adj
SCOTCH_Num *vlblloctab = NULL; // Vertex label array
SCOTCH_Num *edgegsttab = NULL; // Array for ghost vertices
// Get weight info.
SCOTCH_Num *velotab = NULL; // Vertex weights
SCOTCH_Num *edlotab = NULL; // Edge weights
int nVwgts = model->getNumWeightsPerVertex();
int nEwgts = model->getNumWeightsPerEdge();
if (nVwgts > 1 && me == 0) {
std::cerr << "Warning: NumWeightsPerVertex is " << nVwgts
<< " but Scotch allows only one weight. "
<< " Zoltan2 will use only the first weight per vertex."
<< std::endl;
}
if (nEwgts > 1 && me == 0) {
std::cerr << "Warning: NumWeightsPerEdge is " << nEwgts
<< " but Scotch allows only one weight. "
<< " Zoltan2 will use only the first weight per edge."
<< std::endl;
}
if (nVwgts) {
velotab = new SCOTCH_Num[nVtx+1]; // +1 since Scotch wants all procs
// to have non-NULL arrays
scale_weights(nVtx, vwgts[0], velotab);
}
if (nEwgts) {
edlotab = new SCOTCH_Num[nEdge+1]; // +1 since Scotch wants all procs
// to have non-NULL arrays
scale_weights(nEdge, ewgts[0], edlotab);
}
// Build PTScotch distributed data structure
ierr = SCOTCH_dgraphBuild(gr, baseval, vertlocnbr, vertlocmax,
vertloctab, vendloctab, velotab, vlblloctab,
edgelocnbr, edgelocsize,
edgeloctab, edgegsttab, edlotab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphBuild",
!ierr, BASIC_ASSERTION, problemComm);
// Create array for Scotch to return results in.
ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx,true);
SCOTCH_Num *partloctab = NULL;
if (nVtx && (sizeof(SCOTCH_Num) == sizeof(part_t))) {
// Can write directly into the solution's memory
partloctab = (SCOTCH_Num *) partList.getRawPtr();
}
else {
// Can't use solution memory directly; will have to copy later.
// Note: Scotch does not like NULL arrays, so add 1 to always have non-null.
// ParMETIS has this same "feature." See Zoltan bug 4299.
partloctab = new SCOTCH_Num[nVtx+1];
}
// Get target part sizes
float *partsizes = new float[numGlobalParts];
if (!solution->criteriaHasUniformPartSizes(0))
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = solution->getCriteriaPartSize(0, i);
else
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = 1.0 / float(numGlobalParts);
// Allocate and initialize PTScotch target architecture data structure
SCOTCH_Arch archdat;
SCOTCH_archInit(&archdat);
SCOTCH_Num velosum = 0;
SCOTCH_dgraphSize (gr, &velosum, NULL, NULL, NULL);
SCOTCH_Num *goalsizes = new SCOTCH_Num[partnbr];
// TODO: The goalsizes are set as in Zoltan; not sure it is correct there
// or here.
// It appears velosum is global NUMBER of vertices, not global total
// vertex weight. I think we should use the latter.
// Fix this when we add vertex weights.
for (SCOTCH_Num i = 0; i < partnbr; i++)
goalsizes[i] = SCOTCH_Num(ceil(velosum * partsizes[i]));
delete [] partsizes;
SCOTCH_archCmpltw(&archdat, partnbr, goalsizes);
// Call partitioning; result returned in partloctab.
ierr = SCOTCH_dgraphMap(gr, &archdat, &stratstr, partloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphMap",
!ierr, BASIC_ASSERTION, problemComm);
SCOTCH_archExit(&archdat);
delete [] goalsizes;
// TODO - metrics
#ifdef SHOW_ZOLTAN2_SCOTCH_MEMORY
int me = env->comm_->getRank();
#endif
#ifdef HAVE_SCOTCH_ZOLTAN2_GETMEMORYMAX
if (me == 0){
size_t scotchBytes = SCOTCH_getMemoryMax();
std::cout << "Rank " << me << ": Maximum bytes used by Scotch: ";
std::cout << scotchBytes << std::endl;
}
#endif
// Clean up PTScotch
SCOTCH_dgraphExit(gr);
free(gr);
SCOTCH_stratExit(&stratstr);
// Load answer into the solution.
if ((sizeof(SCOTCH_Num) != sizeof(part_t)) || (nVtx == 0)) {
for (size_t i = 0; i < nVtx; i++) partList[i] = partloctab[i];
delete [] partloctab;
}
solution->setParts(partList);
env->memory("Zoltan2-Scotch: After creating solution");
// Clean up copies made due to differing data sizes.
TPL_Traits<SCOTCH_Num, lno_t>::DELETE_TPL_T_ARRAY(&vertloctab);
TPL_Traits<SCOTCH_Num, gno_t>::DELETE_TPL_T_ARRAY(&edgeloctab);
if (nVwgts) delete [] velotab;
if (nEwgts) delete [] edlotab;
#else // DO NOT HAVE_MPI
// TODO: Handle serial case with calls to Scotch.
// TODO: For now, assign everything to rank 0 and assume only one part.
// TODO: Can probably use the code above for loading solution,
// TODO: instead of duplicating it here.
// TODO
// TODO: Actual logic should call Scotch when number of processes == 1.
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
size_t nVtx = model->getVertexList(vtxID, xyz, vwgts);
ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx, true);
for (size_t i = 0; i < nVtx; i++) partList[i] = 0;
solution->setParts(partList);
#endif // DO NOT HAVE_MPI
}
int
main (
int argc,
char * argv[])
{
SCOTCH_Dgraph grafdat;
SCOTCH_Dordering ordedat;
SCOTCH_Strat stradat;
SCOTCH_Num straval;
char * straptr;
int flagval;
int procglbnbr;
int proclocnum;
int protglbnum; /* Root process */
Clock runtime[2]; /* Timing variables */
double reduloctab[12]; /* 3 * (min, max, sum) */
double reduglbtab[12];
MPI_Datatype redutype;
MPI_Op reduop;
int i, j;
#ifdef SCOTCH_PTHREAD
int thrdlvlreqval;
int thrdlvlproval;
#endif /* SCOTCH_PTHREAD */
errorProg ("dgord");
#ifdef SCOTCH_PTHREAD
thrdlvlreqval = MPI_THREAD_MULTIPLE;
if (MPI_Init_thread (&argc, &argv, thrdlvlreqval, &thrdlvlproval) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (1)");
if (thrdlvlreqval > thrdlvlproval)
errorPrint ("main: MPI implementation is not thread-safe: recompile without SCOTCH_PTHREAD");
#else /* SCOTCH_PTHREAD */
if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (2)");
#endif /* SCOTCH_PTHREAD */
MPI_Comm_size (MPI_COMM_WORLD, &procglbnbr); /* Get communicator data */
MPI_Comm_rank (MPI_COMM_WORLD, &proclocnum);
protglbnum = 0; /* Assume root process is process 0 */
if ((argc >= 2) && (argv[1][0] == '?')) { /* If need for help */
usagePrint (stdout, C_usageList);
return (0);
}
SCOTCH_randomProc (proclocnum); /* Record process number to initialize pseudo-random seed */
flagval = C_FLAGNONE; /* Default behavior */
straval = 0; /* No strategy flags */
straptr = NULL;
SCOTCH_stratInit (&stradat);
fileBlockInit (C_fileTab, C_FILENBR); /* Set default stream pointers */
for (i = 1; i < argc; i ++) { /* Loop for all option codes */
if ((argv[i][0] != '-') || (argv[i][1] == '\0') || (argv[i][1] == '.')) { /* If found a file name */
if (C_fileNum < C_FILEARGNBR) /* File name has been given */
fileBlockName (C_fileTab, C_fileNum ++) = argv[i];
else
errorPrint ("main: too many file names given");
}
else { /* If found an option name */
switch (argv[i][1]) {
case 'B' :
case 'b' :
flagval |= C_FLAGBLOCK;
break;
case 'C' :
case 'c' : /* Strategy selection parameters */
for (j = 2; argv[i][j] != '\0'; j ++) {
switch (argv[i][j]) {
case 'B' :
case 'b' :
straval |= SCOTCH_STRATBALANCE;
break;
case 'Q' :
case 'q' :
straval |= SCOTCH_STRATQUALITY;
break;
case 'S' :
case 's' :
straval |= SCOTCH_STRATSPEED;
break;
case 'T' :
case 't' :
straval |= SCOTCH_STRATSAFETY;
break;
case 'X' :
case 'x' :
straval |= SCOTCH_STRATSCALABILITY;
break;
default :
errorPrint ("main: invalid strategy selection option '%c'", argv[i][j]);
}
}
break;
#ifdef SCOTCH_DEBUG_ALL
case 'D' :
case 'd' :
flagval |= C_FLAGDEBUG;
break;
#endif /* SCOTCH_DEBUG_ALL */
case 'H' : /* Give the usage message */
case 'h' :
usagePrint (stdout, C_usageList);
return (0);
case 'M' : /* Output separator mapping */
case 'm' :
flagval |= C_FLAGMAPOUT;
if (argv[i][2] != '\0')
C_filenamemapout = &argv[i][2];
break;
case 'O' : /* Ordering strategy */
case 'o' :
straptr = &argv[i][2];
SCOTCH_stratExit (&stradat);
SCOTCH_stratInit (&stradat);
SCOTCH_stratDgraphOrder (&stradat, straptr);
break;
case 'R' : /* Root process (if necessary) */
case 'r' :
protglbnum = atoi (&argv[i][2]);
if ((protglbnum < 0) ||
(protglbnum >= procglbnbr) ||
((protglbnum == 0) && (argv[i][2] != '0')))
errorPrint ("main: invalid root process number");
break;
case 'T' : /* Output separator tree */
case 't' :
flagval |= C_FLAGTREOUT;
if (argv[i][2] != '\0')
C_filenametreout = &argv[i][2];
break;
case 'V' :
fprintf (stderr, "dgord, version " SCOTCH_VERSION_STRING "\n");
fprintf (stderr, "Copyright 2007-2012,2014 IPB, Universite de Bordeaux, INRIA & CNRS, France\n");
fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
return (0);
case 'v' : /* Output control info */
for (j = 2; argv[i][j] != '\0'; j ++) {
switch (argv[i][j]) {
case 'A' :
case 'a' :
#ifdef COMMON_MEMORY_TRACE
flagval |= C_FLAGVERBMEM;
#else /* COMMON_MEMORY_TRACE */
errorPrint ("main: not compiled with COMMON_MEMORY_TRACE");
#endif /* COMMON_MEMORY_TRACE */
break;
case 'S' :
case 's' :
flagval |= C_FLAGVERBSTR;
break;
case 'T' :
case 't' :
flagval |= C_FLAGVERBTIM;
break;
default :
errorPrint ("main: unprocessed parameter '%c' in '%s'", argv[i][j], argv[i]);
}
}
break;
default :
errorPrint ("main: unprocessed option '%s'", argv[i]);
}
}
}
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0) {
fprintf (stderr, "Proc %4d of %d, pid %d\n", proclocnum, procglbnbr, getpid ());
if (proclocnum == protglbnum) { /* Synchronize on keybord input */
char c;
printf ("Waiting for key press...\n");
scanf ("%c", &c);
}
MPI_Barrier (MPI_COMM_WORLD);
}
#endif /* SCOTCH_DEBUG_ALL */
fileBlockOpenDist (C_fileTab, C_FILENBR, procglbnbr, proclocnum, protglbnum); /* Open all files */
clockInit (&runtime[0]);
clockStart (&runtime[0]);
SCOTCH_dgraphInit (&grafdat, MPI_COMM_WORLD);
SCOTCH_dgraphLoad (&grafdat, C_filepntrsrcinp, -1, 0);
if (straval != 0) {
if (straptr != NULL)
errorPrint ("main: options '-c' and '-o' are exclusive");
SCOTCH_stratDgraphOrderBuild (&stradat, straval, (SCOTCH_Num) procglbnbr, 0, 0.2);
}
clockStop (&runtime[0]); /* Get input time */
clockInit (&runtime[1]);
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0)
MPI_Barrier (MPI_COMM_WORLD);
#endif /* SCOTCH_DEBUG_ALL */
clockStart (&runtime[1]);
SCOTCH_dgraphGhst (&grafdat); /* Compute it once for good */
SCOTCH_dgraphOrderInit (&grafdat, &ordedat);
SCOTCH_dgraphOrderCompute (&grafdat, &ordedat, &stradat);
clockStop (&runtime[1]); /* Get ordering time */
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0)
MPI_Barrier (MPI_COMM_WORLD);
#endif /* SCOTCH_DEBUG_ALL */
clockStart (&runtime[0]);
if (proclocnum == protglbnum) {
if ((flagval & C_FLAGBLOCK) == 0)
SCOTCH_dgraphOrderSave (&grafdat, &ordedat, C_filepntrordout);
else
SCOTCH_dgraphOrderSaveBlock (&grafdat, &ordedat, C_filepntrordout);
if ((flagval & C_FLAGMAPOUT) != 0) /* If mapping wanted */
SCOTCH_dgraphOrderSaveMap (&grafdat, &ordedat, C_filepntrmapout); /* Write mapping */
if ((flagval & C_FLAGTREOUT) != 0) /* If separator tree wanted */
SCOTCH_dgraphOrderSaveTree (&grafdat, &ordedat, C_filepntrtreout); /* Write tree */
}
else {
if ((flagval & C_FLAGBLOCK) == 0)
SCOTCH_dgraphOrderSave (&grafdat, &ordedat, NULL);
else
SCOTCH_dgraphOrderSaveBlock (&grafdat, &ordedat, NULL);
if ((flagval & C_FLAGMAPOUT) != 0)
SCOTCH_dgraphOrderSaveMap (&grafdat, &ordedat, NULL);
if ((flagval & C_FLAGTREOUT) != 0)
SCOTCH_dgraphOrderSaveTree (&grafdat, &ordedat, NULL);
}
clockStop (&runtime[0]);
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0)
MPI_Barrier (MPI_COMM_WORLD);
#endif /* SCOTCH_DEBUG_ALL */
MPI_Type_contiguous (3, MPI_DOUBLE, &redutype);
MPI_Type_commit (&redutype);
MPI_Op_create ((MPI_User_function *) dgordStatReduceOp, 1, &reduop);
if ((flagval & C_FLAGVERBTIM) != 0) {
reduloctab[0] =
reduloctab[1] =
reduloctab[2] = (double) clockVal (&runtime[1]);
reduloctab[3] =
reduloctab[4] =
reduloctab[5] = (double) clockVal (&runtime[0]);
reduloctab[6] =
reduloctab[7] =
reduloctab[8] = reduloctab[0] + reduloctab[3];
MPI_Allreduce (&reduloctab[0], &reduglbtab[0], 3, redutype, reduop, MPI_COMM_WORLD);
}
#ifdef COMMON_MEMORY_TRACE
if ((flagval & C_FLAGVERBMEM) != 0) {
reduloctab[9] =
reduloctab[10] =
reduloctab[11] = (double) memMax ();
MPI_Allreduce (&reduloctab[9], &reduglbtab[9], 1, redutype, reduop, MPI_COMM_WORLD);
}
#endif /* COMMON_MEMORY_TRACE */
MPI_Op_free (&reduop);
MPI_Type_free (&redutype);
if (C_filepntrlogout != NULL) {
if ((flagval & C_FLAGVERBSTR) != 0) {
fprintf (C_filepntrlogout, "S\tStrat=");
SCOTCH_stratSave (&stradat, C_filepntrlogout);
putc ('\n', C_filepntrlogout);
}
if ((flagval & C_FLAGVERBTIM) != 0) {
fprintf (C_filepntrlogout, "T\tOrder\tmin=%g\tmax=%g\tavg=%g\nT\tI/O\tmin=%g\tmax=%g\tavg=%g\nT\tTotal\tmin=%g\tmax=%g\tavg=%g\n",
reduglbtab[0], reduglbtab[1], reduglbtab[2] / (double) procglbnbr,
reduglbtab[3], reduglbtab[4], reduglbtab[5] / (double) procglbnbr,
reduglbtab[6], reduglbtab[7], reduglbtab[8] / (double) procglbnbr);
}
#ifdef COMMON_MEMORY_TRACE
if ((flagval & C_FLAGVERBMEM) != 0)
fprintf (C_filepntrlogout, "A\tMemory\tmin=%g\tmax=%g\tavg=%g\n",
reduglbtab[9], reduglbtab[10], reduglbtab[11] / (double) procglbnbr);
#endif /* COMMON_MEMORY_TRACE */
}
fileBlockClose (C_fileTab, C_FILENBR); /* Always close explicitely to end eventual (un)compression tasks */
SCOTCH_dgraphOrderExit (&grafdat, &ordedat);
SCOTCH_dgraphExit (&grafdat);
SCOTCH_stratExit (&stradat);
MPI_Finalize ();
#ifdef COMMON_PTHREAD
pthread_exit ((void *) 0); /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
return (0);
}
void AlgPTScotch(
const RCP<const Environment> &env, // parameters & app comm
const RCP<const Comm<int> > &problemComm, // problem comm
#ifdef HAVE_ZOLTAN2_MPI
MPI_Comm mpicomm,
#endif
const RCP<GraphModel<typename Adapter::base_adapter_t> > &model, // the graph
RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
typedef typename Adapter::lno_t lno_t;
typedef typename Adapter::gno_t gno_t;
typedef typename Adapter::scalar_t scalar_t;
int ierr = 0;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
SCOTCH_Num partnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(partnbr, numGlobalParts, env);
#ifdef HAVE_ZOLTAN2_MPI
const SCOTCH_Num baseval = 0; // Base value for array indexing.
// GraphModel returns GNOs from base 0.
SCOTCH_Strat stratstr; // Strategy string
// TODO: Set from parameters
SCOTCH_stratInit(&stratstr);
// Allocate & initialize PTScotch data structure.
SCOTCH_Dgraph *gr = SCOTCH_dgraphAlloc(); // Scotch distributed graph
ierr = SCOTCH_dgraphInit(gr, mpicomm);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphInit",
!ierr, BASIC_ASSERTION, problemComm);
// Get vertex info
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vtxWt;
size_t nVtx = model->getVertexList(vtxID, xyz, vtxWt);
SCOTCH_Num vertlocnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(vertlocnbr, nVtx, env);
SCOTCH_Num vertlocmax = vertlocnbr; // Assumes no holes in global nums.
// Get edge info
ArrayView<const gno_t> edgeIds;
ArrayView<const int> procIds;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
size_t nEdges = model->getEdgeList(edgeIds, procIds, offsets, ewgts);
SCOTCH_Num edgelocnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(edgelocnbr, nEdges, env);
const SCOTCH_Num edgelocsize = edgelocnbr; // Assumes adj array is compact.
SCOTCH_Num *vertloctab; // starting adj/vtx
SCOTCH_Num_Traits<lno_t>::ASSIGN_SCOTCH_NUM_ARRAY(&vertloctab, offsets, env);
SCOTCH_Num *edgeloctab; // adjacencies
SCOTCH_Num_Traits<gno_t>::ASSIGN_SCOTCH_NUM_ARRAY(&edgeloctab, edgeIds, env);
// We don't use these arrays, but we need them as arguments to Scotch.
SCOTCH_Num *vendloctab = NULL; // Assume consecutive storage for adj
SCOTCH_Num *vlblloctab = NULL; // Vertex label array
SCOTCH_Num *edgegsttab = NULL; // Array for ghost vertices
// Get weight info.
// TODO: Actually get the weights; for now, not using weights.
SCOTCH_Num *veloloctab = NULL; // Vertex weights
SCOTCH_Num *edloloctab = NULL; // Edge weights
//TODO int vwtdim = model->getVertexWeightDim();
//TODO int ewtdim = model->getEdgeWeightDim();
//TODO if (vwtdim) veloloctab = new SCOTCH_Num[nVtx];
//TODO if (ewtdim) edloloctab = new SCOTCH_Num[nEdges];
//TODO scale weights to SCOTCH_Nums.
// Build PTScotch distributed data structure
ierr = SCOTCH_dgraphBuild(gr, baseval, vertlocnbr, vertlocmax,
vertloctab, vendloctab, veloloctab, vlblloctab,
edgelocnbr, edgelocsize,
edgeloctab, edgegsttab, edloloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphBuild",
!ierr, BASIC_ASSERTION, problemComm);
// Create array for Scotch to return results in.
ArrayRCP<partId_t> partList(new partId_t [nVtx], 0, nVtx,true);
SCOTCH_Num *partloctab = NULL;
if (nVtx && (sizeof(SCOTCH_Num) == sizeof(partId_t))) {
// Can write directly into the solution's memory
partloctab = (SCOTCH_Num *) partList.getRawPtr();
}
else {
// Can't use solution memory directly; will have to copy later.
// Note: Scotch does not like NULL arrays, so add 1 to always have non-null.
// ParMETIS has this same "feature." See Zoltan bug 4299.
partloctab = new SCOTCH_Num[nVtx+1];
}
// Call partitioning; result returned in partloctab.
// TODO: Use SCOTCH_dgraphMap so can include a machine model in partitioning
ierr = SCOTCH_dgraphPart(gr, partnbr, &stratstr, partloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphPart",
!ierr, BASIC_ASSERTION, problemComm);
// TODO - metrics
#ifdef SHOW_ZOLTAN2_SCOTCH_MEMORY
int me = env->comm_->getRank();
#endif
#ifdef HAVE_SCOTCH_ZOLTAN2_GETMEMORYMAX
if (me == 0){
size_t scotchBytes = SCOTCH_getMemoryMax();
std::cout << "Rank " << me << ": Maximum bytes used by Scotch: ";
std::cout << scotchBytes << std::endl;
}
#endif
// Clean up PTScotch
SCOTCH_dgraphExit(gr);
delete gr;
SCOTCH_stratExit(&stratstr);
// Load answer into the solution.
if ((sizeof(SCOTCH_Num) != sizeof(partId_t)) || (nVtx == 0)) {
for (size_t i = 0; i < nVtx; i++) partList[i] = partloctab[i];
delete [] partloctab;
}
ArrayRCP<const gno_t> gnos = arcpFromArrayView(vtxID);
solution->setParts(gnos, partList, true);
env->memory("Zoltan2-Scotch: After creating solution");
//if (me == 0) cout << " done." << endl;
// Clean up Zoltan2
//TODO if (vwtdim) delete [] velotab;
//TODO if (ewtdim) delete [] edlotab;
// Clean up copies made due to differing data sizes.
SCOTCH_Num_Traits<lno_t>::DELETE_SCOTCH_NUM_ARRAY(&vertloctab);
SCOTCH_Num_Traits<gno_t>::DELETE_SCOTCH_NUM_ARRAY(&edgeloctab);
#else // DO NOT HAVE_MPI
// TODO: Handle serial case with calls to Scotch.
// TODO: For now, assign everything to rank 0 and assume only one part.
// TODO: Can probably use the code above for loading solution,
// TODO: instead of duplicating it here.
// TODO
// TODO: Actual logic should call Scotch when number of processes == 1.
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vtxWt;
size_t nVtx = model->getVertexList(vtxID, xyz, vtxWt);
for (size_t i = 0; i < nVtx; i++) partList[i] = 0;
#endif // DO NOT HAVE_MPI
}
void
METISNAMEU(ParMETIS_V3_PartKway) (
const SCOTCH_Num * const vtxdist,
SCOTCH_Num * const xadj,
SCOTCH_Num * const adjncy,
SCOTCH_Num * const vwgt,
SCOTCH_Num * const adjwgt,
const SCOTCH_Num * const wgtflag,
const SCOTCH_Num * const numflag,
const SCOTCH_Num * const ncon, /* Not used */
const SCOTCH_Num * const nparts,
const float * const tpwgts,
const float * const ubvec, /* Not used */
const SCOTCH_Num * const options, /* Not used */
SCOTCH_Num * const edgecut,
SCOTCH_Num * const part,
MPI_Comm * comm)
{
MPI_Comm proccomm;
int procglbnbr;
int proclocnum;
SCOTCH_Num baseval;
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat; /* Scotch distributed graph object to interface with libScotch */
SCOTCH_Dmapping mappdat; /* Scotch distributed mapping object to interface with libScotch */
SCOTCH_Strat stradat;
SCOTCH_Num vertlocnbr;
SCOTCH_Num * veloloctab;
SCOTCH_Num edgelocnbr;
SCOTCH_Num * edloloctab;
SCOTCH_Num * velotab;
double * vwgttab;
SCOTCH_Num i;
if ((vwgttab = malloc (*nparts * sizeof (double))) == NULL)
return;
if ((velotab = malloc (*nparts * sizeof (SCOTCH_Num))) == NULL) {
free (vwgttab);
return;
}
for (i = 0; i < *nparts; i ++)
vwgttab[i] = (double) tpwgts[i] * (double) (*nparts);
for (i = 0; i < *nparts; i ++) {
double deltval;
deltval = fabs (vwgttab[i] - floor (vwgttab[i] + 0.5));
if (deltval > 0.01) {
SCOTCH_Num j;
deltval = 1.0 / deltval;
for (j = 0; j < *nparts; j ++)
vwgttab[j] *= deltval;
}
}
for (i = 0; i < *nparts; i ++)
velotab[i] = (SCOTCH_Num) (vwgttab[i] + 0.5);
proccomm = *comm;
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0)
return;
MPI_Comm_size (proccomm, &procglbnbr);
MPI_Comm_rank (proccomm, &proclocnum);
baseval = *numflag;
vertlocnbr = vtxdist[proclocnum + 1] - vtxdist[proclocnum];
edgelocnbr = xadj[vertlocnbr] - baseval;
veloloctab = ((vwgt != NULL) && ((*wgtflag & 2) != 0)) ? vwgt : NULL;
edloloctab = ((adjwgt != NULL) && ((*wgtflag & 1) != 0)) ? adjwgt : NULL;
if (SCOTCH_dgraphBuild (&grafdat, baseval,
vertlocnbr, vertlocnbr, xadj, xadj + 1, veloloctab, NULL,
edgelocnbr, edgelocnbr, adjncy, NULL, edloloctab) == 0) {
SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_dgraphCheck (&grafdat) == 0) /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
{
SCOTCH_archInit (&archdat);
if ((SCOTCH_archCmpltw (&archdat, *nparts, velotab) == 0) &&
(SCOTCH_dgraphMapInit (&grafdat, &mappdat, &archdat, part) == 0)) {
SCOTCH_dgraphMapCompute (&grafdat, &mappdat, &stradat);
SCOTCH_dgraphMapExit (&grafdat, &mappdat);
}
SCOTCH_archExit (&archdat);
}
SCOTCH_stratExit (&stradat);
}
SCOTCH_dgraphExit (&grafdat);
*edgecut = 0; /* TODO : compute real edge cut for people who might want it */
free (vwgttab);
free (velotab);
if (baseval != 0) { /* MeTiS part array is based, Scotch is not */
SCOTCH_Num vertlocnum;
for (vertlocnum = 0; vertlocnum < vertlocnbr; vertlocnum ++)
part[vertlocnum] += baseval;
}
}
// Call scotch with options from dictionary.
Foam::label Foam::ptscotchDecomp::decompose
(
const fileName& meshPath,
const List<int>& adjncy,
const List<int>& xadj,
const scalarField& cWeights,
List<int>& finalDecomp
) const
{
if (debug)
{
Pout<< "ptscotchDecomp : entering with xadj:" << xadj.size() << endl;
}
// Dump graph
if (decompositionDict_.found("ptscotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("ptscotchCoeffs");
if (scotchCoeffs.lookupOrDefault("writeGraph", false))
{
OFstream str
(
meshPath + "_" + Foam::name(Pstream::myProcNo()) + ".dgr"
);
Pout<< "Dumping Scotch graph file to " << str.name() << endl
<< "Use this in combination with dgpart." << endl;
globalIndex globalCells(xadj.size()-1);
// Distributed graph file (.grf)
label version = 2;
str << version << nl;
// Number of files (procglbnbr)
str << Pstream::nProcs();
// My file number (procloc)
str << ' ' << Pstream::myProcNo() << nl;
// Total number of vertices (vertglbnbr)
str << globalCells.size();
// Total number of connections (edgeglbnbr)
str << ' ' << returnReduce(xadj[xadj.size()-1], sumOp<label>())
<< nl;
// Local number of vertices (vertlocnbr)
str << xadj.size()-1;
// Local number of connections (edgelocnbr)
str << ' ' << xadj[xadj.size()-1] << nl;
// Numbering starts from 0
label baseval = 0;
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs
str << baseval << ' ' << "000" << nl;
for (label cellI = 0; cellI < xadj.size()-1; cellI++)
{
label start = xadj[cellI];
label end = xadj[cellI+1];
str << end-start;
for (label i = start; i < end; i++)
{
str << ' ' << adjncy[i];
}
str << nl;
}
}
}
// Strategy
// ~~~~~~~~
// Default.
SCOTCH_Strat stradat;
check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
string strategy;
if (scotchCoeffs.readIfPresent("strategy", strategy))
{
if (debug)
{
Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
}
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
//fprintf(stdout, "\n");
}
}
// Graph
// ~~~~~
List<int> velotab;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningIn
(
"ptscotchDecomp::decompose(..)"
) << "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != xadj.size()-1)
{
FatalErrorIn
(
"ptscotchDecomp::decompose(..)"
) << "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << xadj.size()-1
<< exit(FatalError);
}
// Convert to integers.
velotab.setSize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int(cWeights[i]/minWeights);
}
}
if (debug)
{
Pout<< "SCOTCH_dgraphInit" << endl;
}
SCOTCH_Dgraph grafdat;
check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
if (debug)
{
Pout<< "SCOTCH_dgraphBuild with:" << nl
<< "xadj.size()-1 : " << xadj.size()-1 << nl
<< "xadj : " << long(xadj.begin()) << nl
<< "velotab : " << long(velotab.begin()) << nl
<< "adjncy.size() : " << adjncy.size() << nl
<< "adjncy : " << long(adjncy.begin()) << nl
<< endl;
}
check
(
SCOTCH_dgraphBuild
(
&grafdat, // grafdat
0, // baseval, c-style numbering
xadj.size()-1, // vertlocnbr, nCells
xadj.size()-1, // vertlocmax
const_cast<SCOTCH_Num*>(xadj.begin()),
// vertloctab, start index per cell into
// adjncy
const_cast<SCOTCH_Num*>(&xadj[1]),// vendloctab, end index ,,
const_cast<SCOTCH_Num*>(velotab.begin()),// veloloctab, vtx weights
NULL, // vlblloctab
adjncy.size(), // edgelocnbr, number of arcs
adjncy.size(), // edgelocsiz
const_cast<SCOTCH_Num*>(adjncy.begin()), // edgeloctab
NULL, // edgegsttab
NULL // edlotab, edge weights
),
"SCOTCH_dgraphBuild"
);
if (debug)
{
Pout<< "SCOTCH_dgraphCheck" << endl;
}
check(SCOTCH_dgraphCheck(&grafdat), "SCOTCH_dgraphCheck");
// Architecture
// ~~~~~~~~~~~~
// (fully connected network topology since using switch)
if (debug)
{
Pout<< "SCOTCH_archInit" << endl;
}
SCOTCH_Arch archdat;
check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
List<label> processorWeights;
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
scotchCoeffs.readIfPresent("processorWeights", processorWeights);
}
if (processorWeights.size())
{
if (debug)
{
Info<< "ptscotchDecomp : Using procesor weights "
<< processorWeights
<< endl;
}
check
(
SCOTCH_archCmpltw(&archdat, nProcessors_, processorWeights.begin()),
"SCOTCH_archCmpltw"
);
}
else
{
if (debug)
{
Pout<< "SCOTCH_archCmplt" << endl;
}
check
(
SCOTCH_archCmplt(&archdat, nProcessors_),
"SCOTCH_archCmplt"
);
}
//SCOTCH_Mapping mapdat;
//SCOTCH_dgraphMapInit(&grafdat, &mapdat, &archdat, NULL);
//SCOTCH_dgraphMapCompute(&grafdat, &mapdat, &stradat); /*Perform mapping*/
//SCOTCHdgraphMapExit(&grafdat, &mapdat);
// Hack:switch off fpu error trapping
# ifdef LINUX_GNUC
int oldExcepts = fedisableexcept
(
FE_DIVBYZERO
| FE_INVALID
| FE_OVERFLOW
);
# endif
if (debug)
{
Pout<< "SCOTCH_dgraphMap" << endl;
}
finalDecomp.setSize(xadj.size()-1);
finalDecomp = 0;
check
(
SCOTCH_dgraphMap
(
&grafdat,
&archdat,
&stradat, // const SCOTCH_Strat *
finalDecomp.begin() // parttab
),
"SCOTCH_graphMap"
);
# ifdef LINUX_GNUC
feenableexcept(oldExcepts);
# endif
//finalDecomp.setSize(xadj.size()-1);
//check
//(
// SCOTCH_dgraphPart
// (
// &grafdat,
// nProcessors_, // partnbr
// &stradat, // const SCOTCH_Strat *
// finalDecomp.begin() // parttab
// ),
// "SCOTCH_graphPart"
//);
if (debug)
{
Pout<< "SCOTCH_dgraphExit" << endl;
}
// Release storage for graph
SCOTCH_dgraphExit(&grafdat);
// Release storage for strategy
SCOTCH_stratExit(&stradat);
// Release storage for network topology
SCOTCH_archExit(&archdat);
return 0;
}
int
main (
int argc,
char * argv[])
{
SCOTCH_Dgraph grafdat;
int procglbnbr;
int proclocnum;
int protglbnum; /* Root process */
SCOTCH_Num vertnbr;
SCOTCH_Num velomin;
SCOTCH_Num velomax;
SCOTCH_Num velosum;
double veloavg;
double velodlt;
SCOTCH_Num degrmin;
SCOTCH_Num degrmax;
double degravg;
double degrdlt;
SCOTCH_Num edgenbr;
SCOTCH_Num edlomin;
SCOTCH_Num edlomax;
SCOTCH_Num edlosum;
double edloavg;
double edlodlt;
int flagval;
int i;
#ifdef SCOTCH_PTHREAD
int thrdlvlreqval;
int thrdlvlproval;
#endif /* SCOTCH_PTHREAD */
errorProg ("dgtst");
#ifdef SCOTCH_PTHREAD
thrdlvlreqval = MPI_THREAD_MULTIPLE;
if (MPI_Init_thread (&argc, &argv, thrdlvlreqval, &thrdlvlproval) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (1)");
if (thrdlvlreqval > thrdlvlproval)
errorPrint ("main: MPI implementation is not thread-safe: recompile without SCOTCH_PTHREAD");
#else /* SCOTCH_PTHREAD */
if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (2)");
#endif /* SCOTCH_PTHREAD */
MPI_Comm_size (MPI_COMM_WORLD, &procglbnbr); /* Get communicator data */
MPI_Comm_rank (MPI_COMM_WORLD, &proclocnum);
protglbnum = 0; /* Assume root process is process 0 */
if ((argc >= 2) && (argv[1][0] == '?')) { /* If need for help */
usagePrint (stdout, C_usageList);
return (0);
}
flagval = C_FLAGNONE;
for (i = 0; i < C_FILENBR; i ++) /* Set default stream pointers */
C_fileTab[i].pntr = (C_fileTab[i].mode[0] == 'r') ? stdin : stdout;
for (i = 1; i < argc; i ++) { /* Loop for all option codes */
if ((argv[i][0] != '+') && /* If found a file name */
((argv[i][0] != '-') || (argv[i][1] == '\0'))) {
if (C_fileNum < C_FILEARGNBR) /* A file name has been given */
C_fileTab[C_fileNum ++].name = argv[i];
else
errorPrint ("main: too many file names given");
}
else { /* If found an option name */
switch (argv[i][1]) {
#ifdef SCOTCH_DEBUG_ALL
case 'D' :
case 'd' :
flagval |= C_FLAGDEBUG;
break;
#endif /* SCOTCH_DEBUG_ALL */
case 'H' : /* Give the usage message */
case 'h' :
usagePrint (stdout, C_usageList);
return (0);
case 'R' : /* Root process (if necessary) */
case 'r' :
protglbnum = atoi (&argv[i][2]);
if ((protglbnum < 0) ||
(protglbnum >= procglbnbr) ||
((protglbnum == 0) && (argv[i][2] != '0')))
errorPrint ("main: invalid root process number");
break;
case 'V' :
case 'v' :
fprintf (stderr, "dgtst, version " SCOTCH_VERSION_STRING "\n");
fprintf (stderr, "Copyright 2007,2008,2010,2011 ENSEIRB, INRIA & CNRS, France\n");
fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
return (0);
default :
errorPrint ("main: unprocessed option '%s'", argv[i]);
}
}
}
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0) {
fprintf (stderr, "Proc %4d of %d, pid %d\n", proclocnum, procglbnbr, getpid ());
if (proclocnum == protglbnum) { /* Synchronize on keybord input */
char c;
printf ("Waiting for key press...\n");
scanf ("%c", &c);
}
MPI_Barrier (MPI_COMM_WORLD);
}
#endif /* SCOTCH_DEBUG_ALL */
fileBlockOpenDist (C_fileTab, C_FILENBR, procglbnbr, proclocnum, protglbnum); /* Open all files */
SCOTCH_dgraphInit (&grafdat, MPI_COMM_WORLD);
SCOTCH_dgraphLoad (&grafdat, C_filepntrsrcinp, -1, 0);
SCOTCH_dgraphCheck (&grafdat);
SCOTCH_dgraphSize (&grafdat, &vertnbr, NULL, &edgenbr, NULL);
SCOTCH_dgraphStat (&grafdat, &velomin, &velomax, &velosum, &veloavg, &velodlt,
°rmin, °rmax, °ravg, °rdlt,
&edlomin, &edlomax, &edlosum, &edloavg, &edlodlt);
if (C_filepntrdatout != NULL) {
fprintf (C_filepntrdatout, "S\tVertex\tnbr=" SCOTCH_NUMSTRING "\n",
(SCOTCH_Num) vertnbr);
fprintf (C_filepntrdatout, "S\tVertex load\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) velomin, (SCOTCH_Num) velomax, (SCOTCH_Num) velosum, veloavg, velodlt);
fprintf (C_filepntrdatout, "S\tVertex degree\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) degrmin, (SCOTCH_Num) degrmax, (SCOTCH_Num) edgenbr, degravg, degrdlt);
fprintf (C_filepntrdatout, "S\tEdge\tnbr=" SCOTCH_NUMSTRING "\n",
(SCOTCH_Num) (edgenbr / 2));
fprintf (C_filepntrdatout, "S\tEdge load\tmin=" SCOTCH_NUMSTRING "\tmax=" SCOTCH_NUMSTRING "\tsum=" SCOTCH_NUMSTRING "\tavg=%g\tdlt=%g\n",
(SCOTCH_Num) edlomin, (SCOTCH_Num) edlomax, (SCOTCH_Num) edlosum, edloavg, edlodlt);
}
fileBlockClose (C_fileTab, C_FILENBR); /* Always close explicitely to end eventual (un)compression tasks */
SCOTCH_dgraphExit (&grafdat);
MPI_Finalize ();
#ifdef COMMON_PTHREAD
pthread_exit ((void *) 0); /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
return (0);
}
int
main (
int argc,
char * argv[])
{
SCOTCH_Graph * cgrfptr;
SCOTCH_Graph cgrfdat;
SCOTCH_Dgraph dgrfdat;
int procglbnbr;
int proclocnum;
int protglbnum; /* Root process */
int flagval;
int i;
int reduloctab[2];
int reduglbtab[2];
#ifdef SCOTCH_PTHREAD
int thrdlvlreqval;
int thrdlvlproval;
#endif /* SCOTCH_PTHREAD */
errorProg ("dggath");
#ifdef SCOTCH_PTHREAD
thrdlvlreqval = MPI_THREAD_MULTIPLE;
if (MPI_Init_thread (&argc, &argv, thrdlvlreqval, &thrdlvlproval) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (1)");
if (thrdlvlreqval > thrdlvlproval)
errorPrint ("main: MPI implementation is not thread-safe: recompile without SCOTCH_PTHREAD");
#else /* SCOTCH_PTHREAD */
if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (2)");
#endif /* SCOTCH_PTHREAD */
MPI_Comm_size (MPI_COMM_WORLD, &procglbnbr); /* Get communicator data */
MPI_Comm_rank (MPI_COMM_WORLD, &proclocnum);
protglbnum = 0; /* Assume root process is process 0 */
if ((argc >= 2) && (argv[1][0] == '?')) { /* If need for help */
usagePrint (stdout, C_usageList);
return (0);
}
flagval = C_FLAGNONE;
for (i = 0; i < C_FILENBR; i ++) /* Set default stream pointers */
C_fileTab[i].pntr = (C_fileTab[i].mode[0] == 'r') ? stdin : stdout;
for (i = 1; i < argc; i ++) { /* Loop for all option codes */
if ((argv[i][0] != '-') || (argv[i][1] == '\0') || (argv[i][1] == '.')) { /* If found a file name */
if (C_fileNum < C_FILEARGNBR) /* File name has been given */
C_fileTab[C_fileNum ++].name = argv[i];
else
errorPrint ("main: too many file names given");
}
else { /* If found an option name */
switch (argv[i][1]) {
case 'C' :
case 'c' :
flagval |= C_FLAGCHECK;
break;
#ifdef SCOTCH_DEBUG_ALL
case 'D' :
case 'd' :
flagval |= C_FLAGDEBUG;
break;
#endif /* SCOTCH_DEBUG_ALL */
case 'H' : /* Give the usage message */
case 'h' :
usagePrint (stdout, C_usageList);
return (0);
case 'R' : /* Root process (if necessary) */
case 'r' :
protglbnum = atoi (&argv[i][2]);
if ((protglbnum < 0) ||
(protglbnum >= procglbnbr) ||
((protglbnum == 0) && (argv[i][2] != '0'))) {
errorPrint ("main: invalid root process number");
}
break;
case 'V' :
case 'v' :
fprintf (stderr, "dggath, version " SCOTCH_VERSION_STRING "\n");
fprintf (stderr, "Copyright 2008,2010 ENSEIRB, INRIA & CNRS, France\n");
fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
return (0);
default :
errorPrint ("main: unprocessed option (\"%s\")", argv[i]);
}
}
}
#ifdef SCOTCH_DEBUG_ALL
if ((flagval & C_FLAGDEBUG) != 0) {
fprintf (stderr, "Proc %4d of %d, pid %d\n", proclocnum, procglbnbr, getpid ());
if (proclocnum == protglbnum) { /* Synchronize on keybord input */
char c;
printf ("Waiting for key press...\n");
scanf ("%c", &c);
}
MPI_Barrier (MPI_COMM_WORLD);
}
#endif /* SCOTCH_DEBUG_ALL */
fileBlockOpenDist (C_fileTab, C_FILENBR, procglbnbr, proclocnum, protglbnum); /* Open all files */
if (C_filepntrsrcout == NULL) {
cgrfptr = NULL;
reduloctab[0] =
reduloctab[1] = 0;
}
else {
cgrfptr = &cgrfdat;
reduloctab[0] = 1;
reduloctab[1] = proclocnum;
}
if (MPI_Allreduce (reduloctab, reduglbtab, 2, MPI_INT, MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS)
errorPrint ("main: communication error");
if (reduglbtab[0] != 1)
errorPrint ("main: should have only one root");
if (reduglbtab[1] != protglbnum)
errorPrint ("main: root process mismatch");
SCOTCH_dgraphInit (&dgrfdat, MPI_COMM_WORLD);
SCOTCH_dgraphLoad (&dgrfdat, C_filepntrsrcinp, -1, 0);
if ((flagval & C_FLAGCHECK) != 0)
SCOTCH_dgraphCheck (&dgrfdat);
SCOTCH_graphInit (&cgrfdat);
SCOTCH_dgraphGather (&dgrfdat, cgrfptr);
if (cgrfptr != NULL)
SCOTCH_graphSave (cgrfptr, C_filepntrsrcout);
fileBlockClose (C_fileTab, C_FILENBR); /* Always close explicitely to end eventual (un)compression tasks */
SCOTCH_graphExit (&cgrfdat);
SCOTCH_dgraphExit (&dgrfdat);
MPI_Finalize ();
#ifdef COMMON_PTHREAD
pthread_exit ((void *) 0); /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
return (0);
}
int
main (
int argc,
char * argv[])
{
MPI_Comm proccomm;
int procglbnbr; /* Number of processes sharing graph data */
int proclocnum; /* Number of this process */
SCOTCH_Num vertglbnbr;
SCOTCH_Num vertlocnbr;
SCOTCH_Num baseval;
SCOTCH_Dgraph grafdat;
SCOTCH_Dgraph coargrafdat;
SCOTCH_Num coarvertglbnbr;
SCOTCH_Num coarvertlocnbr;
double coarrat;
FILE * file;
#ifdef SCOTCH_PTHREAD
int thrdlvlreqval;
int thrdlvlproval;
#endif /* SCOTCH_PTHREAD */
int i;
errorProg (argv[0]);
#ifdef SCOTCH_PTHREAD
thrdlvlreqval = MPI_THREAD_MULTIPLE;
if (MPI_Init_thread (&argc, &argv, thrdlvlreqval, &thrdlvlproval) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (1)");
if (thrdlvlreqval > thrdlvlproval)
errorPrint ("main: MPI implementation is not thread-safe: recompile without SCOTCH_PTHREAD");
#else /* SCOTCH_PTHREAD */
if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
errorPrint ("main: Cannot initialize (2)");
#endif /* SCOTCH_PTHREAD */
if (argc != 2) {
errorPrint ("main: invalid number of parameters");
exit (1);
}
proccomm = MPI_COMM_WORLD;
MPI_Comm_size (proccomm, &procglbnbr); /* Get communicator data */
MPI_Comm_rank (proccomm, &proclocnum);
fprintf (stderr, "Proc %2d of %2d, pid %d\n", proclocnum, procglbnbr, getpid ());
#ifndef SCOTCH_CHECK_AUTO
if (proclocnum == 0) { /* Synchronize on keybord input */
char c;
printf ("Waiting for key press...\n");
scanf ("%c", &c);
}
#endif /* SCOTCH_CHECK_AUTO */
if (MPI_Barrier (proccomm) != MPI_SUCCESS) { /* Synchronize for debug */
errorPrint ("main: cannot communicate");
return (1);
}
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0) { /* Initialize source graph */
errorPrint ("main: cannot initialize graph (1)");
return (1);
}
file = NULL;
if ((proclocnum == 0) &&
((file = fopen (argv[1], "r")) == NULL)) {
errorPrint ("main: cannot open graph file");
return (1);
}
if (SCOTCH_dgraphLoad (&grafdat, file, 0, 0) != 0) {
errorPrint ("main: cannot load graph");
return (1);
}
if (file != NULL)
fclose (file);
if (MPI_Barrier (proccomm) != MPI_SUCCESS) { /* Synchronize for debug */
errorPrint ("main: cannot communicate");
return (1);
}
SCOTCH_dgraphData (&grafdat, NULL, &vertglbnbr, &vertlocnbr, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
coarrat = 0.8; /* Lazy coarsening ratio */
for (i = 0; i < 3; i ++) { /* For all test cases */
SCOTCH_Num * multloctab;
SCOTCH_Num multlocsiz;
SCOTCH_Num foldval;
char * foldstr;
char * coarstr;
int procnum;
int o;
switch (i) {
case 0 :
multlocsiz = vertlocnbr;
foldval = SCOTCH_COARSENNONE;
foldstr = "Plain coarsening";
break;
case 1 :
multlocsiz = (SCOTCH_Num) (((double) vertglbnbr * coarrat) / (double) (procglbnbr / 2)) + 1;
foldval = SCOTCH_COARSENFOLD;
foldstr = "Folding";
break;
case 2 :
multlocsiz = (SCOTCH_Num) (((double) vertglbnbr * coarrat) / (double) (procglbnbr / 2)) + 1;
foldval = SCOTCH_COARSENFOLDDUP;
foldstr = "Folding with duplication";
break;
}
if (proclocnum == 0)
printf ("%s\n", foldstr);
if ((multloctab = malloc (multlocsiz * 2 * sizeof (SCOTCH_Num))) == NULL) {
errorPrint ("main: cannot allocate multinode array");
return (1);
}
if (SCOTCH_dgraphInit (&coargrafdat, proccomm) != 0) { /* Initialize band graph */
errorPrint ("main: cannot initialize graph (2)");
return (1);
}
o = SCOTCH_dgraphCoarsen (&grafdat, 0, coarrat, foldval, &coargrafdat, multloctab);
SCOTCH_dgraphData (&coargrafdat, NULL, &coarvertglbnbr, &coarvertlocnbr, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
for (procnum = 0; procnum < procglbnbr; procnum ++) {
switch (o) {
case 0 :
coarstr = "coarse graph created";
break;
case 1 :
coarstr = "graph could not be coarsened";
break;
case 2 :
coarstr = "folded graph not created here";
break;
case 3 :
coarstr = "cannot create coarse graph";
break;
}
if (procnum == proclocnum)
printf ("%d: %s (%ld / %ld / %ld)\n", procnum, coarstr, (long) multlocsiz, (long) coarvertlocnbr, (long) vertlocnbr);
MPI_Barrier (proccomm);
}
if (coarvertlocnbr > multlocsiz) {
errorPrint ("main: invalid local multinode array size");
return (1);
}
SCOTCH_dgraphExit (&coargrafdat);
free (multloctab);
}
SCOTCH_dgraphExit (&grafdat);
MPI_Finalize ();
exit (0);
}
int
main (
int argc,
char * argv[])
{
MPI_Comm proccomm;
int procglbnbr; /* Number of processes sharing graph data */
int proclocnum; /* Number of this process */
SCOTCH_Num vertglbnbr;
SCOTCH_Num vertlocnbr;
SCOTCH_Num * fronloctab;
SCOTCH_Num baseval;
SCOTCH_Dgraph grafdat;
SCOTCH_Dgraph bandgrafdat;
SCOTCH_Num bandvertglbnbr;
SCOTCH_Num bandvertlocnbr;
SCOTCH_Num * bandvlblloctab;
FILE * file;
int procnum;
#ifdef SCOTCH_PTHREAD
int thrdlvlreqval;
int thrdlvlproval;
#endif /* SCOTCH_PTHREAD */
SCOTCH_errorProg (argv[0]);
#ifdef SCOTCH_PTHREAD
thrdlvlreqval = MPI_THREAD_MULTIPLE;
if (MPI_Init_thread (&argc, &argv, thrdlvlreqval, &thrdlvlproval) != MPI_SUCCESS)
SCOTCH_errorPrint ("main: Cannot initialize (1)");
if (thrdlvlreqval > thrdlvlproval)
SCOTCH_errorPrint ("main: MPI implementation is not thread-safe: recompile without SCOTCH_PTHREAD");
#else /* SCOTCH_PTHREAD */
if (MPI_Init (&argc, &argv) != MPI_SUCCESS)
SCOTCH_errorPrint ("main: Cannot initialize (2)");
#endif /* SCOTCH_PTHREAD */
if (argc != 2) {
SCOTCH_errorPrint ("usage: %s graph_file", argv[0]);
exit (EXIT_FAILURE);
}
proccomm = MPI_COMM_WORLD;
MPI_Comm_size (proccomm, &procglbnbr); /* Get communicator data */
MPI_Comm_rank (proccomm, &proclocnum);
fprintf (stderr, "Proc %2d of %2d, pid %d\n", proclocnum, procglbnbr, getpid ());
#ifdef SCOTCH_CHECK_NOAUTO
if (proclocnum == 0) { /* Synchronize on keybord input */
char c;
printf ("Waiting for key press...\n");
scanf ("%c", &c);
}
#endif /* SCOTCH_CHECK_NOAUTO */
if (MPI_Barrier (proccomm) != MPI_SUCCESS) { /* Synchronize for debug */
SCOTCH_errorPrint ("main: cannot communicate (1)");
exit (EXIT_FAILURE);
}
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0) { /* Initialize source graph */
SCOTCH_errorPrint ("main: cannot initialize graph (1)");
exit (EXIT_FAILURE);
}
file = NULL;
if ((proclocnum == 0) &&
((file = fopen (argv[1], "r")) == NULL)) {
SCOTCH_errorPrint ("main: cannot open graph file");
exit (EXIT_FAILURE);
}
if (SCOTCH_dgraphLoad (&grafdat, file, 0, 0) != 0) {
SCOTCH_errorPrint ("main: cannot load graph");
exit (EXIT_FAILURE);
}
if (file != NULL)
fclose (file);
if (MPI_Barrier (proccomm) != MPI_SUCCESS) { /* Synchronize for debug */
SCOTCH_errorPrint ("main: cannot communicate (2)");
exit (EXIT_FAILURE);
}
SCOTCH_dgraphData (&grafdat, NULL, &vertglbnbr, &vertlocnbr, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
if ((fronloctab = malloc (vertlocnbr * sizeof (SCOTCH_Num))) == NULL) {
SCOTCH_errorPrint ("main: cannot allocate frontier array");
exit (EXIT_FAILURE);
}
if (SCOTCH_dgraphInit (&bandgrafdat, proccomm) != 0) { /* Initialize band graph */
SCOTCH_errorPrint ("main: cannot initialize graph (2)");
exit (EXIT_FAILURE);
}
fronloctab[0] = 0;
if (SCOTCH_dgraphBand (&grafdat, (proclocnum == 1) ? 1 : 0, fronloctab, 4, &bandgrafdat) != 0) {
SCOTCH_errorPrint ("main: cannot compute band graph");
exit (EXIT_FAILURE);
}
free (fronloctab);
SCOTCH_dgraphData (&bandgrafdat, &baseval, &bandvertglbnbr, &bandvertlocnbr, NULL, NULL, NULL, NULL, NULL, &bandvlblloctab, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
for (procnum = 0; procnum < procglbnbr; procnum ++) {
SCOTCH_Num bandvertlocnum;
MPI_Barrier (proccomm);
if (procnum == proclocnum) {
if ((file = fopen ("/tmp/test_scotch_dgraph_band.map", (procnum == 0) ? "w" : "a+")) == NULL) {
SCOTCH_errorPrint ("main: cannot open mapping file");
exit (EXIT_FAILURE);
}
if (procnum == 0)
fprintf (file, "%ld\n", (long) bandvertglbnbr);
for (bandvertlocnum = 0; bandvertlocnum < bandvertlocnbr; bandvertlocnum ++)
fprintf (file, "%ld\t1\n", (long) bandvlblloctab[bandvertlocnum]);
fclose (file);
}
}
SCOTCH_dgraphExit (&bandgrafdat);
SCOTCH_dgraphExit (&grafdat);
MPI_Finalize ();
exit (EXIT_SUCCESS);
}
