void AlgPTScotch<Adapter>::partition(
const RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
SCOTCH_Num partnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(partnbr, numGlobalParts, env);
#ifdef HAVE_ZOLTAN2_MPI
int ierr = 0;
int me = problemComm->getRank();
const SCOTCH_Num baseval = 0; // Base value for array indexing.
// GraphModel returns GNOs from base 0.
SCOTCH_Strat stratstr; // Strategy string
// TODO: Set from parameters
SCOTCH_stratInit(&stratstr);
// Allocate and initialize PTScotch Graph data structure.
SCOTCH_Dgraph *gr = SCOTCH_dgraphAlloc(); // Scotch distributed graph
ierr = SCOTCH_dgraphInit(gr, mpicomm);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphInit",
!ierr, BASIC_ASSERTION, problemComm);
// Get vertex info
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
size_t nVtx = model->getVertexList(vtxID, xyz, vwgts);
SCOTCH_Num vertlocnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(vertlocnbr, nVtx, env);
SCOTCH_Num vertlocmax = vertlocnbr; // Assumes no holes in global nums.
// Get edge info
ArrayView<const gno_t> edgeIds;
ArrayView<const int> procIds;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
size_t nEdge = model->getEdgeList(edgeIds, procIds, offsets, ewgts);
SCOTCH_Num edgelocnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(edgelocnbr, nEdge, env);
const SCOTCH_Num edgelocsize = edgelocnbr; // Assumes adj array is compact.
SCOTCH_Num *vertloctab; // starting adj/vtx
TPL_Traits<SCOTCH_Num, lno_t>::ASSIGN_TPL_T_ARRAY(&vertloctab, offsets, env);
SCOTCH_Num *edgeloctab; // adjacencies
TPL_Traits<SCOTCH_Num, gno_t>::ASSIGN_TPL_T_ARRAY(&edgeloctab, edgeIds, env);
// We don't use these arrays, but we need them as arguments to Scotch.
SCOTCH_Num *vendloctab = NULL; // Assume consecutive storage for adj
SCOTCH_Num *vlblloctab = NULL; // Vertex label array
SCOTCH_Num *edgegsttab = NULL; // Array for ghost vertices
// Get weight info.
SCOTCH_Num *velotab = NULL; // Vertex weights
SCOTCH_Num *edlotab = NULL; // Edge weights
int nVwgts = model->getNumWeightsPerVertex();
int nEwgts = model->getNumWeightsPerEdge();
if (nVwgts > 1 && me == 0) {
std::cerr << "Warning: NumWeightsPerVertex is " << nVwgts
<< " but Scotch allows only one weight. "
<< " Zoltan2 will use only the first weight per vertex."
<< std::endl;
}
if (nEwgts > 1 && me == 0) {
std::cerr << "Warning: NumWeightsPerEdge is " << nEwgts
<< " but Scotch allows only one weight. "
<< " Zoltan2 will use only the first weight per edge."
<< std::endl;
}
if (nVwgts) {
velotab = new SCOTCH_Num[nVtx+1]; // +1 since Scotch wants all procs
// to have non-NULL arrays
scale_weights(nVtx, vwgts[0], velotab);
}
if (nEwgts) {
edlotab = new SCOTCH_Num[nEdge+1]; // +1 since Scotch wants all procs
// to have non-NULL arrays
scale_weights(nEdge, ewgts[0], edlotab);
}
// Build PTScotch distributed data structure
ierr = SCOTCH_dgraphBuild(gr, baseval, vertlocnbr, vertlocmax,
vertloctab, vendloctab, velotab, vlblloctab,
edgelocnbr, edgelocsize,
edgeloctab, edgegsttab, edlotab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphBuild",
!ierr, BASIC_ASSERTION, problemComm);
// Create array for Scotch to return results in.
ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx,true);
SCOTCH_Num *partloctab = NULL;
if (nVtx && (sizeof(SCOTCH_Num) == sizeof(part_t))) {
// Can write directly into the solution's memory
partloctab = (SCOTCH_Num *) partList.getRawPtr();
}
else {
// Can't use solution memory directly; will have to copy later.
// Note: Scotch does not like NULL arrays, so add 1 to always have non-null.
// ParMETIS has this same "feature." See Zoltan bug 4299.
partloctab = new SCOTCH_Num[nVtx+1];
}
// Get target part sizes
float *partsizes = new float[numGlobalParts];
if (!solution->criteriaHasUniformPartSizes(0))
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = solution->getCriteriaPartSize(0, i);
else
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = 1.0 / float(numGlobalParts);
// Allocate and initialize PTScotch target architecture data structure
SCOTCH_Arch archdat;
SCOTCH_archInit(&archdat);
SCOTCH_Num velosum = 0;
SCOTCH_dgraphSize (gr, &velosum, NULL, NULL, NULL);
SCOTCH_Num *goalsizes = new SCOTCH_Num[partnbr];
// TODO: The goalsizes are set as in Zoltan; not sure it is correct there
// or here.
// It appears velosum is global NUMBER of vertices, not global total
// vertex weight. I think we should use the latter.
// Fix this when we add vertex weights.
for (SCOTCH_Num i = 0; i < partnbr; i++)
goalsizes[i] = SCOTCH_Num(ceil(velosum * partsizes[i]));
delete [] partsizes;
SCOTCH_archCmpltw(&archdat, partnbr, goalsizes);
// Call partitioning; result returned in partloctab.
ierr = SCOTCH_dgraphMap(gr, &archdat, &stratstr, partloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphMap",
!ierr, BASIC_ASSERTION, problemComm);
SCOTCH_archExit(&archdat);
delete [] goalsizes;
// TODO - metrics
#ifdef SHOW_ZOLTAN2_SCOTCH_MEMORY
int me = env->comm_->getRank();
#endif
#ifdef HAVE_SCOTCH_ZOLTAN2_GETMEMORYMAX
if (me == 0){
size_t scotchBytes = SCOTCH_getMemoryMax();
std::cout << "Rank " << me << ": Maximum bytes used by Scotch: ";
std::cout << scotchBytes << std::endl;
}
#endif
// Clean up PTScotch
SCOTCH_dgraphExit(gr);
free(gr);
SCOTCH_stratExit(&stratstr);
// Load answer into the solution.
if ((sizeof(SCOTCH_Num) != sizeof(part_t)) || (nVtx == 0)) {
for (size_t i = 0; i < nVtx; i++) partList[i] = partloctab[i];
delete [] partloctab;
}
solution->setParts(partList);
env->memory("Zoltan2-Scotch: After creating solution");
// Clean up copies made due to differing data sizes.
TPL_Traits<SCOTCH_Num, lno_t>::DELETE_TPL_T_ARRAY(&vertloctab);
TPL_Traits<SCOTCH_Num, gno_t>::DELETE_TPL_T_ARRAY(&edgeloctab);
if (nVwgts) delete [] velotab;
if (nEwgts) delete [] edlotab;
#else // DO NOT HAVE_MPI
// TODO: Handle serial case with calls to Scotch.
// TODO: For now, assign everything to rank 0 and assume only one part.
// TODO: Can probably use the code above for loading solution,
// TODO: instead of duplicating it here.
// TODO
// TODO: Actual logic should call Scotch when number of processes == 1.
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
size_t nVtx = model->getVertexList(vtxID, xyz, vwgts);
ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx, true);
for (size_t i = 0; i < nVtx; i++) partList[i] = 0;
solution->setParts(partList);
#endif // DO NOT HAVE_MPI
}
static PetscErrorCode MatPartitioningApply_PTScotch_Private(MatPartitioning part, PetscBool useND, IS *partitioning)
{
MPI_Comm pcomm,comm;
MatPartitioning_PTScotch *scotch = (MatPartitioning_PTScotch*)part->data;
PetscErrorCode ierr;
PetscMPIInt rank;
Mat mat = part->adj;
Mat_MPIAdj *adj = (Mat_MPIAdj*)mat->data;
PetscBool flg,distributed;
PetscBool proc_weight_flg;
PetscInt i,j,p,bs=1,nold;
PetscInt *NDorder = NULL;
PetscReal *vwgttab,deltval;
SCOTCH_Num *locals,*velotab,*veloloctab,*edloloctab,vertlocnbr,edgelocnbr,nparts=part->n;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)part,&pcomm);CHKERRQ(ierr);
/* Duplicate the communicator to be sure that PTSCOTCH attribute caching does not interfere with PETSc. */
ierr = MPI_Comm_dup(pcomm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr);
if (!flg) {
/* bs indicates if the converted matrix is "reduced" from the original and hence the
resulting partition results need to be stretched to match the original matrix */
nold = mat->rmap->n;
ierr = MatConvert(mat,MATMPIADJ,MAT_INITIAL_MATRIX,&mat);CHKERRQ(ierr);
if (mat->rmap->n > 0) bs = nold/mat->rmap->n;
adj = (Mat_MPIAdj*)mat->data;
}
proc_weight_flg = PETSC_TRUE;
ierr = PetscOptionsGetBool(NULL, NULL, "-mat_partitioning_ptscotch_proc_weight", &proc_weight_flg, NULL);CHKERRQ(ierr);
ierr = PetscMalloc1(mat->rmap->n+1,&locals);CHKERRQ(ierr);
if (useND) {
#if defined(PETSC_HAVE_SCOTCH_PARMETIS_V3_NODEND)
PetscInt *sizes, *seps, log2size, subd, *level, base = 0;
PetscMPIInt size;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
log2size = PetscLog2Real(size);
subd = PetscPowInt(2,log2size);
if (subd != size) SETERRQ(comm,PETSC_ERR_SUP,"Only power of 2 communicator sizes");
ierr = PetscMalloc1(mat->rmap->n,&NDorder);CHKERRQ(ierr);
ierr = PetscMalloc3(2*size,&sizes,4*size,&seps,size,&level);CHKERRQ(ierr);
SCOTCH_ParMETIS_V3_NodeND(mat->rmap->range,adj->i,adj->j,&base,NULL,NDorder,sizes,&comm);
ierr = MatPartitioningSizesToSep_Private(subd,sizes,seps,level);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
PetscInt loc;
ierr = PetscFindInt(NDorder[i],2*subd,seps,&loc);CHKERRQ(ierr);
if (loc < 0) {
loc = -(loc+1);
if (loc%2) { /* part of subdomain */
locals[i] = loc/2;
} else {
ierr = PetscFindInt(NDorder[i],2*(subd-1),seps+2*subd,&loc);CHKERRQ(ierr);
loc = loc < 0 ? -(loc+1)/2 : loc/2;
locals[i] = level[loc];
}
} else locals[i] = loc/2;
}
ierr = PetscFree3(sizes,seps,level);CHKERRQ(ierr);
#else
SETERRQ(pcomm,PETSC_ERR_SUP,"Need libptscotchparmetis.a compiled with -DSCOTCH_METIS_PREFIX");
#endif
} else {
velotab = NULL;
if (proc_weight_flg) {
ierr = PetscMalloc1(nparts,&vwgttab);CHKERRQ(ierr);
ierr = PetscMalloc1(nparts,&velotab);CHKERRQ(ierr);
for (j=0; j<nparts; j++) {
if (part->part_weights) vwgttab[j] = part->part_weights[j]*nparts;
else vwgttab[j] = 1.0;
}
for (i=0; i<nparts; i++) {
deltval = PetscAbsReal(vwgttab[i]-PetscFloorReal(vwgttab[i]+0.5));
if (deltval>0.01) {
for (j=0; j<nparts; j++) vwgttab[j] /= deltval;
}
}
for (i=0; i<nparts; i++) velotab[i] = (SCOTCH_Num)(vwgttab[i] + 0.5);
ierr = PetscFree(vwgttab);CHKERRQ(ierr);
}
vertlocnbr = mat->rmap->range[rank+1] - mat->rmap->range[rank];
edgelocnbr = adj->i[vertlocnbr];
veloloctab = part->vertex_weights;
edloloctab = adj->values;
/* detect whether all vertices are located at the same process in original graph */
for (p = 0; !mat->rmap->range[p+1] && p < nparts; ++p);
distributed = (mat->rmap->range[p+1] == mat->rmap->N) ? PETSC_FALSE : PETSC_TRUE;
if (distributed) {
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat;
SCOTCH_Dmapping mappdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_dgraphInit(&grafdat,comm);CHKERRQ(ierr);
ierr = SCOTCH_dgraphBuild(&grafdat,0,vertlocnbr,vertlocnbr,adj->i,adj->i+1,veloloctab,
NULL,edgelocnbr,edgelocnbr,adj->j,NULL,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_dgraphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_archInit(&archdat);CHKERRQ(ierr);
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratDgraphMapBuild(&stradat,scotch->strategy,nparts,nparts,scotch->imbalance);CHKERRQ(ierr);
if (velotab) {
ierr = SCOTCH_archCmpltw(&archdat,nparts,velotab);CHKERRQ(ierr);
} else {
ierr = SCOTCH_archCmplt( &archdat,nparts);CHKERRQ(ierr);
}
ierr = SCOTCH_dgraphMapInit(&grafdat,&mappdat,&archdat,locals);CHKERRQ(ierr);
ierr = SCOTCH_dgraphMapCompute(&grafdat,&mappdat,&stradat);CHKERRQ(ierr);
SCOTCH_dgraphMapExit(&grafdat,&mappdat);
SCOTCH_archExit(&archdat);
SCOTCH_stratExit(&stradat);
SCOTCH_dgraphExit(&grafdat);
} else if (rank == p) {
SCOTCH_Graph grafdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_graphInit(&grafdat);CHKERRQ(ierr);
ierr = SCOTCH_graphBuild(&grafdat,0,vertlocnbr,adj->i,adj->i+1,veloloctab,NULL,edgelocnbr,adj->j,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_graphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratGraphMapBuild(&stradat,scotch->strategy,nparts,scotch->imbalance);CHKERRQ(ierr);
ierr = SCOTCH_graphPart(&grafdat,nparts,&stradat,locals);CHKERRQ(ierr);
SCOTCH_stratExit(&stradat);
SCOTCH_graphExit(&grafdat);
}
ierr = PetscFree(velotab);CHKERRQ(ierr);
}
ierr = MPI_Comm_free(&comm);CHKERRQ(ierr);
if (bs > 1) {
PetscInt *newlocals;
ierr = PetscMalloc1(bs*mat->rmap->n,&newlocals);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
for (j=0;j<bs;j++) {
newlocals[bs*i+j] = locals[i];
}
}
ierr = PetscFree(locals);CHKERRQ(ierr);
ierr = ISCreateGeneral(pcomm,bs*mat->rmap->n,newlocals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,locals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
}
if (useND) {
IS ndis;
if (bs > 1) {
ierr = ISCreateBlock(pcomm,bs,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
}
ierr = ISSetPermutation(ndis);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject)(*partitioning),"_petsc_matpartitioning_ndorder",(PetscObject)ndis);CHKERRQ(ierr);
ierr = ISDestroy(&ndis);CHKERRQ(ierr);
}
if (!flg) {
ierr = MatDestroy(&mat);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
void
METISNAMEU(ParMETIS_V3_PartKway) (
const SCOTCH_Num * const vtxdist,
SCOTCH_Num * const xadj,
SCOTCH_Num * const adjncy,
SCOTCH_Num * const vwgt,
SCOTCH_Num * const adjwgt,
const SCOTCH_Num * const wgtflag,
const SCOTCH_Num * const numflag,
const SCOTCH_Num * const ncon, /* Not used */
const SCOTCH_Num * const nparts,
const float * const tpwgts,
const float * const ubvec, /* Not used */
const SCOTCH_Num * const options, /* Not used */
SCOTCH_Num * const edgecut,
SCOTCH_Num * const part,
MPI_Comm * comm)
{
MPI_Comm proccomm;
int procglbnbr;
int proclocnum;
SCOTCH_Num baseval;
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat; /* Scotch distributed graph object to interface with libScotch */
SCOTCH_Dmapping mappdat; /* Scotch distributed mapping object to interface with libScotch */
SCOTCH_Strat stradat;
SCOTCH_Num vertlocnbr;
SCOTCH_Num * veloloctab;
SCOTCH_Num edgelocnbr;
SCOTCH_Num * edloloctab;
SCOTCH_Num * velotab;
double * vwgttab;
SCOTCH_Num i;
if ((vwgttab = malloc (*nparts * sizeof (double))) == NULL)
return;
if ((velotab = malloc (*nparts * sizeof (SCOTCH_Num))) == NULL) {
free (vwgttab);
return;
}
for (i = 0; i < *nparts; i ++)
vwgttab[i] = (double) tpwgts[i] * (double) (*nparts);
for (i = 0; i < *nparts; i ++) {
double deltval;
deltval = fabs (vwgttab[i] - floor (vwgttab[i] + 0.5));
if (deltval > 0.01) {
SCOTCH_Num j;
deltval = 1.0 / deltval;
for (j = 0; j < *nparts; j ++)
vwgttab[j] *= deltval;
}
}
for (i = 0; i < *nparts; i ++)
velotab[i] = (SCOTCH_Num) (vwgttab[i] + 0.5);
proccomm = *comm;
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0)
return;
MPI_Comm_size (proccomm, &procglbnbr);
MPI_Comm_rank (proccomm, &proclocnum);
baseval = *numflag;
vertlocnbr = vtxdist[proclocnum + 1] - vtxdist[proclocnum];
edgelocnbr = xadj[vertlocnbr] - baseval;
veloloctab = ((vwgt != NULL) && ((*wgtflag & 2) != 0)) ? vwgt : NULL;
edloloctab = ((adjwgt != NULL) && ((*wgtflag & 1) != 0)) ? adjwgt : NULL;
if (SCOTCH_dgraphBuild (&grafdat, baseval,
vertlocnbr, vertlocnbr, xadj, xadj + 1, veloloctab, NULL,
edgelocnbr, edgelocnbr, adjncy, NULL, edloloctab) == 0) {
SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_dgraphCheck (&grafdat) == 0) /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
{
SCOTCH_archInit (&archdat);
if ((SCOTCH_archCmpltw (&archdat, *nparts, velotab) == 0) &&
(SCOTCH_dgraphMapInit (&grafdat, &mappdat, &archdat, part) == 0)) {
SCOTCH_dgraphMapCompute (&grafdat, &mappdat, &stradat);
SCOTCH_dgraphMapExit (&grafdat, &mappdat);
}
SCOTCH_archExit (&archdat);
}
SCOTCH_stratExit (&stradat);
}
SCOTCH_dgraphExit (&grafdat);
*edgecut = 0; /* TODO : compute real edge cut for people who might want it */
free (vwgttab);
free (velotab);
if (baseval != 0) { /* MeTiS part array is based, Scotch is not */
SCOTCH_Num vertlocnum;
for (vertlocnum = 0; vertlocnum < vertlocnbr; vertlocnum ++)
part[vertlocnum] += baseval;
}
}
void AlgPTScotch(
const RCP<const Environment> &env, // parameters & app comm
const RCP<const Comm<int> > &problemComm, // problem comm
#ifdef HAVE_ZOLTAN2_MPI
MPI_Comm mpicomm,
#endif
const RCP<GraphModel<typename Adapter::base_adapter_t> > &model, // the graph
RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
typedef typename Adapter::lno_t lno_t;
typedef typename Adapter::gno_t gno_t;
typedef typename Adapter::scalar_t scalar_t;
int ierr = 0;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
SCOTCH_Num partnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(partnbr, numGlobalParts, env);
#ifdef HAVE_ZOLTAN2_MPI
const SCOTCH_Num baseval = 0; // Base value for array indexing.
// GraphModel returns GNOs from base 0.
SCOTCH_Strat stratstr; // Strategy string
// TODO: Set from parameters
SCOTCH_stratInit(&stratstr);
// Allocate & initialize PTScotch data structure.
SCOTCH_Dgraph *gr = SCOTCH_dgraphAlloc(); // Scotch distributed graph
ierr = SCOTCH_dgraphInit(gr, mpicomm);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphInit",
!ierr, BASIC_ASSERTION, problemComm);
// Get vertex info
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vtxWt;
size_t nVtx = model->getVertexList(vtxID, xyz, vtxWt);
SCOTCH_Num vertlocnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(vertlocnbr, nVtx, env);
SCOTCH_Num vertlocmax = vertlocnbr; // Assumes no holes in global nums.
// Get edge info
ArrayView<const gno_t> edgeIds;
ArrayView<const int> procIds;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
size_t nEdges = model->getEdgeList(edgeIds, procIds, offsets, ewgts);
SCOTCH_Num edgelocnbr;
SCOTCH_Num_Traits<size_t>::ASSIGN_TO_SCOTCH_NUM(edgelocnbr, nEdges, env);
const SCOTCH_Num edgelocsize = edgelocnbr; // Assumes adj array is compact.
SCOTCH_Num *vertloctab; // starting adj/vtx
SCOTCH_Num_Traits<lno_t>::ASSIGN_SCOTCH_NUM_ARRAY(&vertloctab, offsets, env);
SCOTCH_Num *edgeloctab; // adjacencies
SCOTCH_Num_Traits<gno_t>::ASSIGN_SCOTCH_NUM_ARRAY(&edgeloctab, edgeIds, env);
// We don't use these arrays, but we need them as arguments to Scotch.
SCOTCH_Num *vendloctab = NULL; // Assume consecutive storage for adj
SCOTCH_Num *vlblloctab = NULL; // Vertex label array
SCOTCH_Num *edgegsttab = NULL; // Array for ghost vertices
// Get weight info.
// TODO: Actually get the weights; for now, not using weights.
SCOTCH_Num *veloloctab = NULL; // Vertex weights
SCOTCH_Num *edloloctab = NULL; // Edge weights
//TODO int vwtdim = model->getVertexWeightDim();
//TODO int ewtdim = model->getEdgeWeightDim();
//TODO if (vwtdim) veloloctab = new SCOTCH_Num[nVtx];
//TODO if (ewtdim) edloloctab = new SCOTCH_Num[nEdges];
//TODO scale weights to SCOTCH_Nums.
// Build PTScotch distributed data structure
ierr = SCOTCH_dgraphBuild(gr, baseval, vertlocnbr, vertlocmax,
vertloctab, vendloctab, veloloctab, vlblloctab,
edgelocnbr, edgelocsize,
edgeloctab, edgegsttab, edloloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphBuild",
!ierr, BASIC_ASSERTION, problemComm);
// Create array for Scotch to return results in.
ArrayRCP<partId_t> partList(new partId_t [nVtx], 0, nVtx,true);
SCOTCH_Num *partloctab = NULL;
if (nVtx && (sizeof(SCOTCH_Num) == sizeof(partId_t))) {
// Can write directly into the solution's memory
partloctab = (SCOTCH_Num *) partList.getRawPtr();
}
else {
// Can't use solution memory directly; will have to copy later.
// Note: Scotch does not like NULL arrays, so add 1 to always have non-null.
// ParMETIS has this same "feature." See Zoltan bug 4299.
partloctab = new SCOTCH_Num[nVtx+1];
}
// Call partitioning; result returned in partloctab.
// TODO: Use SCOTCH_dgraphMap so can include a machine model in partitioning
ierr = SCOTCH_dgraphPart(gr, partnbr, &stratstr, partloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphPart",
!ierr, BASIC_ASSERTION, problemComm);
// TODO - metrics
#ifdef SHOW_ZOLTAN2_SCOTCH_MEMORY
int me = env->comm_->getRank();
#endif
#ifdef HAVE_SCOTCH_ZOLTAN2_GETMEMORYMAX
if (me == 0){
size_t scotchBytes = SCOTCH_getMemoryMax();
std::cout << "Rank " << me << ": Maximum bytes used by Scotch: ";
std::cout << scotchBytes << std::endl;
}
#endif
// Clean up PTScotch
SCOTCH_dgraphExit(gr);
delete gr;
SCOTCH_stratExit(&stratstr);
// Load answer into the solution.
if ((sizeof(SCOTCH_Num) != sizeof(partId_t)) || (nVtx == 0)) {
for (size_t i = 0; i < nVtx; i++) partList[i] = partloctab[i];
delete [] partloctab;
}
ArrayRCP<const gno_t> gnos = arcpFromArrayView(vtxID);
solution->setParts(gnos, partList, true);
env->memory("Zoltan2-Scotch: After creating solution");
//if (me == 0) cout << " done." << endl;
// Clean up Zoltan2
//TODO if (vwtdim) delete [] velotab;
//TODO if (ewtdim) delete [] edlotab;
// Clean up copies made due to differing data sizes.
SCOTCH_Num_Traits<lno_t>::DELETE_SCOTCH_NUM_ARRAY(&vertloctab);
SCOTCH_Num_Traits<gno_t>::DELETE_SCOTCH_NUM_ARRAY(&edgeloctab);
#else // DO NOT HAVE_MPI
// TODO: Handle serial case with calls to Scotch.
// TODO: For now, assign everything to rank 0 and assume only one part.
// TODO: Can probably use the code above for loading solution,
// TODO: instead of duplicating it here.
// TODO
// TODO: Actual logic should call Scotch when number of processes == 1.
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > xyz;
ArrayView<StridedData<lno_t, scalar_t> > vtxWt;
size_t nVtx = model->getVertexList(vtxID, xyz, vtxWt);
for (size_t i = 0; i < nVtx; i++) partList[i] = 0;
#endif // DO NOT HAVE_MPI
}
// Call scotch with options from dictionary.
Foam::label Foam::ptscotchDecomp::decompose
(
const fileName& meshPath,
const List<int>& adjncy,
const List<int>& xadj,
const scalarField& cWeights,
List<int>& finalDecomp
) const
{
if (debug)
{
Pout<< "ptscotchDecomp : entering with xadj:" << xadj.size() << endl;
}
// Dump graph
if (decompositionDict_.found("ptscotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("ptscotchCoeffs");
if (scotchCoeffs.lookupOrDefault("writeGraph", false))
{
OFstream str
(
meshPath + "_" + Foam::name(Pstream::myProcNo()) + ".dgr"
);
Pout<< "Dumping Scotch graph file to " << str.name() << endl
<< "Use this in combination with dgpart." << endl;
globalIndex globalCells(xadj.size()-1);
// Distributed graph file (.grf)
label version = 2;
str << version << nl;
// Number of files (procglbnbr)
str << Pstream::nProcs();
// My file number (procloc)
str << ' ' << Pstream::myProcNo() << nl;
// Total number of vertices (vertglbnbr)
str << globalCells.size();
// Total number of connections (edgeglbnbr)
str << ' ' << returnReduce(xadj[xadj.size()-1], sumOp<label>())
<< nl;
// Local number of vertices (vertlocnbr)
str << xadj.size()-1;
// Local number of connections (edgelocnbr)
str << ' ' << xadj[xadj.size()-1] << nl;
// Numbering starts from 0
label baseval = 0;
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs
str << baseval << ' ' << "000" << nl;
for (label cellI = 0; cellI < xadj.size()-1; cellI++)
{
label start = xadj[cellI];
label end = xadj[cellI+1];
str << end-start;
for (label i = start; i < end; i++)
{
str << ' ' << adjncy[i];
}
str << nl;
}
}
}
// Strategy
// ~~~~~~~~
// Default.
SCOTCH_Strat stradat;
check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
string strategy;
if (scotchCoeffs.readIfPresent("strategy", strategy))
{
if (debug)
{
Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
}
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
//fprintf(stdout, "\n");
}
}
// Graph
// ~~~~~
List<int> velotab;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningIn
(
"ptscotchDecomp::decompose(..)"
) << "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != xadj.size()-1)
{
FatalErrorIn
(
"ptscotchDecomp::decompose(..)"
) << "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << xadj.size()-1
<< exit(FatalError);
}
// Convert to integers.
velotab.setSize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int(cWeights[i]/minWeights);
}
}
if (debug)
{
Pout<< "SCOTCH_dgraphInit" << endl;
}
SCOTCH_Dgraph grafdat;
check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
if (debug)
{
Pout<< "SCOTCH_dgraphBuild with:" << nl
<< "xadj.size()-1 : " << xadj.size()-1 << nl
<< "xadj : " << long(xadj.begin()) << nl
<< "velotab : " << long(velotab.begin()) << nl
<< "adjncy.size() : " << adjncy.size() << nl
<< "adjncy : " << long(adjncy.begin()) << nl
<< endl;
}
check
(
SCOTCH_dgraphBuild
(
&grafdat, // grafdat
0, // baseval, c-style numbering
xadj.size()-1, // vertlocnbr, nCells
xadj.size()-1, // vertlocmax
const_cast<SCOTCH_Num*>(xadj.begin()),
// vertloctab, start index per cell into
// adjncy
const_cast<SCOTCH_Num*>(&xadj[1]),// vendloctab, end index ,,
const_cast<SCOTCH_Num*>(velotab.begin()),// veloloctab, vtx weights
NULL, // vlblloctab
adjncy.size(), // edgelocnbr, number of arcs
adjncy.size(), // edgelocsiz
const_cast<SCOTCH_Num*>(adjncy.begin()), // edgeloctab
NULL, // edgegsttab
NULL // edlotab, edge weights
),
"SCOTCH_dgraphBuild"
);
if (debug)
{
Pout<< "SCOTCH_dgraphCheck" << endl;
}
check(SCOTCH_dgraphCheck(&grafdat), "SCOTCH_dgraphCheck");
// Architecture
// ~~~~~~~~~~~~
// (fully connected network topology since using switch)
if (debug)
{
Pout<< "SCOTCH_archInit" << endl;
}
SCOTCH_Arch archdat;
check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
List<label> processorWeights;
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
scotchCoeffs.readIfPresent("processorWeights", processorWeights);
}
if (processorWeights.size())
{
if (debug)
{
Info<< "ptscotchDecomp : Using procesor weights "
<< processorWeights
<< endl;
}
check
(
SCOTCH_archCmpltw(&archdat, nProcessors_, processorWeights.begin()),
"SCOTCH_archCmpltw"
);
}
else
{
if (debug)
{
Pout<< "SCOTCH_archCmplt" << endl;
}
check
(
SCOTCH_archCmplt(&archdat, nProcessors_),
"SCOTCH_archCmplt"
);
}
//SCOTCH_Mapping mapdat;
//SCOTCH_dgraphMapInit(&grafdat, &mapdat, &archdat, NULL);
//SCOTCH_dgraphMapCompute(&grafdat, &mapdat, &stradat); /*Perform mapping*/
//SCOTCHdgraphMapExit(&grafdat, &mapdat);
// Hack:switch off fpu error trapping
# ifdef LINUX_GNUC
int oldExcepts = fedisableexcept
(
FE_DIVBYZERO
| FE_INVALID
| FE_OVERFLOW
);
# endif
if (debug)
{
Pout<< "SCOTCH_dgraphMap" << endl;
}
finalDecomp.setSize(xadj.size()-1);
finalDecomp = 0;
check
(
SCOTCH_dgraphMap
(
&grafdat,
&archdat,
&stradat, // const SCOTCH_Strat *
finalDecomp.begin() // parttab
),
"SCOTCH_graphMap"
);
# ifdef LINUX_GNUC
feenableexcept(oldExcepts);
# endif
//finalDecomp.setSize(xadj.size()-1);
//check
//(
// SCOTCH_dgraphPart
// (
// &grafdat,
// nProcessors_, // partnbr
// &stradat, // const SCOTCH_Strat *
// finalDecomp.begin() // parttab
// ),
// "SCOTCH_graphPart"
//);
if (debug)
{
Pout<< "SCOTCH_dgraphExit" << endl;
}
// Release storage for graph
SCOTCH_dgraphExit(&grafdat);
// Release storage for strategy
SCOTCH_stratExit(&stradat);
// Release storage for network topology
SCOTCH_archExit(&archdat);
return 0;
}
void
METISNAMEU(ParMETIS_V3_NodeND) (
const int * const vtxdist,
int * const xadj,
int * const adjncy,
const int * const numflag,
const int * const options, /* Not used */
int * const order,
int * const sizes, /* Of size twice the number of processors ; not used */
MPI_Comm * comm)
{
MPI_Comm proccomm;
int procglbnbr;
int proclocnum;
SCOTCH_Num baseval;
SCOTCH_Dgraph grafdat; /* Scotch distributed graph object to interface with libScotch */
SCOTCH_Dordering ordedat; /* Scotch distributed ordering object to interface with libScotch */
SCOTCH_Strat stradat;
SCOTCH_Num vertlocnbr;
SCOTCH_Num edgelocnbr;
if (sizeof (SCOTCH_Num) != sizeof (int)) {
SCOTCH_errorPrint ("ParMETIS_V3_NodeND (as of SCOTCH): SCOTCH_Num type should equate to int");
return;
}
proccomm = *comm;
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0)
return;
MPI_Comm_size (proccomm, &procglbnbr);
MPI_Comm_rank (proccomm, &proclocnum);
baseval = *numflag;
vertlocnbr = vtxdist[proclocnum + 1] - vtxdist[proclocnum];
edgelocnbr = xadj[vertlocnbr] - baseval;
if (sizes != NULL)
memSet (sizes, ~0, (2 * procglbnbr - 1) * sizeof (int)); /* Array not used if procglbnbr is not a power of 2 or if error */
if (SCOTCH_dgraphBuild (&grafdat, baseval,
vertlocnbr, vertlocnbr, xadj, xadj + 1, NULL, NULL,
edgelocnbr, edgelocnbr, adjncy, NULL, NULL) == 0) {
SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_dgraphCheck (&grafdat) == 0) /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
{
if (SCOTCH_dgraphOrderInit (&grafdat, &ordedat) == 0) {
int levlmax;
int bitsnbr;
SCOTCH_Num proctmp;
SCOTCH_dgraphOrderCompute (&grafdat, &ordedat, &stradat);
SCOTCH_dgraphOrderPerm (&grafdat, &ordedat, order);
for (levlmax = -1, bitsnbr = 0, proctmp = procglbnbr; /* Count number of bits set to 1 in procglbnbr */
proctmp != 0; levlmax ++, proctmp >>= 1)
bitsnbr += proctmp & 1;
if (bitsnbr == 1) {
SCOTCH_Num cblkglbnbr;
if ((cblkglbnbr = SCOTCH_dgraphOrderCblkDist (&grafdat, &ordedat)) >= 0) {
SCOTCH_Num * treeglbtab;
SCOTCH_Num * sizeglbtab;
SCOTCH_Num * sepaglbtab;
if (memAllocGroup ((void **) (void *)
&treeglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num)),
&sizeglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num)),
&sepaglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num) * 3), NULL) != NULL) {
if (SCOTCH_dgraphOrderTreeDist (&grafdat, &ordedat, treeglbtab, sizeglbtab) == 0) {
SCOTCH_Num rootnum;
SCOTCH_Num cblknum;
memSet (sepaglbtab, ~0, cblkglbnbr * sizeof (SCOTCH_Num) * 3);
for (rootnum = -1, cblknum = 0; cblknum < cblkglbnbr; cblknum ++) {
SCOTCH_Num fathnum;
fathnum = treeglbtab[cblknum] - baseval; /* Use un-based indices */
if (fathnum < 0) { /* If father index indicates root */
if (rootnum != -1) { /* If another root already found */
rootnum = -1; /* Indicate an error */
break;
}
rootnum = cblknum; /* Record index of root node */
}
else {
int i;
for (i = 0; i < 3; i ++) {
int j;
j = 3 * fathnum + i; /* Slot number of prospective son */
if (sepaglbtab[j] < 0) { /* If potentially empty slot found */
if (sepaglbtab[j] == -1) /* If we don't have too many sons */
sepaglbtab[j] = cblknum; /* Add link to son in slot */
break;
}
}
if (i == 3) { /* If no empty slot found */
sepaglbtab[3 * fathnum] = -2; /* Indicate there are too many sons */
break;
}
}
}
if ((rootnum >= 0) && (sizes != NULL)) { /* If no error above, go on processing separator tree */
memSet (sizes, 0, (2 * procglbnbr - 1) * sizeof (int)); /* Set array of sizes to 0 by default */
_SCOTCH_ParMETIS_V3_NodeNDTree (sizes + (2 * procglbnbr - 1), sizeglbtab, sepaglbtab, levlmax, 0, rootnum, 1);
}
}
memFree (treeglbtab); /* Free group leader */
}
}
}
SCOTCH_dgraphOrderExit (&grafdat, &ordedat);
}
}
/*
**
*/
FORTRAN ( \
SCOTCHFDGRAPHBUILD, scotchfdgraphbuild, ( \
SCOTCH_Dgraph * const grafptr, \
const SCOTCH_Num * const baseptr, \
const SCOTCH_Num * const vertlocptr, \
const SCOTCH_Num * const vertlocmpt, \
SCOTCH_Num * const vertloctab, \
SCOTCH_Num * const vendloctab, \
SCOTCH_Num * const veloloctab, \
SCOTCH_Num * const vlblloctab, \
const SCOTCH_Num * const edgelocptr, \
const SCOTCH_Num * const edgelocptz, \
SCOTCH_Num * const edgeloctab, \
SCOTCH_Num * const edgegsttab, \
SCOTCH_Num * const edloloctab, \
int * const revaptr), \
(grafptr, baseptr, vertlocptr, vertlocmpt, \
vertloctab, vendloctab, veloloctab, \
vlblloctab, edgelocptr, edgelocptz, \
edgeloctab, edgegsttab, edloloctab, \
revaptr))
{
*revaptr = SCOTCH_dgraphBuild (grafptr, *baseptr, *vertlocptr, *vertlocmpt, vertloctab,
vendloctab, veloloctab, vlblloctab,
*edgelocptr, *edgelocptz, edgeloctab, edgegsttab, edloloctab);
}
