int
SCOTCH_dgraphMap (
SCOTCH_Dgraph * const grafptr, /*+ Graph to map +*/
const SCOTCH_Arch * const archptr, /*+ Target architecture +*/
SCOTCH_Strat * const stratptr, /*+ Mapping strategy +*/
SCOTCH_Num * const termloctab) /*+ Mapping array +*/
{
SCOTCH_Dmapping mappdat;
int o;
SCOTCH_dgraphMapInit (grafptr, &mappdat, archptr, termloctab);
o = SCOTCH_dgraphMapCompute (grafptr, &mappdat, stratptr);
SCOTCH_dgraphMapExit (grafptr, &mappdat);
return (o);
}
static PetscErrorCode MatPartitioningApply_PTScotch_Private(MatPartitioning part, PetscBool useND, IS *partitioning)
{
MPI_Comm pcomm,comm;
MatPartitioning_PTScotch *scotch = (MatPartitioning_PTScotch*)part->data;
PetscErrorCode ierr;
PetscMPIInt rank;
Mat mat = part->adj;
Mat_MPIAdj *adj = (Mat_MPIAdj*)mat->data;
PetscBool flg,distributed;
PetscBool proc_weight_flg;
PetscInt i,j,p,bs=1,nold;
PetscInt *NDorder = NULL;
PetscReal *vwgttab,deltval;
SCOTCH_Num *locals,*velotab,*veloloctab,*edloloctab,vertlocnbr,edgelocnbr,nparts=part->n;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)part,&pcomm);CHKERRQ(ierr);
/* Duplicate the communicator to be sure that PTSCOTCH attribute caching does not interfere with PETSc. */
ierr = MPI_Comm_dup(pcomm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr);
if (!flg) {
/* bs indicates if the converted matrix is "reduced" from the original and hence the
resulting partition results need to be stretched to match the original matrix */
nold = mat->rmap->n;
ierr = MatConvert(mat,MATMPIADJ,MAT_INITIAL_MATRIX,&mat);CHKERRQ(ierr);
if (mat->rmap->n > 0) bs = nold/mat->rmap->n;
adj = (Mat_MPIAdj*)mat->data;
}
proc_weight_flg = PETSC_TRUE;
ierr = PetscOptionsGetBool(NULL, NULL, "-mat_partitioning_ptscotch_proc_weight", &proc_weight_flg, NULL);CHKERRQ(ierr);
ierr = PetscMalloc1(mat->rmap->n+1,&locals);CHKERRQ(ierr);
if (useND) {
#if defined(PETSC_HAVE_SCOTCH_PARMETIS_V3_NODEND)
PetscInt *sizes, *seps, log2size, subd, *level, base = 0;
PetscMPIInt size;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
log2size = PetscLog2Real(size);
subd = PetscPowInt(2,log2size);
if (subd != size) SETERRQ(comm,PETSC_ERR_SUP,"Only power of 2 communicator sizes");
ierr = PetscMalloc1(mat->rmap->n,&NDorder);CHKERRQ(ierr);
ierr = PetscMalloc3(2*size,&sizes,4*size,&seps,size,&level);CHKERRQ(ierr);
SCOTCH_ParMETIS_V3_NodeND(mat->rmap->range,adj->i,adj->j,&base,NULL,NDorder,sizes,&comm);
ierr = MatPartitioningSizesToSep_Private(subd,sizes,seps,level);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
PetscInt loc;
ierr = PetscFindInt(NDorder[i],2*subd,seps,&loc);CHKERRQ(ierr);
if (loc < 0) {
loc = -(loc+1);
if (loc%2) { /* part of subdomain */
locals[i] = loc/2;
} else {
ierr = PetscFindInt(NDorder[i],2*(subd-1),seps+2*subd,&loc);CHKERRQ(ierr);
loc = loc < 0 ? -(loc+1)/2 : loc/2;
locals[i] = level[loc];
}
} else locals[i] = loc/2;
}
ierr = PetscFree3(sizes,seps,level);CHKERRQ(ierr);
#else
SETERRQ(pcomm,PETSC_ERR_SUP,"Need libptscotchparmetis.a compiled with -DSCOTCH_METIS_PREFIX");
#endif
} else {
velotab = NULL;
if (proc_weight_flg) {
ierr = PetscMalloc1(nparts,&vwgttab);CHKERRQ(ierr);
ierr = PetscMalloc1(nparts,&velotab);CHKERRQ(ierr);
for (j=0; j<nparts; j++) {
if (part->part_weights) vwgttab[j] = part->part_weights[j]*nparts;
else vwgttab[j] = 1.0;
}
for (i=0; i<nparts; i++) {
deltval = PetscAbsReal(vwgttab[i]-PetscFloorReal(vwgttab[i]+0.5));
if (deltval>0.01) {
for (j=0; j<nparts; j++) vwgttab[j] /= deltval;
}
}
for (i=0; i<nparts; i++) velotab[i] = (SCOTCH_Num)(vwgttab[i] + 0.5);
ierr = PetscFree(vwgttab);CHKERRQ(ierr);
}
vertlocnbr = mat->rmap->range[rank+1] - mat->rmap->range[rank];
edgelocnbr = adj->i[vertlocnbr];
veloloctab = part->vertex_weights;
edloloctab = adj->values;
/* detect whether all vertices are located at the same process in original graph */
for (p = 0; !mat->rmap->range[p+1] && p < nparts; ++p);
distributed = (mat->rmap->range[p+1] == mat->rmap->N) ? PETSC_FALSE : PETSC_TRUE;
if (distributed) {
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat;
SCOTCH_Dmapping mappdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_dgraphInit(&grafdat,comm);CHKERRQ(ierr);
ierr = SCOTCH_dgraphBuild(&grafdat,0,vertlocnbr,vertlocnbr,adj->i,adj->i+1,veloloctab,
NULL,edgelocnbr,edgelocnbr,adj->j,NULL,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_dgraphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_archInit(&archdat);CHKERRQ(ierr);
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratDgraphMapBuild(&stradat,scotch->strategy,nparts,nparts,scotch->imbalance);CHKERRQ(ierr);
if (velotab) {
ierr = SCOTCH_archCmpltw(&archdat,nparts,velotab);CHKERRQ(ierr);
} else {
ierr = SCOTCH_archCmplt( &archdat,nparts);CHKERRQ(ierr);
}
ierr = SCOTCH_dgraphMapInit(&grafdat,&mappdat,&archdat,locals);CHKERRQ(ierr);
ierr = SCOTCH_dgraphMapCompute(&grafdat,&mappdat,&stradat);CHKERRQ(ierr);
SCOTCH_dgraphMapExit(&grafdat,&mappdat);
SCOTCH_archExit(&archdat);
SCOTCH_stratExit(&stradat);
SCOTCH_dgraphExit(&grafdat);
} else if (rank == p) {
SCOTCH_Graph grafdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_graphInit(&grafdat);CHKERRQ(ierr);
ierr = SCOTCH_graphBuild(&grafdat,0,vertlocnbr,adj->i,adj->i+1,veloloctab,NULL,edgelocnbr,adj->j,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_graphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratGraphMapBuild(&stradat,scotch->strategy,nparts,scotch->imbalance);CHKERRQ(ierr);
ierr = SCOTCH_graphPart(&grafdat,nparts,&stradat,locals);CHKERRQ(ierr);
SCOTCH_stratExit(&stradat);
SCOTCH_graphExit(&grafdat);
}
ierr = PetscFree(velotab);CHKERRQ(ierr);
}
ierr = MPI_Comm_free(&comm);CHKERRQ(ierr);
if (bs > 1) {
PetscInt *newlocals;
ierr = PetscMalloc1(bs*mat->rmap->n,&newlocals);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
for (j=0;j<bs;j++) {
newlocals[bs*i+j] = locals[i];
}
}
ierr = PetscFree(locals);CHKERRQ(ierr);
ierr = ISCreateGeneral(pcomm,bs*mat->rmap->n,newlocals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,locals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
}
if (useND) {
IS ndis;
if (bs > 1) {
ierr = ISCreateBlock(pcomm,bs,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
}
ierr = ISSetPermutation(ndis);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject)(*partitioning),"_petsc_matpartitioning_ndorder",(PetscObject)ndis);CHKERRQ(ierr);
ierr = ISDestroy(&ndis);CHKERRQ(ierr);
}
if (!flg) {
ierr = MatDestroy(&mat);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
void
METISNAMEU(ParMETIS_V3_PartKway) (
const SCOTCH_Num * const vtxdist,
SCOTCH_Num * const xadj,
SCOTCH_Num * const adjncy,
SCOTCH_Num * const vwgt,
SCOTCH_Num * const adjwgt,
const SCOTCH_Num * const wgtflag,
const SCOTCH_Num * const numflag,
const SCOTCH_Num * const ncon, /* Not used */
const SCOTCH_Num * const nparts,
const float * const tpwgts,
const float * const ubvec, /* Not used */
const SCOTCH_Num * const options, /* Not used */
SCOTCH_Num * const edgecut,
SCOTCH_Num * const part,
MPI_Comm * comm)
{
MPI_Comm proccomm;
int procglbnbr;
int proclocnum;
SCOTCH_Num baseval;
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat; /* Scotch distributed graph object to interface with libScotch */
SCOTCH_Dmapping mappdat; /* Scotch distributed mapping object to interface with libScotch */
SCOTCH_Strat stradat;
SCOTCH_Num vertlocnbr;
SCOTCH_Num * veloloctab;
SCOTCH_Num edgelocnbr;
SCOTCH_Num * edloloctab;
SCOTCH_Num * velotab;
double * vwgttab;
SCOTCH_Num i;
if ((vwgttab = malloc (*nparts * sizeof (double))) == NULL)
return;
if ((velotab = malloc (*nparts * sizeof (SCOTCH_Num))) == NULL) {
free (vwgttab);
return;
}
for (i = 0; i < *nparts; i ++)
vwgttab[i] = (double) tpwgts[i] * (double) (*nparts);
for (i = 0; i < *nparts; i ++) {
double deltval;
deltval = fabs (vwgttab[i] - floor (vwgttab[i] + 0.5));
if (deltval > 0.01) {
SCOTCH_Num j;
deltval = 1.0 / deltval;
for (j = 0; j < *nparts; j ++)
vwgttab[j] *= deltval;
}
}
for (i = 0; i < *nparts; i ++)
velotab[i] = (SCOTCH_Num) (vwgttab[i] + 0.5);
proccomm = *comm;
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0)
return;
MPI_Comm_size (proccomm, &procglbnbr);
MPI_Comm_rank (proccomm, &proclocnum);
baseval = *numflag;
vertlocnbr = vtxdist[proclocnum + 1] - vtxdist[proclocnum];
edgelocnbr = xadj[vertlocnbr] - baseval;
veloloctab = ((vwgt != NULL) && ((*wgtflag & 2) != 0)) ? vwgt : NULL;
edloloctab = ((adjwgt != NULL) && ((*wgtflag & 1) != 0)) ? adjwgt : NULL;
if (SCOTCH_dgraphBuild (&grafdat, baseval,
vertlocnbr, vertlocnbr, xadj, xadj + 1, veloloctab, NULL,
edgelocnbr, edgelocnbr, adjncy, NULL, edloloctab) == 0) {
SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_dgraphCheck (&grafdat) == 0) /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
{
SCOTCH_archInit (&archdat);
if ((SCOTCH_archCmpltw (&archdat, *nparts, velotab) == 0) &&
(SCOTCH_dgraphMapInit (&grafdat, &mappdat, &archdat, part) == 0)) {
SCOTCH_dgraphMapCompute (&grafdat, &mappdat, &stradat);
SCOTCH_dgraphMapExit (&grafdat, &mappdat);
}
SCOTCH_archExit (&archdat);
}
SCOTCH_stratExit (&stradat);
}
SCOTCH_dgraphExit (&grafdat);
*edgecut = 0; /* TODO : compute real edge cut for people who might want it */
free (vwgttab);
free (velotab);
if (baseval != 0) { /* MeTiS part array is based, Scotch is not */
SCOTCH_Num vertlocnum;
for (vertlocnum = 0; vertlocnum < vertlocnbr; vertlocnum ++)
part[vertlocnum] += baseval;
}
}
