static
int
_SCOTCH_METIS_PartGraph (
const int * const n,
const int * const xadj,
const int * const adjncy,
const int * const vwgt,
const int * const adjwgt,
const int * const numflag,
const int * const nparts,
int * const part)
{
SCOTCH_Graph grafdat; /* Scotch graph object to interface with libScotch */
SCOTCH_Strat stradat;
SCOTCH_Num baseval;
SCOTCH_Num vertnbr;
int o;
if (sizeof (SCOTCH_Num) != sizeof (int)) {
errorPrint ("METIS_PartGraph* (as of SCOTCH): SCOTCH_Num type should equate to int");
return (1);
}
SCOTCH_graphInit (&grafdat);
baseval = *numflag;
vertnbr = *n;
o = 1; /* Assume something will go wrong */
if (SCOTCH_graphBuild (&grafdat,
baseval, vertnbr, xadj, xadj + 1, vwgt, NULL,
xadj[vertnbr] - baseval, adjncy, adjwgt) == 0) {
SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_graphCheck (&grafdat) == 0) /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
o = SCOTCH_graphPart (&grafdat, *nparts, &stradat, part);
SCOTCH_stratExit (&stradat);
}
SCOTCH_graphExit (&grafdat);
if (baseval != 0) { /* MeTiS part array is based, Scotch is not */
SCOTCH_Num vertnum;
for (vertnum = 0; vertnum < vertnbr; vertnum ++)
part[vertnum] += baseval;
}
return (o);
}
void ScotchRefineLB::work(LDStats *stats) {
/** ========================== INITIALIZATION ============================= */
ProcArray *parr = new ProcArray(stats);
ObjGraph *ogr = new ObjGraph(stats);
int cost_array[10] = {64, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536};
/** ============================= STRATEGY ================================ */
// convert ObjGraph to the Scotch graph
SCOTCH_Num baseval = 0; // starting index of vertices
SCOTCH_Num vertnbr = ogr->vertices.size(); // number of vertices
SCOTCH_Num edgenbr = 0; // number of edges
SCOTCH_Num *oldpemap = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * vertnbr);
double maxLoad = 0.0;
double minLoad = 0.0;
if (vertnbr > 0) {
minLoad = ogr->vertices[baseval].getVertexLoad();
}
long maxBytes = 1;
int i, j, k, vert;
/** remove duplicate edges from recvFrom */
for(i = baseval; i < vertnbr; i++) {
for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
vert = ogr->vertices[i].sendToList[j].getNeighborId();
for(k = 0; k < ogr->vertices[i].recvFromList.size(); k++) {
if(ogr->vertices[i].recvFromList[k].getNeighborId() == vert) {
ogr->vertices[i].sendToList[j].setNumBytes(ogr->vertices[i].sendToList[j].getNumBytes() + ogr->vertices[i].recvFromList[k].getNumBytes());
ogr->vertices[i].recvFromList.erase(ogr->vertices[i].recvFromList.begin() + k);
}
}
}
}
/** the object load is normalized to an integer between 0 and 256 */
for(i = baseval; i < vertnbr; i++) {
if(ogr->vertices[i].getVertexLoad() > maxLoad)
maxLoad = ogr->vertices[i].getVertexLoad();
if (ogr->vertices[i].getVertexLoad() < minLoad) {
minLoad = ogr->vertices[i].getVertexLoad();
}
edgenbr += ogr->vertices[i].sendToList.size() + ogr->vertices[i].recvFromList.size();
oldpemap[i] = ogr->vertices[i].getCurrentPe();
}
for(i = baseval; i < vertnbr; i++) {
for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
if (ogr->vertices[i].sendToList[j].getNumBytes() > maxBytes) {
maxBytes = ogr->vertices[i].sendToList[j].getNumBytes();
}
}
for(j = 0; j < ogr->vertices[i].recvFromList.size(); j++) {
if (ogr->vertices[i].recvFromList[j].getNumBytes() > maxBytes) {
maxBytes = ogr->vertices[i].recvFromList[j].getNumBytes();
}
}
}
/* adjacency list */
SCOTCH_Num *verttab = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * (vertnbr+1));
/* loads of vertices */
SCOTCH_Num *velotab = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * vertnbr);
/* id of the neighbors */
SCOTCH_Num *edgetab = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * edgenbr);
/* number of bytes exchanged */
SCOTCH_Num *edlotab = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * edgenbr);
int edgeNum = 0;
double ratio = 256.0/maxLoad;
double byteRatio = 1024.0/maxBytes;
for(i = baseval; i < vertnbr; i++) {
verttab[i] = edgeNum;
velotab[i] = (int)ceil(ogr->vertices[i].getVertexLoad() * ratio);
for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
edgetab[edgeNum] = ogr->vertices[i].sendToList[j].getNeighborId();
edlotab[edgeNum] = (int) ceil(ogr->vertices[i].sendToList[j].getNumBytes()
* byteRatio);
edgeNum++;
}
for(j = 0; j < ogr->vertices[i].recvFromList.size(); j++) {
edgetab[edgeNum] = ogr->vertices[i].recvFromList[j].getNeighborId();
edlotab[edgeNum] = (int)
ceil(ogr->vertices[i].recvFromList[j].getNumBytes() * byteRatio);
edgeNum++;
}
}
verttab[i] = edgeNum;
CkAssert(edgeNum == edgenbr);
SCOTCH_Graph graph; // Graph to partition
SCOTCH_Strat strat; // Strategy to achieve partitioning
/* Initialize data structures */
SCOTCH_graphInit (&graph);
SCOTCH_stratInit (&strat);
SCOTCH_graphBuild (&graph, baseval, vertnbr, verttab, NULL, velotab, NULL, edgenbr, edgetab, edlotab);
SCOTCH_graphCheck (&graph);
double migration_cost = 1024.0;
if (step() == 0) {
SCOTCH_stratGraphMapBuild (&strat, SCOTCH_STRATBALANCE, parr->procs.size (), 0.01);
} else {
SCOTCH_stratGraphMapBuild (&strat, SCOTCH_STRATBALANCE | SCOTCH_STRATREMAP, parr->procs.size (), 0.01);
}
SCOTCH_Num *pemap = (SCOTCH_Num *)malloc(sizeof(SCOTCH_Num) * vertnbr);
// Takes as input the graph, arch graph, strategy, migration cost in
// double, old mapping and new mapping
if (step() == 0) {
SCOTCH_graphPart(&graph, parr->procs.size(), &strat, pemap);
} else {
SCOTCH_graphRepart(&graph, parr->procs.size(), oldpemap, migration_cost, NULL, &strat, pemap);
}
SCOTCH_graphExit (&graph);
SCOTCH_stratExit (&strat);
free(verttab);
free(velotab);
free(edgetab);
free(edlotab);
for(i = baseval; i < vertnbr; i++) {
if(pemap[i] != ogr->vertices[i].getCurrentPe())
ogr->vertices[i].setNewPe(pemap[i]);
}
free(pemap);
free(oldpemap);
/** ============================== CLEANUP ================================ */
ogr->convertDecisions(stats);
delete parr;
delete ogr;
}
static PetscErrorCode MatPartitioningApply_PTScotch_Private(MatPartitioning part, PetscBool useND, IS *partitioning)
{
MPI_Comm pcomm,comm;
MatPartitioning_PTScotch *scotch = (MatPartitioning_PTScotch*)part->data;
PetscErrorCode ierr;
PetscMPIInt rank;
Mat mat = part->adj;
Mat_MPIAdj *adj = (Mat_MPIAdj*)mat->data;
PetscBool flg,distributed;
PetscBool proc_weight_flg;
PetscInt i,j,p,bs=1,nold;
PetscInt *NDorder = NULL;
PetscReal *vwgttab,deltval;
SCOTCH_Num *locals,*velotab,*veloloctab,*edloloctab,vertlocnbr,edgelocnbr,nparts=part->n;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)part,&pcomm);CHKERRQ(ierr);
/* Duplicate the communicator to be sure that PTSCOTCH attribute caching does not interfere with PETSc. */
ierr = MPI_Comm_dup(pcomm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIADJ,&flg);CHKERRQ(ierr);
if (!flg) {
/* bs indicates if the converted matrix is "reduced" from the original and hence the
resulting partition results need to be stretched to match the original matrix */
nold = mat->rmap->n;
ierr = MatConvert(mat,MATMPIADJ,MAT_INITIAL_MATRIX,&mat);CHKERRQ(ierr);
if (mat->rmap->n > 0) bs = nold/mat->rmap->n;
adj = (Mat_MPIAdj*)mat->data;
}
proc_weight_flg = PETSC_TRUE;
ierr = PetscOptionsGetBool(NULL, NULL, "-mat_partitioning_ptscotch_proc_weight", &proc_weight_flg, NULL);CHKERRQ(ierr);
ierr = PetscMalloc1(mat->rmap->n+1,&locals);CHKERRQ(ierr);
if (useND) {
#if defined(PETSC_HAVE_SCOTCH_PARMETIS_V3_NODEND)
PetscInt *sizes, *seps, log2size, subd, *level, base = 0;
PetscMPIInt size;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
log2size = PetscLog2Real(size);
subd = PetscPowInt(2,log2size);
if (subd != size) SETERRQ(comm,PETSC_ERR_SUP,"Only power of 2 communicator sizes");
ierr = PetscMalloc1(mat->rmap->n,&NDorder);CHKERRQ(ierr);
ierr = PetscMalloc3(2*size,&sizes,4*size,&seps,size,&level);CHKERRQ(ierr);
SCOTCH_ParMETIS_V3_NodeND(mat->rmap->range,adj->i,adj->j,&base,NULL,NDorder,sizes,&comm);
ierr = MatPartitioningSizesToSep_Private(subd,sizes,seps,level);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
PetscInt loc;
ierr = PetscFindInt(NDorder[i],2*subd,seps,&loc);CHKERRQ(ierr);
if (loc < 0) {
loc = -(loc+1);
if (loc%2) { /* part of subdomain */
locals[i] = loc/2;
} else {
ierr = PetscFindInt(NDorder[i],2*(subd-1),seps+2*subd,&loc);CHKERRQ(ierr);
loc = loc < 0 ? -(loc+1)/2 : loc/2;
locals[i] = level[loc];
}
} else locals[i] = loc/2;
}
ierr = PetscFree3(sizes,seps,level);CHKERRQ(ierr);
#else
SETERRQ(pcomm,PETSC_ERR_SUP,"Need libptscotchparmetis.a compiled with -DSCOTCH_METIS_PREFIX");
#endif
} else {
velotab = NULL;
if (proc_weight_flg) {
ierr = PetscMalloc1(nparts,&vwgttab);CHKERRQ(ierr);
ierr = PetscMalloc1(nparts,&velotab);CHKERRQ(ierr);
for (j=0; j<nparts; j++) {
if (part->part_weights) vwgttab[j] = part->part_weights[j]*nparts;
else vwgttab[j] = 1.0;
}
for (i=0; i<nparts; i++) {
deltval = PetscAbsReal(vwgttab[i]-PetscFloorReal(vwgttab[i]+0.5));
if (deltval>0.01) {
for (j=0; j<nparts; j++) vwgttab[j] /= deltval;
}
}
for (i=0; i<nparts; i++) velotab[i] = (SCOTCH_Num)(vwgttab[i] + 0.5);
ierr = PetscFree(vwgttab);CHKERRQ(ierr);
}
vertlocnbr = mat->rmap->range[rank+1] - mat->rmap->range[rank];
edgelocnbr = adj->i[vertlocnbr];
veloloctab = part->vertex_weights;
edloloctab = adj->values;
/* detect whether all vertices are located at the same process in original graph */
for (p = 0; !mat->rmap->range[p+1] && p < nparts; ++p);
distributed = (mat->rmap->range[p+1] == mat->rmap->N) ? PETSC_FALSE : PETSC_TRUE;
if (distributed) {
SCOTCH_Arch archdat;
SCOTCH_Dgraph grafdat;
SCOTCH_Dmapping mappdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_dgraphInit(&grafdat,comm);CHKERRQ(ierr);
ierr = SCOTCH_dgraphBuild(&grafdat,0,vertlocnbr,vertlocnbr,adj->i,adj->i+1,veloloctab,
NULL,edgelocnbr,edgelocnbr,adj->j,NULL,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_dgraphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_archInit(&archdat);CHKERRQ(ierr);
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratDgraphMapBuild(&stradat,scotch->strategy,nparts,nparts,scotch->imbalance);CHKERRQ(ierr);
if (velotab) {
ierr = SCOTCH_archCmpltw(&archdat,nparts,velotab);CHKERRQ(ierr);
} else {
ierr = SCOTCH_archCmplt( &archdat,nparts);CHKERRQ(ierr);
}
ierr = SCOTCH_dgraphMapInit(&grafdat,&mappdat,&archdat,locals);CHKERRQ(ierr);
ierr = SCOTCH_dgraphMapCompute(&grafdat,&mappdat,&stradat);CHKERRQ(ierr);
SCOTCH_dgraphMapExit(&grafdat,&mappdat);
SCOTCH_archExit(&archdat);
SCOTCH_stratExit(&stradat);
SCOTCH_dgraphExit(&grafdat);
} else if (rank == p) {
SCOTCH_Graph grafdat;
SCOTCH_Strat stradat;
ierr = SCOTCH_graphInit(&grafdat);CHKERRQ(ierr);
ierr = SCOTCH_graphBuild(&grafdat,0,vertlocnbr,adj->i,adj->i+1,veloloctab,NULL,edgelocnbr,adj->j,edloloctab);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
ierr = SCOTCH_graphCheck(&grafdat);CHKERRQ(ierr);
#endif
ierr = SCOTCH_stratInit(&stradat);CHKERRQ(ierr);
ierr = SCOTCH_stratGraphMapBuild(&stradat,scotch->strategy,nparts,scotch->imbalance);CHKERRQ(ierr);
ierr = SCOTCH_graphPart(&grafdat,nparts,&stradat,locals);CHKERRQ(ierr);
SCOTCH_stratExit(&stradat);
SCOTCH_graphExit(&grafdat);
}
ierr = PetscFree(velotab);CHKERRQ(ierr);
}
ierr = MPI_Comm_free(&comm);CHKERRQ(ierr);
if (bs > 1) {
PetscInt *newlocals;
ierr = PetscMalloc1(bs*mat->rmap->n,&newlocals);CHKERRQ(ierr);
for (i=0;i<mat->rmap->n;i++) {
for (j=0;j<bs;j++) {
newlocals[bs*i+j] = locals[i];
}
}
ierr = PetscFree(locals);CHKERRQ(ierr);
ierr = ISCreateGeneral(pcomm,bs*mat->rmap->n,newlocals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,locals,PETSC_OWN_POINTER,partitioning);CHKERRQ(ierr);
}
if (useND) {
IS ndis;
if (bs > 1) {
ierr = ISCreateBlock(pcomm,bs,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
} else {
ierr = ISCreateGeneral(pcomm,mat->rmap->n,NDorder,PETSC_OWN_POINTER,&ndis);CHKERRQ(ierr);
}
ierr = ISSetPermutation(ndis);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject)(*partitioning),"_petsc_matpartitioning_ndorder",(PetscObject)ndis);CHKERRQ(ierr);
ierr = ISDestroy(&ndis);CHKERRQ(ierr);
}
if (!flg) {
ierr = MatDestroy(&mat);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode MatPartitioningApply_Scotch(MatPartitioning part, IS * partitioning)
{
PetscErrorCode ierr;
int *parttab, *locals = PETSC_NULL, rank, i, size;
size_t j;
Mat mat = part->adj, matMPI, matSeq;
int nb_locals = mat->rmap->n;
Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
PetscTruth flg;
#ifdef PETSC_HAVE_UNISTD_H
int fd_stdout, fd_pipe[2], count,err;
#endif
PetscFunctionBegin;
/* check if the matrix is sequential, use MatGetSubMatrices if necessary */
ierr = MPI_Comm_size(((PetscObject)mat)->comm, &size);CHKERRQ(ierr);
ierr = PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);CHKERRQ(ierr);
if (size > 1) {
int M, N;
IS isrow, iscol;
Mat *A;
if (flg) {
SETERRQ(0, "Distributed matrix format MPIAdj is not supported for sequential partitioners");
}
PetscPrintf(((PetscObject)part)->comm, "Converting distributed matrix to sequential: this could be a performance loss\n");CHKERRQ(ierr);
ierr = MatGetSize(mat, &M, &N);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF, M, 0, 1, &isrow);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF, N, 0, 1, &iscol);CHKERRQ(ierr);
ierr = MatGetSubMatrices(mat, 1, &isrow, &iscol, MAT_INITIAL_MATRIX, &A);CHKERRQ(ierr);
matSeq = *A;
ierr = PetscFree(A);CHKERRQ(ierr);
ierr = ISDestroy(isrow);CHKERRQ(ierr);
ierr = ISDestroy(iscol);CHKERRQ(ierr);
} else
matSeq = mat;
/* convert the the matrix to MPIADJ type if necessary */
if (!flg) {
ierr = MatConvert(matSeq, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);CHKERRQ(ierr);
} else {
matMPI = matSeq;
}
adj = (Mat_MPIAdj *) matMPI->data; /* finaly adj contains adjacency graph */
ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
{
/* definition of Scotch library arguments */
SCOTCH_Strat stratptr; /* scotch strategy */
SCOTCH_Graph grafptr; /* scotch graph */
#if defined(DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
int vertnbr = mat->rmap->N; /* number of vertices in full graph */
int *verttab = adj->i; /* start of edge list for each vertex */
int *edgetab = adj->j; /* edge list data */
int edgenbr = adj->nz; /* number of edges */
int *velotab = NULL; /* not used by petsc interface */
int *vlbltab = NULL;
int *edlotab = NULL;
int flagval = 3; /* (cf doc scotch no weight edge & vertices) */
#endif
int baseval = 0; /* 0 for C array indexing */
char strategy[256];
ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &parttab);CHKERRQ(ierr);
/* redirect output to buffer scotch -> mesg_log */
#ifdef PETSC_HAVE_UNISTD_H
fd_stdout = dup(1);
pipe(fd_pipe);
close(1);
dup2(fd_pipe[1], 1);
ierr = PetscMalloc(SIZE_LOG * sizeof(char), &(scotch->mesg_log));CHKERRQ(ierr);
#endif
/* library call */
/* Construction of the scotch graph object */
ierr = SCOTCH_graphInit(&grafptr);
#if defined(DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
ierr = SCOTCH_graphBuild((SCOTCH_Graph *) &grafptr,
(const SCOTCH_Num) vertnbr,
(const SCOTCH_Num) verttab,
(const SCOTCH_Num *)velotab,
(const SCOTCH_Num *)vlbltab,
(const SCOTCH_Num *)edgenbr,
(const SCOTCH_Num *)edgetab,
(const SCOTCH_Num) edlotab,
(const SCOTCH_Num *)baseval,
(const SCOTCH_Num *)flagval);CHKERRQ(ierr);
#else
SETERRQ(PETSC_ERR_SUP,"Scotch interface currently broken");
#endif
ierr = SCOTCH_graphCheck(&grafptr);CHKERRQ(ierr);
/* Construction of the strategy */
if (scotch->strategy[0] != 0) {
ierr = PetscStrcpy(strategy, scotch->strategy);CHKERRQ(ierr);
} else {
PetscStrcpy(strategy, "b{strat=");
if (scotch->multilevel) {
/* PetscStrcat(strategy,"m{vert=");
sprintf(strategy+strlen(strategy),"%d",scotch->nbvtxcoarsed);
PetscStrcat(strategy,",asc="); */
sprintf(strategy, "b{strat=m{vert=%d,asc=",
scotch->nbvtxcoarsed);
} else
PetscStrcpy(strategy, "b{strat=");
switch (scotch->global_method) {
case MP_SCOTCH_GREEDY:
PetscStrcat(strategy, "h");
break;
case MP_SCOTCH_GPS:
PetscStrcat(strategy, "g");
break;
case MP_SCOTCH_GR_GPS:
PetscStrcat(strategy, "g|h");
}
switch (scotch->local_method) {
case MP_SCOTCH_KERNIGHAN_LIN:
if (scotch->multilevel)
PetscStrcat(strategy, ",low=f}");
else
PetscStrcat(strategy, " f");
break;
case MP_SCOTCH_NONE:
if (scotch->multilevel)
PetscStrcat(strategy, ",asc=x}");
default:
break;
}
PetscStrcat(strategy, " x}");
}
PetscPrintf(((PetscObject)part)->comm, "strategy=[%s]\n", strategy);
ierr = SCOTCH_stratInit(&stratptr);CHKERRQ(ierr);
/*
TODO: Correct this part
Commented because this doesn't exists anymore
ierr = SCOTCH_stratMap(&stratptr, strategy);CHKERRQ(ierr);
*/
/* check for option mapping */
if (!scotch->map) {
/* ********************************************
* *
* TODO: Correct this part *
* *
* Won't work with this tmp SCOTCH_Strat... *
* *
* I just modified it to make scotch compile, *
* to be able to use PaStiX... *
* *
**********************************************/
#if defined (DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
SCOTCH_Strat tmp;
ierr = SCOTCH_graphPart((const SCOTCH_Graph *)&grafptr,
(const SCOTCH_Num) &stratptr,
(const SCOTCH_Strat *)&tmp, /* The Argument changed from scotch 3.04 it was part->n, */
(SCOTCH_Num *) parttab);CHKERRQ(ierr);
#else
SETERRQ(PETSC_ERR_SUP,"Scotch interface currently broken");
#endif
ierr = PetscPrintf(PETSC_COMM_SELF, "Partition simple without mapping\n");
} else {
SCOTCH_Graph grafarch;
SCOTCH_Num *listtab;
SCOTCH_Num listnbr = 0;
SCOTCH_Arch archptr; /* file in scotch architecture format */
SCOTCH_Strat archstrat;
int arch_total_size, *parttab_tmp,err;
int cpt;
char buf[256];
FILE *file1, *file2;
char host_buf[256];
/* generate the graph that represents the arch */
file1 = fopen(scotch->arch, "r");
if (!file1) SETERRQ1(PETSC_ERR_FILE_OPEN, "Scotch: unable to open architecture file %s", scotch->arch);
ierr = SCOTCH_graphInit(&grafarch);CHKERRQ(ierr);
ierr = SCOTCH_graphLoad(&grafarch, file1, baseval, 3);CHKERRQ(ierr);
ierr = SCOTCH_graphCheck(&grafarch);CHKERRQ(ierr);
SCOTCH_graphSize(&grafarch, &arch_total_size, &cpt);
err = fclose(file1);
if (err) SETERRQ(PETSC_ERR_SYS,"fclose() failed on file");
printf("total size = %d\n", arch_total_size);
/* generate the list of nodes currently working */
ierr = PetscGetHostName(host_buf, 256);CHKERRQ(ierr);
ierr = PetscStrlen(host_buf, &j);CHKERRQ(ierr);
file2 = fopen(scotch->host_list, "r");
if (!file2) SETERRQ1(PETSC_ERR_FILE_OPEN, "Scotch: unable to open host list file %s", scotch->host_list);
i = -1;
flg = PETSC_FALSE;
while (!feof(file2) && !flg) {
i++;
fgets(buf, 256, file2);
PetscStrncmp(buf, host_buf, j, &flg);
}
err = fclose(file2);
if (err) SETERRQ(PETSC_ERR_SYS,"fclose() failed on file");
if (!flg) SETERRQ1(PETSC_ERR_LIB, "Scotch: unable to find '%s' in host list file", host_buf);
listnbr = size;
ierr = PetscMalloc(sizeof(SCOTCH_Num) * listnbr, &listtab);CHKERRQ(ierr);
ierr = MPI_Allgather(&i, 1, MPI_INT, listtab, 1, MPI_INT, ((PetscObject)part)->comm);CHKERRQ(ierr);
printf("listnbr = %d, listtab = ", listnbr);
for (i = 0; i < listnbr; i++)
printf("%d ", listtab[i]);
printf("\n");
err = fflush(stdout);
if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on file");
ierr = SCOTCH_stratInit(&archstrat);CHKERRQ(ierr);
/**************************************************************
* *
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* *
* ierr = SCOTCH_stratBipart(&archstrat, "fx");CHKERRQ(ierr); *
**************************************************************/
ierr = SCOTCH_archInit(&archptr);CHKERRQ(ierr);
ierr = SCOTCH_archBuild(&archptr, &grafarch, listnbr, listtab,
&archstrat);CHKERRQ(ierr);
ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &parttab_tmp);CHKERRQ(ierr);
/************************************************************************************
* *
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* *
* ierr = SCOTCH_mapInit(&mappptr, &grafptr, &archptr, parttab_tmp);CHKERRQ(ierr); *
* *
* ierr = SCOTCH_mapCompute(&mappptr, &stratptr);CHKERRQ(ierr); *
* *
* ierr = SCOTCH_mapView(&mappptr, stdout);CHKERRQ(ierr); *
************************************************************************************/
/* now we have to set in the real parttab at the good place */
/* because the ranks order are different than position in */
/* the arch graph */
for (i = 0; i < mat->rmap->N; i++) {
parttab[i] = parttab_tmp[i];
}
ierr = PetscFree(listtab);CHKERRQ(ierr);
SCOTCH_archExit(&archptr);
/*************************************************
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* SCOTCH_mapExit(&mappptr); *
*************************************************/
SCOTCH_stratExit(&archstrat);
}
/* dump to mesg_log... */
#ifdef PETSC_HAVE_UNISTD_H
err = fflush(stdout);
if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on stdout");
count = read(fd_pipe[0], scotch->mesg_log, (SIZE_LOG - 1) * sizeof(char));
if (count < 0)
count = 0;
scotch->mesg_log[count] = 0;
close(1);
dup2(fd_stdout, 1);
close(fd_stdout);
close(fd_pipe[0]);
close(fd_pipe[1]);
#endif
SCOTCH_graphExit(&grafptr);
SCOTCH_stratExit(&stratptr);
}
if (ierr)
SETERRQ(PETSC_ERR_LIB, scotch->mesg_log);
/* Creation of the index set */
ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)part)->comm, &size);CHKERRQ(ierr);
nb_locals = mat->rmap->N / size;
locals = parttab + rank * nb_locals;
if (rank < mat->rmap->N % size) {
nb_locals++;
locals += rank;
} else
locals += mat->rmap->N % size;
ierr = ISCreateGeneral(((PetscObject)part)->comm, nb_locals, locals, partitioning);CHKERRQ(ierr);
/* destroying old objects */
ierr = PetscFree(parttab);CHKERRQ(ierr);
if (matSeq != mat) {
ierr = MatDestroy(matSeq);CHKERRQ(ierr);
}
if (matMPI != mat) {
ierr = MatDestroy(matMPI);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/* Internal function : biPartBoxCompute
* it computes a new numbering of graph vertices, using a bipartitioning.
*
* - graf : the input graph
* - vertNbr : the number of vertices
* - boxVertNbr : the number of vertices of each box
* - permVrtTab : the new numbering
*
* returning 0 if OK, 1 else
*/
int biPartBoxCompute(SCOTCH_Graph graf, int vertNbr, int boxVertNbr, SCOTCH_Num *permVrtTab) {
int boxNbr, vertIdx, boxIdx;
SCOTCH_Num tmp, tmp2, *partTab, *partNumTab, *partPrmTab;
SCOTCH_Strat strat ;
/* Computing the number of boxes */
boxNbr = vertNbr / boxVertNbr;
if (boxNbr * boxVertNbr != vertNbr) {
boxNbr = boxNbr + 1;
}
/* Initializing SCOTCH functions */
CHECK_SCOTCH(SCOTCH_stratInit(&strat), "scotch_stratInit", 0) ;
CHECK_SCOTCH(SCOTCH_stratGraphMap(&strat, "r{job=t,map=t,poli=S,sep=m{type=h,vert=80,low=h{pass=10}f{bal=0.005,move=0},asc=b{bnd=f{bal=0.05,move=0},org=f{bal=0.05,move=0}}}|m{type=h,vert=80,low=h{pass=10}f{bal=0.005,move=0},asc=b{bnd=f{bal=0.05,move=0},org=f{bal=0.05,move=0}}}}"), "scotch_stratGraphMap", 0) ;
partTab = (SCOTCH_Num *)M_calloc(vertNbr, sizeof(SCOTCH_Num), "boxCompute");
/* Partionning the graph */
CHECK_SCOTCH(SCOTCH_graphPart(&graf, boxNbr, &strat, partTab), "scotch_graphPart", 0);
partNumTab = (SCOTCH_Num *)M_calloc(boxNbr, sizeof(SCOTCH_Num), "boxCompute");
if (!memset(partNumTab, 0, boxNbr*sizeof(SCOTCH_Num))) {
perror("memset");
return 0;
}
/* Computing the number of elements of each box */
for( vertIdx = 0 ; vertIdx< vertNbr ;vertIdx++)
partNumTab[partTab[vertIdx]] += 1;
/* partition permutation tabular */
partPrmTab = (SCOTCH_Num *)M_calloc(vertNbr + 1, sizeof(SCOTCH_Num), "boxCompute");
/* Copying the previous tabular in order to have the index of the first
* element of each box
* */
tmp = partNumTab[0];
partNumTab[0] = 0;
for(boxIdx = 1; boxIdx < boxNbr ; boxIdx++) {
tmp2 = partNumTab[boxIdx];
partNumTab[boxIdx] = partNumTab[boxIdx-1] + tmp;
tmp = tmp2;
}
/* partPrmTab is built such as each vertex belongs to his box */
for( vertIdx = 0;vertIdx< vertNbr;vertIdx++)
partPrmTab[partNumTab[partTab[vertIdx]]++] = vertIdx;
/* Infering the new numbering */
for (vertIdx = 0; vertIdx < vertNbr ; vertIdx++)
permVrtTab[partPrmTab[vertIdx] + 1] = vertIdx + 1;
M_free(partTab);
M_free(partNumTab);
M_free(partPrmTab);
SCOTCH_stratExit(&strat) ;
return 0;
}
