// Call scotch with options from dictionary.
Foam::label Foam::ptscotchDecomp::decompose
(
const fileName& meshPath,
const label adjncySize,
const label adjncy[],
const label xadjSize,
const label xadj[],
const scalarField& cWeights,
List<label>& finalDecomp
) const
{
if (debug)
{
Pout<< "ptscotchDecomp : entering with xadj:" << xadjSize << endl;
}
// Dump graph
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
if (scotchCoeffs.lookupOrDefault("writeGraph", false))
{
OFstream str
(
meshPath + "_" + Foam::name(Pstream::myProcNo()) + ".dgr"
);
Pout<< "Dumping Scotch graph file to " << str.name() << endl
<< "Use this in combination with dgpart." << endl;
globalIndex globalCells(xadjSize-1);
// Distributed graph file (.grf)
label version = 2;
str << version << nl;
// Number of files (procglbnbr)
str << Pstream::nProcs();
// My file number (procloc)
str << ' ' << Pstream::myProcNo() << nl;
// Total number of vertices (vertglbnbr)
str << globalCells.size();
// Total number of connections (edgeglbnbr)
str << ' ' << returnReduce(xadj[xadjSize-1], sumOp<label>())
<< nl;
// Local number of vertices (vertlocnbr)
str << xadjSize-1;
// Local number of connections (edgelocnbr)
str << ' ' << xadj[xadjSize-1] << nl;
// Numbering starts from 0
label baseval = 0;
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs
str << baseval << ' ' << "000" << nl;
for (label celli = 0; celli < xadjSize-1; celli++)
{
label start = xadj[celli];
label end = xadj[celli+1];
str << end-start;
for (label i = start; i < end; i++)
{
str << ' ' << adjncy[i];
}
str << nl;
}
}
}
// Strategy
// ~~~~~~~~
// Default.
SCOTCH_Strat stradat;
check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
string strategy;
if (scotchCoeffs.readIfPresent("strategy", strategy))
{
if (debug)
{
Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
}
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
//fprintf(stdout, "\n");
}
}
// Graph
// ~~~~~
List<label> velotab;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
scalar maxWeights = gMax(cWeights);
if (maxWeights > minWeights)
{
if (minWeights <= 0)
{
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != xadjSize-1)
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << xadjSize-1
<< exit(FatalError);
}
}
scalar velotabSum = gSum(cWeights)/minWeights;
scalar rangeScale(1.0);
if (Pstream::master())
{
if (velotabSum > scalar(labelMax - 1))
{
// 0.9 factor of safety to avoid floating point round-off in
// rangeScale tipping the subsequent sum over the integer limit.
rangeScale = 0.9*scalar(labelMax - 1)/velotabSum;
WarningInFunction
<< "Sum of weights has overflowed integer: " << velotabSum
<< ", compressing weight scale by a factor of " << rangeScale
<< endl;
}
}
Pstream::scatter(rangeScale);
if (maxWeights > minWeights)
{
if (cWeights.size())
{
// Convert to integers.
velotab.setSize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int((cWeights[i]/minWeights - 1)*rangeScale) + 1;
}
}
static PetscErrorCode MatPartitioningApply_Scotch(MatPartitioning part, IS * partitioning)
{
PetscErrorCode ierr;
int *parttab, *locals = PETSC_NULL, rank, i, size;
size_t j;
Mat mat = part->adj, matMPI, matSeq;
int nb_locals = mat->rmap->n;
Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
PetscTruth flg;
#ifdef PETSC_HAVE_UNISTD_H
int fd_stdout, fd_pipe[2], count,err;
#endif
PetscFunctionBegin;
/* check if the matrix is sequential, use MatGetSubMatrices if necessary */
ierr = MPI_Comm_size(((PetscObject)mat)->comm, &size);CHKERRQ(ierr);
ierr = PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);CHKERRQ(ierr);
if (size > 1) {
int M, N;
IS isrow, iscol;
Mat *A;
if (flg) {
SETERRQ(0, "Distributed matrix format MPIAdj is not supported for sequential partitioners");
}
PetscPrintf(((PetscObject)part)->comm, "Converting distributed matrix to sequential: this could be a performance loss\n");CHKERRQ(ierr);
ierr = MatGetSize(mat, &M, &N);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF, M, 0, 1, &isrow);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF, N, 0, 1, &iscol);CHKERRQ(ierr);
ierr = MatGetSubMatrices(mat, 1, &isrow, &iscol, MAT_INITIAL_MATRIX, &A);CHKERRQ(ierr);
matSeq = *A;
ierr = PetscFree(A);CHKERRQ(ierr);
ierr = ISDestroy(isrow);CHKERRQ(ierr);
ierr = ISDestroy(iscol);CHKERRQ(ierr);
} else
matSeq = mat;
/* convert the the matrix to MPIADJ type if necessary */
if (!flg) {
ierr = MatConvert(matSeq, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);CHKERRQ(ierr);
} else {
matMPI = matSeq;
}
adj = (Mat_MPIAdj *) matMPI->data; /* finaly adj contains adjacency graph */
ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
{
/* definition of Scotch library arguments */
SCOTCH_Strat stratptr; /* scotch strategy */
SCOTCH_Graph grafptr; /* scotch graph */
#if defined(DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
int vertnbr = mat->rmap->N; /* number of vertices in full graph */
int *verttab = adj->i; /* start of edge list for each vertex */
int *edgetab = adj->j; /* edge list data */
int edgenbr = adj->nz; /* number of edges */
int *velotab = NULL; /* not used by petsc interface */
int *vlbltab = NULL;
int *edlotab = NULL;
int flagval = 3; /* (cf doc scotch no weight edge & vertices) */
#endif
int baseval = 0; /* 0 for C array indexing */
char strategy[256];
ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &parttab);CHKERRQ(ierr);
/* redirect output to buffer scotch -> mesg_log */
#ifdef PETSC_HAVE_UNISTD_H
fd_stdout = dup(1);
pipe(fd_pipe);
close(1);
dup2(fd_pipe[1], 1);
ierr = PetscMalloc(SIZE_LOG * sizeof(char), &(scotch->mesg_log));CHKERRQ(ierr);
#endif
/* library call */
/* Construction of the scotch graph object */
ierr = SCOTCH_graphInit(&grafptr);
#if defined(DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
ierr = SCOTCH_graphBuild((SCOTCH_Graph *) &grafptr,
(const SCOTCH_Num) vertnbr,
(const SCOTCH_Num) verttab,
(const SCOTCH_Num *)velotab,
(const SCOTCH_Num *)vlbltab,
(const SCOTCH_Num *)edgenbr,
(const SCOTCH_Num *)edgetab,
(const SCOTCH_Num) edlotab,
(const SCOTCH_Num *)baseval,
(const SCOTCH_Num *)flagval);CHKERRQ(ierr);
#else
SETERRQ(PETSC_ERR_SUP,"Scotch interface currently broken");
#endif
ierr = SCOTCH_graphCheck(&grafptr);CHKERRQ(ierr);
/* Construction of the strategy */
if (scotch->strategy[0] != 0) {
ierr = PetscStrcpy(strategy, scotch->strategy);CHKERRQ(ierr);
} else {
PetscStrcpy(strategy, "b{strat=");
if (scotch->multilevel) {
/* PetscStrcat(strategy,"m{vert=");
sprintf(strategy+strlen(strategy),"%d",scotch->nbvtxcoarsed);
PetscStrcat(strategy,",asc="); */
sprintf(strategy, "b{strat=m{vert=%d,asc=",
scotch->nbvtxcoarsed);
} else
PetscStrcpy(strategy, "b{strat=");
switch (scotch->global_method) {
case MP_SCOTCH_GREEDY:
PetscStrcat(strategy, "h");
break;
case MP_SCOTCH_GPS:
PetscStrcat(strategy, "g");
break;
case MP_SCOTCH_GR_GPS:
PetscStrcat(strategy, "g|h");
}
switch (scotch->local_method) {
case MP_SCOTCH_KERNIGHAN_LIN:
if (scotch->multilevel)
PetscStrcat(strategy, ",low=f}");
else
PetscStrcat(strategy, " f");
break;
case MP_SCOTCH_NONE:
if (scotch->multilevel)
PetscStrcat(strategy, ",asc=x}");
default:
break;
}
PetscStrcat(strategy, " x}");
}
PetscPrintf(((PetscObject)part)->comm, "strategy=[%s]\n", strategy);
ierr = SCOTCH_stratInit(&stratptr);CHKERRQ(ierr);
/*
TODO: Correct this part
Commented because this doesn't exists anymore
ierr = SCOTCH_stratMap(&stratptr, strategy);CHKERRQ(ierr);
*/
/* check for option mapping */
if (!scotch->map) {
/* ********************************************
* *
* TODO: Correct this part *
* *
* Won't work with this tmp SCOTCH_Strat... *
* *
* I just modified it to make scotch compile, *
* to be able to use PaStiX... *
* *
**********************************************/
#if defined (DOES_NOT_COMPILE_DUE_TO_BROKEN_INTERFACE)
SCOTCH_Strat tmp;
ierr = SCOTCH_graphPart((const SCOTCH_Graph *)&grafptr,
(const SCOTCH_Num) &stratptr,
(const SCOTCH_Strat *)&tmp, /* The Argument changed from scotch 3.04 it was part->n, */
(SCOTCH_Num *) parttab);CHKERRQ(ierr);
#else
SETERRQ(PETSC_ERR_SUP,"Scotch interface currently broken");
#endif
ierr = PetscPrintf(PETSC_COMM_SELF, "Partition simple without mapping\n");
} else {
SCOTCH_Graph grafarch;
SCOTCH_Num *listtab;
SCOTCH_Num listnbr = 0;
SCOTCH_Arch archptr; /* file in scotch architecture format */
SCOTCH_Strat archstrat;
int arch_total_size, *parttab_tmp,err;
int cpt;
char buf[256];
FILE *file1, *file2;
char host_buf[256];
/* generate the graph that represents the arch */
file1 = fopen(scotch->arch, "r");
if (!file1) SETERRQ1(PETSC_ERR_FILE_OPEN, "Scotch: unable to open architecture file %s", scotch->arch);
ierr = SCOTCH_graphInit(&grafarch);CHKERRQ(ierr);
ierr = SCOTCH_graphLoad(&grafarch, file1, baseval, 3);CHKERRQ(ierr);
ierr = SCOTCH_graphCheck(&grafarch);CHKERRQ(ierr);
SCOTCH_graphSize(&grafarch, &arch_total_size, &cpt);
err = fclose(file1);
if (err) SETERRQ(PETSC_ERR_SYS,"fclose() failed on file");
printf("total size = %d\n", arch_total_size);
/* generate the list of nodes currently working */
ierr = PetscGetHostName(host_buf, 256);CHKERRQ(ierr);
ierr = PetscStrlen(host_buf, &j);CHKERRQ(ierr);
file2 = fopen(scotch->host_list, "r");
if (!file2) SETERRQ1(PETSC_ERR_FILE_OPEN, "Scotch: unable to open host list file %s", scotch->host_list);
i = -1;
flg = PETSC_FALSE;
while (!feof(file2) && !flg) {
i++;
fgets(buf, 256, file2);
PetscStrncmp(buf, host_buf, j, &flg);
}
err = fclose(file2);
if (err) SETERRQ(PETSC_ERR_SYS,"fclose() failed on file");
if (!flg) SETERRQ1(PETSC_ERR_LIB, "Scotch: unable to find '%s' in host list file", host_buf);
listnbr = size;
ierr = PetscMalloc(sizeof(SCOTCH_Num) * listnbr, &listtab);CHKERRQ(ierr);
ierr = MPI_Allgather(&i, 1, MPI_INT, listtab, 1, MPI_INT, ((PetscObject)part)->comm);CHKERRQ(ierr);
printf("listnbr = %d, listtab = ", listnbr);
for (i = 0; i < listnbr; i++)
printf("%d ", listtab[i]);
printf("\n");
err = fflush(stdout);
if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on file");
ierr = SCOTCH_stratInit(&archstrat);CHKERRQ(ierr);
/**************************************************************
* *
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* *
* ierr = SCOTCH_stratBipart(&archstrat, "fx");CHKERRQ(ierr); *
**************************************************************/
ierr = SCOTCH_archInit(&archptr);CHKERRQ(ierr);
ierr = SCOTCH_archBuild(&archptr, &grafarch, listnbr, listtab,
&archstrat);CHKERRQ(ierr);
ierr = PetscMalloc((mat->rmap->N) * sizeof(int), &parttab_tmp);CHKERRQ(ierr);
/************************************************************************************
* *
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* *
* ierr = SCOTCH_mapInit(&mappptr, &grafptr, &archptr, parttab_tmp);CHKERRQ(ierr); *
* *
* ierr = SCOTCH_mapCompute(&mappptr, &stratptr);CHKERRQ(ierr); *
* *
* ierr = SCOTCH_mapView(&mappptr, stdout);CHKERRQ(ierr); *
************************************************************************************/
/* now we have to set in the real parttab at the good place */
/* because the ranks order are different than position in */
/* the arch graph */
for (i = 0; i < mat->rmap->N; i++) {
parttab[i] = parttab_tmp[i];
}
ierr = PetscFree(listtab);CHKERRQ(ierr);
SCOTCH_archExit(&archptr);
/*************************************************
* TODO: Correct this part *
* *
* Commented because this doesn't exists anymore *
* SCOTCH_mapExit(&mappptr); *
*************************************************/
SCOTCH_stratExit(&archstrat);
}
/* dump to mesg_log... */
#ifdef PETSC_HAVE_UNISTD_H
err = fflush(stdout);
if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on stdout");
count = read(fd_pipe[0], scotch->mesg_log, (SIZE_LOG - 1) * sizeof(char));
if (count < 0)
count = 0;
scotch->mesg_log[count] = 0;
close(1);
dup2(fd_stdout, 1);
close(fd_stdout);
close(fd_pipe[0]);
close(fd_pipe[1]);
#endif
SCOTCH_graphExit(&grafptr);
SCOTCH_stratExit(&stratptr);
}
if (ierr)
SETERRQ(PETSC_ERR_LIB, scotch->mesg_log);
/* Creation of the index set */
ierr = MPI_Comm_rank(((PetscObject)part)->comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)part)->comm, &size);CHKERRQ(ierr);
nb_locals = mat->rmap->N / size;
locals = parttab + rank * nb_locals;
if (rank < mat->rmap->N % size) {
nb_locals++;
locals += rank;
} else
locals += mat->rmap->N % size;
ierr = ISCreateGeneral(((PetscObject)part)->comm, nb_locals, locals, partitioning);CHKERRQ(ierr);
/* destroying old objects */
ierr = PetscFree(parttab);CHKERRQ(ierr);
if (matSeq != mat) {
ierr = MatDestroy(matSeq);CHKERRQ(ierr);
}
if (matMPI != mat) {
ierr = MatDestroy(matMPI);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
void AlgPTScotch<Adapter>::partition(
const RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
SCOTCH_Num partnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(partnbr, numGlobalParts);
#ifdef HAVE_ZOLTAN2_MPI
int ierr = 0;
int me = problemComm->getRank();
const SCOTCH_Num baseval = 0; // Base value for array indexing.
// GraphModel returns GNOs from base 0.
SCOTCH_Strat stratstr; // Strategy string
// TODO: Set from parameters
SCOTCH_stratInit(&stratstr);
// Allocate and initialize PTScotch Graph data structure.
SCOTCH_Dgraph *gr = SCOTCH_dgraphAlloc(); // Scotch distributed graph
ierr = SCOTCH_dgraphInit(gr, mpicomm);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphInit",
!ierr, BASIC_ASSERTION, problemComm);
// Get vertex info
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
size_t nVtx = model->getVertexList(vtxID, vwgts);
SCOTCH_Num vertlocnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(vertlocnbr, nVtx);
SCOTCH_Num vertlocmax = vertlocnbr; // Assumes no holes in global nums.
// Get edge info
ArrayView<const gno_t> edgeIds;
ArrayView<const lno_t> offsets;
ArrayView<StridedData<lno_t, scalar_t> > ewgts;
size_t nEdge = model->getEdgeList(edgeIds, offsets, ewgts);
SCOTCH_Num edgelocnbr=0;
TPL_Traits<SCOTCH_Num, size_t>::ASSIGN_TPL_T(edgelocnbr, nEdge);
const SCOTCH_Num edgelocsize = edgelocnbr; // Assumes adj array is compact.
SCOTCH_Num *vertloctab; // starting adj/vtx
TPL_Traits<SCOTCH_Num, const lno_t>::ASSIGN_TPL_T_ARRAY(&vertloctab, offsets);
SCOTCH_Num *edgeloctab; // adjacencies
TPL_Traits<SCOTCH_Num, const gno_t>::ASSIGN_TPL_T_ARRAY(&edgeloctab, edgeIds);
// We don't use these arrays, but we need them as arguments to Scotch.
SCOTCH_Num *vendloctab = NULL; // Assume consecutive storage for adj
SCOTCH_Num *vlblloctab = NULL; // Vertex label array
SCOTCH_Num *edgegsttab = NULL; // Array for ghost vertices
// Get weight info.
SCOTCH_Num *velotab = NULL; // Vertex weights
SCOTCH_Num *edlotab = NULL; // Edge weights
int nVwgts = model->getNumWeightsPerVertex();
int nEwgts = model->getNumWeightsPerEdge();
if (nVwgts > 1 && me == 0) {
std::cerr << "Warning: NumWeightsPerVertex is " << nVwgts
<< " but Scotch allows only one weight. "
<< " Zoltan2 will use only the first weight per vertex."
<< std::endl;
}
if (nEwgts > 1 && me == 0) {
std::cerr << "Warning: NumWeightsPerEdge is " << nEwgts
<< " but Scotch allows only one weight. "
<< " Zoltan2 will use only the first weight per edge."
<< std::endl;
}
if (nVwgts) {
velotab = new SCOTCH_Num[nVtx+1]; // +1 since Scotch wants all procs
// to have non-NULL arrays
scale_weights(nVtx, vwgts[0], velotab);
}
if (nEwgts) {
edlotab = new SCOTCH_Num[nEdge+1]; // +1 since Scotch wants all procs
// to have non-NULL arrays
scale_weights(nEdge, ewgts[0], edlotab);
}
// Build PTScotch distributed data structure
ierr = SCOTCH_dgraphBuild(gr, baseval, vertlocnbr, vertlocmax,
vertloctab, vendloctab, velotab, vlblloctab,
edgelocnbr, edgelocsize,
edgeloctab, edgegsttab, edlotab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphBuild",
!ierr, BASIC_ASSERTION, problemComm);
// Create array for Scotch to return results in.
ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx,true);
SCOTCH_Num *partloctab = NULL;
if (nVtx && (sizeof(SCOTCH_Num) == sizeof(part_t))) {
// Can write directly into the solution's memory
partloctab = (SCOTCH_Num *) partList.getRawPtr();
}
else {
// Can't use solution memory directly; will have to copy later.
// Note: Scotch does not like NULL arrays, so add 1 to always have non-null.
// ParMETIS has this same "feature." See Zoltan bug 4299.
partloctab = new SCOTCH_Num[nVtx+1];
}
// Get target part sizes
float *partsizes = new float[numGlobalParts];
if (!solution->criteriaHasUniformPartSizes(0))
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = solution->getCriteriaPartSize(0, i);
else
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = 1.0 / float(numGlobalParts);
// Allocate and initialize PTScotch target architecture data structure
SCOTCH_Arch archdat;
SCOTCH_archInit(&archdat);
SCOTCH_Num velosum = 0;
SCOTCH_dgraphSize (gr, &velosum, NULL, NULL, NULL);
SCOTCH_Num *goalsizes = new SCOTCH_Num[partnbr];
// TODO: The goalsizes are set as in Zoltan; not sure it is correct there
// or here.
// It appears velosum is global NUMBER of vertices, not global total
// vertex weight. I think we should use the latter.
// Fix this when we add vertex weights.
for (SCOTCH_Num i = 0; i < partnbr; i++)
goalsizes[i] = SCOTCH_Num(ceil(velosum * partsizes[i]));
delete [] partsizes;
SCOTCH_archCmpltw(&archdat, partnbr, goalsizes);
// Call partitioning; result returned in partloctab.
ierr = SCOTCH_dgraphMap(gr, &archdat, &stratstr, partloctab);
env->globalInputAssertion(__FILE__, __LINE__, "SCOTCH_dgraphMap",
!ierr, BASIC_ASSERTION, problemComm);
SCOTCH_archExit(&archdat);
delete [] goalsizes;
// TODO - metrics
#ifdef SHOW_ZOLTAN2_SCOTCH_MEMORY
int me = env->comm_->getRank();
#endif
#ifdef HAVE_SCOTCH_ZOLTAN2_GETMEMORYMAX
if (me == 0){
size_t scotchBytes = SCOTCH_getMemoryMax();
std::cout << "Rank " << me << ": Maximum bytes used by Scotch: ";
std::cout << scotchBytes << std::endl;
}
#endif
// Clean up PTScotch
SCOTCH_dgraphExit(gr);
free(gr);
SCOTCH_stratExit(&stratstr);
// Load answer into the solution.
if ((sizeof(SCOTCH_Num) != sizeof(part_t)) || (nVtx == 0)) {
for (size_t i = 0; i < nVtx; i++) partList[i] = partloctab[i];
delete [] partloctab;
}
solution->setParts(partList);
env->memory("Zoltan2-Scotch: After creating solution");
// Clean up copies made due to differing data sizes.
TPL_Traits<SCOTCH_Num, const lno_t>::DELETE_TPL_T_ARRAY(&vertloctab);
TPL_Traits<SCOTCH_Num, const gno_t>::DELETE_TPL_T_ARRAY(&edgeloctab);
if (nVwgts) delete [] velotab;
if (nEwgts) delete [] edlotab;
#else // DO NOT HAVE MPI
// TODO: Handle serial case with calls to Scotch.
// TODO: For now, assign everything to rank 0 and assume only one part.
// TODO: Can probably use the code above for loading solution,
// TODO: instead of duplicating it here.
// TODO
// TODO: Actual logic should call Scotch when number of processes == 1.
ArrayView<const gno_t> vtxID;
ArrayView<StridedData<lno_t, scalar_t> > vwgts;
size_t nVtx = model->getVertexList(vtxID, vwgts);
ArrayRCP<part_t> partList(new part_t[nVtx], 0, nVtx, true);
for (size_t i = 0; i < nVtx; i++) partList[i] = 0;
solution->setParts(partList);
#endif // DO NOT HAVE MPI
}
static
void run_scotch( Container &c,
MapContainer &mapping,
const std::size_t cores,
weight_function_t weight_func,
void *weight )
{
#if 0
static_assert( std::is_signed<
std::remove_reference< decltype( c.end() ) >::type >::value,
"Container must have signed types so that -1 may signify no mapping" );
#endif
raftgraph_t raft_graph;
get_graph_info( c,
raft_graph,
weight_func,
nullptr );
SCOTCH_Graph graph;
if( SCOTCH_graphInit( &graph ) != 0 )
{
/** TODO, add RaftLib Exception **/
std::cerr << "Failed to initialize graph!!\n";
exit( EXIT_FAILURE );
}
auto table( raft_graph.getScotchTables() );
if( SCOTCH_graphBuild(
&graph /** graph ptr **/,
0 /** base value **/,
table.num_vertices /** vertex nmbr (zero indexed) **/,
table.vtable /** vertex tab **/,
&table.vtable[ 1 ] /** vendtab **/,
nullptr /** velotab **/,
nullptr /** vlbltab **/,
table.num_edges /** edge number **/,
table.etable /** edge tab **/,
table.eweight /** edlotab **/
) != 0 )
{
/** TODO, add RaftLib Exception **/
std::cerr << "Failed to build graph\n";
exit( EXIT_FAILURE );
}
if( SCOTCH_graphCheck( &graph ) != 0 )
{
/** TODO, add RaftLib Exception **/
std::cerr << "Graph is inconsistent\n";
std::remove_reference< decltype( table ) >::type::print( std::cerr, table );
std::cerr << "\n";
raft_graph.print( std::cerr );
exit( EXIT_FAILURE );
}
/** TODO, we can do much more with this arch file **/
SCOTCH_Arch archdat;
if( SCOTCH_archInit( &archdat ) != 0 )
{
/** TODO, add RaftLib Exception **/
std::cerr << "Architecture initialization failed\n";
exit( EXIT_FAILURE );
}
/** core are equal **/
if( SCOTCH_archCmplt( &archdat, cores /** num cores **/) != 0 )
{
/** TODO, add RaftLib Exception **/
std::cerr << "Failed to create architecture file\n";
exit( EXIT_FAILURE );
}
/** strategy **/
SCOTCH_Strat stradat;
if( SCOTCH_stratInit( &stradat ) != 0 )
{
/** TODO, add RaftLib Exception **/
std::cerr << "Failed to init strategy!!\n";
exit( EXIT_FAILURE );
}
/** build recursive strategy **/
if( SCOTCH_stratGraphClusterBuild(
&stradat,
SCOTCH_STRATSPEED,
cores,
.75,
.01) != 0 )
{
/** TODO, add RaftLib Exception **/
std::cerr << "Failed to map strategy graph!!\n";
exit( EXIT_FAILURE );
}
if( SCOTCH_graphMap(
&graph /** graph ptr **/,
&archdat,
&stradat,
table.partition /** parttab **/
) != 0 )
{
/** TODO, add RaftLib Exception **/
std::cerr << "Failed to map!!\n";
exit( EXIT_FAILURE );
}
/**
* first case is for if we've mapped all vertices,
* second is for when some of the kernels are innactive
* in which case the number of vertices in the
* table will be less than the size of c in which case
* we need to get which vertices (the actual number id
* from the application) are mapped and to where, the
* returned table in mapping must include even the
* vertices that aren't active (indicated by a -1) so
* that the returning loop can be as simple as possible
*/
if( c.size() == table.num_vertices )
{
/** copy mapping **/
for( auto i( 0 ); i < table.num_vertices; i++ )
{
mapping.emplace_back( table.partition[ i ] );
}
}
else
{
const auto &vmapping( raft_graph.getVertexNumbersAtIndicies() );
auto it_map_index( vmapping.cbegin() );
auto table_index( 0 );
const auto size( c.size() );
for( auto i( 0 ); i < size; i++ )
{
if( i == (*it_map_index) && it_map_index != vmapping.cend() )
{
mapping.emplace_back( table.partition[ table_index++ ] );
++it_map_index;
}
else
{
mapping.emplace_back( -1 );
}
}
}
/** call exit graph **/
SCOTCH_graphExit( &graph );
SCOTCH_stratExit( &stradat );
SCOTCH_archExit ( &archdat );
return;
}
void computeOrdering(const ordinal_type treecut = 0) {
int ierr = 0;
// pointers for global graph ordering
ordinal_type *perm = _perm.ptr_on_device();
ordinal_type *peri = _peri.ptr_on_device();
ordinal_type *range = _range.ptr_on_device();
ordinal_type *tree = _tree.ptr_on_device();
{
// set desired tree level
if (_strat & SCOTCH_STRATLEVELMAX ||
_strat & SCOTCH_STRATLEVELMIN) {
TACHO_TEST_FOR_ABORT(_level == 0, "SCOTCH_STRATLEVEL(MIN/MAX) is used but level is not specified");
}
const int level = Util::max(1, _level-treecut);
SCOTCH_Strat stradat;
SCOTCH_Num straval = _strat;
ierr = SCOTCH_stratInit(&stradat);TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_stratInit");
// if both are zero, do not build strategy
if (_strat || _level) {
std::cout << "GraphTools_Scotch:: User provide a strategy and/or level" << std::endl
<< " strategy = " << _strat << ", level = " << _level << ", treecut = " << treecut << std::endl
<< " strategy & SCOTCH_STRATLEVELMAX = " << (_strat & SCOTCH_STRATLEVELMAX) << std::endl
<< " strategy & SCOTCH_STRATLEVELMIN = " << (_strat & SCOTCH_STRATLEVELMIN) << std::endl
<< " strategy & SCOTCH_STRATLEAFSIMPLE = " << (_strat & SCOTCH_STRATLEAFSIMPLE) << std::endl
<< " strategy & SCOTCH_STRATSEPASIMPLE = " << (_strat & SCOTCH_STRATSEPASIMPLE) << std::endl
<< std::endl;
ierr = SCOTCH_stratGraphOrderBuild(&stradat, straval, level, 0.2);
TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_stratGraphOrderBuild");
}
ierr = SCOTCH_graphOrder(&_graph,
&stradat,
perm,
peri,
&_cblk,
range,
tree);
TACHO_TEST_FOR_ABORT(ierr, "Failed in SCOTCH_graphOrder");
SCOTCH_stratExit(&stradat);
}
{
ordinal_type nroot = 0;
for (ordinal_type i=0;i<_cblk;++i)
nroot += (_tree[i] == -1);
if (nroot > 1) {
std::cout << "GraphTools_Scotch:: # of roots " << nroot << std::endl
<< " a fake root is created to complete the tree" << std::endl
<< std::endl;
_tree [_cblk] = -1; // dummy root
_range[_cblk+1] = _range[_cblk]; // zero range for the dummy root
for (ordinal_type i=0;i<_cblk;++i)
if (_tree[i] == -1) // multiple roots becomes children of the dummy root
_tree[i] = (_cblk+1);
++_cblk; // include the dummy root
}
}
_is_ordered = true;
//std::cout << "SCOTCH level = " << level << std::endl;
//std::cout << "Range Tree " << std::endl;
//for (int i=0;i<_cblk;++i)
// std::cout << _range[i] << " :: " << i << " " << _tree[i] << std::endl;
}
int
main (
int argc,
char * argv[])
{
SCOTCH_Num vnodnbr; /* Number of nodes */
SCOTCH_Mesh meshdat; /* Source graph */
SCOTCH_Ordering ordedat; /* Graph ordering */
SCOTCH_Num * permtab; /* Permutation array */
SCOTCH_Strat stradat; /* Ordering strategy */
SCOTCH_Num straval;
char * straptr;
int flagval;
Clock runtime[2]; /* Timing variables */
int i, j;
errorProg ("mord");
intRandInit ();
if ((argc >= 2) && (argv[1][0] == '?')) { /* If need for help */
usagePrint (stdout, C_usageList);
return (0);
}
flagval = C_FLAGNONE; /* Default behavior */
straval = 0; /* No strategy flags */
straptr = NULL;
SCOTCH_stratInit (&stradat);
for (i = 0; i < C_FILENBR; i ++) /* Set default stream pointers */
C_fileTab[i].pntr = (C_fileTab[i].mode[0] == 'r') ? stdin : stdout;
for (i = 1; i < argc; i ++) { /* Loop for all option codes */
if ((argv[i][0] != '-') || (argv[i][1] == '\0') || (argv[i][1] == '.')) { /* If found a file name */
if (C_fileNum < C_FILEARGNBR) /* File name has been given */
C_fileTab[C_fileNum ++].name = argv[i];
else {
errorPrint ("main: too many file names given");
return (1);
}
}
else { /* If found an option name */
switch (argv[i][1]) {
case 'C' :
case 'c' : /* Strategy selection parameters */
for (j = 2; argv[i][j] != '\0'; j ++) {
switch (argv[i][j]) {
case 'B' :
case 'b' :
straval |= SCOTCH_STRATBALANCE;
break;
case 'Q' :
case 'q' :
straval |= SCOTCH_STRATQUALITY;
break;
case 'S' :
case 's' :
straval |= SCOTCH_STRATSPEED;
break;
case 'T' :
case 't' :
straval |= SCOTCH_STRATSAFETY;
break;
default :
errorPrint ("main: invalid strategy selection option (\"%c\")", argv[i][j]);
}
}
break;
case 'H' : /* Give the usage message */
case 'h' :
usagePrint (stdout, C_usageList);
return (0);
case 'M' : /* Output separator mapping */
case 'm' :
flagval |= C_FLAGMAPOUT;
if (argv[i][2] != '\0')
C_filenamemapout = &argv[i][2];
break;
case 'O' : /* Ordering strategy */
case 'o' :
straptr = &argv[i][2];
SCOTCH_stratExit (&stradat);
SCOTCH_stratInit (&stradat);
if ((SCOTCH_stratMeshOrder (&stradat, straptr)) != 0) {
errorPrint ("main: invalid ordering strategy");
return (1);
}
break;
case 'V' :
fprintf (stderr, "mord, version " SCOTCH_VERSION_STRING "\n");
fprintf (stderr, "Copyright 2004,2007,2008,2010 ENSEIRB, INRIA & CNRS, France\n");
fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
return (0);
break;
case 'v' : /* Output control info */
for (j = 2; argv[i][j] != '\0'; j ++) {
switch (argv[i][j]) {
case 'S' :
case 's' :
flagval |= C_FLAGVERBSTR;
break;
case 'T' :
case 't' :
flagval |= C_FLAGVERBTIM;
break;
default :
errorPrint ("main: unprocessed parameter \"%c\" in \"%s\"",
argv[i][j], argv[i]);
return (1);
}
}
break;
default :
errorPrint ("main: unprocessed option (\"%s\")", argv[i]);
return (1);
}
}
}
fileBlockOpen (C_fileTab, C_FILENBR); /* Open all files */
clockInit (&runtime[0]);
clockStart (&runtime[0]);
SCOTCH_meshInit (&meshdat); /* Create mesh structure */
SCOTCH_meshLoad (&meshdat, C_filepntrsrcinp, -1); /* Read source mesh */
SCOTCH_meshSize (&meshdat, NULL, &vnodnbr, NULL); /* Get number of nodes */
if (straval != 0) {
if (straptr != NULL)
errorPrint ("main: options '-c' and '-o' are exclusive");
SCOTCH_stratMeshOrderBuild (&stradat, straval, 0.1);
}
clockStop (&runtime[0]); /* Get input time */
clockInit (&runtime[1]);
clockStart (&runtime[1]);
if ((permtab = (SCOTCH_Num *) memAlloc (vnodnbr * sizeof (SCOTCH_Num))) == NULL) {
errorPrint ("main: out of memory");
return (1);
}
SCOTCH_meshOrderInit (&meshdat, &ordedat, permtab, NULL, NULL, NULL, NULL); /* Create ordering */
SCOTCH_meshOrderCompute (&meshdat, &ordedat, &stradat); /* Perform ordering */
clockStop (&runtime[1]); /* Get ordering time */
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_meshOrderCheck (&meshdat, &ordedat) != 0)
return (1);
#endif /* SCOTCH_DEBUG_ALL */
clockStart (&runtime[0]);
SCOTCH_meshOrderSave (&meshdat, &ordedat, C_filepntrordout); /* Write ordering */
if (flagval & C_FLAGMAPOUT) /* If mapping wanted */
SCOTCH_meshOrderSaveMap (&meshdat, &ordedat, C_filepntrmapout); /* Write mapping */
clockStop (&runtime[0]); /* Get output time */
if (flagval & C_FLAGVERBSTR) {
fprintf (C_filepntrlogout, "S\tStrat=");
SCOTCH_stratSave (&stradat, C_filepntrlogout);
putc ('\n', C_filepntrlogout);
}
if (flagval & C_FLAGVERBTIM) {
fprintf (C_filepntrlogout, "T\tOrder\t\t%g\nT\tI/O\t\t%g\nT\tTotal\t\t%g\n",
(double) clockVal (&runtime[1]),
(double) clockVal (&runtime[0]),
(double) clockVal (&runtime[0]) +
(double) clockVal (&runtime[1]));
}
fileBlockClose (C_fileTab, C_FILENBR); /* Always close explicitely to end eventual (un)compression tasks */
SCOTCH_meshOrderExit (&meshdat, &ordedat);
SCOTCH_stratExit (&stradat);
SCOTCH_meshExit (&meshdat);
memFree (permtab);
#ifdef COMMON_PTHREAD
pthread_exit ((void *) 0); /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
return (0);
}
int
main (
int argc,
char * argv[])
{
SCOTCH_Strat bipastrat; /* Bipartitioning strategy */
SCOTCH_Arch archdat; /* Target (terminal) architecture */
SCOTCH_Graph grafdat; /* Source graph to turn into architecture */
SCOTCH_Num vertnbr; /* Number of vertices in graph */
SCOTCH_Num * vlbltab; /* Pointer to vertex label array, if present */
SCOTCH_Num listnbr; /* Size of list array */
SCOTCH_Num * listtab; /* Pointer to list array */
C_VertSort * sorttab; /* Vertex label sort area */
SCOTCH_Num baseval;
SCOTCH_Num vertnum;
SCOTCH_Num listnum;
int flag; /* Process flags */
int i;
errorProg ("amk_grf");
intRandInit ();
if ((argc >= 2) && (argv[1][0] == '?')) { /* If need for help */
usagePrint (stdout, C_usageList);
return (0);
}
flag = C_FLAGNONE;
SCOTCH_stratInit (&bipastrat);
for (i = 0; i < C_FILENBR; i ++) /* Set default stream pointers */
C_fileTab[i].pntr = (C_fileTab[i].mode[0] == 'r') ? stdin : stdout;
for (i = 1; i < argc; i ++) { /* Loop for all option codes */
if ((argv[i][0] != '-') || (argv[i][1] == '\0') || (argv[i][1] == '.')) { /* If found a file name */
if (C_fileNum < C_FILEARGNBR) /* File name has been given */
C_fileTab[C_fileNum ++].name = argv[i];
else {
errorPrint ("main: too many file names given");
return (1);
}
}
else { /* If found an option name */
switch (argv[i][1]) {
case 'B' : /* Bipartitioning strategy */
case 'b' :
SCOTCH_stratExit (&bipastrat);
SCOTCH_stratInit (&bipastrat);
if ((SCOTCH_stratGraphBipart (&bipastrat, &argv[i][2])) != 0) {
errorPrint ("main: invalid bipartitioning strategy");
return (1);
}
break;
case 'H' : /* Give the usage message */
case 'h' :
usagePrint (stdout, C_usageList);
return (0);
case 'L' : /* Input vertex list */
case 'l' :
flag |= C_FLAGVRTINP;
if (argv[i][2] != '\0')
C_filenamevrtinp = &argv[i][2];
break;
case 'V' :
fprintf (stderr, "amk_grf, version " SCOTCH_VERSION_STRING "\n");
fprintf (stderr, "Copyright 2004,2007,2008,2010 ENSEIRB, INRIA & CNRS, France\n");
fprintf (stderr, "This software is libre/free software under CeCILL-C -- see the user's manual for more information\n");
return (0);
default :
errorPrint ("main: unprocessed option (\"%s\")", argv[i]);
return (1);
}
}
}
fileBlockOpen (C_fileTab, C_FILENBR); /* Open all files */
SCOTCH_graphInit (&grafdat); /* Create graph structure */
SCOTCH_graphLoad (&grafdat, C_filepntrgrfinp, -1, 0); /* Load source graph */
SCOTCH_graphData (&grafdat, &baseval, &vertnbr, NULL, NULL, NULL, /* Get graph data */
&vlbltab, NULL, NULL, NULL);
listnbr = 0; /* Initialize vertex list */
listtab = NULL;
if (flag & C_FLAGVRTINP) { /* If list of vertices provided */
if ((intLoad (C_filepntrvrtinp, &listnbr) != 1) || /* Read list size */
(listnbr < 0) ||
(listnbr > vertnbr)) {
errorPrint ("main: bad list input (1)");
return (1);
}
if ((listtab = (SCOTCH_Num *) memAlloc (vertnbr * sizeof (SCOTCH_Num) + 1)) == NULL) {
errorPrint ("main: out of memory (1)");
return (1);
}
for (vertnum = 0; vertnum < vertnbr; vertnum ++) { /* Read list data */
if (intLoad (C_filepntrvrtinp, &listtab[vertnum]) != 1) {
errorPrint ("main: bad list input (2)");
return (1);
}
}
intSort1asc1 (listtab, vertnbr);
for (vertnum = 0; vertnum < vertnbr - 1; vertnum ++) { /* Search for duplicates */
if (listtab[vertnum] == listtab[vertnum + 1]) {
errorPrint ("main: duplicate list labels");
memFree (listtab);
return (1);
}
}
if (vlbltab != NULL) { /* If graph has vertex labels */
if ((sorttab = (C_VertSort *) memAlloc (vertnbr * sizeof (C_VertSort))) == NULL) {
errorPrint ("main: out of memory (2)");
memFree (listtab);
return (1);
}
for (vertnum = 0; vertnum < vertnbr; vertnum ++) { /* Initialize sort area */
sorttab[vertnum].labl = vlbltab[vertnum];
sorttab[vertnum].num = vertnum;
}
intSort2asc1 (sorttab, vertnbr); /* Sort by ascending labels */
for (listnum = 0, vertnum = 0; listnum < listnbr; listnum ++) { /* For all labels in list */
while ((vertnum < vertnbr) && (sorttab[vertnum].labl < listtab[listnum]))
vertnum ++; /* Search vertex graph with corresponding label */
if ((vertnum >= vertnbr) || /* If label not found */
(sorttab[vertnum].labl > listtab[listnum])) {
errorPrint ("main: list label not in graph (" SCOTCH_NUMSTRING ")", (SCOTCH_Num) listtab[listnum]);
memFree (sorttab);
memFree (listtab);
return (1);
}
listtab[listnum] = sorttab[vertnum ++].num; /* Replace label by number */
}
memFree (sorttab); /* Free sort area */
}
}
SCOTCH_archInit (&archdat); /* Initialize target architecture */
SCOTCH_archBuild (&archdat, &grafdat, listnbr, listtab, &bipastrat); /* Compute architecture */
SCOTCH_archSave (&archdat, C_filepntrtgtout); /* Write target architecture */
fileBlockClose (C_fileTab, C_FILENBR); /* Always close explicitely to end eventual (un)compression tasks */
SCOTCH_graphExit (&grafdat); /* Free target graph */
SCOTCH_archExit (&archdat); /* Free target architecture */
SCOTCH_stratExit (&bipastrat); /* Free strategy string */
if (listtab != NULL) /* If vertex list provided */
memFree (listtab); /* Free it */
#ifdef COMMON_PTHREAD
pthread_exit ((void *) 0); /* Allow potential (un)compression tasks to complete */
#endif /* COMMON_PTHREAD */
return (0);
}
/* Internal function : biPartBoxCompute
* it computes a new numbering of graph vertices, using a bipartitioning.
*
* - graf : the input graph
* - vertNbr : the number of vertices
* - boxVertNbr : the number of vertices of each box
* - permVrtTab : the new numbering
*
* returning 0 if OK, 1 else
*/
int biPartBoxCompute(SCOTCH_Graph graf, int vertNbr, int boxVertNbr, SCOTCH_Num *permVrtTab) {
int boxNbr, vertIdx, boxIdx;
SCOTCH_Num tmp, tmp2, *partTab, *partNumTab, *partPrmTab;
SCOTCH_Strat strat ;
/* Computing the number of boxes */
boxNbr = vertNbr / boxVertNbr;
if (boxNbr * boxVertNbr != vertNbr) {
boxNbr = boxNbr + 1;
}
/* Initializing SCOTCH functions */
CHECK_SCOTCH(SCOTCH_stratInit(&strat), "scotch_stratInit", 0) ;
CHECK_SCOTCH(SCOTCH_stratGraphMap(&strat, "r{job=t,map=t,poli=S,sep=m{type=h,vert=80,low=h{pass=10}f{bal=0.005,move=0},asc=b{bnd=f{bal=0.05,move=0},org=f{bal=0.05,move=0}}}|m{type=h,vert=80,low=h{pass=10}f{bal=0.005,move=0},asc=b{bnd=f{bal=0.05,move=0},org=f{bal=0.05,move=0}}}}"), "scotch_stratGraphMap", 0) ;
partTab = (SCOTCH_Num *)M_calloc(vertNbr, sizeof(SCOTCH_Num), "boxCompute");
/* Partionning the graph */
CHECK_SCOTCH(SCOTCH_graphPart(&graf, boxNbr, &strat, partTab), "scotch_graphPart", 0);
partNumTab = (SCOTCH_Num *)M_calloc(boxNbr, sizeof(SCOTCH_Num), "boxCompute");
if (!memset(partNumTab, 0, boxNbr*sizeof(SCOTCH_Num))) {
perror("memset");
return 0;
}
/* Computing the number of elements of each box */
for( vertIdx = 0 ; vertIdx< vertNbr ;vertIdx++)
partNumTab[partTab[vertIdx]] += 1;
/* partition permutation tabular */
partPrmTab = (SCOTCH_Num *)M_calloc(vertNbr + 1, sizeof(SCOTCH_Num), "boxCompute");
/* Copying the previous tabular in order to have the index of the first
* element of each box
* */
tmp = partNumTab[0];
partNumTab[0] = 0;
for(boxIdx = 1; boxIdx < boxNbr ; boxIdx++) {
tmp2 = partNumTab[boxIdx];
partNumTab[boxIdx] = partNumTab[boxIdx-1] + tmp;
tmp = tmp2;
}
/* partPrmTab is built such as each vertex belongs to his box */
for( vertIdx = 0;vertIdx< vertNbr;vertIdx++)
partPrmTab[partNumTab[partTab[vertIdx]]++] = vertIdx;
/* Infering the new numbering */
for (vertIdx = 0; vertIdx < vertNbr ; vertIdx++)
permVrtTab[partPrmTab[vertIdx] + 1] = vertIdx + 1;
M_free(partTab);
M_free(partNumTab);
M_free(partPrmTab);
SCOTCH_stratExit(&strat) ;
return 0;
}
/* Internal function : kPartBoxCompute
* it computes a new numbering of graph vertices, using a k-partitioning.
* Assuming that baseval of the graph is 1
*
* - graf : the input graph
* - vertNbr : the number of vertices
* - boxVertNbr : the number of vertices of each box
* - permVrtTab : the new numbering
*
* returning 0 if OK, 1 else
*/
int kPartBoxCompute(SCOTCH_Graph graf, int vertNbr, int boxVertNbr, SCOTCH_Num *permVrtTab) {
int boxNbr, vertIdx;
SCOTCH_Num logMaxVal, SupMaxVal, InfMaxVal, maxVal;
char s[200];
SCOTCH_Num *sortPartTb;
SCOTCH_Strat strat ;
SCOTCH_Arch arch;
/* Computing the number of boxes */
boxNbr = vertNbr / boxVertNbr;
if (boxNbr * boxVertNbr != vertNbr) {
boxNbr = boxNbr + 1;
}
/* Initializing SCOTCH functions */
CHECK_SCOTCH(SCOTCH_stratInit(&strat), "scotch_stratInit", 0) ;
CHECK_SCOTCH(SCOTCH_archVcmplt(&arch), "scotch_archVcmplt", 0) ;
sprintf(s, "m{vert=%d,low=r{job=t,map=t,poli=S,sep=m{type=h,vert=80,low=h{pass=10}f{bal=0.0005,move=80},asc=f{bal=0.005,move=80}}}}", vertNbr / boxVertNbr);
CHECK_SCOTCH(SCOTCH_stratGraphMap(&strat, s), "scotch_stratGraphMap", 0) ;
sortPartTb= (SCOTCH_Num *)M_calloc(2*vertNbr, sizeof(SCOTCH_Num), "boxCompute");
/* Partionning the graph */
CHECK_SCOTCH(SCOTCH_graphMap(&graf, &arch, &strat, sortPartTb), "scotch_graphMap", 0);
// Looking for the max value in sortPartTb and computing sortPartTb as
// followed :
// - sortPartTb[2i] is the box value
// - sortPartTb[2i+1] is the vertex number
maxVal = sortPartTb[0];
for (vertIdx = vertNbr - 1 ; vertIdx >= 0 ; vertIdx--) {
sortPartTb[2*vertIdx] = sortPartTb[vertIdx];
sortPartTb[2*vertIdx+1] = vertIdx + 1;
if (sortPartTb[vertIdx] > maxVal)
maxVal = sortPartTb[vertIdx];
}
// Determining the log of MaxVal
logMaxVal = 0;
while ( maxVal > 0) {
logMaxVal++;
maxVal >>= 1;
}
// Infering the interval in which box values will be
InfMaxVal = logMaxVal << logMaxVal;
SupMaxVal = (logMaxVal << (logMaxVal + 1)) - 1;
// Increasing box values until they are in the previous interval
for (vertIdx = 0 ; vertIdx < vertNbr ; vertIdx++) {
while (!(sortPartTb[2*vertIdx] >= InfMaxVal && sortPartTb[2*vertIdx] <= SupMaxVal)) {
sortPartTb[2*vertIdx] <<= 1;
}
}
// Sorting the tabular, which contains box values and vertex numbers
_SCOTCHintSort2asc1(sortPartTb, vertNbr);
/* Infering the new numbering */
for (vertIdx = 0; vertIdx < vertNbr ; vertIdx++) {
permVrtTab[sortPartTb[2*vertIdx + 1]] = vertIdx + 1;
}
SCOTCH_stratExit(&strat) ;
SCOTCH_archExit(&arch) ;
M_free(sortPartTb);
return 0;
}
void
METISNAMEU(ParMETIS_V3_NodeND) (
const int * const vtxdist,
int * const xadj,
int * const adjncy,
const int * const numflag,
const int * const options, /* Not used */
int * const order,
int * const sizes, /* Of size twice the number of processors ; not used */
MPI_Comm * comm)
{
MPI_Comm proccomm;
int procglbnbr;
int proclocnum;
SCOTCH_Num baseval;
SCOTCH_Dgraph grafdat; /* Scotch distributed graph object to interface with libScotch */
SCOTCH_Dordering ordedat; /* Scotch distributed ordering object to interface with libScotch */
SCOTCH_Strat stradat;
SCOTCH_Num vertlocnbr;
SCOTCH_Num edgelocnbr;
if (sizeof (SCOTCH_Num) != sizeof (int)) {
SCOTCH_errorPrint ("ParMETIS_V3_NodeND (as of SCOTCH): SCOTCH_Num type should equate to int");
return;
}
proccomm = *comm;
if (SCOTCH_dgraphInit (&grafdat, proccomm) != 0)
return;
MPI_Comm_size (proccomm, &procglbnbr);
MPI_Comm_rank (proccomm, &proclocnum);
baseval = *numflag;
vertlocnbr = vtxdist[proclocnum + 1] - vtxdist[proclocnum];
edgelocnbr = xadj[vertlocnbr] - baseval;
if (sizes != NULL)
memSet (sizes, ~0, (2 * procglbnbr - 1) * sizeof (int)); /* Array not used if procglbnbr is not a power of 2 or if error */
if (SCOTCH_dgraphBuild (&grafdat, baseval,
vertlocnbr, vertlocnbr, xadj, xadj + 1, NULL, NULL,
edgelocnbr, edgelocnbr, adjncy, NULL, NULL) == 0) {
SCOTCH_stratInit (&stradat);
#ifdef SCOTCH_DEBUG_ALL
if (SCOTCH_dgraphCheck (&grafdat) == 0) /* TRICK: next instruction called only if graph is consistent */
#endif /* SCOTCH_DEBUG_ALL */
{
if (SCOTCH_dgraphOrderInit (&grafdat, &ordedat) == 0) {
int levlmax;
int bitsnbr;
SCOTCH_Num proctmp;
SCOTCH_dgraphOrderCompute (&grafdat, &ordedat, &stradat);
SCOTCH_dgraphOrderPerm (&grafdat, &ordedat, order);
for (levlmax = -1, bitsnbr = 0, proctmp = procglbnbr; /* Count number of bits set to 1 in procglbnbr */
proctmp != 0; levlmax ++, proctmp >>= 1)
bitsnbr += proctmp & 1;
if (bitsnbr == 1) {
SCOTCH_Num cblkglbnbr;
if ((cblkglbnbr = SCOTCH_dgraphOrderCblkDist (&grafdat, &ordedat)) >= 0) {
SCOTCH_Num * treeglbtab;
SCOTCH_Num * sizeglbtab;
SCOTCH_Num * sepaglbtab;
if (memAllocGroup ((void **) (void *)
&treeglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num)),
&sizeglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num)),
&sepaglbtab, (size_t) (cblkglbnbr * sizeof (SCOTCH_Num) * 3), NULL) != NULL) {
if (SCOTCH_dgraphOrderTreeDist (&grafdat, &ordedat, treeglbtab, sizeglbtab) == 0) {
SCOTCH_Num rootnum;
SCOTCH_Num cblknum;
memSet (sepaglbtab, ~0, cblkglbnbr * sizeof (SCOTCH_Num) * 3);
for (rootnum = -1, cblknum = 0; cblknum < cblkglbnbr; cblknum ++) {
SCOTCH_Num fathnum;
fathnum = treeglbtab[cblknum] - baseval; /* Use un-based indices */
if (fathnum < 0) { /* If father index indicates root */
if (rootnum != -1) { /* If another root already found */
rootnum = -1; /* Indicate an error */
break;
}
rootnum = cblknum; /* Record index of root node */
}
else {
int i;
for (i = 0; i < 3; i ++) {
int j;
j = 3 * fathnum + i; /* Slot number of prospective son */
if (sepaglbtab[j] < 0) { /* If potentially empty slot found */
if (sepaglbtab[j] == -1) /* If we don't have too many sons */
sepaglbtab[j] = cblknum; /* Add link to son in slot */
break;
}
}
if (i == 3) { /* If no empty slot found */
sepaglbtab[3 * fathnum] = -2; /* Indicate there are too many sons */
break;
}
}
}
if ((rootnum >= 0) && (sizes != NULL)) { /* If no error above, go on processing separator tree */
memSet (sizes, 0, (2 * procglbnbr - 1) * sizeof (int)); /* Set array of sizes to 0 by default */
_SCOTCH_ParMETIS_V3_NodeNDTree (sizes + (2 * procglbnbr - 1), sizeglbtab, sepaglbtab, levlmax, 0, rootnum, 1);
}
}
memFree (treeglbtab); /* Free group leader */
}
}
}
SCOTCH_dgraphOrderExit (&grafdat, &ordedat);
}
}
