int CLAPS_sparse_lu::factor(int N_, int NNZ, int COLPTR[], int ROWIDX[],
double ANZ[], int scale_flag_)
{
//
// Input:
// N_ = number of equations
// NNZ = number of nonzeros in full matrix
// ROWIDX[COLPTR[i]:COLPTR[i+1]-1] = nonzero row numbers for column i
// ANZ[ COLPTR[i]:COLPTR[i+1]-1] = nonzero values in column i
// Note: C-style numbering of inputs is assumed (i.e. row and column
// numbers are all between 0 and N_-1
//
// solver parameters
//
DEFBLK = 1;
int order_opt(2);
max_small = 4;
//
N = N_;
if (N == 0) return 0;
if (N <= max_small) {
int INFO;
INFO = small_factor(COLPTR, ROWIDX, ANZ);
return INFO;
}
scale_flag = scale_flag_;
//
// scale matrix entries if requested
//
double *ANZ_SCALED;
if (scale_flag == 1) {
int i, j;
SCALE = new double[N];
for (i=0; i<N; i++) SCALE[i] = 0;
for (i=0; i<N; i++) {
for (j=COLPTR[i]; j<COLPTR[i+1]; j++)
if (ROWIDX[j] == i) SCALE[i] = sqrt(fabs(ANZ[j]));
assert (SCALE[i] != 0);
}
ANZ_SCALED = new double[NNZ];
for (i=0; i<N; i++) SCALE[i] = 1/SCALE[i];
for (i=0; i<N; i++) {
for (j=COLPTR[i]; j<COLPTR[i+1]; j++) {
ANZ_SCALED[j] = SCALE[i]*SCALE[ROWIDX[j]]*ANZ[j];
}
}
}
else {
ANZ_SCALED = ANZ;
}
int NNZA, NADJ, IWMAX, IWSIZE, IFLAG, MAXSUP, NTOT, RWSIZE, LDNS;
int NNZL, NSUB, NLNZ, TMPSIZ, MAXDEF, ASDEF;
double ANORM, EPS, TOL;
int OPTIONS[8];
//
OPTIONS[0]=0; // use default values for options
OPTIONS[1]=0;
MAXSUP=150; // maximum supernode size
NTOT=NNZ; // total number of nonzeros in matrix
NADJ=NTOT-N; // number of nonzeros in full matrix minus those on diagonal
NNZA=NADJ/2+N;// number of nonzeros in lower triangle including diagonal
IWMAX=7*N+3; // dimension for integer working array
if ((MAXSUP+2*N+1) > IWMAX) IWMAX=MAXSUP+2*N+1;
if ((3*N+2*MAXSUP) > IWMAX) IWMAX=3*N+2*MAXSUP;
MAXDEF=10;
ASDEF=0;
// std::cout << "N = " << N << std::endl;
// std::cout << "NTOT = " << NTOT << std::endl;
// std::cout << "NADJ = " << NADJ << std::endl;
// std::cout << "NNZA = " << NNZA << std::endl;
int* ADJ = new int[NTOT];
int* XADJ = new int[N+1];
LINDX = new int[NTOT];
PERM = new int[N];
INVP = new int[N];
int* IWORK = new int[IWMAX];
int* COLCNT = new int[N];
int* SNODE = new int[N];
XSUPER = new int[N+1];
XLINDX = new int[N+1];
XLNZ = new int[N+1];
DEF = new int[N];
IPROW = new int[N];
IPCOL = new int[N];
//
// determine adjacency structure of matrix
// ADJ[XADJ[i]:XADJ[i+1]-1] = dofs adjacent to dof i (but not including
// dof i itself)
//
XADJ[0]=0;
int nnzADJ=0;
for (int i=0; i<N; i++) {
for (int j=COLPTR[i]; j<COLPTR[i+1]; j++) {
int row=ROWIDX[j];
if (row != i) {
ADJ[nnzADJ]=row;
nnzADJ++;
}
}
XADJ[i+1]=nnzADJ;
}
//
// compute L-infinity norm of matrix
//
getnrm(N, COLPTR, ROWIDX, ANZ_SCALED, ANORM);
//
// convert COLPTR, ROWIDX, XADJ, and ADJ to Fortran numbering
//
for (int i=0; i<=N; i++) COLPTR[i]++;
for (int i=0; i<NNZ; i++) ROWIDX[i]++;
for (int i=0; i<=N; i++) XADJ[i]++;
for (int i=0; i<nnzADJ; i++) ADJ[i]++;
//
NADJ=XADJ[N]-1;
for (int i=0; i<=N; i++) XLINDX[i]=XADJ[i];
for (int i=0; i<nnzADJ; i++) LINDX[i]=ADJ[i];
//
// multiple minimum degree ordering
//
IWSIZE=4*N;
if (order_opt == 1) {
ORDMMD2_F77(N,XLINDX,LINDX,INVP,PERM,IWSIZE,IWORK,NSUB,IFLAG);
if (IFLAG != 0) {
std::cout << "error in call to ordmmd2 in CLAPS_sparse_lu::factor" << std::endl;
std::cout << "ORDMMD2 IFLAG=" << IFLAG << std::endl;
return -1;
}
}
//
// Metis ordering
//
if (order_opt == 2) {
int numflag=1;
std::vector<idx_t> options(METIS_NOPTIONS,0);
METIS_SetDefaultOptions(&options[0]);
options[METIS_OPTION_NUMBERING] = numflag;
METIS_NodeND(&N,XLINDX,LINDX,0,&options[0],PERM,INVP);
}
//
// symbolic factorization initialization
//
IWSIZE=7*N+3;
SFINIT_F77(N,NADJ,XADJ,ADJ,PERM,INVP,MAXSUP,DEFBLK,COLCNT,
NNZL,NSUB,NSUPER,XSUPER,SNODE,IWSIZE,IWORK,IFLAG);
if (IFLAG != 0) {
std::cout << "error in call to sfinit in CLAPS_sparse_lu::factor" << std::endl;
std::cout << "SFINIT IFLAG=" << IFLAG << std::endl;
return -1;
}
//
// supernodal symbolic factorization
//
IWSIZE=NSUPER+2*N+1;
if (NSUB>NTOT) {
delete [] LINDX;
LINDX = new int[NSUB];
}
SYMFCT_F77(N,NADJ,XADJ,ADJ,PERM,INVP,COLCNT,NSUPER,XSUPER,
SNODE,NSUB,XLINDX,LINDX,XLNZ,IWSIZE,IWORK,IFLAG);
if (IFLAG != 0) {
std::cout << "error in call to symfct in CLAPS_sparse_lu::factor" << std::endl;
std::cout << "SYMFCT IFLAG=" << IFLAG << std::endl;
return -1;
}
//
// input numerical values into data structures
//
NLNZ=XLNZ[N];
// std::cout << "number of nonzeros in LU factorization = " << NLNZ << std::endl;
// std::cout << "NLNZ = " << NLNZ << std::endl;
// later, sparsepak will call dgemm on some of the "panels" in this data. We need to have
// memory on the end of the array to ensure we don't overrun. Without the extra "N", we
// may not have all of the last column allocated.
LNZ = new double[NLNZ+N];
for (int i=0; i<NLNZ; i++) LNZ[i]=0;
for (int i=0; i<N; i++) DEF[i]=0;
NDEF=0;
LBDEF=0;
inpnv(N,COLPTR,ROWIDX,ANZ_SCALED,PERM,INVP,NSUPER,XSUPER,XLINDX,LINDX,XLNZ,
LNZ,IWORK);
if (scale_flag == 1) delete [] ANZ_SCALED;
//
// numerical factorization
//
BFINIT_F77(NSUPER,XSUPER,SNODE,XLINDX,LINDX,TMPSIZ,RWSIZE);
if (TMPSIZ < 1) TMPSIZ=1;
TMPSIZ=2*TMPSIZ;
double* TMPVEC = new double[TMPSIZ];
int RWORKdim=N;
if (RWSIZE > N) RWORKdim=RWSIZE;
double* RWORK = new double[RWORKdim];
// EPS=1e-10;
EPS = 1e-12;
// EPS=DLAMCH('EPS');
TOL=EPS*ANORM;
IWSIZE = 3*N + 2*NSUPER;
BLKLDL_F77(NSUPER,XSUPER,SNODE,XLINDX,LINDX,XLNZ,LNZ,DEFBLK,ASDEF,NDEF,
LBDEF,DEF,TOL,IPROW,IPCOL,TMPSIZ,TMPVEC,IWSIZE,IWORK,
RWSIZE,RWORK,IFLAG);
if (IFLAG != 0) {
std::cout << "error in call to blkldl in CLAPS_sparse_lu::factor" << std::endl;
std::cout << "BLKLDL IFLAG=" << IFLAG << std::endl;
return -1;
}
// if (DEFBLK == 0) {
// LBDEF=0;
// NDEF=0;
// }
if ((NDEF != 0) && (NDEF <= MAXDEF)) {
//
// compute null space
//
NS = new double [N*NDEF];
LDNS=N;
BLKNS_F77(NSUPER,XSUPER,XLINDX,LINDX,XLNZ,LNZ,DEFBLK,NDEF,LBDEF,
DEF,IPCOL,INVP,NS,LDNS,RWORK);
std::cout << "null space dimension = " << NDEF << std::endl;
/*
std::cout << "NS = " << std::endl;
for (int i=0;i<NDEF;i++) {
for (int j=0;j<N;j++) std::cout << NS[j+N*i] << " ";
std::cout << std::endl;
}
*/
}
delete [] ADJ;
ADJ=0;
delete [] XADJ;
XADJ=0;
delete [] IWORK;
IWORK=0;
delete [] COLCNT;
COLCNT=0;
delete [] SNODE;
SNODE=0;
delete [] TMPVEC;
TMPVEC=0;
delete [] RWORK;
RWORK=0;
//
// convert COLPTR and ROWIDX back to C numbering
//
for (int i=0; i<=N; i++) COLPTR[i]--;
for (int i=0; i<NNZ; i++) ROWIDX[i]--;
return 0;
}
int gpartition(const vector< set<int> > &graph, int npartitions, int partition_method, vector<int> &decomp){
// If no partitioning method is set, choose a default.
if(partition_method<0){
if(npartitions<=8)
partition_method = 0; // METIS PartGraphRecursive
else
partition_method = 1; // METIS PartGraphKway
}
int nnodes = graph.size();
// Compress graph
vector<idxtype> xadj(nnodes+1), adjncy;
int pos=0;
xadj[0]=1;
for(int i=0;i<nnodes;i++){
for(set<int>::iterator jt=graph[i].begin();jt!=graph[i].end();jt++){
adjncy.push_back(*jt);
pos++;
}
xadj[i+1] = pos+1;
}
// Partition graph
decomp.resize(nnodes);
int wgtflag=0, numflag=1, edgecut=0;
#ifdef PARMETIS_V3
int options[] = {0};
#else
idx_t nbc=1;
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING]=1;
#endif
if(partition_method){
#ifdef PARMETIS_V3
METIS_PartGraphKway(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag,
&numflag, &npartitions, options, &edgecut, &(decomp[0]));
#else
METIS_PartGraphKway(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL,
&npartitions, NULL, NULL,options,&edgecut,
&(decomp[0]));
#endif
}else{
#ifdef PARMETIS_V3
METIS_PartGraphRecursive(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag,
&numflag, &npartitions, options, &edgecut, &(decomp[0]));
#else
METIS_PartGraphRecursive(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL,
&npartitions, NULL, NULL,options,&edgecut,
&(decomp[0]));
#endif
}
// number from zero
for(int i=0;i<nnodes;i++)
decomp[i]--;
return edgecut;
}
void metis_wrapper(int *n, int *xadj, int *adjncy, int *nparts, int *part)
{
#ifndef METIS_VER_MAJOR
/* Metis 4.X (or METISLib in ParaMetis 3.X) */
int edgecut;
int wgtflag = 0;
int numflag = 1; /* fortran-stype numbering */
int options[5] = { 0, 3, 1, 1, 0 }; /* default values */
METIS_PartGraphRecursive(n, xadj, adjncy, NULL, NULL, &wgtflag, &numflag,
nparts, options, &edgecut, part);
#elif METIS_VER_MAJOR == 5
/* Metis 5.X */
int edgecut;
int ncon = 1;
int options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING] = 1; /* fortran-stype numbering */
METIS_PartGraphRecursive(n, &ncon, xadj, adjncy, NULL, NULL, NULL,
nparts, NULL, NULL, options, &edgecut, part);
#else
#error unknown Metis version
#endif
}
std::vector<bool>balanced_min_cut(const ListGraph&g){
assert(g.is_valid());
#ifndef USE_CUT_ORDER
return {};
#else
std::vector<int>out_begin;
std::vector<int>out_dest;
build_adj_array(out_begin, out_dest,
g.node_count(), g.arc.size(),
[&](int x){return g.arc[x].source;},
[&](int x){return g.arc[x].target;}
);
std::vector<int>part(g.node_count());
// Call METIS
int one = 1, two = 2, ignore, node_count = g.node_count();
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING] = 0;
options[METIS_OPTION_CONTIG] = 1;
options[METIS_OPTION_UFACTOR] = 100;
//options[METIS_OPTION_NCUTS] = 10;
METIS_PartGraphKway(
(idx_t*)&node_count,
(idx_t*)&one,
(idx_t*)&out_begin[0],
(idx_t*)&out_dest[0],
nullptr,
nullptr,
nullptr,
(idx_t*)&two,
nullptr,
nullptr,
options,
(idx_t*)&ignore,
(idx_t*)&part[0]
);
std::vector<bool>part_ret(g.node_count());
for(int i=0; i<g.node_count(); ++i)
part_ret[i] = part[i] == 0;
return std::move(part_ret);
#endif
}
/* Interface for metis k-way partitioning with 64-bit ints */
void MUMPS_CALL
MUMPS_METIS_KWAY_64(MUMPS_INT8 *n, MUMPS_INT8 *iptr,
MUMPS_INT8 *jcn, MUMPS_INT8 *k,
MUMPS_INT8 *part)
/* n -- the size of the graph to be partitioned
iptr -- pointer to the beginning of each node's adjacency list
jcn -- jcn[iptr[i]:iptr[i+1]-1] contains the list of neighbors of node i
k -- the number of parts
part -- part[i] is the part node i belongs to */
/* SELECTIVE I8 FIXME: add an argument *ierr, check it on exit */
{
#if defined(metis4) || defined(parmetis3)
MUMPS_INT numflag, edgecut, wgtflag, options[8];
MUMPS_INT kINT, nINT;
options[0] = 0;
/* unweighted partitioning */
wgtflag = 0;
/* Use 1-based fortran numbering */
numflag = 1;
/* n and k are MUMPS_INT */
nINT=(MUMPS_INT)(*n);
kINT=(MUMPS_INT)(*k);
/* void METIS_PartGraphKway(int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, int *, int *, int *, idxtype *); */
METIS_PartGraphKway(&nINT, iptr, jcn,
NULL, NULL, &wgtflag,
&numflag, &kINT,
options, &edgecut,
part);
#else /* METIS >= 5 */
int ierr;
# if (IDXTYPEWIDTH == 64)
MUMPS_INT8 ncon, edgecut, options[40];
ierr=METIS_SetDefaultOptions(options);
options[0] = 0;
/* Use 1-based fortran numbering */
options[17] = 1;
ncon = 1;
ierr = METIS_PartGraphKway(n, &ncon, iptr, jcn,
NULL, NULL, NULL,
k, NULL, NULL, options,
&edgecut, part);
# else
printf("** Error: METIS version >= 4, IDXTYPE WIDTH !=64, but MUMPS_METIS_KWAY_64 was called\n");
ierr=1;
# endif
#endif
return;
}
/*! \brief It set Metis on test
*
* \param testing set to true to disable the testing
*
* At the moment disable the seed randomness to keep the result
* reproducible
*
*/
void onTest(bool testing)
{
if (testing == false)
return;
if (Mg.options == NULL)
{
// allocate
Mg.options = new idx_t[METIS_NOPTIONS];
// set default options
METIS_SetDefaultOptions(Mg.options);
}
Mg.options[METIS_OPTION_SEED] = 0;
}
std::vector<int> MetisMeshPartition(const CMesh* mesh, int nPart)
{
int vertCount = mesh->vertCount();
std::vector<int> vPart(vertCount);
int ncon = 1;
std::vector<int> xadj, adjncy;
ZGeom::getMeshGraphCSR(*mesh, xadj, adjncy);
int objval;
int options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_CONTIG] = 1;
options[METIS_OPTION_NUMBERING] = 0;
METIS_PartGraphKway(&vertCount, &ncon, &xadj[0], &adjncy[0], NULL, NULL, NULL, &nPart, NULL, NULL, NULL, &objval, &vPart[0]);
return vPart;
}
int do_metis_kway_partition(network_t *network, options_t *opt, idx_t *part, idx_t mode) {
int ret = ORCC_OK;
idx_t ncon = 1;
idx_t metis_opt[METIS_NOPTIONS];
idx_t objval;
adjacency_list *graph, *metis_graph;
assert(network != NULL);
assert(opt != NULL);
assert(part != NULL);
print_orcc_trace(ORCC_VL_VERBOSE_1, "Applying METIS Kway partition for mapping");
METIS_SetDefaultOptions(metis_opt);
metis_opt[METIS_OPTION_OBJTYPE] = mode;
metis_opt[METIS_OPTION_CONTIG] = 0; /* 0 or 1 */
graph = set_graph_from_network(network);
metis_graph = fix_graph_for_metis(graph);
ret = METIS_PartGraphKway(&metis_graph->nb_vertices, /* idx_t *nvtxs */
&ncon, /*idx_t *ncon*/
metis_graph->xadj, /*idx_t *xadj*/
metis_graph->adjncy, /*idx_t *adjncy*/
metis_graph->vwgt, /*idx_t *vwgt*/
NULL, /*idx_t *vsize*/
metis_graph->adjwgt, /*idx_t *adjwgt*/
&opt->nb_processors, /*idx_t *nparts*/
NULL, /*real t *tpwgts*/
NULL, /*real t ubvec*/
metis_opt, /*idx_t *options*/
&objval, /*idx_t *objval*/
part); /*idx_t *part*/
check_metis_error(ret);
print_orcc_trace(ORCC_VL_VERBOSE_2, "METIS Edgecut : %d", objval);
delete_graph(graph);
delete_graph(metis_graph);
return ret;
}
/***********************************************************************************
* This function is the entry point of the parallel kmetis algorithm that uses
* coordinates to compute an initial graph distribution.
************************************************************************************/
int ParMETIS_V3_PartGeomKway(idx_t *vtxdist, idx_t *xadj, idx_t *adjncy,
idx_t *vwgt, idx_t *adjwgt, idx_t *wgtflag, idx_t *numflag, idx_t *ndims,
real_t *xyz, idx_t *ncon, idx_t *nparts, real_t *tpwgts, real_t *ubvec,
idx_t *options, idx_t *edgecut, idx_t *part, MPI_Comm *comm)
{
idx_t h, i, j, npes, mype, status, nvtxs, seed, dbglvl;
idx_t cut, gcut, maxnvtxs;
idx_t moptions[METIS_NOPTIONS];
ctrl_t *ctrl;
graph_t *graph, *mgraph;
real_t balance;
size_t curmem;
/* Check the input parameters and return if an error */
status = CheckInputsPartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag,
numflag, ndims, xyz, ncon, nparts, tpwgts, ubvec, options,
edgecut, part, comm);
if (GlobalSEMinComm(*comm, status) == 0)
return METIS_ERROR;
status = METIS_OK;
gk_malloc_init();
curmem = gk_GetCurMemoryUsed();
/* Setup the ctrl */
ctrl = SetupCtrl(PARMETIS_OP_GKMETIS, options, *ncon, *nparts, tpwgts, ubvec, *comm);
npes = ctrl->npes;
mype = ctrl->mype;
/* Take care the nparts == 1 case */
if (*nparts == 1) {
iset(vtxdist[mype+1]-vtxdist[mype], (*numflag == 0 ? 0 : 1), part);
*edgecut = 0;
goto DONE;
}
/* Take care of npes == 1 case */
if (npes == 1) {
nvtxs = vtxdist[1] - vtxdist[0]; /* subtraction is required when numflag==1 */
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_NUMBERING] = *numflag;
status = METIS_PartGraphKway(&nvtxs, ncon, xadj, adjncy, vwgt, NULL, adjwgt,
nparts, tpwgts, ubvec, moptions, edgecut, part);
goto DONE;
}
/* Setup the graph */
if (*numflag > 0)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
graph = SetupGraph(ctrl, *ncon, vtxdist, xadj, vwgt, NULL, adjncy, adjwgt, *wgtflag);
gk_free((void **)&graph->nvwgt, LTERM);
/* Allocate the workspace */
AllocateWSpace(ctrl, 10*graph->nvtxs);
/* Compute the initial npes-way partitioning geometric partitioning */
STARTTIMER(ctrl, ctrl->TotalTmr);
Coordinate_Partition(ctrl, graph, *ndims, xyz, 1);
STOPTIMER(ctrl, ctrl->TotalTmr);
/* Move the graph according to the partitioning */
STARTTIMER(ctrl, ctrl->MoveTmr);
ctrl->nparts = npes;
mgraph = MoveGraph(ctrl, graph);
ctrl->nparts = *nparts;
SetupGraph_nvwgts(ctrl, mgraph); /* compute nvwgts for the moved graph */
if (ctrl->dbglvl&DBG_INFO) {
CommInterfaceData(ctrl, graph, graph->where, graph->where+graph->nvtxs);
for (cut=0, i=0; i<graph->nvtxs; i++) {
for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++) {
if (graph->where[i] != graph->where[graph->adjncy[j]])
cut += graph->adjwgt[j];
}
}
gcut = GlobalSESum(ctrl, cut)/2;
maxnvtxs = GlobalSEMax(ctrl, mgraph->nvtxs);
balance = (real_t)(maxnvtxs)/((real_t)(graph->gnvtxs)/(real_t)(npes));
rprintf(ctrl, "XYZ Cut: %6"PRIDX" \tBalance: %6.3"PRREAL" [%"PRIDX" %"PRIDX" %"PRIDX"]\n",
gcut, balance, maxnvtxs, graph->gnvtxs, npes);
}
STOPTIMER(ctrl, ctrl->MoveTmr);
/* Compute the partition of the moved graph */
STARTTIMER(ctrl, ctrl->TotalTmr);
ctrl->CoarsenTo = gk_min(vtxdist[npes]+1, 25*(*ncon)*gk_max(npes, *nparts));
if (vtxdist[npes] < SMALLGRAPH
|| vtxdist[npes] < npes*20
|| GlobalSESum(ctrl, mgraph->nedges) == 0) { /* serially */
IFSET(ctrl->dbglvl, DBG_INFO,
rprintf(ctrl, "Partitioning a graph of size %"PRIDX" serially\n", vtxdist[npes]));
PartitionSmallGraph(ctrl, mgraph);
}
else { /* in parallel */
Global_Partition(ctrl, mgraph);
}
ParallelReMapGraph(ctrl, mgraph);
/* Invert the ordering back to the original graph */
ctrl->nparts = npes;
ProjectInfoBack(ctrl, graph, part, mgraph->where);
ctrl->nparts = *nparts;
*edgecut = mgraph->mincut;
STOPTIMER(ctrl, ctrl->TotalTmr);
/* Print some stats */
IFSET(ctrl->dbglvl, DBG_TIME, PrintTimingInfo(ctrl));
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->gcomm));
IFSET(ctrl->dbglvl, DBG_INFO, PrintPostPartInfo(ctrl, mgraph, 0));
FreeGraph(mgraph);
FreeInitialGraphAndRemap(graph);
if (*numflag > 0)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
DONE:
FreeCtrl(&ctrl);
if (gk_GetCurMemoryUsed() - curmem > 0) {
printf("ParMETIS appears to have a memory leak of %zdbytes. Report this.\n",
(ssize_t)(gk_GetCurMemoryUsed() - curmem));
}
gk_malloc_cleanup(0);
return (int)status;
}
void FC_FUNC_(oct_metis_setdefaultoptions, OCT_METIS_SETDEFAULTOPTIONS)
(idx_t *options)
{
METIS_SetDefaultOptions(options);
}
/*
* Assign loop iterations to tiles carving partitions out of /seedLoop/ using
* the METIS library.
*/
static int* metis(loop_t* seedLoop, int tileSize, map_list* meshMaps,
int* nCore, int* nExec, int* nNonExec, int nThreads)
{
int i;
int setCore = seedLoop->set->core;
int setSize = seedLoop->set->size;
// use the mesh description to find a suitable map for partitioning through METIS
map_t* map = NULL;
map_list::const_iterator it, end;
for (it = meshMaps->begin(), end = meshMaps->end(); it != end; it++) {
if (set_eq(seedLoop->set, (*it)->inSet) || set_eq(seedLoop->set, (*it)->outSet)) {
map = *it;
break;
}
}
if (! map) {
// unfortunate scenario: the user provided a mesh description, but the loop picked
// as seed has an iteration space which is not part of the mesh description.
// will have to revert to chunk partitioning
return NULL;
}
// now partition through METIS:
// ... mesh geometry
int nElements = map->inSet->size;
int nNodes = map->outSet->size;
int nParts = nElements / tileSize;
int arity = map->size / nElements;
// ... data needed for partitioning
int* indMap = new int[nElements];
int* indNodesMap = new int[nNodes];
int* adjncy = map->values;
int* offsets = new int[nElements+1]();
for (i = 1; i < nElements+1; i++) {
offsets[i] = offsets[i-1] + arity;
}
// ... options
int result, objval, ncon = 1;
int options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING] = 0;
options[METIS_OPTION_CONTIG] = 1;
// ... do partition!
result = (arity == 2) ?
METIS_PartGraphKway (&nNodes, &ncon, offsets, adjncy, NULL, NULL, NULL,
&nParts, NULL, NULL, options, &objval, indMap) :
METIS_PartMeshNodal (&nElements, &nNodes, offsets, adjncy, NULL, NULL,
&nParts, NULL, options, &objval, indMap, indNodesMap);
ASSERT(result == METIS_OK, "Invalid METIS partitioning");
// what's the target iteration set ?
if (set_eq(seedLoop->set, map->inSet)) {
delete[] indNodesMap;
}
else {
// note: must be set_eq(seedLoop->set, map-outSet)
delete[] indMap;
indMap = indNodesMap;
}
delete[] offsets;
// restrict partitions to the core region
std::fill (indMap + setCore, indMap + setSize, 0);
std::set<int> partitions (indMap, indMap + setCore);
// ensure the set of partitions IDs is compact (i.e., if we have a partitioning
// 0: {0,1,...}, 1: {4,5,...}, 2: {}, 3: {6,10,...} ...
// we instead want to have
// 0: {0,1,...}, 1: {4,5,...}, 2: {6,10,...}, ...
std::map<int, int> mapper;
std::set<int>::const_iterator sIt, sEnd;
for (i = 0, sIt = partitions.begin(), sEnd = partitions.end(); sIt != sEnd; sIt++, i++) {
mapper[*sIt] = i;
}
for (i = 0; i < setCore; i++) {
indMap[i] = mapper[indMap[i]];
}
*nCore = partitions.size();
// partition the exec halo region
chunk_halo (seedLoop, tileSize, *nCore - 1, indMap, nExec, nNonExec, nThreads);
return indMap;
}
int main(int argc, char *argv[])
{
idx_t options[METIS_NOPTIONS];
bigraph_t *bigraph;
idx_t *perm, *iperm;
params_t *params;
int status, i, j;
/* rdiags[i][0] and cdiags[i][0] saves the length of each array
* excluding the first value */
idx_t **rdiags, **cdiags;
idx_t ndiags;
params = parse_cmdline(argc, argv);
gk_startcputimer(params->iotimer);
bigraph = ReadBiGraph(params);
gk_stopcputimer(params->iotimer);
if(bigraph == NULL){
printf("Input Error : nrows + ncols != nvtxs\n");
printf("\n***Metis returned with an error.\n");
return -1;
}
BDFPrintInfo(params, bigraph);
METIS_SetDefaultOptions(options);
/*User specific parameters*/
options[METIS_OPTION_CTYPE] = params->ctype;
options[METIS_OPTION_IPTYPE] = params->iptype;
options[METIS_OPTION_RTYPE] = params->rtype;
options[METIS_OPTION_CCORDER] = params->ccorder;
options[METIS_OPTION_SEED] = params->seed;
options[METIS_OPTION_DBGLVL] = params->dbglvl;
options[METIS_OPTION_DENSITY] = params->density * DIVIDER;
options[METIS_OPTION_NROWS] = params->nrows;
options[METIS_OPTION_NCOLS] = params->ncols;
options[METIS_OPTION_KAPPA] = params->kappa;
options[METIS_OPTION_NDIAGS] = params->ndiags;
/*Inner parameters*/
options[METIS_OPTION_COMPRESS] = params->compress;
options[METIS_OPTION_UFACTOR] = params->ufactor;
options[METIS_OPTION_PFACTOR] = params->pfactor;
options[METIS_OPTION_NCUTS] = params->ncuts;
options[METIS_OPTION_NSEPS] = params->nseps;
options[METIS_OPTION_NITER] = params->niter;
options[METIS_OPTION_OBJTYPE] = params->objtype;
perm = imalloc(bigraph->super->nvtxs, "main: perm");
iperm = imalloc(bigraph->super->nvtxs, "main: iperm");
gk_malloc_init();
gk_startcputimer(params->parttimer);
/* Initialize my global paramters */
gk_clearcputimer(_parttimer);
gk_clearcputimer(_nztimer);
_totalcheck = 0;
_firsthit = 0;
_maxarea = -1;
_maxnz = -1;
_minarea = 700000000000;
_minnz = 300000000;
_avgarea = 0;
_avgnz = 0;
_maxdense = 0;
_mindense = 0;
/* All the memory that is not allocated in this file should be allocated after
* gk_malloc_init() and be freed before gk_GetCurMemoryUsed().
* Memory that is allocated in this file should be free in the end of main()*/
status = METIS_NodeBDF(&bigraph->super->nvtxs, bigraph->super->xadj, bigraph->super->adjncy,
bigraph->super->vwgt, bigraph->nrows, bigraph->ncols,
options, bigraph->rlabel->label, bigraph->rlabel->ref, bigraph->clabel->label, bigraph->clabel->ref,
&rdiags, &cdiags, &ndiags, perm, iperm);
gk_stopcputimer(params->parttimer);
if (gk_GetCurMemoryUsed() != 0)
printf("***It seems that Metis did not free all of its memory!\n");
params->maxmemory = gk_GetMaxMemoryUsed();
gk_malloc_cleanup(0);
if (status != METIS_OK) {
printf("\n***Metis returned with an error.\n");
}
else {
if (! params->nooutput) {
/* Write the permutation */
gk_startcputimer(params->iotimer);
WritePermutation(params->filename, iperm, bigraph->super->nvtxs);
WriteDiags(params->filename, rdiags, cdiags, ndiags);
gk_stopcputimer(params->iotimer);
}
BDFReportResults(params, bigraph);
}
/* free inner function memory */
for (i = 0; i < ndiags; i++) {
free((void*)rdiags[i]);
free((void*)cdiags[i]);
}
free((void*)rdiags);
free((void*)cdiags);
/* free memroy allocated in this function */
FreeBiGraph((ctrl_t*)NULL, &bigraph);
gk_free((void **)&perm, &iperm, LTERM);
gk_free((void **)¶ms->filename, ¶ms->tpwgtsfile, ¶ms->tpwgts,
¶ms->ubvec, ¶ms, LTERM);
return status;
}
int
main (int argc, char **argv)
{
// see Metis documentation (p.20 ff for Metis 5.x)
/*----------------------------------------------------------------------------*/
// example graph
// 0 -- 3
// | |
// 1 4
// | / |
// 2 5
// number of vertices
idx_t n_vertex = 6;
// size of xadj is n_vertex+1.
// This array stores for every vertex i in xadj[i] the index
// where the adjacent vertices are found in adjncy[].
// Looking at the example graph:
// Neighbors of vertex 0 are found starting at adjncy[0] and ending in adjncy[2-1],
// i.e. one less than the starting point of the next vertex.
// Neighbors of vertex 1 are found starting at adjncy[2] and ending in adjncy[4-1],
// i.e. one less than the starting point of the next vertex.
// etc.
idx_t xadj[] = { 0, 2, 4, 6, 8, 11, 12 };
// number of edges (see below: adjcny stores every edge twice as (u,v) and (v,u))
idx_t n_edge = 6;
// size of adjncy is 2 * n_edge (every edge appears twice in undirected graph)
idx_t adjncy[] = {
1, 3, // neighbors of vertex 0
0, 2, // neighbors of vertex 1
1, 4, // neighbors of vertex 2
0, 4, // neighbors of vertex 3
2, 3, 5, // neighbors of vertex 4
4 // neighbors of vertex 5
};
// vertex weights (size of vwgt is n_vertex)
idx_t vwgt[] = { 20, 30, 80, 60, 100, 90 };
// edge weights (size of adjwgt is n_edge*2)
idx_t adjwgt[] = { 10, 20, 10, 10, 10, 30, 20, 10, 30, 10, 10, 10 };
// the resulting partition:
// for every vertex i we find in part[i] afterwards the partition number for that vertex
idx_t part[n_vertex];
// Metis options (initialized to default values)
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions (options);
// partitioning method: k-way partitioning
options[METIS_OPTION_PTYPE] = METIS_PTYPE_KWAY;
// edge cut minimization
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT;
// C-style numbering
options[METIS_OPTION_NUMBERING] = 0;
// number of balancing constraints
idx_t ncon = 1;
// edge cut after partitioning
idx_t edgecut;
// !!!!!!!!!! you have to change this !!!!!!!!!!!!
// number of partitions we want to partition to
idx_t nparts = 2;
// !!!!!!!!!! you have to change this !!!!!!!!!!!!
/*--------------------------------------------------------------------------*/
// print header and input graph
printf ("partitioning graph with %ld vertices and %ld edges for %ld partitions\n", (long)n_vertex, (long)n_edge, (long)nparts);
print_graph (n_vertex, vwgt, xadj, adjncy, adjwgt);
// call graph partitioning
double t0 = gettime ();
int rc = METIS_PartGraphKway (&n_vertex, // number of vertices
&ncon, // number of balancing constraints
xadj, // adjacency structure of graph
adjncy, // adjacency structure of graph
vwgt, // vertex weights
NULL, // only for total communication volume
adjwgt, // edge weights
&nparts, // number of partitions wanted
NULL, // tpwgts: no desired partition weights
NULL, // ubvec: allowed load imbalance
options, // special options
&edgecut, // objective value (edge cut)
part // vector with partition information for each vertex
);
t0 = gettime () - t0;
// check Metis return value
switch (rc)
{
case METIS_OK:
break;
case METIS_ERROR_INPUT:
printf ("error in Metis input\n");
exit (1);
case METIS_ERROR_MEMORY:
printf ("no more memory in Metis\n");
exit (1);
case METIS_ERROR:
printf ("some error in Metis\n");
exit (1);
default:
printf ("unknown return code ffrom Metis\n");
exit (1);
}
/*--------------------------------------------------------------------------*/
/* print results */
// time for partitioning
printf ("partitioning time: %.9f s\n", t0);
// print parition
print_partition (n_vertex, vwgt, xadj, adjncy, adjwgt, part, nparts, edgecut);
return 0;
}
void MetisLB::work(LDStats* stats)
{
/** ========================== INITIALIZATION ============================= */
ProcArray *parr = new ProcArray(stats);
ObjGraph *ogr = new ObjGraph(stats);
/** ============================= STRATEGY ================================ */
if (_lb_args.debug() >= 2) {
CkPrintf("[%d] In MetisLB Strategy...\n", CkMyPe());
}
// convert ObjGraph to the adjacency structure
int numVertices = ogr->vertices.size(); // number of vertices
int numEdges = 0; // number of edges
double maxLoad = 0.0;
int i, j, k, vert;
/** remove duplicate edges from recvFrom */
for(i = 0; i < numVertices; i++) {
for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
vert = ogr->vertices[i].sendToList[j].getNeighborId();
for(k = 0; k < ogr->vertices[i].recvFromList.size(); k++) {
if(ogr->vertices[i].recvFromList[k].getNeighborId() == vert) {
ogr->vertices[i].sendToList[j].setNumBytes(ogr->vertices[i].sendToList[j].getNumBytes() + ogr->vertices[i].recvFromList[k].getNumBytes());
ogr->vertices[i].recvFromList.erase(ogr->vertices[i].recvFromList.begin() + k);
}
}
}
}
/** the object load is normalized to an integer between 0 and 256 */
for(i = 0; i < numVertices; i++) {
if(ogr->vertices[i].getVertexLoad() > maxLoad)
maxLoad = ogr->vertices[i].getVertexLoad();
numEdges = numEdges + ogr->vertices[i].sendToList.size() + ogr->vertices[i].recvFromList.size();
}
/* adjacency list */
idx_t *xadj = new idx_t[numVertices + 1];
/* id of the neighbors */
idx_t *adjncy = new idx_t[numEdges];
/* weights of the vertices */
idx_t *vwgt = new idx_t[numVertices];
/* weights of the edges */
idx_t *adjwgt = new idx_t[numEdges];
int edgeNum = 0;
double ratio = 256.0/maxLoad;
for(i = 0; i < numVertices; i++) {
xadj[i] = edgeNum;
vwgt[i] = (int)ceil(ogr->vertices[i].getVertexLoad() * ratio);
for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
adjncy[edgeNum] = ogr->vertices[i].sendToList[j].getNeighborId();
adjwgt[edgeNum] = ogr->vertices[i].sendToList[j].getNumBytes();
edgeNum++;
}
for(j = 0; j < ogr->vertices[i].recvFromList.size(); j++) {
adjncy[edgeNum] = ogr->vertices[i].recvFromList[j].getNeighborId();
adjwgt[edgeNum] = ogr->vertices[i].recvFromList[j].getNumBytes();
edgeNum++;
}
}
xadj[i] = edgeNum;
CkAssert(edgeNum == numEdges);
idx_t edgecut; // number of edges cut by the partitioning
idx_t *pemap;
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
//options[METIS_OPTION_PTYPE] = METIS_PTYPE_RB;
// C style numbering
options[METIS_OPTION_NUMBERING] = 0;
// number of constrains
idx_t ncon = 1;
// number of partitions
idx_t numPes = parr->procs.size();
real_t ubvec[ncon];
// allow 10% imbalance
ubvec[0] = 1.1;
// mapping of objs to partitions
pemap = new idx_t[numVertices];
// Specifies size of vertices for computing the total communication volume
idx_t *vsize = NULL;
// This array of size nparts specifies the desired weight for each partition
// and setting it to NULL indicates graph should be equally divided among
// partitions
real_t *tpwgts = NULL;
int option = 0;
if (WEIGHTED == option) {
// set up the different weights between 0 and 1
tpwgts = new real_t[numPes];
for (i = 0; i < numPes; i++) {
tpwgts[i] = 1.0/(real_t)numPes;
}
} else if (MULTI_CONSTRAINT == option) {
CkAbort("Multiple constraints not implemented.\n");
}
// numVertices: num vertices in the graph; ncon: num balancing constrains
// xadj, adjncy: of size n+1 and adjncy of 2m, adjncy[xadj[i]] through and
// including adjncy[xadj[i+1]-1];
// vwgt: weight of the vertices; vsize: amt of data that needs to be sent
// for ith vertex is vsize[i]
// adjwght: the weight of edges; numPes: total parts
// tpwghts: target partition weight, can pass NULL to equally divide
// ubvec: of size ncon to indicate allowed load imbalance tolerance (> 1.0)
// options: array of options; edgecut: stores the edgecut; pemap: mapping
METIS_PartGraphRecursive(&numVertices, &ncon, xadj, adjncy, vwgt, vsize, adjwgt,
&numPes, tpwgts, ubvec, options, &edgecut, pemap);
delete[] xadj;
delete[] adjncy;
delete[] vwgt;
delete[] adjwgt;
delete[] vsize;
delete[] tpwgts;
if (_lb_args.debug() >= 1) {
CkPrintf("[%d] MetisLB done! \n", CkMyPe());
}
for(i = 0; i < numVertices; i++) {
if(pemap[i] != ogr->vertices[i].getCurrentPe())
ogr->vertices[i].setNewPe(pemap[i]);
}
delete[] pemap;
/** ============================== CLEANUP ================================ */
ogr->convertDecisions(stats);
delete parr;
delete ogr;
}
void Cluster::subdivide(unsigned bmin)
{
const unsigned size(_end - _beg);
if (size > bmin) {
idx_t n_rows(static_cast<idx_t>(_l_adj_mat->getNRows()));
idx_t *xadj(new idx_t[n_rows+1]);
unsigned const*const original_row_ptr(_l_adj_mat->getRowPtrArray());
for(idx_t k(0); k<=n_rows; k++) {
xadj[k] = original_row_ptr[k];
}
unsigned nnz(_l_adj_mat->getNNZ());
idx_t *adjncy(new idx_t[nnz]);
unsigned const*const original_adjncy(_l_adj_mat->getColIdxArray());
for(unsigned k(0); k<nnz; k++) {
adjncy[k] = original_adjncy[k];
}
// unsigned nparts = 2;
idx_t options[METIS_NOPTIONS]; // for METIS
METIS_SetDefaultOptions(options);
// options[METIS OPTION PTYPE] = METIS PTYPE RB;
// options[METIS OPTION OBJTYPE] = METIS OBJTYPE CUT;
// options[METIS OPTION CTYPE] = METIS CTYPE SHEM;
// options[] = ;
// options[] = ;
// options[] = ;
// unsigned sepsize(0); // for METIS
idx_t *vwgt(new idx_t[n_rows + 1]);
// const unsigned nnz(xadj[n_rows]);
// unsigned *adjwgt(new unsigned[nnz]);
for (idx_t k(0); k < n_rows + 1; k++)
vwgt[k] = 1;
// for (unsigned k(0); k < nnz; k++)
// adjwgt[k] = 1;
// unsigned *part(new unsigned[n_rows + 1]);
// subdivide the index set into three parts employing METIS
// METIS_ComputeVertexSeparator(&n_rows, xadj, adjncy, vwgt, &options,
// &sepsize, part);
idx_t *loc_op_perm(new idx_t[n_rows]);
idx_t *loc_po_perm(new idx_t[n_rows]);
for (idx_t k(0); k<n_rows; k++) {
loc_op_perm[k] = _g_op_perm[k];
}
for (idx_t k(0); k<n_rows; k++) {
loc_po_perm[k] = _g_po_perm[k];
}
METIS_NodeND(&n_rows, xadj, adjncy, vwgt, options, loc_op_perm, loc_po_perm);
for (idx_t k(0); k<n_rows; k++) {
_g_op_perm[k] = loc_op_perm[k];
}
for (idx_t k(0); k<n_rows; k++) {
_g_po_perm[k] = loc_po_perm[k];
}
delete [] loc_op_perm;
delete [] loc_po_perm;
delete [] vwgt;
delete [] adjncy;
delete [] xadj;
// // create and init local permutations
// unsigned *l_op_perm(new unsigned[size]);
// unsigned *l_po_perm(new unsigned[size]);
// for (unsigned i = 0; i < size; ++i)
// l_op_perm[i] = l_po_perm[i] = i;
//
// unsigned isep1, isep2;
// updatePerm(part, isep1, isep2, l_op_perm, l_po_perm);
// delete[] part;
//
// // update global permutation
// unsigned *t_op_perm = new unsigned[size];
// for (unsigned k = 0; k < size; ++k)
// t_op_perm[k] = _g_op_perm[_beg + l_op_perm[k]];
//
// for (unsigned k = _beg; k < _end; ++k) {
// _g_op_perm[k] = t_op_perm[k - _beg];
// _g_po_perm[_g_op_perm[k]] = k;
// }
// delete[] t_op_perm;
//
// // next recursion step
// if ((isep1 >= bmin) && (isep2 - isep1 >= bmin)) {
// // construct adj matrices for [0, isep1), [isep1,isep2), [isep2, _end)
// AdjMat *l_adj0(_l_adj_mat->getMat(0, isep1, l_op_perm, l_po_perm));
// AdjMat *l_adj1(_l_adj_mat->getMat(isep1, isep2, l_op_perm, l_po_perm));
// AdjMat *l_adj2(_l_adj_mat->getMat(isep2, size, l_op_perm, l_po_perm));
//
// delete[] l_op_perm;
// delete[] l_po_perm;
// delete _l_adj_mat;
// _l_adj_mat = NULL;
//
// _n_sons = 3;
// _sons = new ClusterBase*[_n_sons];
//
// isep1 += _beg;
// isep2 += _beg;
//
// // constructing child nodes for index cluster tree
// _sons[0] = new Cluster(this, _beg, isep1, _g_op_perm, _g_po_perm, _g_adj_mat, l_adj0);
// _sons[1] = new Cluster(this, isep1, isep2, _g_op_perm, _g_po_perm, _g_adj_mat, l_adj1);
// _sons[2] = new Separator(this, isep2, _end, _g_op_perm, _g_po_perm, _g_adj_mat, l_adj2);
//
// dynamic_cast<Cluster*>(_sons[0])->subdivide(bmin);
// dynamic_cast<Cluster*>(_sons[1])->subdivide(bmin);
//
// } else {
// delete _l_adj_mat;
// _l_adj_mat = NULL;
// } // end if next recursion step
} // end if ( connected && size () > bmin )
}
int Zoltan_ParMetis_Order(
ZZ *zz, /* Zoltan structure */
int num_obj, /* Number of (local) objects to order. */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */
int *iperm,
ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */
)
{
static char *yo = "Zoltan_ParMetis_Order";
int i, n, ierr;
ZOLTAN_Output_Order ord;
ZOLTAN_Third_Graph gr;
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't want to risk letting external
packages changing our communicator */
#endif
indextype numflag = 0;
int timer_p = 0;
int get_times = 0;
int use_timers = 0;
double times[5];
ZOLTAN_ID_PTR l_gids = NULL;
ZOLTAN_ID_PTR l_lids = NULL;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
ZOLTAN_TRACE_ENTER(zz, yo);
#ifdef ZOLTAN_PARMETIS
#if TPL_USE_DATATYPE != TPL_METIS_DATATYPES
#ifdef TPL_FLOAT_WEIGHT
i = 1;
#else
i = 0;
#endif
if ((sizeof(indextype) != sizeof(idxtype)) ||
(sizeof(weighttype) != sizeof(idxtype)) || i){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,
"Not supported: Multiple 3rd party libraries with incompatible "
"data types.");
return ZOLTAN_FATAL;
}
#endif
#endif
memset(&gr, 0, sizeof(ZOLTAN_Third_Graph));
memset(&ord, 0, sizeof(ZOLTAN_Output_Order));
memset(times, 0, sizeof(times));
ord.order_opt = order_opt;
if (!order_opt){
/* If for some reason order_opt is NULL, allocate a new ZOOS here. */
/* This should really never happen. */
order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
strcpy(order_opt->method,"PARMETIS");
}
ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord);
/* ParMetis only computes the rank vector */
order_opt->return_args = RETURN_RANK;
/* Check that num_obj equals the number of objects on this proc. */
/* This constraint may be removed in the future. */
n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){
/* Return error code */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
return(ZOLTAN_FATAL);
}
if (n != num_obj){
/* Currently this is a fatal error. */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the "
"number of objects.");
return(ZOLTAN_FATAL);
}
/* Do not use weights for ordering */
gr.obj_wgt_dim = -1;
gr.edge_wgt_dim = -1;
gr.num_obj = num_obj;
/* Check what ordering type is requested */
if (order_opt){
SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */
#ifdef ZOLTAN_PARMETIS
if ((strcmp(order_opt->method, "METIS") == 0))
#endif /* ZOLTAN_PARMETIS */
SET_LOCAL_GRAPH(&gr.graph_type);
}
gr.get_data = 1;
if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: "
"set ParMetis instead.");
return(ZOLTAN_FATAL);
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
return (ierr);
}
/* Allocate space for separator sizes */
if (IS_GLOBAL_GRAPH(gr.graph_type)) {
if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) {
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype));
if (ord.sep_sizes == NULL) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */
}
/* Allocate space for direct perm */
ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.rank){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
if (IS_LOCAL_GRAPH(gr.graph_type)){
/* Allocate space for inverse perm */
ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.iperm){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
}
else
ord.iperm = NULL;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_PARMETIS
if (IS_GLOBAL_GRAPH(gr.graph_type)){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library");
ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy,
&numflag, options, ord.rank, ord.sep_sizes, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else
#endif /* ZOLTAN_PARMETIS */
#if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS)
if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */
indextype numobj = gr.num_obj;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library");
order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0; /* Use default options for METIS. */
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options,
ord.iperm, ord.rank);
#else
METIS_SetDefaultOptions(options);
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, NULL, options,
ord.iperm, ord.rank); /* NULL is vwgt -- new interface in v4 */
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */
int numbloc;
int start;
int leaf;
int *converttab;
int levelmax;
levelmax = mylog2(zz->Num_Proc) + 1;
converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int));
memset(converttab, 0, zz->Num_Proc*2*sizeof(int));
/* Determine the first node in each separator, store it in zz->TPL_Order.start */
for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) {
int father;
father = zz->Num_Proc + numbloc;
converttab[start] = 2*numbloc;
zz->TPL_Order.leaves[leaf++]=start;
zz->TPL_Order.ancestor[start] = start + 2;
converttab[start+1] = 2*numbloc+1;
zz->TPL_Order.leaves[leaf++]=start+1;
zz->TPL_Order.ancestor[start+1] = start + 2;
start+=2;
do {
converttab[start] = father;
if (father %2 == 0) {
int nextoffset;
int level;
level = mylog2(2*zz->Num_Proc - 1 - father);
nextoffset = (1<<(levelmax-level));
zz->TPL_Order.ancestor[start] = start+nextoffset;
start++;
break;
}
else {
zz->TPL_Order.ancestor[start] = start+1;
start++;
father = zz->Num_Proc + father/2;
}
} while (father < 2*zz->Num_Proc - 1);
}
zz->TPL_Order.start[0] = 0;
zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1;
for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) {
int oldblock=converttab[numbloc-1];
zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock];
}
ZOLTAN_FREE(&converttab);
ZOLTAN_FREE(&ord.sep_sizes);
zz->TPL_Order.leaves[zz->Num_Proc] = -1;
zz->TPL_Order.nbr_leaves = zz->Num_Proc;
zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1;
}
else { /* No tree */
zz->TPL_Order.nbr_blocks = 0;
zz->TPL_Order.start = NULL;
zz->TPL_Order.ancestor = NULL;
zz->TPL_Order.leaves = NULL;
}
/* Correct because no redistribution */
memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE));
memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE));
ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL);
ZOLTAN_FREE(&l_gids);
ZOLTAN_FREE(&l_lids);
/* Get a time here */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){
memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i];
}
}
if ((ord.iperm != NULL) && (iperm != NULL)){
if (sizeof(indextype) == sizeof(int)){
memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
iperm[i] = (int)ord.iperm[i];
}
}
}
if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm);
ZOLTAN_FREE(&ord.rank);
/* Free all other "graph" stuff */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
idx_t* gpmetis( int argc, char **argv )
/*************************************************************************/
/*! Let the game begin! */
/*************************************************************************/
//int main(int argc, char *argv[])
{
idx_t i;
char *curptr, *newptr;
idx_t options[METIS_NOPTIONS];
graph_t *graph;
idx_t *part;
idx_t objval;
params_t *params;
int status=0;
gk_optind = 0;
//printf( "argc: %d\n", argc );
//printf( "gk_optind %d\n", gk_optind );
fflush( stdout );
for( i = 0; i < argc; i++ )
{
//printf( "%s*\n", argv[ i ] );
}
params = parse_cmdline(argc, argv);
//printf( "gk_optind %d\n", gk_optind );
//fflush( stdout );
//return NULL;
gk_startcputimer(params->iotimer);
graph = ReadGraph(params);
ReadTPwgts(params, graph->ncon);
gk_stopcputimer(params->iotimer);
/* Check if the graph is contiguous */
if (params->contig && !IsConnected(graph, 0)) {
printf("***The input graph is not contiguous.\n"
"***The specified -contig option will be ignored.\n");
params->contig = 0;
}
/* Get ubvec if supplied */
if (params->ubvecstr) {
params->ubvec = rmalloc(graph->ncon, "main");
curptr = params->ubvecstr;
for (i=0; i<graph->ncon; i++) {
params->ubvec[i] = strtoreal(curptr, &newptr);
if (curptr == newptr)
errexit("Error parsing entry #%"PRIDX" of ubvec [%s] (possibly missing).\n",
i, params->ubvecstr);
curptr = newptr;
}
}
/* Setup iptype */
if (params->iptype == -1) {
if (params->ptype == METIS_PTYPE_RB) {
if (graph->ncon == 1)
params->iptype = METIS_IPTYPE_GROW;
else
params->iptype = METIS_IPTYPE_RANDOM;
}
}
GPPrintInfo(params, graph);
part = imalloc(graph->nvtxs, "main: part");
METIS_SetDefaultOptions(options);
options[METIS_OPTION_OBJTYPE] = params->objtype;
options[METIS_OPTION_CTYPE] = params->ctype;
options[METIS_OPTION_IPTYPE] = params->iptype;
options[METIS_OPTION_RTYPE] = params->rtype;
options[METIS_OPTION_MINCONN] = params->minconn;
options[METIS_OPTION_CONTIG] = params->contig;
options[METIS_OPTION_SEED] = params->seed;
options[METIS_OPTION_NITER] = params->niter;
options[METIS_OPTION_NCUTS] = params->ncuts;
options[METIS_OPTION_UFACTOR] = params->ufactor;
options[METIS_OPTION_DBGLVL] = params->dbglvl;
gk_malloc_init();
gk_startcputimer(params->parttimer);
switch (params->ptype) {
case METIS_PTYPE_RB:
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, part);
break;
case METIS_PTYPE_KWAY:
status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, part);
break;
}
gk_stopcputimer(params->parttimer);
if (gk_GetCurMemoryUsed() != 0)
printf("***It seems that Metis did not free all of its memory! Report this.\n");
params->maxmemory = gk_GetMaxMemoryUsed();
gk_malloc_cleanup(0);
if (status != METIS_OK) {
printf("\n***Metis returned with an error.\n");
}
else {
if (!params->nooutput) {
/* Write the solution */
gk_startcputimer(params->iotimer);
WritePartition(params->filename, part, graph->nvtxs, params->nparts);
gk_stopcputimer(params->iotimer);
}
GPReportResults(params, graph, part, objval);
}
idx_t *r_part = ( idx_t* ) calloc( graph->nvtxs, sizeof( idx_t ) );
for( i = 0; i < graph->nvtxs; i++ )
{
r_part[ i ] = part[ i ];
}
FreeGraph(&graph);
gk_free((void **)&part, LTERM);
gk_free((void **)¶ms->filename, ¶ms->tpwgtsfile, ¶ms->tpwgts,
¶ms->ubvecstr, ¶ms->ubvec, ¶ms, LTERM);
return r_part;
}
void InitKWayPartitioningRB(ctrl_t *ctrl, graph_t *graph)
{
idx_t i, options[METIS_NOPTIONS], curobj=0;
idx_t *bestwhere=NULL;
real_t *ubvec=NULL;
int status;
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NITER] = 10;
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT;
options[METIS_OPTION_NO2HOP] = ctrl->no2hop;
options[METIS_OPTION_ONDISK] = ctrl->ondisk;
ubvec = rmalloc(graph->ncon, "InitKWayPartitioning: ubvec");
for (i=0; i<graph->ncon; i++)
ubvec[i] = (real_t)pow(ctrl->ubfactors[i], 1.0/log(ctrl->nparts));
switch (ctrl->objtype) {
case METIS_OBJTYPE_CUT:
case METIS_OBJTYPE_VOL:
options[METIS_OPTION_NCUTS] = ctrl->nIparts;
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon,
graph->xadj, graph->adjncy, graph->vwgt, graph->vsize,
graph->adjwgt, &ctrl->nparts, ctrl->tpwgts, ubvec,
options, &curobj, graph->where);
if (status != METIS_OK)
gk_errexit(SIGERR, "Failed during initial partitioning\n");
break;
#ifdef XXX /* This does not seem to help */
case METIS_OBJTYPE_VOL:
bestwhere = imalloc(graph->nvtxs, "InitKWayPartitioning: bestwhere");
options[METIS_OPTION_NCUTS] = 2;
ntrials = (ctrl->nIparts+1)/2;
for (i=0; i<ntrials; i++) {
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon,
graph->xadj, graph->adjncy, graph->vwgt, graph->vsize,
graph->adjwgt, &ctrl->nparts, ctrl->tpwgts, ubvec,
options, &curobj, graph->where);
if (status != METIS_OK)
gk_errexit(SIGERR, "Failed during initial partitioning\n");
curobj = ComputeVolume(graph, graph->where);
if (i == 0 || bestobj > curobj) {
bestobj = curobj;
if (i < ntrials-1)
icopy(graph->nvtxs, graph->where, bestwhere);
}
if (bestobj == 0)
break;
}
if (bestobj != curobj)
icopy(graph->nvtxs, bestwhere, graph->where);
break;
#endif
default:
gk_errexit(SIGERR, "Unknown objtype: %d\n", ctrl->objtype);
}
gk_free((void **)&ubvec, &bestwhere, LTERM);
}
int MESH_PartitionWithMetis(Mesh_ptr mesh, int nparts, int **part) {
MEdge_ptr fedge;
MFace_ptr mf, oppf, rface;
MRegion_ptr mr, oppr;
List_ptr fedges, efaces, rfaces, fregions;
int i, ncells, ipos;
int nv, ne, nf, nr, nfe, nef, nfr, nrf, idx, idx2;
#ifdef METIS_5
idx_t ngraphvtx, numflag, nedgecut, numparts, ncons;
idx_t wtflag, metisopts[METIS_NOPTIONS];
idx_t *vsize, *idxpart;
idx_t *xadj, *adjncy, *vwgt, *adjwgt;
real_t *tpwgts, *ubvec;
#else
idxtype ngraphvtx, numflag, nedgecut, numparts;
idxtype wtflag, metisopts[5] = {0,0,0,0,0};
idxtype *xadj, *adjncy, *vwgt, *adjwgt, *idxpart;
#endif
/* First build a nodal graph of the mesh in the format required by
metis */
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
ipos = 0;
if (nr == 0) {
if (nf == 0) {
fprintf(stderr,"Cannot partition wire meshes\n");
exit(-1);
}
#ifdef METIS_5
xadj = (idx_t *) malloc((nf+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*ne*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nf+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*ne*sizeof(idxtype));
#endif
ncells = nf;
/* Surface mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mf = MESH_Next_Face(mesh,&idx))) {
fedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(fedges);
idx2 = 0;
while ((fedge = List_Next_Entry(fedges,&idx2))) {
efaces = ME_Faces(fedge);
nef = List_Num_Entries(efaces);
if (nef == 1) {
continue; /* boundary edge; nothing to do */
}
else {
int j;
for (j = 0; j < nef; j++) {
oppf = List_Entry(efaces,j);
if (oppf != mf) {
adjncy[ipos] = MF_ID(oppf)-1;
ipos++;
}
}
}
List_Delete(efaces);
}
List_Delete(fedges);
i++;
xadj[i] = ipos;
}
}
else {
#ifdef METIS_5
xadj = (idx_t *) malloc((nr+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*nf*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nr+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*nf*sizeof(idxtype));
#endif
ncells = nr;
/* Volume mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mr = MESH_Next_Region(mesh,&idx))) {
rfaces = MR_Faces(mr);
nrf = List_Num_Entries(rfaces);
idx2 = 0;
while ((rface = List_Next_Entry(rfaces,&idx2))) {
fregions = MF_Regions(rface);
nfr = List_Num_Entries(fregions);
if (nfr > 1) {
oppr = List_Entry(fregions,0);
if (oppr == mr)
oppr = List_Entry(fregions,1);
adjncy[ipos] = MR_ID(oppr)-1;
ipos++;
}
List_Delete(fregions);
}
List_Delete(rfaces);
i++;
xadj[i] = ipos;
}
}
/* Partition the graph */
wtflag = 0; /* No weights are specified */
vwgt = adjwgt = NULL;
numflag = 0; /* C style numbering of elements (nodes of the dual graph) */
ngraphvtx = ncells; /* we want the variable to be of type idxtype or idx_t */
numparts = nparts; /* we want the variable to be of type idxtype or idx_t */
#ifdef METIS_5
idxpart = (idx_t *) malloc(ncells*sizeof(idx_t));
ncons = 1; /* Number of constraints */
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
METIS_SetDefaultOptions(metisopts);
metisopts[METIS_OPTION_NUMBERING] = 0;
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,
idxpart);
else
METIS_PartGraphKway(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,idxpart);
#else
idxpart = (idxtype *) malloc(ncells*sizeof(idxtype));
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,
&numflag,&numparts,metisopts,&nedgecut,idxpart);
else
METIS_PartGraphKway(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,&numflag,
&numparts,metisopts,&nedgecut,idxpart);
#endif
free(xadj);
free(adjncy);
*part = (int *) malloc(ncells*sizeof(int));
for (i = 0; i < ncells; i++)
(*part)[i] = (int) idxpart[i];
free(idxpart);
return 1;
}
int main()
{
/* We aim to partition the following mesh into 4 parts:
*
* 0---1---2---3---4
* | | | | |
* 5---6---7---8---9
* | | | | |
* 10--11--12--13--14
*
*/
const int nVertices = 15;
const int nNeighs = 44;
idx_t xadj [nVertices+1] = { 0, 2, 5, 8, 11, 13, 16, 20, 24, 28, 31, 33, 36, 39, 42, 44 };
idx_t adjncy [nNeighs] = { 1, 5, 0, 2, 6, 1, 3, 7, 2, 4, 8, 3, 9, 0, 6, 10, 1, 5, 7, 11,
2, 6, 8, 12, 3, 7, 9, 13, 4, 8, 14, 5, 11, 6, 10, 12, 7, 11,
13, 8, 12, 14, 9, 13 };
std::cout << "Before Partitioning:" << std::endl;
std::cout << std::endl;
std::cout << "0---0---0---0---0" << std::endl;
std::cout << "| | | | |" << std::endl;
std::cout << "0---0---0---0---0" << std::endl;
std::cout << "| | | | |" << std::endl;
std::cout << "0---0---0---0---0" << std::endl;
std::cout << std::endl;
// partition the graph using metis
idx_t nvtxs = nVertices; // number of vertices
idx_t ncon = 1; // number of balancing constraints
idx_t *vwgt = NULL; // vertex weights
idx_t *vsize = NULL; // vertex sizes
idx_t *adjwgt = NULL; // edge weights
idx_t nparts = 4; // number of partitions
real_t *tpwgts = NULL; // target partition weight
real_t *ubvec = NULL; // load balance tolerance for each constraint
idx_t options[METIS_NOPTIONS]; // array of options
METIS_SetDefaultOptions( options ); // use defaults
idx_t objval; // edge-cut on return
std::vector< idx_t > colors( nVertices, 0 ); // ranks
int ierr = METIS_PartGraphRecursive( &nvtxs, &ncon, &xadj[0], &adjncy[0], vwgt, vsize,
adjwgt, &nparts, tpwgts, ubvec, options, &objval, &colors[0] );
if ( ierr != METIS_OK )
throw std::runtime_error( "METIS graph partitioning failed!" );
std::cout << "After Partitioning:" << std::endl;
std::cout << std::endl;
std::cout << colors[0] << "---" << colors[1] << "---" << colors[2] <<
"---" << colors[3] << "---" << colors[4] << std::endl;
std::cout << "| | | | |" << std::endl;
std::cout << colors[5] << "---" << colors[6] << "---" << colors[7] <<
"---" << colors[8] << "---" << colors[9] << std::endl;
std::cout << "| | | | |" << std::endl;
std::cout << colors[10] << "---" << colors[11] << "---" << colors[12] <<
"---" << colors[13] << "---" << colors[14] << std::endl;
std::cout << std::endl;
return 0;
}
int generateLevels(std::vector< std::vector<sbfSElement *> > & selements, std::vector<int> & numTargetByLayers, std::vector<double> &maxImbalance, bool verbouse){
sbfMesh * mesh = selements[0][0]->mesh();
if(!mesh) return 1;
int numRegElems = selements[0].size();
std::vector <double> facesWeigth;
std::vector <int> facesOwners;
std::vector <double> randoms;
randoms.resize(50);
for(size_t ct = 0; ct < randoms.size(); ct++) randoms[ct] = ((double)rand())/RAND_MAX;
facesWeigth.reserve(numRegElems*50);
facesOwners.reserve(numRegElems*50);
for(int elemID = 0; elemID < numRegElems; elemID++){//Loop on elements
sbfElement *elem = mesh->elemPtr(elemID);
double faceWeigth;
int facesOwner = elemID;
std::list<int> inds;
int count;
std::vector< std::vector<int> > facesNodesIndexes = elem->facesNodesIndexes();
for(auto itFace = facesNodesIndexes.begin(); itFace != facesNodesIndexes.end(); itFace++){//Loop on faces
inds.clear();
for(auto itIndex = (*itFace).begin(); itIndex != (*itFace).end(); itIndex++)
inds.push_back(*itIndex);
inds.sort();
count = 0; faceWeigth = 0; for(auto it = inds.begin(); it != inds.end(); it++) {faceWeigth += *it*randoms[count++];}
facesWeigth.push_back(faceWeigth);
facesOwners.push_back(facesOwner);
}//Loop on faces
}//Loop on elements
quickAssociatedSort<double, int>(&facesWeigth[0], &facesOwners[0], 0, facesWeigth.size()-1);
if ( verbouse ) std::cout << "sort done" << std::endl;
for(size_t ctLevel = 0; ctLevel < numTargetByLayers.size(); ctLevel++){//Loop on levels
int numTargetSE = numTargetByLayers[ctLevel];
int numSElems = selements[ctLevel].size();
int vertnbr, edgenbr;
vertnbr = numSElems;
edgenbr = 0;
std::vector< std::list <int> > elemNeibour;
elemNeibour.resize(vertnbr);
int founded = 0, unfounded = 0;
std::vector <double> facesWeigthNextLevel;
std::vector <int> facesOwnersNextLevel;
facesWeigthNextLevel.reserve(facesWeigth.size());
facesOwnersNextLevel.reserve(facesOwners.size());
auto facesWeigthIt = facesWeigth.begin();
auto facesOwnersIt = facesOwners.begin();
auto facesWeigthEndM1 = facesWeigth.end() - 1;
for(; facesWeigthIt < facesWeigthEndM1; facesWeigthIt++, facesOwnersIt++){
if(*facesWeigthIt == *(facesWeigthIt+1)){
//Check if *facesOwnersIt and *(facesOwnersIt+1) are in one SE
int owner0, owner1;
owner0 = *facesOwnersIt; owner1 = *(facesOwnersIt+1);
int ownerSE0 = -1, ownerSE1 = -1;
bool inSame = false;
if(ctLevel == 0){ownerSE0 = owner0; ownerSE1 = owner1;}
else{
sbfSElement * se = selements[0][owner0];
for(size_t ct = 0; ct < ctLevel; ct++) se = se->parent();
ownerSE0 = se->index();
se = selements[0][owner1];
for(size_t ct = 0; ct < ctLevel; ct++) se = se->parent();
ownerSE1 = se->index();
if(ownerSE0 == ownerSE1) inSame = true;
}
if (inSame){ facesWeigthIt++; facesOwnersIt++; continue; }
elemNeibour[ownerSE1].push_back(ownerSE0);
elemNeibour[ownerSE0].push_back(ownerSE1);
facesWeigthNextLevel.push_back(*facesWeigthIt);
facesWeigthNextLevel.push_back(*facesWeigthIt);
facesOwnersNextLevel.push_back(*facesOwnersIt);
facesOwnersNextLevel.push_back(*(facesOwnersIt+1));
facesWeigthIt++; facesOwnersIt++;
founded++;
}
else unfounded++;
}
std::vector<idx_t> verttab, edgetab, edlotab, parttab;
verttab.resize(vertnbr+1);
parttab.resize(vertnbr);
int count = 0;
verttab[0] = count;
for(size_t ct = 0; ct < elemNeibour.size(); ct++){
std::list<int> elemNeibourAll, elemNeibourUnique;
for(auto it = elemNeibour[ct].begin(); it != elemNeibour[ct].end(); it++)
elemNeibourAll.push_back(*it);
elemNeibourAll.sort();
elemNeibourUnique = elemNeibourAll;
elemNeibourUnique.unique();
auto elemNeibourUniqueBegin = elemNeibourUnique.begin();
auto elemNeibourUniqueEnd = elemNeibourUnique.end();
auto elemNeibourAllBegin = elemNeibourAll.begin();
auto elemNeibourAllEnd = elemNeibourAll.end();
auto itA = elemNeibourAllBegin;
for(auto itU = elemNeibourUniqueBegin; itU != elemNeibourUniqueEnd; itU++)
{
if(static_cast<int>(ct) != *itU){
int numAcuarence = 0;
while(*itA == *itU && itA != elemNeibourAllEnd){
numAcuarence++;
itA++;
}
edgetab.push_back(*itU);
edlotab.push_back(numAcuarence);
count++;
edgenbr++;
}
}
verttab[ct+1] = count;
}
idx_t nvtxs = vertnbr, ncon = 1, nparts = numTargetSE, objval;
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_VOL;
options[METIS_OPTION_NUMBERING] = 0;
options[METIS_OPTION_NITER] = 600;
double maxLayerImbalance = maxImbalance.back();
if( ctLevel < maxImbalance.size() ) maxLayerImbalance = maxImbalance.at(ctLevel);
options[METIS_OPTION_UFACTOR] = static_cast<int>((maxLayerImbalance-1)*1000);
options[METIS_OPTION_CONTIG] = 1; //Force contiguous partitions
// options[METIS_OPTION_DBGLVL] = 1;
int rez = METIS_PartGraphKway(&nvtxs,
&ncon,
verttab.data(),
edgetab.data(),
/*idx_t *vwgt*/nullptr,
/*idx_t *vsize*/nullptr,
/*idx_t *adjwgt*/nullptr,
&nparts,
/*real_t *tpwgts*/nullptr,
/*real_t *ubvec*/nullptr,
options,
&objval,
parttab.data()
);
if ( rez != METIS_OK ) throw std::runtime_error("Metis runtime failed :(");
for(int ct = 0; ct < numSElems; ct++){
selements[ctLevel][ct]->setParent(selements[ctLevel+1][parttab[ct]]);
selements[ctLevel+1][parttab[ct]]->addChildren(selements[ctLevel][ct]);
}
//Metis solves following problem, but still it should be checked
//There are SElements with some disconnected clusters of elements.
//For such SE split them to several SEs
for(auto seIT = selements[ctLevel+1].begin(); seIT != selements[ctLevel+1].end(); seIT++){//Loop on SElements including new ones
std::set<int> allElemIndexes;//Indexes of all elements in this SE
for(int ct = 0; ct < (*seIT)->numSElements(); ct++) allElemIndexes.insert((*seIT)->children(ct)->index());
std::set<int> inOneSE;
inOneSE.insert(*(allElemIndexes.begin()));
bool flagChanges = true;
while(flagChanges){
flagChanges = false;
for(auto it = inOneSE.begin(); it != inOneSE.end(); it++){
for(auto itN = elemNeibour[*it].begin(); itN != elemNeibour[*it].end(); itN++){
if(allElemIndexes.count(*itN) && !inOneSE.count(*itN)){
inOneSE.insert(*itN);
flagChanges = true;
}
}
}
}
if(allElemIndexes.size() != inOneSE.size()){
std::set<int> inOtherSE;
(*seIT)->setChildrens(std::vector<sbfSElement *>{});
(*seIT)->numSElements();
for(auto it = inOneSE.begin(); it != inOneSE.end(); it++){
(*seIT)->addChildren(selements[ctLevel][*it]);
selements[ctLevel][*it]->setParent(*seIT);
}
for(auto it = allElemIndexes.begin(); it != allElemIndexes.end(); it++){
if(!inOneSE.count(*it))
inOtherSE.insert(*it);
}
sbfSElement *additionalSE = new sbfSElement((*seIT)->mesh(), selements[ctLevel+1].size());
for(auto it = inOtherSE.begin(); it != inOtherSE.end(); it++){
additionalSE->addChildren(selements[ctLevel][*it]);
selements[ctLevel][*it]->setParent(additionalSE);
}
selements[ctLevel+1].push_back(additionalSE);
if ( verbouse ) std::cout << "Split disconnected SE to two of sizes: " << inOneSE.size() << ", " << inOtherSE.size() << std::endl;
}
}//Loop on SElements including new ones
facesWeigth = facesWeigthNextLevel;
facesOwners = facesOwnersNextLevel;
std::cout << "level " << ctLevel << " done" << std::endl;
//get statistics
std::list<int> selems;
int numAll = 0;
for(auto se : selements[ctLevel+1]){
selems.push_back(se->numSElements());
numAll += se->numSElements();
}
if(numAll != numSElems)
throw std::runtime_error("SElements contain not all elements of previous layer");
selems.sort();
selems.reverse();
if ( verbouse ) { for (auto se : selems ) std::cout << se << "\t"; std::cout << "Imbalance is " << (1.0*selems.front())/selems.back() << std::endl; }
}//Loop on levels
return 0;
}
sparse_matrix* graph_partition(LIST *list , partition_t* partition_info)
{
graph_t* graph;
params_t *params;
idx_t options[METIS_NOPTIONS], status;
idx_t objval;
sparse_matrix* matrix = (sparse_matrix*)malloc(sizeof(sparse_matrix));
graph = ReadGraph(list);
params = bmalloc(sizeof(*params), "Allocating memory for params");
METIS_SetDefaultOptions(options);
METIS_init_params(params, graph->ncon);
options[METIS_OPTION_OBJTYPE] = params->objtype;
options[METIS_OPTION_CTYPE] = params->ctype;
options[METIS_OPTION_IPTYPE] = params->iptype;
options[METIS_OPTION_RTYPE] = params->rtype;
options[METIS_OPTION_NO2HOP] = params->no2hop;
options[METIS_OPTION_MINCONN] = params->minconn;
options[METIS_OPTION_CONTIG] = params->contig;
options[METIS_OPTION_SEED] = params->seed;
options[METIS_OPTION_NITER] = params->niter;
options[METIS_OPTION_NCUTS] = params->ncuts;
options[METIS_OPTION_UFACTOR] = params->ufactor;
options[METIS_OPTION_DBGLVL] = params->dbglvl;
//print_input_data(graph,params);
params->tpwgts = NULL;
params->ubvec = NULL;
partition_info->partion_table = imalloc(graph->nvtxs, "Allocate memory for part");
switch (params->ptype) {
case METIS_PTYPE_RB:
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, partition_info->partion_table);
break;
case METIS_PTYPE_KWAY:
status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, partition_info->partion_table);
break;
}
if (status != METIS_OK) {
fprintf(stderr,"\n***Metis returned with an error.***\n");
return NULL;
}
partition_info->size = graph->nvtxs;
partition_info->noOfParts = params->nparts;
WritePartition(params->filename, partition_info->partion_table, graph->nvtxs, params->nparts);
printf("Objval: %d\n",objval);
matrix->nz = -1;
matrix->n = graph->nvtxs;
matrix->p = graph->xadj;
matrix->i = graph->adjncy;
matrix->x = NULL;
return matrix;
}
void partitionCS( CS *mesh )
{
/*
create compressed storage format (CSR) of mesh and METIS it
*/
CS *self=mesh;
int nVerts=mesh->nVerts;
int nEls=mesh->nEls;
int **e_n=mesh->my_e_n;
int *epart, *vpart;
epart=mesh->epart;
vpart=mesh->vpart;
idx_t *eptr=malloc( (nEls+1)*sizeof(idx_t) );
idx_t *eind=malloc( nEls*8*sizeof(idx_t) );
idx_t *part=malloc( nVerts*sizeof(idx_t) ); // the vert
idx_t ncon=1;
real_t *tpwgts=NULL;
idx_t v_i, nbr_i;
idx_t num_conn=0;
idx_t vtxdist=nVerts;
idx_t elmdist=nEls;
idx_t ncommonnodes = 4; // for hexahedral
// fill in element indices
idx_t num_eind=0;
idx_t e_i, t_i;
for( e_i=0; e_i<nEls; e_i++ ) {
eptr[e_i]=num_eind;
for(nbr_i=0; nbr_i<8; nbr_i++ ) {
eind[num_eind] = e_n[e_i][nbr_i];
num_eind++;
}
}
eptr[e_i]=num_eind; // the final entry
idx_t ncommon=4;
idx_t nparts=4;
tpwgts = malloc( ncon*nparts*sizeof(real_t) );
memset( tpwgts, 1, nparts*sizeof(real_t) );
idx_t options[METIS_NOPTIONS];
idx_t objval;
METIS_SetDefaultOptions(options);
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT;
for( t_i=0; t_i<ncon*nparts; t_i++ ) {
tpwgts[t_i]=1.0/(double)nparts;
}
int err = METIS_PartMeshDual(
(idx_t*)&nEls, // number of elements
(idx_t*)&nVerts, // number of verts
eptr,
eind,
NULL, // vwgt
NULL, // vsize
&ncommon, // 4
&nparts,
NULL, //tpwgts
NULL, //options....
&objval,
(idx_t*)epart,
(idx_t*)vpart );
if( err==METIS_OK ) {
printf("Wow, METIS_OK - the number of edges is %d\n", objval);
} else {
printf("METIS is not happy\n");
}
free( tpwgts );
free( eptr );
free( eind );
free( part );
}
vector<vector<Geometry::ElemId>> Volume::getPartitionsIds(
const size_t nDivisions,
const vector<pair<Geometry::ElemId,int>> idWgt,
const Math::Real* taskPower) const {
// Metis v5 manual:
// [...] take as input the element-node array of the mesh and
// compute a k-way partitioning for both its elements and its nodes
// idWgt contains id and weight pairs.
vector<vector<Geometry::ElemId>> res;
res.resize(nDivisions, vector<Geometry::ElemId>());
// Accounts for the one partition case.
if (nDivisions == 1) {
Geometry::ConstElemRGroup physVol = elems();
physVol.removeMatId(MatId(0));
const size_t nK = physVol.sizeOf<Geometry::VolR>();
res[0].resize(nK, Geometry::ElemId(0));
for (size_t i = 0; i < nK; i++) {
res[0][i] = (elems())(i)->getId();
}
return res;
}
#ifdef MESH_ALLOW_PARTITIONING
// Prepares mesh info.
cout << " - Preparing mesh info... " << flush;
idx_t ne = elems().sizeOf<Geometry::VolR>();
idx_t *eptr, *eind;
eptr = new idx_t[ne+1];
eind = new idx_t[ne*4];
size_t counter = 0;
eptr[0] = counter;
for (idx_t i = 0; i < ne; i++) {
const Geometry::VolR* vol = elem_.tet[i];
for (size_t j = 0; j < vol->numberOfVertices(); j++) {
eind[counter++] = vol->getVertex(j)->id - 1;
}
eptr[i+1] = counter;
}
cout << "OK" << endl;
// Relabels ids, needed by quadratic or linearized meshes.
cout << " - Relabeling... " << flush;
DynMatrix<Math::Int> id(ne*4,3);
for (Math::Int i = 0; i < ne*4; i++) {
id(i,0) = i;
id(i,1) = eind[i];
id(i,2) = 0;
}
id.sortRows_omp(1,1);
Math::Int label = 0;
for (Math::Int i = 1; i < ne*4; i++) {
if (id(i,1) == id(i-1,1)) {
id(i,2) = label;
} else {
id(i,2) = ++label;
}
}
id.sortRows_omp(0,0);
for (Math::Int i = 0; i < ne*4; i++) {
eind[i] = id(i,2);
}
idx_t nn = label+1; // Number of vertices.
cout << "OK" << endl;
// Copies weights.
cout << " - Copying weights... " << flush;
idx_t *vwgt;
if (idWgt.size() == 0) {
vwgt = NULL;
} else {
vwgt = new idx_t[ne];
for (Math::Int e = 0; e < ne; e++) {
vwgt[e] = idWgt[e].second;
}
}
idx_t *vsize = NULL;
idx_t nparts = nDivisions;
idx_t objval;
idx_t *epart;
epart = new idx_t[ne];
idx_t *npart;
npart = new idx_t[nn];
cout << "OK" << endl;
// Computes task computational powers.
real_t *tpwgts = NULL;
if (taskPower != NULL) {
tpwgts = new real_t[nDivisions];
real_t sum = 0.0;
for (size_t i = 0; i < nDivisions; i++) {
tpwgts[i] = taskPower[i];
sum += tpwgts[i];
}
assert(std::abs(sum) - 1.0e-16 < 1.0);
}
// METIS options.
cout << " - Setting Options... " << flush;
idx_t options[METIS_NOPTIONS];
Math::Int status;
status = METIS_SetDefaultOptions(options);
options[METIS_OPTION_PTYPE] = METIS_PTYPE_KWAY;
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT;
options[METIS_OPTION_SEED] = (idx_t) 0;
// options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_VOL;
// c numbering. Starts from 0.
options[METIS_OPTION_NUMBERING] = 0;
cout << "OK" << endl;
// Calls METIS partition function for meshes.
idx_t ncommon = 3; // Number of common vertices per element.
cout << " - Calling Part Mesh Dual... " << flush;
status = METIS_PartMeshDual(
&ne, &nn, eptr, eind, vwgt, vsize, &ncommon, &nparts,
tpwgts, options, &objval, epart, npart);
if (status != METIS_OK) {
throw Error("METIS_PartMeshDual fn failed with error: " + status);
}
cout << "OK" << endl;
// Converts result.
for (size_t i = 0; i < nDivisions; i++) {
res[i].reserve(ne);
}
for (Math::Int i = 0; i < ne; i++) {
size_t id = elem_.tet[i]->getId();
res[epart[i]].push_back(id);
}
// Frees memory.
delete vwgt;
delete epart;
delete npart;
delete eptr;
delete eind;
// Returns result.
return res;
#else
throw logic_error("Mesh partitioning is not allowed.");
#endif
}
int main(int argc, char *argv[]) {
// Allocate stack
stack st[N];
// Create shortest paths matrix
const unsigned seed = atoi(argv[1]);
st->sp = createsp(seed);
// Generate random set of drivers
for (agent i = 0; i < D; i++)
st->dr[i] = 1;
memset(st->dr + D, 0, sizeof(agent) * (N - D));
shuffle(st->dr, N, sizeof(agent));
memcpy(drg, st->dr, N * sizeof(agent));
// Initialise n, s, and cs data structures
st->n[N] = N;
for (agent i = 0; i < N; i++) {
X(sg, i) = X(st->s, i) = 1;
Y(sg, i) = Y(st->s, i) = csg[i] = st->cs[i] = i;
st->l[i] = st->sp[4 * i * N + 2 * i + 1];
min += COST(i, st->dr, st->l);
st->n[st->n[i] = N + i + 1] = i;
}
// Initialise c and r bitmasks
ONES(st->c, E + 1, C);
CLEAR(st->c, 0);
ONES(st->r, E + 1, C);
CLEAR(st->r, 0);
// Create graph
#ifdef M
init(seed);
memset(st->g, 0, sizeof(edge) * N * N);
scalefree(st->g, st->a);
#else
FILE *f = fopen(argv[2], "r");
for (agent e = 1; e <= E; e++) {
agent v1, v2;
fscanf(f, "%u %u", &v1, &v2);
createedge(st->g, st->a, v1, v2, e);
}
fclose(f);
#endif
// Reorder (eventually)
#ifdef REORDER
edge go[N * N] = {0};
agent ao[2 * (E + 1)];
#ifdef METIS
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT;
options[METIS_OPTION_SEED] = seed;
real_t tpwgts[2] = {0.5, 0.5}, ubvec = TOLERANCE;
agent map[N];
edge e = 1;
for (agent i = 0; i < N; i++) map[i] = i;
reorderedges(st->g, map, N, E, go, ao, &e, tpwgts, &ubvec, options);
#else
driversbfs(st->a, st->dr, go, ao);
#endif
memcpy(st->g, go, sizeof(edge) * N * N);
memcpy(st->a, ao, sizeof(agent) * 2 * (E + 1));
#endif
#ifdef TREEDOT
st->dot = fopen(TREEDOT, "w+");
fprintf(st->dot, "digraph TREE {\n");
fprintf(st->dot, "\tnode [color = none; shape = plaintext, width = 0.2, height = 0.2];\n");
#endif
// Solve
sol = *st;
#ifdef LIMIT
value bou = bound(st);
#endif
gettimeofday(&t1, NULL);
srcfss(st, min);
gettimeofday(&t2, NULL);
// Print solution
#ifdef TREEDOT
printf("SOLUTION = %zu\n", sol.id);
fprintf(st->dot, "\t%zu [shape = circle, style = filled, fillcolor = green];\n", sol.id);
#endif
#ifdef PK
FILE *pk = fopen(PK, "w+");
fprintf(pk, "%u\n%u\n%u\n", N, K, seed);
printg(st->g, pk);
printpk(&sol, pk);
fclose(pk);
#endif
#ifdef CSV
printf("%u,%u,%s,%.2f,%f,%zu\n", N, E, argv[1], 0.01 * min,
(double)(t2.tv_usec - t1.tv_usec) / 1e6 + t2.tv_sec - t1.tv_sec, count);
#else
printcs(&sol);
printf("Visited nodes = %zu\n", count);
printf("Elapsed time = %f\n", (double)(t2.tv_usec - t1.tv_usec) / 1e6 + t2.tv_sec - t1.tv_sec);
printf("Solution = %.2f€\n", 0.01 * min);
#ifdef LIMIT
printf("Bound = %.2f€\n", 0.01 * bou);
#endif
#endif
// Free data structures
#ifdef TREEDOT
fprintf(st->dot, "}");
fclose(st->dot);
#endif
free(st->sp);
return 0;
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
realtype itr = 0.0;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
/* change our zz struct. */
#endif
indextype i;
realtype *imb_tols;
indextype ncon;
indextype edgecut;
indextype wgtflag;
indextype numflag = 0;
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
ZOLTAN_TRACE_ENTER(zz, yo);
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
if (sizeof(realtype) != sizeof(float)) {
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));
for (i = 0; i < tmp; i++)
prt.input_part_sizes[i] = (realtype) part_sizes[i];
/* KDD 2/2014: removed re-scaling part sizes so they sum to one.
* part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
* plus, the code here was wrong for multiple object weights.
* similar scaling code did not exist in the Scotch interface.
*/
prt.part_sizes = prt.input_part_sizes;
}
else
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
indextype j;
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
/* Now we can call ParMetis */
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
/* Use default options for METIS */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0;
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
#else
METIS_SetDefaultOptions(options);
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
prt.part_sizes, imb_tols, options,
&edgecut, prt.part);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) {
/* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/*************************************************************************
* This function is the entry point of the initial partition algorithm
* that does recursive bissection.
* This algorithm assembles the graph to all the processors and preceeds
* by parallelizing the recursive bisection step.
**************************************************************************/
void InitPartition(ctrl_t *ctrl, graph_t *graph)
{
idx_t i, j, ncon, mype, npes, gnvtxs, ngroups;
idx_t *xadj, *adjncy, *adjwgt, *vwgt;
idx_t *part, *gwhere0, *gwhere1;
idx_t *tmpwhere, *tmpvwgt, *tmpxadj, *tmpadjncy, *tmpadjwgt;
graph_t *agraph;
idx_t lnparts, fpart, fpe, lnpes;
idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut;
real_t *tpwgts, *tpwgts2, *lbvec, lbsum, min_lbsum, wsum;
MPI_Comm ipcomm;
struct {
double sum;
int rank;
} lpesum, gpesum;
WCOREPUSH;
ncon = graph->ncon;
ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, ctrl->npes), 1);
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
lbvec = rwspacemalloc(ctrl, ncon);
/* assemble the graph to all the processors */
agraph = AssembleAdaptiveGraph(ctrl, graph);
gnvtxs = agraph->nvtxs;
/* make a copy of the graph's structure for later */
xadj = icopy(gnvtxs+1, agraph->xadj, iwspacemalloc(ctrl, gnvtxs+1));
vwgt = icopy(gnvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, gnvtxs*ncon));
adjncy = icopy(agraph->nedges, agraph->adjncy, iwspacemalloc(ctrl, agraph->nedges));
adjwgt = icopy(agraph->nedges, agraph->adjwgt, iwspacemalloc(ctrl, agraph->nedges));
part = iwspacemalloc(ctrl, gnvtxs);
/* create different processor groups */
gkMPI_Comm_split(ctrl->gcomm, ctrl->mype % ngroups, 0, &ipcomm);
gkMPI_Comm_rank(ipcomm, &mype);
gkMPI_Comm_size(ipcomm, &npes);
/* Go into the recursive bisection */
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_SEED] = ctrl->sync + (ctrl->mype % ngroups) + 1;
tpwgts = ctrl->tpwgts;
tpwgts2 = rwspacemalloc(ctrl, 2*ncon);
lnparts = ctrl->nparts;
fpart = fpe = 0;
lnpes = npes;
while (lnpes > 1 && lnparts > 1) {
/* determine the weights of the two partitions as a function of the
weight of the target partition weights */
for (j=(lnparts>>1), i=0; i<ncon; i++) {
tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon);
tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon);
wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]);
tpwgts2[i] *= wsum;
tpwgts2[ncon+i] *= wsum;
}
METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &twoparts, tpwgts2, NULL, moptions,
&edgecut, part);
/* pick one of the branches */
if (mype < fpe+lnpes/2) {
KeepPart(ctrl, agraph, part, 0);
lnpes = lnpes/2;
lnparts = lnparts/2;
}
else {
KeepPart(ctrl, agraph, part, 1);
fpart = fpart + lnparts/2;
fpe = fpe + lnpes/2;
lnpes = lnpes - lnpes/2;
lnparts = lnparts - lnparts/2;
}
}
gwhere0 = iset(gnvtxs, 0, iwspacemalloc(ctrl, gnvtxs));
gwhere1 = iwspacemalloc(ctrl, gnvtxs);
if (lnparts == 1) { /* Case npes is greater than or equal to nparts */
/* Only the first process will assign labels (for the reduction to work) */
if (mype == fpe) {
for (i=0; i<agraph->nvtxs; i++)
gwhere0[agraph->label[i]] = fpart;
}
}
else { /* Case in which npes is smaller than nparts */
/* create the normalized tpwgts for the lnparts from ctrl->tpwgts */
tpwgts = rwspacemalloc(ctrl, lnparts*ncon);
for (j=0; j<ncon; j++) {
for (wsum=0.0, i=0; i<lnparts; i++) {
tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j];
wsum += tpwgts[i*ncon+j];
}
for (wsum=1.0/wsum, i=0; i<lnparts; i++)
tpwgts[i*ncon+j] *= wsum;
}
METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions,
&edgecut, part);
for (i=0; i<agraph->nvtxs; i++)
gwhere0[agraph->label[i]] = fpart + part[i];
}
gkMPI_Allreduce((void *)gwhere0, (void *)gwhere1, gnvtxs, IDX_T, MPI_SUM, ipcomm);
if (ngroups > 1) {
tmpxadj = agraph->xadj;
tmpadjncy = agraph->adjncy;
tmpadjwgt = agraph->adjwgt;
tmpvwgt = agraph->vwgt;
tmpwhere = agraph->where;
agraph->xadj = xadj;
agraph->adjncy = adjncy;
agraph->adjwgt = adjwgt;
agraph->vwgt = vwgt;
agraph->where = gwhere1;
agraph->vwgt = vwgt;
agraph->nvtxs = gnvtxs;
edgecut = ComputeSerialEdgeCut(agraph);
ComputeSerialBalance(ctrl, agraph, gwhere1, lbvec);
lbsum = rsum(ncon, lbvec, 1);
gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ctrl->gcomm);
gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ctrl->gcomm);
lpesum.sum = lbsum;
if (min_lbsum < UNBALANCE_FRACTION*ncon) {
if (lbsum < UNBALANCE_FRACTION*ncon)
lpesum.sum = edgecut;
else
lpesum.sum = max_cut;
}
lpesum.rank = ctrl->mype;
gkMPI_Allreduce((void *)&lpesum, (void *)&gpesum, 1, MPI_DOUBLE_INT,
MPI_MINLOC, ctrl->gcomm);
gkMPI_Bcast((void *)gwhere1, gnvtxs, IDX_T, gpesum.rank, ctrl->gcomm);
agraph->xadj = tmpxadj;
agraph->adjncy = tmpadjncy;
agraph->adjwgt = tmpadjwgt;
agraph->vwgt = tmpvwgt;
agraph->where = tmpwhere;
}
icopy(graph->nvtxs, gwhere1+graph->vtxdist[ctrl->mype], graph->where);
FreeGraph(agraph);
gkMPI_Comm_free(&ipcomm);
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->InitPartTmr));
WCOREPOP;
}
/*************************************************************************
* This function is the entry point of the initial balancing algorithm.
* This algorithm assembles the graph to all the processors and preceeds
* with the balancing step.
**************************************************************************/
void Balance_Partition(ctrl_t *ctrl, graph_t *graph)
{
idx_t i, j, nvtxs, nedges, ncon;
idx_t mype, npes, srnpes, srmype;
idx_t *vtxdist, *xadj, *adjncy, *adjwgt, *vwgt, *vsize;
idx_t *part, *lwhere, *home;
idx_t lnparts, fpart, fpe, lnpes, ngroups;
idx_t *rcounts, *rdispls;
idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut;
idx_t sr_pe, gd_pe, sr, gd, who_wins;
real_t my_cut, my_totalv, my_cost = -1.0, my_balance = -1.0, wsum;
real_t rating, max_rating, your_cost = -1.0, your_balance = -1.0;
real_t lbsum, min_lbsum, *lbvec, *tpwgts, *tpwgts2, buffer[2];
graph_t *agraph, cgraph;
ctrl_t *myctrl;
MPI_Status status;
MPI_Comm ipcomm, srcomm;
struct {
double cost;
int rank;
} lpecost, gpecost;
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
WCOREPUSH;
vtxdist = graph->vtxdist;
agraph = AssembleAdaptiveGraph(ctrl, graph);
nvtxs = cgraph.nvtxs = agraph->nvtxs;
nedges = cgraph.nedges = agraph->nedges;
ncon = cgraph.ncon = agraph->ncon;
xadj = cgraph.xadj = icopy(nvtxs+1, agraph->xadj, iwspacemalloc(ctrl, nvtxs+1));
vwgt = cgraph.vwgt = icopy(nvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, nvtxs*ncon));
vsize = cgraph.vsize = icopy(nvtxs, agraph->vsize, iwspacemalloc(ctrl, nvtxs));
adjncy = cgraph.adjncy = icopy(nedges, agraph->adjncy, iwspacemalloc(ctrl, nedges));
adjwgt = cgraph.adjwgt = icopy(nedges, agraph->adjwgt, iwspacemalloc(ctrl, nedges));
part = cgraph.where = agraph->where = iwspacemalloc(ctrl, nvtxs);
lwhere = iwspacemalloc(ctrl, nvtxs);
home = iwspacemalloc(ctrl, nvtxs);
lbvec = rwspacemalloc(ctrl, graph->ncon);
/****************************************/
/****************************************/
if (ctrl->ps_relation == PARMETIS_PSR_UNCOUPLED) {
WCOREPUSH;
rcounts = iwspacemalloc(ctrl, ctrl->npes);
rdispls = iwspacemalloc(ctrl, ctrl->npes+1);
for (i=0; i<ctrl->npes; i++)
rdispls[i] = rcounts[i] = vtxdist[i+1]-vtxdist[i];
MAKECSR(i, ctrl->npes, rdispls);
gkMPI_Allgatherv((void *)graph->home, graph->nvtxs, IDX_T,
(void *)part, rcounts, rdispls, IDX_T, ctrl->comm);
for (i=0; i<agraph->nvtxs; i++)
home[i] = part[i];
WCOREPOP; /* local frees */
}
else {
for (i=0; i<ctrl->npes; i++) {
for (j=vtxdist[i]; j<vtxdist[i+1]; j++)
part[j] = home[j] = i;
}
}
/* Ensure that the initial partitioning is legal */
for (i=0; i<agraph->nvtxs; i++) {
if (part[i] >= ctrl->nparts)
part[i] = home[i] = part[i] % ctrl->nparts;
if (part[i] < 0)
part[i] = home[i] = (-1*part[i]) % ctrl->nparts;
}
/****************************************/
/****************************************/
IFSET(ctrl->dbglvl, DBG_REFINEINFO,
ComputeSerialBalance(ctrl, agraph, agraph->where, lbvec));
IFSET(ctrl->dbglvl, DBG_REFINEINFO,
rprintf(ctrl, "input cut: %"PRIDX", balance: ", ComputeSerialEdgeCut(agraph)));
for (i=0; i<agraph->ncon; i++)
IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "%.3"PRREAL" ", lbvec[i]));
IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "\n"));
/****************************************/
/* Split the processors into two groups */
/****************************************/
sr = (ctrl->mype % 2 == 0) ? 1 : 0;
gd = (ctrl->mype % 2 == 1) ? 1 : 0;
if (graph->ncon > MAX_NCON_FOR_DIFFUSION || ctrl->npes == 1) {
sr = 1;
gd = 0;
}
sr_pe = 0;
gd_pe = 1;
gkMPI_Comm_split(ctrl->gcomm, sr, 0, &ipcomm);
gkMPI_Comm_rank(ipcomm, &mype);
gkMPI_Comm_size(ipcomm, &npes);
if (sr == 1) { /* Half of the processors do scratch-remap */
ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, npes), 1);
gkMPI_Comm_split(ipcomm, mype % ngroups, 0, &srcomm);
gkMPI_Comm_rank(srcomm, &srmype);
gkMPI_Comm_size(srcomm, &srnpes);
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_SEED] = ctrl->sync + (mype % ngroups) + 1;
tpwgts = ctrl->tpwgts;
tpwgts2 = rwspacemalloc(ctrl, 2*ncon);
iset(nvtxs, 0, lwhere);
lnparts = ctrl->nparts;
fpart = fpe = 0;
lnpes = srnpes;
while (lnpes > 1 && lnparts > 1) {
PASSERT(ctrl, agraph->nvtxs > 1);
/* determine the weights of the two partitions as a function of the
weight of the target partition weights */
for (j=(lnparts>>1), i=0; i<ncon; i++) {
tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon);
tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon);
wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]);
tpwgts2[i] *= wsum;
tpwgts2[ncon+i] *= wsum;
}
METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj,
agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt,
&twoparts, tpwgts2, NULL, moptions, &edgecut, part);
/* pick one of the branches */
if (srmype < fpe+lnpes/2) {
KeepPart(ctrl, agraph, part, 0);
lnpes = lnpes/2;
lnparts = lnparts/2;
}
else {
KeepPart(ctrl, agraph, part, 1);
fpart = fpart + lnparts/2;
fpe = fpe + lnpes/2;
lnpes = lnpes - lnpes/2;
lnparts = lnparts - lnparts/2;
}
}
if (lnparts == 1) { /* Case in which srnpes is greater or equal to nparts */
/* Only the first process will assign labels (for the reduction to work) */
if (srmype == fpe) {
for (i=0; i<agraph->nvtxs; i++)
lwhere[agraph->label[i]] = fpart;
}
}
else { /* Case in which srnpes is smaller than nparts */
/* create the normalized tpwgts for the lnparts from ctrl->tpwgts */
tpwgts = rwspacemalloc(ctrl, lnparts*ncon);
for (j=0; j<ncon; j++) {
for (wsum=0.0, i=0; i<lnparts; i++) {
tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j];
wsum += tpwgts[i*ncon+j];
}
for (wsum=1.0/wsum, i=0; i<lnparts; i++)
tpwgts[i*ncon+j] *= wsum;
}
METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions,
&edgecut, part);
for (i=0; i<agraph->nvtxs; i++)
lwhere[agraph->label[i]] = fpart + part[i];
}
gkMPI_Allreduce((void *)lwhere, (void *)part, nvtxs, IDX_T, MPI_SUM, srcomm);
edgecut = ComputeSerialEdgeCut(&cgraph);
ComputeSerialBalance(ctrl, &cgraph, part, lbvec);
lbsum = rsum(ncon, lbvec, 1);
gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ipcomm);
gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ipcomm);
lpecost.rank = ctrl->mype;
lpecost.cost = lbsum;
if (min_lbsum < UNBALANCE_FRACTION * (real_t)(ncon)) {
if (lbsum < UNBALANCE_FRACTION * (real_t)(ncon))
lpecost.cost = (double)edgecut;
else
lpecost.cost = (double)max_cut + lbsum;
}
gkMPI_Allreduce((void *)&lpecost, (void *)&gpecost, 1, MPI_DOUBLE_INT,
MPI_MINLOC, ipcomm);
if (ctrl->mype == gpecost.rank && ctrl->mype != sr_pe)
gkMPI_Send((void *)part, nvtxs, IDX_T, sr_pe, 1, ctrl->comm);
if (ctrl->mype != gpecost.rank && ctrl->mype == sr_pe)
gkMPI_Recv((void *)part, nvtxs, IDX_T, gpecost.rank, 1, ctrl->comm, &status);
if (ctrl->mype == sr_pe) {
icopy(nvtxs, part, lwhere);
SerialRemap(ctrl, &cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts);
}
gkMPI_Comm_free(&srcomm);
}
