int gpartition(const vector< set<int> > &graph, int npartitions, int partition_method, vector<int> &decomp){
// If no partitioning method is set, choose a default.
if(partition_method<0){
if(npartitions<=8)
partition_method = 0; // METIS PartGraphRecursive
else
partition_method = 1; // METIS PartGraphKway
}
int nnodes = graph.size();
// Compress graph
vector<idxtype> xadj(nnodes+1), adjncy;
int pos=0;
xadj[0]=1;
for(int i=0;i<nnodes;i++){
for(set<int>::iterator jt=graph[i].begin();jt!=graph[i].end();jt++){
adjncy.push_back(*jt);
pos++;
}
xadj[i+1] = pos+1;
}
// Partition graph
decomp.resize(nnodes);
int wgtflag=0, numflag=1, edgecut=0;
#ifdef PARMETIS_V3
int options[] = {0};
#else
idx_t nbc=1;
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING]=1;
#endif
if(partition_method){
#ifdef PARMETIS_V3
METIS_PartGraphKway(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag,
&numflag, &npartitions, options, &edgecut, &(decomp[0]));
#else
METIS_PartGraphKway(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL,
&npartitions, NULL, NULL,options,&edgecut,
&(decomp[0]));
#endif
}else{
#ifdef PARMETIS_V3
METIS_PartGraphRecursive(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag,
&numflag, &npartitions, options, &edgecut, &(decomp[0]));
#else
METIS_PartGraphRecursive(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL,
&npartitions, NULL, NULL,options,&edgecut,
&(decomp[0]));
#endif
}
// number from zero
for(int i=0;i<nnodes;i++)
decomp[i]--;
return edgecut;
}
/* Interface for metis k-way partitioning with 64-bit ints */
void MUMPS_CALL
MUMPS_METIS_KWAY_64(MUMPS_INT8 *n, MUMPS_INT8 *iptr,
MUMPS_INT8 *jcn, MUMPS_INT8 *k,
MUMPS_INT8 *part)
/* n -- the size of the graph to be partitioned
iptr -- pointer to the beginning of each node's adjacency list
jcn -- jcn[iptr[i]:iptr[i+1]-1] contains the list of neighbors of node i
k -- the number of parts
part -- part[i] is the part node i belongs to */
/* SELECTIVE I8 FIXME: add an argument *ierr, check it on exit */
{
#if defined(metis4) || defined(parmetis3)
MUMPS_INT numflag, edgecut, wgtflag, options[8];
MUMPS_INT kINT, nINT;
options[0] = 0;
/* unweighted partitioning */
wgtflag = 0;
/* Use 1-based fortran numbering */
numflag = 1;
/* n and k are MUMPS_INT */
nINT=(MUMPS_INT)(*n);
kINT=(MUMPS_INT)(*k);
/* void METIS_PartGraphKway(int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, int *, int *, int *, idxtype *); */
METIS_PartGraphKway(&nINT, iptr, jcn,
NULL, NULL, &wgtflag,
&numflag, &kINT,
options, &edgecut,
part);
#else /* METIS >= 5 */
int ierr;
# if (IDXTYPEWIDTH == 64)
MUMPS_INT8 ncon, edgecut, options[40];
ierr=METIS_SetDefaultOptions(options);
options[0] = 0;
/* Use 1-based fortran numbering */
options[17] = 1;
ncon = 1;
ierr = METIS_PartGraphKway(n, &ncon, iptr, jcn,
NULL, NULL, NULL,
k, NULL, NULL, options,
&edgecut, part);
# else
printf("** Error: METIS version >= 4, IDXTYPE WIDTH !=64, but MUMPS_METIS_KWAY_64 was called\n");
ierr=1;
# endif
#endif
return;
}
/*Partition this mesh's elements into n chunks,
writing each element's 0-based chunk number to elem2chunk.
*/
void FEM_Mesh_partition(const FEM_Mesh *mesh,int nchunks,int *elem2chunk, bool faceGraph)
{
CkThresholdTimer time("FEM Split> Building graph for metis partitioner",1.0);
int nelems=mesh->nElems();
if (nchunks==1) { //Metis doesn't handle this case (!)
for (int i=0;i<nelems;i++) elem2chunk[i]=0;
return;
}
Graph g(nelems);
if (!faceGraph) {
mesh2graph(mesh,&g);
} else {
mesh2graph_face(mesh,&g);
}
int *adjStart; /*Maps elem # -> start index in adjacency list*/
int *adjList; /*Lists adjacent vertices for each element*/
g.toAdjList(adjStart,adjList);
int ecut,ncon=1;
time.start("FEM Split> Calling metis partitioner");
if (nchunks<8) /*Metis manual says recursive version is higher-quality here*/
METIS_PartGraphRecursive(&nelems, &ncon, adjStart, adjList, NULL, NULL, NULL,
&nchunks, NULL, NULL, NULL, &ecut, elem2chunk);
else /*For many chunks, Kway is supposedly faster */
METIS_PartGraphKway(&nelems, &ncon, adjStart, adjList, NULL, NULL, NULL,
&nchunks, NULL, NULL, NULL, &ecut, elem2chunk);
delete[] adjStart;
delete[] adjList;
}
void FC_FUNC_(oct_metis_partgraphkway, OCT_METIS_PARTGRAPHKWAY)
(idx_t *nvtxs, idx_t *ncon, idx_t *xadj, idx_t *adjncy, idx_t *nparts,
real_t *tpwgts, real_t *ubvec, idx_t *options, idx_t *objval, idx_t *part)
{
METIS_PartGraphKway(nvtxs, ncon, xadj, adjncy, NULL, NULL, NULL, nparts,
tpwgts, ubvec, options, objval, part);
}
std::vector<bool>balanced_min_cut(const ListGraph&g){
assert(g.is_valid());
#ifndef USE_CUT_ORDER
return {};
#else
std::vector<int>out_begin;
std::vector<int>out_dest;
build_adj_array(out_begin, out_dest,
g.node_count(), g.arc.size(),
[&](int x){return g.arc[x].source;},
[&](int x){return g.arc[x].target;}
);
std::vector<int>part(g.node_count());
// Call METIS
int one = 1, two = 2, ignore, node_count = g.node_count();
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING] = 0;
options[METIS_OPTION_CONTIG] = 1;
options[METIS_OPTION_UFACTOR] = 100;
//options[METIS_OPTION_NCUTS] = 10;
METIS_PartGraphKway(
(idx_t*)&node_count,
(idx_t*)&one,
(idx_t*)&out_begin[0],
(idx_t*)&out_dest[0],
nullptr,
nullptr,
nullptr,
(idx_t*)&two,
nullptr,
nullptr,
options,
(idx_t*)&ignore,
(idx_t*)&part[0]
);
std::vector<bool>part_ret(g.node_count());
for(int i=0; i<g.node_count(); ++i)
part_ret[i] = part[i] == 0;
return std::move(part_ret);
#endif
}
std::vector<int> MetisMeshPartition(const CMesh* mesh, int nPart)
{
int vertCount = mesh->vertCount();
std::vector<int> vPart(vertCount);
int ncon = 1;
std::vector<int> xadj, adjncy;
ZGeom::getMeshGraphCSR(*mesh, xadj, adjncy);
int objval;
int options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_CONTIG] = 1;
options[METIS_OPTION_NUMBERING] = 0;
METIS_PartGraphKway(&vertCount, &ncon, &xadj[0], &adjncy[0], NULL, NULL, NULL, &nPart, NULL, NULL, NULL, &objval, &vPart[0]);
return vPart;
}
int do_metis_kway_partition(network_t *network, options_t *opt, idx_t *part, idx_t mode) {
int ret = ORCC_OK;
idx_t ncon = 1;
idx_t metis_opt[METIS_NOPTIONS];
idx_t objval;
adjacency_list *graph, *metis_graph;
assert(network != NULL);
assert(opt != NULL);
assert(part != NULL);
print_orcc_trace(ORCC_VL_VERBOSE_1, "Applying METIS Kway partition for mapping");
METIS_SetDefaultOptions(metis_opt);
metis_opt[METIS_OPTION_OBJTYPE] = mode;
metis_opt[METIS_OPTION_CONTIG] = 0; /* 0 or 1 */
graph = set_graph_from_network(network);
metis_graph = fix_graph_for_metis(graph);
ret = METIS_PartGraphKway(&metis_graph->nb_vertices, /* idx_t *nvtxs */
&ncon, /*idx_t *ncon*/
metis_graph->xadj, /*idx_t *xadj*/
metis_graph->adjncy, /*idx_t *adjncy*/
metis_graph->vwgt, /*idx_t *vwgt*/
NULL, /*idx_t *vsize*/
metis_graph->adjwgt, /*idx_t *adjwgt*/
&opt->nb_processors, /*idx_t *nparts*/
NULL, /*real t *tpwgts*/
NULL, /*real t ubvec*/
metis_opt, /*idx_t *options*/
&objval, /*idx_t *objval*/
part); /*idx_t *part*/
check_metis_error(ret);
print_orcc_trace(ORCC_VL_VERBOSE_2, "METIS Edgecut : %d", objval);
delete_graph(graph);
delete_graph(metis_graph);
return ret;
}
void METIS_PartMeshDual_WV(int *ne, int *nn, idxtype *elmnts, int *etype, int *numflag,
int *nparts, int *edgecut, idxtype *epart, idxtype *npart, idxtype *vwgts)
{
int i, j, k, me;
idxtype *xadj, *adjncy, *pwgts, *nptr, *nind;
int options[10], pnumflag=0, wgtflag=2;
int nnbrs, nbrind[200], nbrwgt[200], maxpwgt;
int esize, esizes[] = {-1, 3, 4, 8, 4};
esize = esizes[*etype];
if (*numflag == 1)
ChangeMesh2CNumbering((*ne)*esize, elmnts);
xadj = idxmalloc(*ne+1, "METIS_MESHPARTNODAL: xadj");
adjncy = idxmalloc(esize*(*ne), "METIS_MESHPARTNODAL: adjncy");
METIS_MeshToDual(ne, nn, elmnts, etype, &pnumflag, xadj, adjncy);
options[0] = 0;
METIS_PartGraphKway(ne, xadj, adjncy, vwgts, NULL, &wgtflag, &pnumflag, nparts, options, edgecut, epart);
/* Construct the node-element list */
nptr = idxsmalloc(*nn+1, 0, "METIS_MESHPARTDUAL: nptr");
for (j=esize*(*ne), i=0; i<j; i++)
nptr[elmnts[i]]++;
MAKECSR(i, *nn, nptr);
nind = idxmalloc(nptr[*nn], "METIS_MESHPARTDUAL: nind");
for (k=i=0; i<(*ne); i++) {
for (j=0; j<esize; j++, k++)
nind[nptr[elmnts[k]]++] = i;
}
for (i=(*nn); i>0; i--)
nptr[i] = nptr[i-1];
nptr[0] = 0;
/* OK, now compute a nodal partition based on the element partition npart */
idxset(*nn, -1, npart);
pwgts = idxsmalloc(*nparts, 0, "METIS_MESHPARTDUAL: pwgts");
for (i=0; i<*nn; i++) {
me = epart[nind[nptr[i]]];
for (j=nptr[i]+1; j<nptr[i+1]; j++) {
if (epart[nind[j]] != me)
break;
}
if (j == nptr[i+1]) {
npart[i] = me;
pwgts[me]++;
}
}
maxpwgt = 1.03*(*nn)/(*nparts);
for (i=0; i<*nn; i++) {
if (npart[i] == -1) { /* Assign the boundary element */
nnbrs = 0;
for (j=nptr[i]; j<nptr[i+1]; j++) {
me = epart[nind[j]];
for (k=0; k<nnbrs; k++) {
if (nbrind[k] == me) {
nbrwgt[k]++;
break;
}
}
if (k == nnbrs) {
nbrind[nnbrs] = me;
nbrwgt[nnbrs++] = 1;
}
}
/* Try to assign it first to the domain with most things in common */
j = iamax(nnbrs, nbrwgt);
if (pwgts[nbrind[j]] < maxpwgt) {
npart[i] = nbrind[j];
}
else {
/* If that fails, assign it to a light domain */
npart[i] = nbrind[0];
for (j=0; j<nnbrs; j++) {
if (pwgts[nbrind[j]] < maxpwgt) {
npart[i] = nbrind[j];
break;
}
}
}
pwgts[npart[i]]++;
}
}
if (*numflag == 1)
ChangeMesh2FNumbering2((*ne)*esize, elmnts, *ne, *nn, epart, npart);
GKfree(&xadj, &adjncy, &pwgts, &nptr, &nind, LTERM);
}
JNIEXPORT jint JNICALL Java_jprime_JMetis_PartitionGraph(
JNIEnv * env,
jobject obj,
jboolean j_force_PartGraphRecursive,
jboolean j_force_PartGraphKway,
jint j_num_vertices,
jintArray j_xadj,
jintArray j_adjncy,
jintArray j_vwgt,
jintArray j_adjwgt,
jint j_wgtflag,
jint j_nparts,
jintArray j_options,
jintArray j_partioned_nodes) {
mysrand(7654321L);
//delcare vars
jint * xadj=NULL, * adjncy=NULL,* vwgt=NULL, * adjwgt=NULL, * options=NULL, * partioned_nodes=NULL;
int idx, edges_cut=-1, num_vertices=0, nparts=0, wgtflag=0, numflag=0;
jsize xadj_len, adjncy_len, vwgt_len, adjwgt_len, options_len, partioned_nodes_len;
//copy the jsize vars
num_vertices=j_num_vertices;
nparts=j_nparts;
wgtflag=j_wgtflag;
//get array lengths
xadj_len = (*env)->GetArrayLength(env,j_xadj);
adjncy_len = (*env)->GetArrayLength(env,j_adjncy);
vwgt_len = (*env)->GetArrayLength(env,j_vwgt);
adjwgt_len = (*env)->GetArrayLength(env,j_adjwgt);
options_len = (*env)->GetArrayLength(env,j_options);
partioned_nodes_len = (*env)->GetArrayLength(env,j_partioned_nodes);
//printf("xadj_len=%i, adjncy_len=%i, vwgt_len=%i, adjwgt_len=%i, options_len=%i, partioned_nodes_len=%i\n",xadj_len, adjncy_len, vwgt_len, adjwgt_len, options_len, partioned_nodes_len);
//create/get local copies
xadj = (*env)->GetIntArrayElements(env,j_xadj,0);
adjncy = (*env)->GetIntArrayElements(env,j_adjncy,0);
if(vwgt_len>0) vwgt = (*env)->GetIntArrayElements(env,j_vwgt,0);
if(adjwgt_len>0) adjwgt = (*env)->GetIntArrayElements(env,j_adjwgt,0);
if(options_len>0) options = (*env)->GetIntArrayElements(env,j_options,0);
partioned_nodes = (int*)malloc(sizeof(int)*partioned_nodes_len);
#if 0
printf("num_vertices=%i,wgtflag=%i, numflag=%i, nparts=%i\n", num_vertices, wgtflag, numflag, nparts);
printf("xadj=%p, adjncy=%p, vwgt=%p, adjwgt=%p, options=%p, partioned_nodes=%p\n",
xadj,
adjncy,
vwgt,
adjwgt,
options,
partioned_nodes);
printf("xadj:");
for(idx=0;idx<xadj_len;idx++) {
printf("%i ",xadj[idx]);
}
printf("\n");
printf("adjncy:");
for(idx=0;idx<adjncy_len;idx++) {
printf("%i ",adjncy[idx]);
}
printf("\n");
#endif
//call func
if((j_nparts<8 || j_force_PartGraphRecursive) && !j_force_PartGraphKway) {
METIS_PartGraphRecursive(
&num_vertices,
xadj,
adjncy,
vwgt,
adjwgt,
&wgtflag,
&numflag,
&nparts,
options,
&edges_cut,
partioned_nodes);
}
else {
METIS_PartGraphKway(
&num_vertices,
xadj,
adjncy,
vwgt,
adjwgt,
&wgtflag,
&numflag,
&nparts,
options,
&edges_cut,
partioned_nodes);
}
//pop partioned_nodes
(*env)->SetIntArrayRegion(env,j_partioned_nodes,0,partioned_nodes_len,partioned_nodes);
//free local copies
free(partioned_nodes);
(*env)->ReleaseIntArrayElements(env, j_xadj, xadj, 0);
(*env)->ReleaseIntArrayElements(env, j_adjncy, adjncy, 0);
if(vwgt_len>0) (*env)->ReleaseIntArrayElements(env, j_vwgt, vwgt, 0);
if(adjwgt_len>0) (*env)->ReleaseIntArrayElements(env, j_adjwgt, adjwgt, 0);
if(options_len>0) (*env)->ReleaseIntArrayElements(env, j_options, options, 0);
return edges_cut;
}
/*
* Assign loop iterations to tiles carving partitions out of /seedLoop/ using
* the METIS library.
*/
static int* metis(loop_t* seedLoop, int tileSize, map_list* meshMaps,
int* nCore, int* nExec, int* nNonExec, int nThreads)
{
int i;
int setCore = seedLoop->set->core;
int setSize = seedLoop->set->size;
// use the mesh description to find a suitable map for partitioning through METIS
map_t* map = NULL;
map_list::const_iterator it, end;
for (it = meshMaps->begin(), end = meshMaps->end(); it != end; it++) {
if (set_eq(seedLoop->set, (*it)->inSet) || set_eq(seedLoop->set, (*it)->outSet)) {
map = *it;
break;
}
}
if (! map) {
// unfortunate scenario: the user provided a mesh description, but the loop picked
// as seed has an iteration space which is not part of the mesh description.
// will have to revert to chunk partitioning
return NULL;
}
// now partition through METIS:
// ... mesh geometry
int nElements = map->inSet->size;
int nNodes = map->outSet->size;
int nParts = nElements / tileSize;
int arity = map->size / nElements;
// ... data needed for partitioning
int* indMap = new int[nElements];
int* indNodesMap = new int[nNodes];
int* adjncy = map->values;
int* offsets = new int[nElements+1]();
for (i = 1; i < nElements+1; i++) {
offsets[i] = offsets[i-1] + arity;
}
// ... options
int result, objval, ncon = 1;
int options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING] = 0;
options[METIS_OPTION_CONTIG] = 1;
// ... do partition!
result = (arity == 2) ?
METIS_PartGraphKway (&nNodes, &ncon, offsets, adjncy, NULL, NULL, NULL,
&nParts, NULL, NULL, options, &objval, indMap) :
METIS_PartMeshNodal (&nElements, &nNodes, offsets, adjncy, NULL, NULL,
&nParts, NULL, options, &objval, indMap, indNodesMap);
ASSERT(result == METIS_OK, "Invalid METIS partitioning");
// what's the target iteration set ?
if (set_eq(seedLoop->set, map->inSet)) {
delete[] indNodesMap;
}
else {
// note: must be set_eq(seedLoop->set, map-outSet)
delete[] indMap;
indMap = indNodesMap;
}
delete[] offsets;
// restrict partitions to the core region
std::fill (indMap + setCore, indMap + setSize, 0);
std::set<int> partitions (indMap, indMap + setCore);
// ensure the set of partitions IDs is compact (i.e., if we have a partitioning
// 0: {0,1,...}, 1: {4,5,...}, 2: {}, 3: {6,10,...} ...
// we instead want to have
// 0: {0,1,...}, 1: {4,5,...}, 2: {6,10,...}, ...
std::map<int, int> mapper;
std::set<int>::const_iterator sIt, sEnd;
for (i = 0, sIt = partitions.begin(), sEnd = partitions.end(); sIt != sEnd; sIt++, i++) {
mapper[*sIt] = i;
}
for (i = 0; i < setCore; i++) {
indMap[i] = mapper[indMap[i]];
}
*nCore = partitions.size();
// partition the exec halo region
chunk_halo (seedLoop, tileSize, *nCore - 1, indMap, nExec, nNonExec, nThreads);
return indMap;
}
/*************************************************************************
* Let the game begin
**************************************************************************/
main(int argc, char *argv[])
{
int i, nparts, options[10];
idxtype *part;
float rubvec[MAXNCON], lbvec[MAXNCON];
GraphType graph;
char filename[256];
int numflag = 0, wgtflag = 0, edgecut;
timer TOTALTmr, METISTmr, IOTmr;
if (argc != 3) {
printf("Usage: %s <GraphFile> <Nparts>\n",argv[0]);
exit(0);
}
strcpy(filename, argv[1]);
nparts = atoi(argv[2]);
if (nparts < 2) {
printf("The number of partitions should be greater than 1!\n");
exit(0);
}
cleartimer(TOTALTmr);
cleartimer(METISTmr);
cleartimer(IOTmr);
starttimer(TOTALTmr);
starttimer(IOTmr);
ReadGraph(&graph, filename, &wgtflag);
if (graph.nvtxs <= 0) {
printf("Empty graph. Nothing to do.\n");
exit(0);
}
stoptimer(IOTmr);
printf("**********************************************************************\n");
printf("%s", METISTITLE);
printf("Graph Information ---------------------------------------------------\n");
printf(" Name: %s, #Vertices: %d, #Edges: %d, #Parts: %d\n", filename, graph.nvtxs, graph.nedges/2, nparts);
if (graph.ncon > 1)
printf(" Balancing Constraints: %d\n", graph.ncon);
printf("\nK-way Partitioning... -----------------------------------------------\n");
part = idxmalloc(graph.nvtxs, "main: part");
options[0] = 0;
starttimer(METISTmr);
if (graph.ncon == 1) {
METIS_PartGraphKway(&graph.nvtxs, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt,
&wgtflag, &numflag, &nparts, options, &edgecut, part);
}
else {
for (i=0; i<graph.ncon; i++)
rubvec[i] = HORIZONTAL_IMBALANCE;
METIS_mCPartGraphKway(&graph.nvtxs, &graph.ncon, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &nparts, rubvec, options, &edgecut, part);
}
stoptimer(METISTmr);
ComputePartitionBalance(&graph, nparts, part, lbvec);
printf(" %d-way Edge-Cut: %7d, Balance: ", nparts, edgecut);
for (i=0; i<graph.ncon; i++)
printf("%5.2f ", lbvec[i]);
printf("\n");
starttimer(IOTmr);
WritePartition(filename, part, graph.nvtxs, nparts);
stoptimer(IOTmr);
stoptimer(TOTALTmr);
printf("\nTiming Information --------------------------------------------------\n");
printf(" I/O: \t\t %7.3f\n", gettimer(IOTmr));
printf(" Partitioning: \t\t %7.3f (KMETIS time)\n", gettimer(METISTmr));
printf(" Total: \t\t %7.3f\n", gettimer(TOTALTmr));
printf("**********************************************************************\n");
GKfree(&graph.xadj, &graph.adjncy, &graph.vwgt, &graph.adjwgt, &part, LTERM);
}
int
main (int argc, char **argv)
{
// see Metis documentation (p.20 ff for Metis 5.x)
/*----------------------------------------------------------------------------*/
// example graph
// 0 -- 3
// | |
// 1 4
// | / |
// 2 5
// number of vertices
idx_t n_vertex = 6;
// size of xadj is n_vertex+1.
// This array stores for every vertex i in xadj[i] the index
// where the adjacent vertices are found in adjncy[].
// Looking at the example graph:
// Neighbors of vertex 0 are found starting at adjncy[0] and ending in adjncy[2-1],
// i.e. one less than the starting point of the next vertex.
// Neighbors of vertex 1 are found starting at adjncy[2] and ending in adjncy[4-1],
// i.e. one less than the starting point of the next vertex.
// etc.
idx_t xadj[] = { 0, 2, 4, 6, 8, 11, 12 };
// number of edges (see below: adjcny stores every edge twice as (u,v) and (v,u))
idx_t n_edge = 6;
// size of adjncy is 2 * n_edge (every edge appears twice in undirected graph)
idx_t adjncy[] = {
1, 3, // neighbors of vertex 0
0, 2, // neighbors of vertex 1
1, 4, // neighbors of vertex 2
0, 4, // neighbors of vertex 3
2, 3, 5, // neighbors of vertex 4
4 // neighbors of vertex 5
};
// vertex weights (size of vwgt is n_vertex)
idx_t vwgt[] = { 20, 30, 80, 60, 100, 90 };
// edge weights (size of adjwgt is n_edge*2)
idx_t adjwgt[] = { 10, 20, 10, 10, 10, 30, 20, 10, 30, 10, 10, 10 };
// the resulting partition:
// for every vertex i we find in part[i] afterwards the partition number for that vertex
idx_t part[n_vertex];
// Metis options (initialized to default values)
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions (options);
// partitioning method: k-way partitioning
options[METIS_OPTION_PTYPE] = METIS_PTYPE_KWAY;
// edge cut minimization
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT;
// C-style numbering
options[METIS_OPTION_NUMBERING] = 0;
// number of balancing constraints
idx_t ncon = 1;
// edge cut after partitioning
idx_t edgecut;
// !!!!!!!!!! you have to change this !!!!!!!!!!!!
// number of partitions we want to partition to
idx_t nparts = 2;
// !!!!!!!!!! you have to change this !!!!!!!!!!!!
/*--------------------------------------------------------------------------*/
// print header and input graph
printf ("partitioning graph with %ld vertices and %ld edges for %ld partitions\n", (long)n_vertex, (long)n_edge, (long)nparts);
print_graph (n_vertex, vwgt, xadj, adjncy, adjwgt);
// call graph partitioning
double t0 = gettime ();
int rc = METIS_PartGraphKway (&n_vertex, // number of vertices
&ncon, // number of balancing constraints
xadj, // adjacency structure of graph
adjncy, // adjacency structure of graph
vwgt, // vertex weights
NULL, // only for total communication volume
adjwgt, // edge weights
&nparts, // number of partitions wanted
NULL, // tpwgts: no desired partition weights
NULL, // ubvec: allowed load imbalance
options, // special options
&edgecut, // objective value (edge cut)
part // vector with partition information for each vertex
);
t0 = gettime () - t0;
// check Metis return value
switch (rc)
{
case METIS_OK:
break;
case METIS_ERROR_INPUT:
printf ("error in Metis input\n");
exit (1);
case METIS_ERROR_MEMORY:
printf ("no more memory in Metis\n");
exit (1);
case METIS_ERROR:
printf ("some error in Metis\n");
exit (1);
default:
printf ("unknown return code ffrom Metis\n");
exit (1);
}
/*--------------------------------------------------------------------------*/
/* print results */
// time for partitioning
printf ("partitioning time: %.9f s\n", t0);
// print parition
print_partition (n_vertex, vwgt, xadj, adjncy, adjwgt, part, nparts, edgecut);
return 0;
}
idx_t* gpmetis( int argc, char **argv )
/*************************************************************************/
/*! Let the game begin! */
/*************************************************************************/
//int main(int argc, char *argv[])
{
idx_t i;
char *curptr, *newptr;
idx_t options[METIS_NOPTIONS];
graph_t *graph;
idx_t *part;
idx_t objval;
params_t *params;
int status=0;
gk_optind = 0;
//printf( "argc: %d\n", argc );
//printf( "gk_optind %d\n", gk_optind );
fflush( stdout );
for( i = 0; i < argc; i++ )
{
//printf( "%s*\n", argv[ i ] );
}
params = parse_cmdline(argc, argv);
//printf( "gk_optind %d\n", gk_optind );
//fflush( stdout );
//return NULL;
gk_startcputimer(params->iotimer);
graph = ReadGraph(params);
ReadTPwgts(params, graph->ncon);
gk_stopcputimer(params->iotimer);
/* Check if the graph is contiguous */
if (params->contig && !IsConnected(graph, 0)) {
printf("***The input graph is not contiguous.\n"
"***The specified -contig option will be ignored.\n");
params->contig = 0;
}
/* Get ubvec if supplied */
if (params->ubvecstr) {
params->ubvec = rmalloc(graph->ncon, "main");
curptr = params->ubvecstr;
for (i=0; i<graph->ncon; i++) {
params->ubvec[i] = strtoreal(curptr, &newptr);
if (curptr == newptr)
errexit("Error parsing entry #%"PRIDX" of ubvec [%s] (possibly missing).\n",
i, params->ubvecstr);
curptr = newptr;
}
}
/* Setup iptype */
if (params->iptype == -1) {
if (params->ptype == METIS_PTYPE_RB) {
if (graph->ncon == 1)
params->iptype = METIS_IPTYPE_GROW;
else
params->iptype = METIS_IPTYPE_RANDOM;
}
}
GPPrintInfo(params, graph);
part = imalloc(graph->nvtxs, "main: part");
METIS_SetDefaultOptions(options);
options[METIS_OPTION_OBJTYPE] = params->objtype;
options[METIS_OPTION_CTYPE] = params->ctype;
options[METIS_OPTION_IPTYPE] = params->iptype;
options[METIS_OPTION_RTYPE] = params->rtype;
options[METIS_OPTION_MINCONN] = params->minconn;
options[METIS_OPTION_CONTIG] = params->contig;
options[METIS_OPTION_SEED] = params->seed;
options[METIS_OPTION_NITER] = params->niter;
options[METIS_OPTION_NCUTS] = params->ncuts;
options[METIS_OPTION_UFACTOR] = params->ufactor;
options[METIS_OPTION_DBGLVL] = params->dbglvl;
gk_malloc_init();
gk_startcputimer(params->parttimer);
switch (params->ptype) {
case METIS_PTYPE_RB:
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, part);
break;
case METIS_PTYPE_KWAY:
status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, part);
break;
}
gk_stopcputimer(params->parttimer);
if (gk_GetCurMemoryUsed() != 0)
printf("***It seems that Metis did not free all of its memory! Report this.\n");
params->maxmemory = gk_GetMaxMemoryUsed();
gk_malloc_cleanup(0);
if (status != METIS_OK) {
printf("\n***Metis returned with an error.\n");
}
else {
if (!params->nooutput) {
/* Write the solution */
gk_startcputimer(params->iotimer);
WritePartition(params->filename, part, graph->nvtxs, params->nparts);
gk_stopcputimer(params->iotimer);
}
GPReportResults(params, graph, part, objval);
}
idx_t *r_part = ( idx_t* ) calloc( graph->nvtxs, sizeof( idx_t ) );
for( i = 0; i < graph->nvtxs; i++ )
{
r_part[ i ] = part[ i ];
}
FreeGraph(&graph);
gk_free((void **)&part, LTERM);
gk_free((void **)¶ms->filename, ¶ms->tpwgtsfile, ¶ms->tpwgts,
¶ms->ubvecstr, ¶ms->ubvec, ¶ms, LTERM);
return r_part;
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
realtype itr = 0.0;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
/* change our zz struct. */
#endif
indextype i;
realtype *imb_tols;
indextype ncon;
indextype edgecut;
indextype wgtflag;
indextype numflag = 0;
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
ZOLTAN_TRACE_ENTER(zz, yo);
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
if (sizeof(realtype) != sizeof(float)) {
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));
for (i = 0; i < tmp; i++)
prt.input_part_sizes[i] = (realtype) part_sizes[i];
/* KDD 2/2014: removed re-scaling part sizes so they sum to one.
* part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
* plus, the code here was wrong for multiple object weights.
* similar scaling code did not exist in the Scotch interface.
*/
prt.part_sizes = prt.input_part_sizes;
}
else
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
indextype j;
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
/* Now we can call ParMetis */
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
/* Use default options for METIS */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0;
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
#else
METIS_SetDefaultOptions(options);
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
prt.part_sizes, imb_tols, options,
&edgecut, prt.part);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) {
/* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
void metis_partgraphkway__(int *nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part)
{
METIS_PartGraphKway(nvtxs, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, part);
}
/*************************************************************************
* This function partitions a finite element mesh by partitioning its nodal
* graph using KMETIS and then assigning elements in a load balanced fashion.
**************************************************************************/
int METIS_PartMeshNodal(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind,
idx_t *vwgt, idx_t *vsize, idx_t *nparts, real_t *tpwgts,
idx_t *options, idx_t *objval, idx_t *epart, idx_t *npart)
{
int sigrval=0, renumber=0, ptype;
idx_t *xadj=NULL, *adjncy=NULL;
idx_t ncon=1, pnumflag=0;
int rstatus=METIS_OK;
/* set up malloc cleaning code and signal catchers */
if (!gk_malloc_init())
return METIS_ERROR_MEMORY;
gk_sigtrap();
if ((sigrval = gk_sigcatch()) != 0)
goto SIGTHROW;
renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0);
ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY);
/* renumber the mesh */
if (renumber) {
ChangeMesh2CNumbering(*ne, eptr, eind);
options[METIS_OPTION_NUMBERING] = 0;
}
/* get the nodal graph */
rstatus = METIS_MeshToNodal(ne, nn, eptr, eind, &pnumflag, &xadj, &adjncy);
if (rstatus != METIS_OK)
raise(SIGERR);
/* partition the graph */
if (ptype == METIS_PTYPE_KWAY)
rstatus = METIS_PartGraphKway(nn, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, npart);
else
rstatus = METIS_PartGraphRecursive(nn, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, npart);
if (rstatus != METIS_OK)
raise(SIGERR);
/* partition the other side of the mesh */
InduceRowPartFromColumnPart(*ne, eptr, eind, epart, npart, *nparts, tpwgts);
SIGTHROW:
if (renumber) {
ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart);
options[METIS_OPTION_NUMBERING] = 1;
}
METIS_Free(xadj);
METIS_Free(adjncy);
gk_siguntrap();
gk_malloc_cleanup(0);
return metis_rcode(sigrval);
}
/*************************************************************************
* This function partitions a finite element mesh by partitioning its nodal
* graph using KMETIS and then assigning elements in a load balanced fashion.
**************************************************************************/
void METIS_PartMeshNodal(int *ne, int *nn, idxtype *elmnts, int *etype, int *numflag,
int *nparts, int *edgecut, idxtype *epart, idxtype *npart)
{
int i, j, k, me;
idxtype *xadj, *adjncy, *pwgts;
int options[10], pnumflag=0, wgtflag=0;
int nnbrs, nbrind[200], nbrwgt[200], maxpwgt;
int esize, esizes[] = {-1, 3, 4, 8, 4};
esize = esizes[*etype];
if (*numflag == 1)
ChangeMesh2CNumbering((*ne)*esize, elmnts);
xadj = idxmalloc(*nn+1, "METIS_MESHPARTNODAL: xadj");
adjncy = idxmalloc(20*(*nn), "METIS_MESHPARTNODAL: adjncy");
METIS_MeshToNodal(ne, nn, elmnts, etype, &pnumflag, xadj, adjncy);
adjncy = realloc(adjncy, xadj[*nn]*sizeof(idxtype));
options[0] = 0;
METIS_PartGraphKway(nn, xadj, adjncy, NULL, NULL, &wgtflag, &pnumflag, nparts, options, edgecut, npart);
/* OK, now compute an element partition based on the nodal partition npart */
idxset(*ne, -1, epart);
pwgts = idxsmalloc(*nparts, 0, "METIS_MESHPARTNODAL: pwgts");
for (i=0; i<*ne; i++) {
me = npart[elmnts[i*esize]];
for (j=1; j<esize; j++) {
if (npart[elmnts[i*esize+j]] != me)
break;
}
if (j == esize) {
epart[i] = me;
pwgts[me]++;
}
}
maxpwgt = 1.03*(*ne)/(*nparts);
for (i=0; i<*ne; i++) {
if (epart[i] == -1) { /* Assign the boundary element */
nnbrs = 0;
for (j=0; j<esize; j++) {
me = npart[elmnts[i*esize+j]];
for (k=0; k<nnbrs; k++) {
if (nbrind[k] == me) {
nbrwgt[k]++;
break;
}
}
if (k == nnbrs) {
nbrind[nnbrs] = me;
nbrwgt[nnbrs++] = 1;
}
}
/* Try to assign it first to the domain with most things in common */
j = iamax(nnbrs, nbrwgt);
if (pwgts[nbrind[j]] < maxpwgt) {
epart[i] = nbrind[j];
}
else {
/* If that fails, assign it to a light domain */
for (j=0; j<nnbrs; j++) {
if (pwgts[nbrind[j]] < maxpwgt) {
epart[i] = nbrind[j];
break;
}
}
if (j == nnbrs)
epart[i] = nbrind[iamax(nnbrs, nbrwgt)];
}
pwgts[epart[i]]++;
}
}
if (*numflag == 1)
ChangeMesh2FNumbering2((*ne)*esize, elmnts, *ne, *nn, epart, npart);
GKfree(&xadj, &adjncy, &pwgts, LTERM);
}
sparse_matrix* graph_partition(LIST *list , partition_t* partition_info)
{
graph_t* graph;
params_t *params;
idx_t options[METIS_NOPTIONS], status;
idx_t objval;
sparse_matrix* matrix = (sparse_matrix*)malloc(sizeof(sparse_matrix));
graph = ReadGraph(list);
params = bmalloc(sizeof(*params), "Allocating memory for params");
METIS_SetDefaultOptions(options);
METIS_init_params(params, graph->ncon);
options[METIS_OPTION_OBJTYPE] = params->objtype;
options[METIS_OPTION_CTYPE] = params->ctype;
options[METIS_OPTION_IPTYPE] = params->iptype;
options[METIS_OPTION_RTYPE] = params->rtype;
options[METIS_OPTION_NO2HOP] = params->no2hop;
options[METIS_OPTION_MINCONN] = params->minconn;
options[METIS_OPTION_CONTIG] = params->contig;
options[METIS_OPTION_SEED] = params->seed;
options[METIS_OPTION_NITER] = params->niter;
options[METIS_OPTION_NCUTS] = params->ncuts;
options[METIS_OPTION_UFACTOR] = params->ufactor;
options[METIS_OPTION_DBGLVL] = params->dbglvl;
//print_input_data(graph,params);
params->tpwgts = NULL;
params->ubvec = NULL;
partition_info->partion_table = imalloc(graph->nvtxs, "Allocate memory for part");
switch (params->ptype) {
case METIS_PTYPE_RB:
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, partition_info->partion_table);
break;
case METIS_PTYPE_KWAY:
status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, partition_info->partion_table);
break;
}
if (status != METIS_OK) {
fprintf(stderr,"\n***Metis returned with an error.***\n");
return NULL;
}
partition_info->size = graph->nvtxs;
partition_info->noOfParts = params->nparts;
WritePartition(params->filename, partition_info->partion_table, graph->nvtxs, params->nparts);
printf("Objval: %d\n",objval);
matrix->nz = -1;
matrix->n = graph->nvtxs;
matrix->p = graph->xadj;
matrix->i = graph->adjncy;
matrix->x = NULL;
return matrix;
}
// Call Metis with options from dictionary.
Foam::label Foam::metisDecomp::decompose
(
const List<int>& adjncy,
const List<int>& xadj,
const scalarField& cWeights,
List<int>& finalDecomp
)
{
// C style numbering
int numFlag = 0;
// Method of decomposition
// recursive: multi-level recursive bisection (default)
// k-way: multi-level k-way
word method("k-way");
int numCells = xadj.size()-1;
// decomposition options. 0 = use defaults
List<int> options(5, 0);
// processor weights initialised with no size, only used if specified in
// a file
Field<floatScalar> processorWeights;
// cell weights (so on the vertices of the dual)
List<int> cellWeights;
// face weights (so on the edges of the dual)
List<int> faceWeights;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningIn
(
"metisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != numCells)
{
FatalErrorIn
(
"metisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << numCells
<< exit(FatalError);
}
// Convert to integers.
cellWeights.setSize(cWeights.size());
forAll(cellWeights, i)
{
cellWeights[i] = int(cWeights[i]/minWeights);
}
}
// Check for user supplied weights and decomp options
if (decompositionDict_.found("metisCoeffs"))
{
const dictionary& metisCoeffs =
decompositionDict_.subDict("metisCoeffs");
word weightsFile;
if (metisCoeffs.readIfPresent("method", method))
{
if (method != "recursive" && method != "k-way")
{
FatalErrorIn("metisDecomp::decompose()")
<< "Method " << method << " in metisCoeffs in dictionary : "
<< decompositionDict_.name()
<< " should be 'recursive' or 'k-way'"
<< exit(FatalError);
}
Info<< "metisDecomp : Using Metis method " << method
<< nl << endl;
}
if (metisCoeffs.readIfPresent("options", options))
{
if (options.size() != 5)
{
FatalErrorIn("metisDecomp::decompose()")
<< "Number of options in metisCoeffs in dictionary : "
<< decompositionDict_.name()
<< " should be 5"
<< exit(FatalError);
}
Info<< "metisDecomp : Using Metis options " << options
<< nl << endl;
}
if (metisCoeffs.readIfPresent("processorWeights", processorWeights))
{
processorWeights /= sum(processorWeights);
if (processorWeights.size() != nProcessors_)
{
FatalErrorIn("metisDecomp::decompose(const pointField&)")
<< "Number of processor weights "
<< processorWeights.size()
<< " does not equal number of domains " << nProcessors_
<< exit(FatalError);
}
}
//if (metisCoeffs.readIfPresent("cellWeightsFile", weightsFile))
//{
// Info<< "metisDecomp : Using cell-based weights." << endl;
//
// IOList<int> cellIOWeights
// (
// IOobject
// (
// weightsFile,
// mesh_.time().timeName(),
// mesh_,
// IOobject::MUST_READ,
// IOobject::AUTO_WRITE
// )
// );
// cellWeights.transfer(cellIOWeights);
//
// if (cellWeights.size() != xadj.size()-1)
// {
// FatalErrorIn("metisDecomp::decompose(const pointField&)")
// << "Number of cell weights " << cellWeights.size()
// << " does not equal number of cells " << xadj.size()-1
// << exit(FatalError);
// }
//}
}
int nProcs = nProcessors_;
// output: cell -> processor addressing
finalDecomp.setSize(numCells);
// output: number of cut edges
int edgeCut = 0;
// Vertex weight info
int wgtFlag = 0;
int* vwgtPtr = NULL;
int* adjwgtPtr = NULL;
if (cellWeights.size())
{
vwgtPtr = cellWeights.begin();
wgtFlag += 2; // Weights on vertices
}
if (faceWeights.size())
{
adjwgtPtr = faceWeights.begin();
wgtFlag += 1; // Weights on edges
}
if (method == "recursive")
{
if (processorWeights.size())
{
METIS_WPartGraphRecursive
(
&numCells, // num vertices in graph
const_cast<List<int>&>(xadj).begin(), // indexing into adjncy
const_cast<List<int>&>(adjncy).begin(), // neighbour info
vwgtPtr, // vertexweights
adjwgtPtr, // no edgeweights
&wgtFlag,
&numFlag,
&nProcs,
processorWeights.begin(),
options.begin(),
&edgeCut,
finalDecomp.begin()
);
}
else
{
METIS_PartGraphRecursive
(
&numCells, // num vertices in graph
const_cast<List<int>&>(xadj).begin(), // indexing into adjncy
const_cast<List<int>&>(adjncy).begin(), // neighbour info
vwgtPtr, // vertexweights
adjwgtPtr, // no edgeweights
&wgtFlag,
&numFlag,
&nProcs,
options.begin(),
&edgeCut,
finalDecomp.begin()
);
}
}
else
{
if (processorWeights.size())
{
METIS_WPartGraphKway
(
&numCells, // num vertices in graph
const_cast<List<int>&>(xadj).begin(), // indexing into adjncy
const_cast<List<int>&>(adjncy).begin(), // neighbour info
vwgtPtr, // vertexweights
adjwgtPtr, // no edgeweights
&wgtFlag,
&numFlag,
&nProcs,
processorWeights.begin(),
options.begin(),
&edgeCut,
finalDecomp.begin()
);
}
else
{
METIS_PartGraphKway
(
&numCells, // num vertices in graph
const_cast<List<int>&>(xadj).begin(), // indexing into adjncy
const_cast<List<int>&>(adjncy).begin(), // neighbour info
vwgtPtr, // vertexweights
adjwgtPtr, // no edgeweights
&wgtFlag,
&numFlag,
&nProcs,
options.begin(),
&edgeCut,
finalDecomp.begin()
);
}
}
return edgeCut;
}
int generateLevels(std::vector< std::vector<sbfSElement *> > & selements, std::vector<int> & numTargetByLayers, std::vector<double> &maxImbalance, bool verbouse){
sbfMesh * mesh = selements[0][0]->mesh();
if(!mesh) return 1;
int numRegElems = selements[0].size();
std::vector <double> facesWeigth;
std::vector <int> facesOwners;
std::vector <double> randoms;
randoms.resize(50);
for(size_t ct = 0; ct < randoms.size(); ct++) randoms[ct] = ((double)rand())/RAND_MAX;
facesWeigth.reserve(numRegElems*50);
facesOwners.reserve(numRegElems*50);
for(int elemID = 0; elemID < numRegElems; elemID++){//Loop on elements
sbfElement *elem = mesh->elemPtr(elemID);
double faceWeigth;
int facesOwner = elemID;
std::list<int> inds;
int count;
std::vector< std::vector<int> > facesNodesIndexes = elem->facesNodesIndexes();
for(auto itFace = facesNodesIndexes.begin(); itFace != facesNodesIndexes.end(); itFace++){//Loop on faces
inds.clear();
for(auto itIndex = (*itFace).begin(); itIndex != (*itFace).end(); itIndex++)
inds.push_back(*itIndex);
inds.sort();
count = 0; faceWeigth = 0; for(auto it = inds.begin(); it != inds.end(); it++) {faceWeigth += *it*randoms[count++];}
facesWeigth.push_back(faceWeigth);
facesOwners.push_back(facesOwner);
}//Loop on faces
}//Loop on elements
quickAssociatedSort<double, int>(&facesWeigth[0], &facesOwners[0], 0, facesWeigth.size()-1);
if ( verbouse ) std::cout << "sort done" << std::endl;
for(size_t ctLevel = 0; ctLevel < numTargetByLayers.size(); ctLevel++){//Loop on levels
int numTargetSE = numTargetByLayers[ctLevel];
int numSElems = selements[ctLevel].size();
int vertnbr, edgenbr;
vertnbr = numSElems;
edgenbr = 0;
std::vector< std::list <int> > elemNeibour;
elemNeibour.resize(vertnbr);
int founded = 0, unfounded = 0;
std::vector <double> facesWeigthNextLevel;
std::vector <int> facesOwnersNextLevel;
facesWeigthNextLevel.reserve(facesWeigth.size());
facesOwnersNextLevel.reserve(facesOwners.size());
auto facesWeigthIt = facesWeigth.begin();
auto facesOwnersIt = facesOwners.begin();
auto facesWeigthEndM1 = facesWeigth.end() - 1;
for(; facesWeigthIt < facesWeigthEndM1; facesWeigthIt++, facesOwnersIt++){
if(*facesWeigthIt == *(facesWeigthIt+1)){
//Check if *facesOwnersIt and *(facesOwnersIt+1) are in one SE
int owner0, owner1;
owner0 = *facesOwnersIt; owner1 = *(facesOwnersIt+1);
int ownerSE0 = -1, ownerSE1 = -1;
bool inSame = false;
if(ctLevel == 0){ownerSE0 = owner0; ownerSE1 = owner1;}
else{
sbfSElement * se = selements[0][owner0];
for(size_t ct = 0; ct < ctLevel; ct++) se = se->parent();
ownerSE0 = se->index();
se = selements[0][owner1];
for(size_t ct = 0; ct < ctLevel; ct++) se = se->parent();
ownerSE1 = se->index();
if(ownerSE0 == ownerSE1) inSame = true;
}
if (inSame){ facesWeigthIt++; facesOwnersIt++; continue; }
elemNeibour[ownerSE1].push_back(ownerSE0);
elemNeibour[ownerSE0].push_back(ownerSE1);
facesWeigthNextLevel.push_back(*facesWeigthIt);
facesWeigthNextLevel.push_back(*facesWeigthIt);
facesOwnersNextLevel.push_back(*facesOwnersIt);
facesOwnersNextLevel.push_back(*(facesOwnersIt+1));
facesWeigthIt++; facesOwnersIt++;
founded++;
}
else unfounded++;
}
std::vector<idx_t> verttab, edgetab, edlotab, parttab;
verttab.resize(vertnbr+1);
parttab.resize(vertnbr);
int count = 0;
verttab[0] = count;
for(size_t ct = 0; ct < elemNeibour.size(); ct++){
std::list<int> elemNeibourAll, elemNeibourUnique;
for(auto it = elemNeibour[ct].begin(); it != elemNeibour[ct].end(); it++)
elemNeibourAll.push_back(*it);
elemNeibourAll.sort();
elemNeibourUnique = elemNeibourAll;
elemNeibourUnique.unique();
auto elemNeibourUniqueBegin = elemNeibourUnique.begin();
auto elemNeibourUniqueEnd = elemNeibourUnique.end();
auto elemNeibourAllBegin = elemNeibourAll.begin();
auto elemNeibourAllEnd = elemNeibourAll.end();
auto itA = elemNeibourAllBegin;
for(auto itU = elemNeibourUniqueBegin; itU != elemNeibourUniqueEnd; itU++)
{
if(static_cast<int>(ct) != *itU){
int numAcuarence = 0;
while(*itA == *itU && itA != elemNeibourAllEnd){
numAcuarence++;
itA++;
}
edgetab.push_back(*itU);
edlotab.push_back(numAcuarence);
count++;
edgenbr++;
}
}
verttab[ct+1] = count;
}
idx_t nvtxs = vertnbr, ncon = 1, nparts = numTargetSE, objval;
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_VOL;
options[METIS_OPTION_NUMBERING] = 0;
options[METIS_OPTION_NITER] = 600;
double maxLayerImbalance = maxImbalance.back();
if( ctLevel < maxImbalance.size() ) maxLayerImbalance = maxImbalance.at(ctLevel);
options[METIS_OPTION_UFACTOR] = static_cast<int>((maxLayerImbalance-1)*1000);
options[METIS_OPTION_CONTIG] = 1; //Force contiguous partitions
// options[METIS_OPTION_DBGLVL] = 1;
int rez = METIS_PartGraphKway(&nvtxs,
&ncon,
verttab.data(),
edgetab.data(),
/*idx_t *vwgt*/nullptr,
/*idx_t *vsize*/nullptr,
/*idx_t *adjwgt*/nullptr,
&nparts,
/*real_t *tpwgts*/nullptr,
/*real_t *ubvec*/nullptr,
options,
&objval,
parttab.data()
);
if ( rez != METIS_OK ) throw std::runtime_error("Metis runtime failed :(");
for(int ct = 0; ct < numSElems; ct++){
selements[ctLevel][ct]->setParent(selements[ctLevel+1][parttab[ct]]);
selements[ctLevel+1][parttab[ct]]->addChildren(selements[ctLevel][ct]);
}
//Metis solves following problem, but still it should be checked
//There are SElements with some disconnected clusters of elements.
//For such SE split them to several SEs
for(auto seIT = selements[ctLevel+1].begin(); seIT != selements[ctLevel+1].end(); seIT++){//Loop on SElements including new ones
std::set<int> allElemIndexes;//Indexes of all elements in this SE
for(int ct = 0; ct < (*seIT)->numSElements(); ct++) allElemIndexes.insert((*seIT)->children(ct)->index());
std::set<int> inOneSE;
inOneSE.insert(*(allElemIndexes.begin()));
bool flagChanges = true;
while(flagChanges){
flagChanges = false;
for(auto it = inOneSE.begin(); it != inOneSE.end(); it++){
for(auto itN = elemNeibour[*it].begin(); itN != elemNeibour[*it].end(); itN++){
if(allElemIndexes.count(*itN) && !inOneSE.count(*itN)){
inOneSE.insert(*itN);
flagChanges = true;
}
}
}
}
if(allElemIndexes.size() != inOneSE.size()){
std::set<int> inOtherSE;
(*seIT)->setChildrens(std::vector<sbfSElement *>{});
(*seIT)->numSElements();
for(auto it = inOneSE.begin(); it != inOneSE.end(); it++){
(*seIT)->addChildren(selements[ctLevel][*it]);
selements[ctLevel][*it]->setParent(*seIT);
}
for(auto it = allElemIndexes.begin(); it != allElemIndexes.end(); it++){
if(!inOneSE.count(*it))
inOtherSE.insert(*it);
}
sbfSElement *additionalSE = new sbfSElement((*seIT)->mesh(), selements[ctLevel+1].size());
for(auto it = inOtherSE.begin(); it != inOtherSE.end(); it++){
additionalSE->addChildren(selements[ctLevel][*it]);
selements[ctLevel][*it]->setParent(additionalSE);
}
selements[ctLevel+1].push_back(additionalSE);
if ( verbouse ) std::cout << "Split disconnected SE to two of sizes: " << inOneSE.size() << ", " << inOtherSE.size() << std::endl;
}
}//Loop on SElements including new ones
facesWeigth = facesWeigthNextLevel;
facesOwners = facesOwnersNextLevel;
std::cout << "level " << ctLevel << " done" << std::endl;
//get statistics
std::list<int> selems;
int numAll = 0;
for(auto se : selements[ctLevel+1]){
selems.push_back(se->numSElements());
numAll += se->numSElements();
}
if(numAll != numSElems)
throw std::runtime_error("SElements contain not all elements of previous layer");
selems.sort();
selems.reverse();
if ( verbouse ) { for (auto se : selems ) std::cout << se << "\t"; std::cout << "Imbalance is " << (1.0*selems.front())/selems.back() << std::endl; }
}//Loop on levels
return 0;
}
int MESH_PartitionWithMetis(Mesh_ptr mesh, int nparts, int **part) {
MEdge_ptr fedge;
MFace_ptr mf, oppf, rface;
MRegion_ptr mr, oppr;
List_ptr fedges, efaces, rfaces, fregions;
int i, ncells, ipos;
int nv, ne, nf, nr, nfe, nef, nfr, nrf, idx, idx2;
#ifdef METIS_5
idx_t ngraphvtx, numflag, nedgecut, numparts, ncons;
idx_t wtflag, metisopts[METIS_NOPTIONS];
idx_t *vsize, *idxpart;
idx_t *xadj, *adjncy, *vwgt, *adjwgt;
real_t *tpwgts, *ubvec;
#else
idxtype ngraphvtx, numflag, nedgecut, numparts;
idxtype wtflag, metisopts[5] = {0,0,0,0,0};
idxtype *xadj, *adjncy, *vwgt, *adjwgt, *idxpart;
#endif
/* First build a nodal graph of the mesh in the format required by
metis */
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
ipos = 0;
if (nr == 0) {
if (nf == 0) {
fprintf(stderr,"Cannot partition wire meshes\n");
exit(-1);
}
#ifdef METIS_5
xadj = (idx_t *) malloc((nf+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*ne*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nf+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*ne*sizeof(idxtype));
#endif
ncells = nf;
/* Surface mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mf = MESH_Next_Face(mesh,&idx))) {
fedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(fedges);
idx2 = 0;
while ((fedge = List_Next_Entry(fedges,&idx2))) {
efaces = ME_Faces(fedge);
nef = List_Num_Entries(efaces);
if (nef == 1) {
continue; /* boundary edge; nothing to do */
}
else {
int j;
for (j = 0; j < nef; j++) {
oppf = List_Entry(efaces,j);
if (oppf != mf) {
adjncy[ipos] = MF_ID(oppf)-1;
ipos++;
}
}
}
List_Delete(efaces);
}
List_Delete(fedges);
i++;
xadj[i] = ipos;
}
}
else {
#ifdef METIS_5
xadj = (idx_t *) malloc((nr+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*nf*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nr+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*nf*sizeof(idxtype));
#endif
ncells = nr;
/* Volume mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mr = MESH_Next_Region(mesh,&idx))) {
rfaces = MR_Faces(mr);
nrf = List_Num_Entries(rfaces);
idx2 = 0;
while ((rface = List_Next_Entry(rfaces,&idx2))) {
fregions = MF_Regions(rface);
nfr = List_Num_Entries(fregions);
if (nfr > 1) {
oppr = List_Entry(fregions,0);
if (oppr == mr)
oppr = List_Entry(fregions,1);
adjncy[ipos] = MR_ID(oppr)-1;
ipos++;
}
List_Delete(fregions);
}
List_Delete(rfaces);
i++;
xadj[i] = ipos;
}
}
/* Partition the graph */
wtflag = 0; /* No weights are specified */
vwgt = adjwgt = NULL;
numflag = 0; /* C style numbering of elements (nodes of the dual graph) */
ngraphvtx = ncells; /* we want the variable to be of type idxtype or idx_t */
numparts = nparts; /* we want the variable to be of type idxtype or idx_t */
#ifdef METIS_5
idxpart = (idx_t *) malloc(ncells*sizeof(idx_t));
ncons = 1; /* Number of constraints */
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
METIS_SetDefaultOptions(metisopts);
metisopts[METIS_OPTION_NUMBERING] = 0;
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,
idxpart);
else
METIS_PartGraphKway(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,idxpart);
#else
idxpart = (idxtype *) malloc(ncells*sizeof(idxtype));
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,
&numflag,&numparts,metisopts,&nedgecut,idxpart);
else
METIS_PartGraphKway(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,&numflag,
&numparts,metisopts,&nedgecut,idxpart);
#endif
free(xadj);
free(adjncy);
*part = (int *) malloc(ncells*sizeof(int));
for (i = 0; i < ncells; i++)
(*part)[i] = (int) idxpart[i];
free(idxpart);
return 1;
}
void METIS_PARTGRAPHKWAY(int *nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part)
{
METIS_PartGraphKway(nvtxs, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, part);
}
//==============================================================================
// NOTE:
// - matrix is supposed to be localized, and passes through the
// singleton filter. This means that I do not have to look
// for Dirichlet nodes (singletons). Also, all rows and columns are
// local.
int Ifpack_METISPartitioner::ComputePartitions()
{
int ierr;
#ifdef HAVE_IFPACK_METIS
int nbytes = 0;
int edgecut;
#endif
Teuchos::RefCountPtr<Epetra_CrsGraph> SymGraph ;
Teuchos::RefCountPtr<Epetra_Map> SymMap;
Teuchos::RefCountPtr<Ifpack_Graph_Epetra_CrsGraph> SymIFPACKGraph;
Teuchos::RefCountPtr<Ifpack_Graph> IFPACKGraph = Teuchos::rcp( (Ifpack_Graph*)Graph_, false );
int Length = 2 * MaxNumEntries();
int NumIndices;
std::vector<int> Indices;
Indices.resize(Length);
/* construct the CSR graph information of the LOCAL matrix
using the get_row function */
std::vector<idxtype> wgtflag;
wgtflag.resize(4);
std::vector<int> options;
options.resize(4);
int numflag;
if (UseSymmetricGraph_) {
#if !defined(EPETRA_NO_32BIT_GLOBAL_INDICES) || !defined(EPETRA_NO_64BIT_GLOBAL_INDICES)
// need to build a symmetric graph.
// I do this in two stages:
// 1.- construct an Epetra_CrsMatrix, symmetric
// 2.- convert the Epetra_CrsMatrix into METIS format
SymMap = Teuchos::rcp( new Epetra_Map(NumMyRows(),0,Graph_->Comm()) );
SymGraph = Teuchos::rcp( new Epetra_CrsGraph(Copy,*SymMap,0) );
#endif
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
if(SymGraph->RowMap().GlobalIndicesInt()) {
for (int i = 0; i < NumMyRows() ; ++i) {
ierr = Graph_->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]);
IFPACK_CHK_ERR(ierr);
for (int j = 0 ; j < NumIndices ; ++j) {
int jj = Indices[j];
if (jj != i) {
SymGraph->InsertGlobalIndices(i,1,&jj);
SymGraph->InsertGlobalIndices(jj,1,&i);
}
}
}
}
else
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
if(SymGraph->RowMap().GlobalIndicesLongLong()) {
for (int i = 0; i < NumMyRows() ; ++i) {
long long i_LL = i;
ierr = Graph_->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]);
IFPACK_CHK_ERR(ierr);
for (int j = 0 ; j < NumIndices ; ++j) {
long long jj = Indices[j];
if (jj != i_LL) {
SymGraph->InsertGlobalIndices(i_LL,1,&jj);
SymGraph->InsertGlobalIndices(jj,1,&i_LL);
}
}
}
}
else
#endif
throw "Ifpack_METISPartitioner::ComputePartitions: GlobalIndices type unknown";
IFPACK_CHK_ERR(SymGraph->FillComplete());
SymIFPACKGraph = Teuchos::rcp( new Ifpack_Graph_Epetra_CrsGraph(SymGraph) );
IFPACKGraph = SymIFPACKGraph;
}
// now work on IFPACKGraph, that can be the symmetric or
// the non-symmetric one
/* set parameters */
wgtflag[0] = 0; /* no weights */
numflag = 0; /* C style */
options[0] = 0; /* default options */
std::vector<idxtype> xadj;
xadj.resize(NumMyRows() + 1);
std::vector<idxtype> adjncy;
adjncy.resize(NumMyNonzeros());
int count = 0;
int count2 = 0;
xadj[0] = 0;
for (int i = 0; i < NumMyRows() ; ++i) {
xadj[count2+1] = xadj[count2]; /* nonzeros in row i-1 */
ierr = IFPACKGraph->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]);
IFPACK_CHK_ERR(ierr);
for (int j = 0 ; j < NumIndices ; ++j) {
int jj = Indices[j];
if (jj != i) {
adjncy[count++] = jj;
xadj[count2+1]++;
}
}
count2++;
}
std::vector<idxtype> NodesInSubgraph;
NodesInSubgraph.resize(NumLocalParts_);
// some cases can be handled separately
int ok;
if (NumLocalParts() == 1) {
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = 0;
} else if (NumLocalParts() == NumMyRows()) {
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = i;
} else {
ok = 0;
// sometimes METIS creates less partitions than specified.
// ok will check this problem, and recall metis, asking
// for NumLocalParts_/2 partitions
while (ok == 0) {
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = -1;
#ifdef HAVE_IFPACK_METIS
int j = NumMyRows();
if (NumLocalParts_ < 8) {
int i = 1; /* optype in the METIS manual */
numflag = 0;
METIS_EstimateMemory(&j, &xadj[0], &adjncy[0],
&numflag, &i, &nbytes );
METIS_PartGraphRecursive(&j, &xadj[0], &adjncy[0],
NULL, NULL,
&wgtflag[0], &numflag, &NumLocalParts_,
&options[0], &edgecut, &Partition_[0]);
} else {
numflag = 0;
METIS_PartGraphKway (&j, &xadj[0], &adjncy[0],
NULL,
NULL, &wgtflag[0], &numflag,
&NumLocalParts_, &options[0],
&edgecut, &Partition_[0]);
}
#else
numflag = numflag * 2; // avoid warning for unused variable
if (Graph_->Comm().MyPID() == 0) {
cerr << "METIS was not linked; now I put all" << endl;
cerr << "the local nodes in the same partition." << endl;
}
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = 0;
NumLocalParts_ = 1;
#endif
ok = 1;
for (int i = 0 ; i < NumLocalParts() ; ++i)
NodesInSubgraph[i] = 0;
for (int i = 0 ; i < NumMyRows() ; ++i) {
int j = Partition_[i];
if ((j < 0) || (j>= NumLocalParts())) {
ok = 0;
break;
}
else NodesInSubgraph[j]++;
}
for (int i = 0 ; i < NumLocalParts() ; ++i) {
if( NodesInSubgraph[i] == 0 ) {
ok = 0;
break;
}
}
if (ok == 0) {
cerr << "Specified number of subgraphs ("
<< NumLocalParts_ << ") generates empty subgraphs." << endl;
cerr << "Now I recall METIS with NumLocalParts_ = "
<< NumLocalParts_ / 2 << "..." << endl;
NumLocalParts_ = NumLocalParts_/2;
}
if (NumLocalParts() == 0) {
IFPACK_CHK_ERR(-10); // something went wrong
}
if (NumLocalParts() == 1) {
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = 0;
ok = 1;
}
} /* while( ok == 0 ) */
} /* if( NumLocalParts_ == 1 ) */
return(0);
}
/*************************************************************************
* This function is the entry point of the initial partition algorithm
* that does recursive bissection.
* This algorithm assembles the graph to all the processors and preceeds
* by parallelizing the recursive bisection step.
**************************************************************************/
void InitPartition(ctrl_t *ctrl, graph_t *graph)
{
idx_t i, j, ncon, mype, npes, gnvtxs, ngroups;
idx_t *xadj, *adjncy, *adjwgt, *vwgt;
idx_t *part, *gwhere0, *gwhere1;
idx_t *tmpwhere, *tmpvwgt, *tmpxadj, *tmpadjncy, *tmpadjwgt;
graph_t *agraph;
idx_t lnparts, fpart, fpe, lnpes;
idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut;
real_t *tpwgts, *tpwgts2, *lbvec, lbsum, min_lbsum, wsum;
MPI_Comm ipcomm;
struct {
double sum;
int rank;
} lpesum, gpesum;
WCOREPUSH;
ncon = graph->ncon;
ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, ctrl->npes), 1);
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
lbvec = rwspacemalloc(ctrl, ncon);
/* assemble the graph to all the processors */
agraph = AssembleAdaptiveGraph(ctrl, graph);
gnvtxs = agraph->nvtxs;
/* make a copy of the graph's structure for later */
xadj = icopy(gnvtxs+1, agraph->xadj, iwspacemalloc(ctrl, gnvtxs+1));
vwgt = icopy(gnvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, gnvtxs*ncon));
adjncy = icopy(agraph->nedges, agraph->adjncy, iwspacemalloc(ctrl, agraph->nedges));
adjwgt = icopy(agraph->nedges, agraph->adjwgt, iwspacemalloc(ctrl, agraph->nedges));
part = iwspacemalloc(ctrl, gnvtxs);
/* create different processor groups */
gkMPI_Comm_split(ctrl->gcomm, ctrl->mype % ngroups, 0, &ipcomm);
gkMPI_Comm_rank(ipcomm, &mype);
gkMPI_Comm_size(ipcomm, &npes);
/* Go into the recursive bisection */
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_SEED] = ctrl->sync + (ctrl->mype % ngroups) + 1;
tpwgts = ctrl->tpwgts;
tpwgts2 = rwspacemalloc(ctrl, 2*ncon);
lnparts = ctrl->nparts;
fpart = fpe = 0;
lnpes = npes;
while (lnpes > 1 && lnparts > 1) {
/* determine the weights of the two partitions as a function of the
weight of the target partition weights */
for (j=(lnparts>>1), i=0; i<ncon; i++) {
tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon);
tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon);
wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]);
tpwgts2[i] *= wsum;
tpwgts2[ncon+i] *= wsum;
}
METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &twoparts, tpwgts2, NULL, moptions,
&edgecut, part);
/* pick one of the branches */
if (mype < fpe+lnpes/2) {
KeepPart(ctrl, agraph, part, 0);
lnpes = lnpes/2;
lnparts = lnparts/2;
}
else {
KeepPart(ctrl, agraph, part, 1);
fpart = fpart + lnparts/2;
fpe = fpe + lnpes/2;
lnpes = lnpes - lnpes/2;
lnparts = lnparts - lnparts/2;
}
}
gwhere0 = iset(gnvtxs, 0, iwspacemalloc(ctrl, gnvtxs));
gwhere1 = iwspacemalloc(ctrl, gnvtxs);
if (lnparts == 1) { /* Case npes is greater than or equal to nparts */
/* Only the first process will assign labels (for the reduction to work) */
if (mype == fpe) {
for (i=0; i<agraph->nvtxs; i++)
gwhere0[agraph->label[i]] = fpart;
}
}
else { /* Case in which npes is smaller than nparts */
/* create the normalized tpwgts for the lnparts from ctrl->tpwgts */
tpwgts = rwspacemalloc(ctrl, lnparts*ncon);
for (j=0; j<ncon; j++) {
for (wsum=0.0, i=0; i<lnparts; i++) {
tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j];
wsum += tpwgts[i*ncon+j];
}
for (wsum=1.0/wsum, i=0; i<lnparts; i++)
tpwgts[i*ncon+j] *= wsum;
}
METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions,
&edgecut, part);
for (i=0; i<agraph->nvtxs; i++)
gwhere0[agraph->label[i]] = fpart + part[i];
}
gkMPI_Allreduce((void *)gwhere0, (void *)gwhere1, gnvtxs, IDX_T, MPI_SUM, ipcomm);
if (ngroups > 1) {
tmpxadj = agraph->xadj;
tmpadjncy = agraph->adjncy;
tmpadjwgt = agraph->adjwgt;
tmpvwgt = agraph->vwgt;
tmpwhere = agraph->where;
agraph->xadj = xadj;
agraph->adjncy = adjncy;
agraph->adjwgt = adjwgt;
agraph->vwgt = vwgt;
agraph->where = gwhere1;
agraph->vwgt = vwgt;
agraph->nvtxs = gnvtxs;
edgecut = ComputeSerialEdgeCut(agraph);
ComputeSerialBalance(ctrl, agraph, gwhere1, lbvec);
lbsum = rsum(ncon, lbvec, 1);
gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ctrl->gcomm);
gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ctrl->gcomm);
lpesum.sum = lbsum;
if (min_lbsum < UNBALANCE_FRACTION*ncon) {
if (lbsum < UNBALANCE_FRACTION*ncon)
lpesum.sum = edgecut;
else
lpesum.sum = max_cut;
}
lpesum.rank = ctrl->mype;
gkMPI_Allreduce((void *)&lpesum, (void *)&gpesum, 1, MPI_DOUBLE_INT,
MPI_MINLOC, ctrl->gcomm);
gkMPI_Bcast((void *)gwhere1, gnvtxs, IDX_T, gpesum.rank, ctrl->gcomm);
agraph->xadj = tmpxadj;
agraph->adjncy = tmpadjncy;
agraph->adjwgt = tmpadjwgt;
agraph->vwgt = tmpvwgt;
agraph->where = tmpwhere;
}
icopy(graph->nvtxs, gwhere1+graph->vtxdist[ctrl->mype], graph->where);
FreeGraph(agraph);
gkMPI_Comm_free(&ipcomm);
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->InitPartTmr));
WCOREPOP;
}
/***********************************************************************************
* This function is the entry point of the parallel kmetis algorithm that uses
* coordinates to compute an initial graph distribution.
************************************************************************************/
int ParMETIS_V3_PartGeomKway(idx_t *vtxdist, idx_t *xadj, idx_t *adjncy,
idx_t *vwgt, idx_t *adjwgt, idx_t *wgtflag, idx_t *numflag, idx_t *ndims,
real_t *xyz, idx_t *ncon, idx_t *nparts, real_t *tpwgts, real_t *ubvec,
idx_t *options, idx_t *edgecut, idx_t *part, MPI_Comm *comm)
{
idx_t h, i, j, npes, mype, status, nvtxs, seed, dbglvl;
idx_t cut, gcut, maxnvtxs;
idx_t moptions[METIS_NOPTIONS];
ctrl_t *ctrl;
graph_t *graph, *mgraph;
real_t balance;
size_t curmem;
/* Check the input parameters and return if an error */
status = CheckInputsPartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag,
numflag, ndims, xyz, ncon, nparts, tpwgts, ubvec, options,
edgecut, part, comm);
if (GlobalSEMinComm(*comm, status) == 0)
return METIS_ERROR;
status = METIS_OK;
gk_malloc_init();
curmem = gk_GetCurMemoryUsed();
/* Setup the ctrl */
ctrl = SetupCtrl(PARMETIS_OP_GKMETIS, options, *ncon, *nparts, tpwgts, ubvec, *comm);
npes = ctrl->npes;
mype = ctrl->mype;
/* Take care the nparts == 1 case */
if (*nparts == 1) {
iset(vtxdist[mype+1]-vtxdist[mype], (*numflag == 0 ? 0 : 1), part);
*edgecut = 0;
goto DONE;
}
/* Take care of npes == 1 case */
if (npes == 1) {
nvtxs = vtxdist[1] - vtxdist[0]; /* subtraction is required when numflag==1 */
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_NUMBERING] = *numflag;
status = METIS_PartGraphKway(&nvtxs, ncon, xadj, adjncy, vwgt, NULL, adjwgt,
nparts, tpwgts, ubvec, moptions, edgecut, part);
goto DONE;
}
/* Setup the graph */
if (*numflag > 0)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1);
graph = SetupGraph(ctrl, *ncon, vtxdist, xadj, vwgt, NULL, adjncy, adjwgt, *wgtflag);
gk_free((void **)&graph->nvwgt, LTERM);
/* Allocate the workspace */
AllocateWSpace(ctrl, 10*graph->nvtxs);
/* Compute the initial npes-way partitioning geometric partitioning */
STARTTIMER(ctrl, ctrl->TotalTmr);
Coordinate_Partition(ctrl, graph, *ndims, xyz, 1);
STOPTIMER(ctrl, ctrl->TotalTmr);
/* Move the graph according to the partitioning */
STARTTIMER(ctrl, ctrl->MoveTmr);
ctrl->nparts = npes;
mgraph = MoveGraph(ctrl, graph);
ctrl->nparts = *nparts;
SetupGraph_nvwgts(ctrl, mgraph); /* compute nvwgts for the moved graph */
if (ctrl->dbglvl&DBG_INFO) {
CommInterfaceData(ctrl, graph, graph->where, graph->where+graph->nvtxs);
for (cut=0, i=0; i<graph->nvtxs; i++) {
for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++) {
if (graph->where[i] != graph->where[graph->adjncy[j]])
cut += graph->adjwgt[j];
}
}
gcut = GlobalSESum(ctrl, cut)/2;
maxnvtxs = GlobalSEMax(ctrl, mgraph->nvtxs);
balance = (real_t)(maxnvtxs)/((real_t)(graph->gnvtxs)/(real_t)(npes));
rprintf(ctrl, "XYZ Cut: %6"PRIDX" \tBalance: %6.3"PRREAL" [%"PRIDX" %"PRIDX" %"PRIDX"]\n",
gcut, balance, maxnvtxs, graph->gnvtxs, npes);
}
STOPTIMER(ctrl, ctrl->MoveTmr);
/* Compute the partition of the moved graph */
STARTTIMER(ctrl, ctrl->TotalTmr);
ctrl->CoarsenTo = gk_min(vtxdist[npes]+1, 25*(*ncon)*gk_max(npes, *nparts));
if (vtxdist[npes] < SMALLGRAPH
|| vtxdist[npes] < npes*20
|| GlobalSESum(ctrl, mgraph->nedges) == 0) { /* serially */
IFSET(ctrl->dbglvl, DBG_INFO,
rprintf(ctrl, "Partitioning a graph of size %"PRIDX" serially\n", vtxdist[npes]));
PartitionSmallGraph(ctrl, mgraph);
}
else { /* in parallel */
Global_Partition(ctrl, mgraph);
}
ParallelReMapGraph(ctrl, mgraph);
/* Invert the ordering back to the original graph */
ctrl->nparts = npes;
ProjectInfoBack(ctrl, graph, part, mgraph->where);
ctrl->nparts = *nparts;
*edgecut = mgraph->mincut;
STOPTIMER(ctrl, ctrl->TotalTmr);
/* Print some stats */
IFSET(ctrl->dbglvl, DBG_TIME, PrintTimingInfo(ctrl));
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->gcomm));
IFSET(ctrl->dbglvl, DBG_INFO, PrintPostPartInfo(ctrl, mgraph, 0));
FreeGraph(mgraph);
FreeInitialGraphAndRemap(graph);
if (*numflag > 0)
ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0);
DONE:
FreeCtrl(&ctrl);
if (gk_GetCurMemoryUsed() - curmem > 0) {
printf("ParMETIS appears to have a memory leak of %zdbytes. Report this.\n",
(ssize_t)(gk_GetCurMemoryUsed() - curmem));
}
gk_malloc_cleanup(0);
return (int)status;
}
/*************************************************************************
* This function partitions a finite element mesh by partitioning its dual
* graph using KMETIS and then assigning nodes in a load balanced fashion.
**************************************************************************/
int METIS_PartMeshDual(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind,
idx_t *vwgt, idx_t *vsize, idx_t *ncommon, idx_t *nparts,
real_t *tpwgts, idx_t *options, idx_t *objval, idx_t *epart,
idx_t *npart)
{
int sigrval=0, renumber=0, ptype;
idx_t i, j;
idx_t *xadj=NULL, *adjncy=NULL, *nptr=NULL, *nind=NULL;
idx_t ncon=1, pnumflag=0;
int rstatus = METIS_OK;
/* set up malloc cleaning code and signal catchers */
if (!gk_malloc_init())
return METIS_ERROR_MEMORY;
gk_sigtrap();
if ((sigrval = gk_sigcatch()) != 0)
goto SIGTHROW;
renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0);
ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY);
/* renumber the mesh */
if (renumber) {
ChangeMesh2CNumbering(*ne, eptr, eind);
options[METIS_OPTION_NUMBERING] = 0;
}
/* get the dual graph */
rstatus = METIS_MeshToDual(ne, nn, eptr, eind, ncommon, &pnumflag, &xadj, &adjncy);
if (rstatus != METIS_OK)
raise(SIGERR);
/* partition the graph */
if (ptype == METIS_PTYPE_KWAY)
rstatus = METIS_PartGraphKway(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, epart);
else
rstatus = METIS_PartGraphRecursive(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, epart);
if (rstatus != METIS_OK)
raise(SIGERR);
/* construct the node-element list */
nptr = ismalloc(*nn+1, 0, "METIS_PartMeshDual: nptr");
nind = imalloc(eptr[*ne], "METIS_PartMeshDual: nind");
for (i=0; i<*ne; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++)
nptr[eind[j]]++;
}
MAKECSR(i, *nn, nptr);
for (i=0; i<*ne; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++)
nind[nptr[eind[j]]++] = i;
}
SHIFTCSR(i, *nn, nptr);
/* partition the other side of the mesh */
InduceRowPartFromColumnPart(*nn, nptr, nind, npart, epart, *nparts, tpwgts);
gk_free((void **)&nptr, &nind, LTERM);
SIGTHROW:
if (renumber) {
ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart);
options[METIS_OPTION_NUMBERING] = 1;
}
METIS_Free(xadj);
METIS_Free(adjncy);
gk_siguntrap();
gk_malloc_cleanup(0);
return metis_rcode(sigrval);
}
/*************************************************************************
* This function is the entry point of the initial balancing algorithm.
* This algorithm assembles the graph to all the processors and preceeds
* with the balancing step.
**************************************************************************/
void Balance_Partition(ctrl_t *ctrl, graph_t *graph)
{
idx_t i, j, nvtxs, nedges, ncon;
idx_t mype, npes, srnpes, srmype;
idx_t *vtxdist, *xadj, *adjncy, *adjwgt, *vwgt, *vsize;
idx_t *part, *lwhere, *home;
idx_t lnparts, fpart, fpe, lnpes, ngroups;
idx_t *rcounts, *rdispls;
idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut;
idx_t sr_pe, gd_pe, sr, gd, who_wins;
real_t my_cut, my_totalv, my_cost = -1.0, my_balance = -1.0, wsum;
real_t rating, max_rating, your_cost = -1.0, your_balance = -1.0;
real_t lbsum, min_lbsum, *lbvec, *tpwgts, *tpwgts2, buffer[2];
graph_t *agraph, cgraph;
ctrl_t *myctrl;
MPI_Status status;
MPI_Comm ipcomm, srcomm;
struct {
double cost;
int rank;
} lpecost, gpecost;
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
WCOREPUSH;
vtxdist = graph->vtxdist;
agraph = AssembleAdaptiveGraph(ctrl, graph);
nvtxs = cgraph.nvtxs = agraph->nvtxs;
nedges = cgraph.nedges = agraph->nedges;
ncon = cgraph.ncon = agraph->ncon;
xadj = cgraph.xadj = icopy(nvtxs+1, agraph->xadj, iwspacemalloc(ctrl, nvtxs+1));
vwgt = cgraph.vwgt = icopy(nvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, nvtxs*ncon));
vsize = cgraph.vsize = icopy(nvtxs, agraph->vsize, iwspacemalloc(ctrl, nvtxs));
adjncy = cgraph.adjncy = icopy(nedges, agraph->adjncy, iwspacemalloc(ctrl, nedges));
adjwgt = cgraph.adjwgt = icopy(nedges, agraph->adjwgt, iwspacemalloc(ctrl, nedges));
part = cgraph.where = agraph->where = iwspacemalloc(ctrl, nvtxs);
lwhere = iwspacemalloc(ctrl, nvtxs);
home = iwspacemalloc(ctrl, nvtxs);
lbvec = rwspacemalloc(ctrl, graph->ncon);
/****************************************/
/****************************************/
if (ctrl->ps_relation == PARMETIS_PSR_UNCOUPLED) {
WCOREPUSH;
rcounts = iwspacemalloc(ctrl, ctrl->npes);
rdispls = iwspacemalloc(ctrl, ctrl->npes+1);
for (i=0; i<ctrl->npes; i++)
rdispls[i] = rcounts[i] = vtxdist[i+1]-vtxdist[i];
MAKECSR(i, ctrl->npes, rdispls);
gkMPI_Allgatherv((void *)graph->home, graph->nvtxs, IDX_T,
(void *)part, rcounts, rdispls, IDX_T, ctrl->comm);
for (i=0; i<agraph->nvtxs; i++)
home[i] = part[i];
WCOREPOP; /* local frees */
}
else {
for (i=0; i<ctrl->npes; i++) {
for (j=vtxdist[i]; j<vtxdist[i+1]; j++)
part[j] = home[j] = i;
}
}
/* Ensure that the initial partitioning is legal */
for (i=0; i<agraph->nvtxs; i++) {
if (part[i] >= ctrl->nparts)
part[i] = home[i] = part[i] % ctrl->nparts;
if (part[i] < 0)
part[i] = home[i] = (-1*part[i]) % ctrl->nparts;
}
/****************************************/
/****************************************/
IFSET(ctrl->dbglvl, DBG_REFINEINFO,
ComputeSerialBalance(ctrl, agraph, agraph->where, lbvec));
IFSET(ctrl->dbglvl, DBG_REFINEINFO,
rprintf(ctrl, "input cut: %"PRIDX", balance: ", ComputeSerialEdgeCut(agraph)));
for (i=0; i<agraph->ncon; i++)
IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "%.3"PRREAL" ", lbvec[i]));
IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "\n"));
/****************************************/
/* Split the processors into two groups */
/****************************************/
sr = (ctrl->mype % 2 == 0) ? 1 : 0;
gd = (ctrl->mype % 2 == 1) ? 1 : 0;
if (graph->ncon > MAX_NCON_FOR_DIFFUSION || ctrl->npes == 1) {
sr = 1;
gd = 0;
}
sr_pe = 0;
gd_pe = 1;
gkMPI_Comm_split(ctrl->gcomm, sr, 0, &ipcomm);
gkMPI_Comm_rank(ipcomm, &mype);
gkMPI_Comm_size(ipcomm, &npes);
if (sr == 1) { /* Half of the processors do scratch-remap */
ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, npes), 1);
gkMPI_Comm_split(ipcomm, mype % ngroups, 0, &srcomm);
gkMPI_Comm_rank(srcomm, &srmype);
gkMPI_Comm_size(srcomm, &srnpes);
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_SEED] = ctrl->sync + (mype % ngroups) + 1;
tpwgts = ctrl->tpwgts;
tpwgts2 = rwspacemalloc(ctrl, 2*ncon);
iset(nvtxs, 0, lwhere);
lnparts = ctrl->nparts;
fpart = fpe = 0;
lnpes = srnpes;
while (lnpes > 1 && lnparts > 1) {
PASSERT(ctrl, agraph->nvtxs > 1);
/* determine the weights of the two partitions as a function of the
weight of the target partition weights */
for (j=(lnparts>>1), i=0; i<ncon; i++) {
tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon);
tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon);
wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]);
tpwgts2[i] *= wsum;
tpwgts2[ncon+i] *= wsum;
}
METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj,
agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt,
&twoparts, tpwgts2, NULL, moptions, &edgecut, part);
/* pick one of the branches */
if (srmype < fpe+lnpes/2) {
KeepPart(ctrl, agraph, part, 0);
lnpes = lnpes/2;
lnparts = lnparts/2;
}
else {
KeepPart(ctrl, agraph, part, 1);
fpart = fpart + lnparts/2;
fpe = fpe + lnpes/2;
lnpes = lnpes - lnpes/2;
lnparts = lnparts - lnparts/2;
}
}
if (lnparts == 1) { /* Case in which srnpes is greater or equal to nparts */
/* Only the first process will assign labels (for the reduction to work) */
if (srmype == fpe) {
for (i=0; i<agraph->nvtxs; i++)
lwhere[agraph->label[i]] = fpart;
}
}
else { /* Case in which srnpes is smaller than nparts */
/* create the normalized tpwgts for the lnparts from ctrl->tpwgts */
tpwgts = rwspacemalloc(ctrl, lnparts*ncon);
for (j=0; j<ncon; j++) {
for (wsum=0.0, i=0; i<lnparts; i++) {
tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j];
wsum += tpwgts[i*ncon+j];
}
for (wsum=1.0/wsum, i=0; i<lnparts; i++)
tpwgts[i*ncon+j] *= wsum;
}
METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions,
&edgecut, part);
for (i=0; i<agraph->nvtxs; i++)
lwhere[agraph->label[i]] = fpart + part[i];
}
gkMPI_Allreduce((void *)lwhere, (void *)part, nvtxs, IDX_T, MPI_SUM, srcomm);
edgecut = ComputeSerialEdgeCut(&cgraph);
ComputeSerialBalance(ctrl, &cgraph, part, lbvec);
lbsum = rsum(ncon, lbvec, 1);
gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ipcomm);
gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ipcomm);
lpecost.rank = ctrl->mype;
lpecost.cost = lbsum;
if (min_lbsum < UNBALANCE_FRACTION * (real_t)(ncon)) {
if (lbsum < UNBALANCE_FRACTION * (real_t)(ncon))
lpecost.cost = (double)edgecut;
else
lpecost.cost = (double)max_cut + lbsum;
}
gkMPI_Allreduce((void *)&lpecost, (void *)&gpecost, 1, MPI_DOUBLE_INT,
MPI_MINLOC, ipcomm);
if (ctrl->mype == gpecost.rank && ctrl->mype != sr_pe)
gkMPI_Send((void *)part, nvtxs, IDX_T, sr_pe, 1, ctrl->comm);
if (ctrl->mype != gpecost.rank && ctrl->mype == sr_pe)
gkMPI_Recv((void *)part, nvtxs, IDX_T, gpecost.rank, 1, ctrl->comm, &status);
if (ctrl->mype == sr_pe) {
icopy(nvtxs, part, lwhere);
SerialRemap(ctrl, &cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts);
}
gkMPI_Comm_free(&srcomm);
}
