int gpartition(const vector< set<int> > &graph, int npartitions, int partition_method, vector<int> &decomp){ // If no partitioning method is set, choose a default. if(partition_method<0){ if(npartitions<=8) partition_method = 0; // METIS PartGraphRecursive else partition_method = 1; // METIS PartGraphKway } int nnodes = graph.size(); // Compress graph vector<idxtype> xadj(nnodes+1), adjncy; int pos=0; xadj[0]=1; for(int i=0;i<nnodes;i++){ for(set<int>::iterator jt=graph[i].begin();jt!=graph[i].end();jt++){ adjncy.push_back(*jt); pos++; } xadj[i+1] = pos+1; } // Partition graph decomp.resize(nnodes); int wgtflag=0, numflag=1, edgecut=0; #ifdef PARMETIS_V3 int options[] = {0}; #else idx_t nbc=1; idx_t options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); options[METIS_OPTION_NUMBERING]=1; #endif if(partition_method){ #ifdef PARMETIS_V3 METIS_PartGraphKway(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag, &numflag, &npartitions, options, &edgecut, &(decomp[0])); #else METIS_PartGraphKway(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL, &npartitions, NULL, NULL,options,&edgecut, &(decomp[0])); #endif }else{ #ifdef PARMETIS_V3 METIS_PartGraphRecursive(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag, &numflag, &npartitions, options, &edgecut, &(decomp[0])); #else METIS_PartGraphRecursive(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL, &npartitions, NULL, NULL,options,&edgecut, &(decomp[0])); #endif } // number from zero for(int i=0;i<nnodes;i++) decomp[i]--; return edgecut; }
/* Interface for metis k-way partitioning with 64-bit ints */ void MUMPS_CALL MUMPS_METIS_KWAY_64(MUMPS_INT8 *n, MUMPS_INT8 *iptr, MUMPS_INT8 *jcn, MUMPS_INT8 *k, MUMPS_INT8 *part) /* n -- the size of the graph to be partitioned iptr -- pointer to the beginning of each node's adjacency list jcn -- jcn[iptr[i]:iptr[i+1]-1] contains the list of neighbors of node i k -- the number of parts part -- part[i] is the part node i belongs to */ /* SELECTIVE I8 FIXME: add an argument *ierr, check it on exit */ { #if defined(metis4) || defined(parmetis3) MUMPS_INT numflag, edgecut, wgtflag, options[8]; MUMPS_INT kINT, nINT; options[0] = 0; /* unweighted partitioning */ wgtflag = 0; /* Use 1-based fortran numbering */ numflag = 1; /* n and k are MUMPS_INT */ nINT=(MUMPS_INT)(*n); kINT=(MUMPS_INT)(*k); /* void METIS_PartGraphKway(int *, idxtype *, idxtype *, idxtype *, idxtype *, int *, int *, int *, int *, int *, idxtype *); */ METIS_PartGraphKway(&nINT, iptr, jcn, NULL, NULL, &wgtflag, &numflag, &kINT, options, &edgecut, part); #else /* METIS >= 5 */ int ierr; # if (IDXTYPEWIDTH == 64) MUMPS_INT8 ncon, edgecut, options[40]; ierr=METIS_SetDefaultOptions(options); options[0] = 0; /* Use 1-based fortran numbering */ options[17] = 1; ncon = 1; ierr = METIS_PartGraphKway(n, &ncon, iptr, jcn, NULL, NULL, NULL, k, NULL, NULL, options, &edgecut, part); # else printf("** Error: METIS version >= 4, IDXTYPE WIDTH !=64, but MUMPS_METIS_KWAY_64 was called\n"); ierr=1; # endif #endif return; }
/*Partition this mesh's elements into n chunks, writing each element's 0-based chunk number to elem2chunk. */ void FEM_Mesh_partition(const FEM_Mesh *mesh,int nchunks,int *elem2chunk, bool faceGraph) { CkThresholdTimer time("FEM Split> Building graph for metis partitioner",1.0); int nelems=mesh->nElems(); if (nchunks==1) { //Metis doesn't handle this case (!) for (int i=0;i<nelems;i++) elem2chunk[i]=0; return; } Graph g(nelems); if (!faceGraph) { mesh2graph(mesh,&g); } else { mesh2graph_face(mesh,&g); } int *adjStart; /*Maps elem # -> start index in adjacency list*/ int *adjList; /*Lists adjacent vertices for each element*/ g.toAdjList(adjStart,adjList); int ecut,ncon=1; time.start("FEM Split> Calling metis partitioner"); if (nchunks<8) /*Metis manual says recursive version is higher-quality here*/ METIS_PartGraphRecursive(&nelems, &ncon, adjStart, adjList, NULL, NULL, NULL, &nchunks, NULL, NULL, NULL, &ecut, elem2chunk); else /*For many chunks, Kway is supposedly faster */ METIS_PartGraphKway(&nelems, &ncon, adjStart, adjList, NULL, NULL, NULL, &nchunks, NULL, NULL, NULL, &ecut, elem2chunk); delete[] adjStart; delete[] adjList; }
void FC_FUNC_(oct_metis_partgraphkway, OCT_METIS_PARTGRAPHKWAY) (idx_t *nvtxs, idx_t *ncon, idx_t *xadj, idx_t *adjncy, idx_t *nparts, real_t *tpwgts, real_t *ubvec, idx_t *options, idx_t *objval, idx_t *part) { METIS_PartGraphKway(nvtxs, ncon, xadj, adjncy, NULL, NULL, NULL, nparts, tpwgts, ubvec, options, objval, part); }
std::vector<bool>balanced_min_cut(const ListGraph&g){ assert(g.is_valid()); #ifndef USE_CUT_ORDER return {}; #else std::vector<int>out_begin; std::vector<int>out_dest; build_adj_array(out_begin, out_dest, g.node_count(), g.arc.size(), [&](int x){return g.arc[x].source;}, [&](int x){return g.arc[x].target;} ); std::vector<int>part(g.node_count()); // Call METIS int one = 1, two = 2, ignore, node_count = g.node_count(); idx_t options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); options[METIS_OPTION_NUMBERING] = 0; options[METIS_OPTION_CONTIG] = 1; options[METIS_OPTION_UFACTOR] = 100; //options[METIS_OPTION_NCUTS] = 10; METIS_PartGraphKway( (idx_t*)&node_count, (idx_t*)&one, (idx_t*)&out_begin[0], (idx_t*)&out_dest[0], nullptr, nullptr, nullptr, (idx_t*)&two, nullptr, nullptr, options, (idx_t*)&ignore, (idx_t*)&part[0] ); std::vector<bool>part_ret(g.node_count()); for(int i=0; i<g.node_count(); ++i) part_ret[i] = part[i] == 0; return std::move(part_ret); #endif }
std::vector<int> MetisMeshPartition(const CMesh* mesh, int nPart) { int vertCount = mesh->vertCount(); std::vector<int> vPart(vertCount); int ncon = 1; std::vector<int> xadj, adjncy; ZGeom::getMeshGraphCSR(*mesh, xadj, adjncy); int objval; int options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); options[METIS_OPTION_CONTIG] = 1; options[METIS_OPTION_NUMBERING] = 0; METIS_PartGraphKway(&vertCount, &ncon, &xadj[0], &adjncy[0], NULL, NULL, NULL, &nPart, NULL, NULL, NULL, &objval, &vPart[0]); return vPart; }
int do_metis_kway_partition(network_t *network, options_t *opt, idx_t *part, idx_t mode) { int ret = ORCC_OK; idx_t ncon = 1; idx_t metis_opt[METIS_NOPTIONS]; idx_t objval; adjacency_list *graph, *metis_graph; assert(network != NULL); assert(opt != NULL); assert(part != NULL); print_orcc_trace(ORCC_VL_VERBOSE_1, "Applying METIS Kway partition for mapping"); METIS_SetDefaultOptions(metis_opt); metis_opt[METIS_OPTION_OBJTYPE] = mode; metis_opt[METIS_OPTION_CONTIG] = 0; /* 0 or 1 */ graph = set_graph_from_network(network); metis_graph = fix_graph_for_metis(graph); ret = METIS_PartGraphKway(&metis_graph->nb_vertices, /* idx_t *nvtxs */ &ncon, /*idx_t *ncon*/ metis_graph->xadj, /*idx_t *xadj*/ metis_graph->adjncy, /*idx_t *adjncy*/ metis_graph->vwgt, /*idx_t *vwgt*/ NULL, /*idx_t *vsize*/ metis_graph->adjwgt, /*idx_t *adjwgt*/ &opt->nb_processors, /*idx_t *nparts*/ NULL, /*real t *tpwgts*/ NULL, /*real t ubvec*/ metis_opt, /*idx_t *options*/ &objval, /*idx_t *objval*/ part); /*idx_t *part*/ check_metis_error(ret); print_orcc_trace(ORCC_VL_VERBOSE_2, "METIS Edgecut : %d", objval); delete_graph(graph); delete_graph(metis_graph); return ret; }
void METIS_PartMeshDual_WV(int *ne, int *nn, idxtype *elmnts, int *etype, int *numflag, int *nparts, int *edgecut, idxtype *epart, idxtype *npart, idxtype *vwgts) { int i, j, k, me; idxtype *xadj, *adjncy, *pwgts, *nptr, *nind; int options[10], pnumflag=0, wgtflag=2; int nnbrs, nbrind[200], nbrwgt[200], maxpwgt; int esize, esizes[] = {-1, 3, 4, 8, 4}; esize = esizes[*etype]; if (*numflag == 1) ChangeMesh2CNumbering((*ne)*esize, elmnts); xadj = idxmalloc(*ne+1, "METIS_MESHPARTNODAL: xadj"); adjncy = idxmalloc(esize*(*ne), "METIS_MESHPARTNODAL: adjncy"); METIS_MeshToDual(ne, nn, elmnts, etype, &pnumflag, xadj, adjncy); options[0] = 0; METIS_PartGraphKway(ne, xadj, adjncy, vwgts, NULL, &wgtflag, &pnumflag, nparts, options, edgecut, epart); /* Construct the node-element list */ nptr = idxsmalloc(*nn+1, 0, "METIS_MESHPARTDUAL: nptr"); for (j=esize*(*ne), i=0; i<j; i++) nptr[elmnts[i]]++; MAKECSR(i, *nn, nptr); nind = idxmalloc(nptr[*nn], "METIS_MESHPARTDUAL: nind"); for (k=i=0; i<(*ne); i++) { for (j=0; j<esize; j++, k++) nind[nptr[elmnts[k]]++] = i; } for (i=(*nn); i>0; i--) nptr[i] = nptr[i-1]; nptr[0] = 0; /* OK, now compute a nodal partition based on the element partition npart */ idxset(*nn, -1, npart); pwgts = idxsmalloc(*nparts, 0, "METIS_MESHPARTDUAL: pwgts"); for (i=0; i<*nn; i++) { me = epart[nind[nptr[i]]]; for (j=nptr[i]+1; j<nptr[i+1]; j++) { if (epart[nind[j]] != me) break; } if (j == nptr[i+1]) { npart[i] = me; pwgts[me]++; } } maxpwgt = 1.03*(*nn)/(*nparts); for (i=0; i<*nn; i++) { if (npart[i] == -1) { /* Assign the boundary element */ nnbrs = 0; for (j=nptr[i]; j<nptr[i+1]; j++) { me = epart[nind[j]]; for (k=0; k<nnbrs; k++) { if (nbrind[k] == me) { nbrwgt[k]++; break; } } if (k == nnbrs) { nbrind[nnbrs] = me; nbrwgt[nnbrs++] = 1; } } /* Try to assign it first to the domain with most things in common */ j = iamax(nnbrs, nbrwgt); if (pwgts[nbrind[j]] < maxpwgt) { npart[i] = nbrind[j]; } else { /* If that fails, assign it to a light domain */ npart[i] = nbrind[0]; for (j=0; j<nnbrs; j++) { if (pwgts[nbrind[j]] < maxpwgt) { npart[i] = nbrind[j]; break; } } } pwgts[npart[i]]++; } } if (*numflag == 1) ChangeMesh2FNumbering2((*ne)*esize, elmnts, *ne, *nn, epart, npart); GKfree(&xadj, &adjncy, &pwgts, &nptr, &nind, LTERM); }
JNIEXPORT jint JNICALL Java_jprime_JMetis_PartitionGraph( JNIEnv * env, jobject obj, jboolean j_force_PartGraphRecursive, jboolean j_force_PartGraphKway, jint j_num_vertices, jintArray j_xadj, jintArray j_adjncy, jintArray j_vwgt, jintArray j_adjwgt, jint j_wgtflag, jint j_nparts, jintArray j_options, jintArray j_partioned_nodes) { mysrand(7654321L); //delcare vars jint * xadj=NULL, * adjncy=NULL,* vwgt=NULL, * adjwgt=NULL, * options=NULL, * partioned_nodes=NULL; int idx, edges_cut=-1, num_vertices=0, nparts=0, wgtflag=0, numflag=0; jsize xadj_len, adjncy_len, vwgt_len, adjwgt_len, options_len, partioned_nodes_len; //copy the jsize vars num_vertices=j_num_vertices; nparts=j_nparts; wgtflag=j_wgtflag; //get array lengths xadj_len = (*env)->GetArrayLength(env,j_xadj); adjncy_len = (*env)->GetArrayLength(env,j_adjncy); vwgt_len = (*env)->GetArrayLength(env,j_vwgt); adjwgt_len = (*env)->GetArrayLength(env,j_adjwgt); options_len = (*env)->GetArrayLength(env,j_options); partioned_nodes_len = (*env)->GetArrayLength(env,j_partioned_nodes); //printf("xadj_len=%i, adjncy_len=%i, vwgt_len=%i, adjwgt_len=%i, options_len=%i, partioned_nodes_len=%i\n",xadj_len, adjncy_len, vwgt_len, adjwgt_len, options_len, partioned_nodes_len); //create/get local copies xadj = (*env)->GetIntArrayElements(env,j_xadj,0); adjncy = (*env)->GetIntArrayElements(env,j_adjncy,0); if(vwgt_len>0) vwgt = (*env)->GetIntArrayElements(env,j_vwgt,0); if(adjwgt_len>0) adjwgt = (*env)->GetIntArrayElements(env,j_adjwgt,0); if(options_len>0) options = (*env)->GetIntArrayElements(env,j_options,0); partioned_nodes = (int*)malloc(sizeof(int)*partioned_nodes_len); #if 0 printf("num_vertices=%i,wgtflag=%i, numflag=%i, nparts=%i\n", num_vertices, wgtflag, numflag, nparts); printf("xadj=%p, adjncy=%p, vwgt=%p, adjwgt=%p, options=%p, partioned_nodes=%p\n", xadj, adjncy, vwgt, adjwgt, options, partioned_nodes); printf("xadj:"); for(idx=0;idx<xadj_len;idx++) { printf("%i ",xadj[idx]); } printf("\n"); printf("adjncy:"); for(idx=0;idx<adjncy_len;idx++) { printf("%i ",adjncy[idx]); } printf("\n"); #endif //call func if((j_nparts<8 || j_force_PartGraphRecursive) && !j_force_PartGraphKway) { METIS_PartGraphRecursive( &num_vertices, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag, &nparts, options, &edges_cut, partioned_nodes); } else { METIS_PartGraphKway( &num_vertices, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag, &nparts, options, &edges_cut, partioned_nodes); } //pop partioned_nodes (*env)->SetIntArrayRegion(env,j_partioned_nodes,0,partioned_nodes_len,partioned_nodes); //free local copies free(partioned_nodes); (*env)->ReleaseIntArrayElements(env, j_xadj, xadj, 0); (*env)->ReleaseIntArrayElements(env, j_adjncy, adjncy, 0); if(vwgt_len>0) (*env)->ReleaseIntArrayElements(env, j_vwgt, vwgt, 0); if(adjwgt_len>0) (*env)->ReleaseIntArrayElements(env, j_adjwgt, adjwgt, 0); if(options_len>0) (*env)->ReleaseIntArrayElements(env, j_options, options, 0); return edges_cut; }
/* * Assign loop iterations to tiles carving partitions out of /seedLoop/ using * the METIS library. */ static int* metis(loop_t* seedLoop, int tileSize, map_list* meshMaps, int* nCore, int* nExec, int* nNonExec, int nThreads) { int i; int setCore = seedLoop->set->core; int setSize = seedLoop->set->size; // use the mesh description to find a suitable map for partitioning through METIS map_t* map = NULL; map_list::const_iterator it, end; for (it = meshMaps->begin(), end = meshMaps->end(); it != end; it++) { if (set_eq(seedLoop->set, (*it)->inSet) || set_eq(seedLoop->set, (*it)->outSet)) { map = *it; break; } } if (! map) { // unfortunate scenario: the user provided a mesh description, but the loop picked // as seed has an iteration space which is not part of the mesh description. // will have to revert to chunk partitioning return NULL; } // now partition through METIS: // ... mesh geometry int nElements = map->inSet->size; int nNodes = map->outSet->size; int nParts = nElements / tileSize; int arity = map->size / nElements; // ... data needed for partitioning int* indMap = new int[nElements]; int* indNodesMap = new int[nNodes]; int* adjncy = map->values; int* offsets = new int[nElements+1](); for (i = 1; i < nElements+1; i++) { offsets[i] = offsets[i-1] + arity; } // ... options int result, objval, ncon = 1; int options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); options[METIS_OPTION_NUMBERING] = 0; options[METIS_OPTION_CONTIG] = 1; // ... do partition! result = (arity == 2) ? METIS_PartGraphKway (&nNodes, &ncon, offsets, adjncy, NULL, NULL, NULL, &nParts, NULL, NULL, options, &objval, indMap) : METIS_PartMeshNodal (&nElements, &nNodes, offsets, adjncy, NULL, NULL, &nParts, NULL, options, &objval, indMap, indNodesMap); ASSERT(result == METIS_OK, "Invalid METIS partitioning"); // what's the target iteration set ? if (set_eq(seedLoop->set, map->inSet)) { delete[] indNodesMap; } else { // note: must be set_eq(seedLoop->set, map-outSet) delete[] indMap; indMap = indNodesMap; } delete[] offsets; // restrict partitions to the core region std::fill (indMap + setCore, indMap + setSize, 0); std::set<int> partitions (indMap, indMap + setCore); // ensure the set of partitions IDs is compact (i.e., if we have a partitioning // 0: {0,1,...}, 1: {4,5,...}, 2: {}, 3: {6,10,...} ... // we instead want to have // 0: {0,1,...}, 1: {4,5,...}, 2: {6,10,...}, ... std::map<int, int> mapper; std::set<int>::const_iterator sIt, sEnd; for (i = 0, sIt = partitions.begin(), sEnd = partitions.end(); sIt != sEnd; sIt++, i++) { mapper[*sIt] = i; } for (i = 0; i < setCore; i++) { indMap[i] = mapper[indMap[i]]; } *nCore = partitions.size(); // partition the exec halo region chunk_halo (seedLoop, tileSize, *nCore - 1, indMap, nExec, nNonExec, nThreads); return indMap; }
/************************************************************************* * Let the game begin **************************************************************************/ main(int argc, char *argv[]) { int i, nparts, options[10]; idxtype *part; float rubvec[MAXNCON], lbvec[MAXNCON]; GraphType graph; char filename[256]; int numflag = 0, wgtflag = 0, edgecut; timer TOTALTmr, METISTmr, IOTmr; if (argc != 3) { printf("Usage: %s <GraphFile> <Nparts>\n",argv[0]); exit(0); } strcpy(filename, argv[1]); nparts = atoi(argv[2]); if (nparts < 2) { printf("The number of partitions should be greater than 1!\n"); exit(0); } cleartimer(TOTALTmr); cleartimer(METISTmr); cleartimer(IOTmr); starttimer(TOTALTmr); starttimer(IOTmr); ReadGraph(&graph, filename, &wgtflag); if (graph.nvtxs <= 0) { printf("Empty graph. Nothing to do.\n"); exit(0); } stoptimer(IOTmr); printf("**********************************************************************\n"); printf("%s", METISTITLE); printf("Graph Information ---------------------------------------------------\n"); printf(" Name: %s, #Vertices: %d, #Edges: %d, #Parts: %d\n", filename, graph.nvtxs, graph.nedges/2, nparts); if (graph.ncon > 1) printf(" Balancing Constraints: %d\n", graph.ncon); printf("\nK-way Partitioning... -----------------------------------------------\n"); part = idxmalloc(graph.nvtxs, "main: part"); options[0] = 0; starttimer(METISTmr); if (graph.ncon == 1) { METIS_PartGraphKway(&graph.nvtxs, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &nparts, options, &edgecut, part); } else { for (i=0; i<graph.ncon; i++) rubvec[i] = HORIZONTAL_IMBALANCE; METIS_mCPartGraphKway(&graph.nvtxs, &graph.ncon, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &nparts, rubvec, options, &edgecut, part); } stoptimer(METISTmr); ComputePartitionBalance(&graph, nparts, part, lbvec); printf(" %d-way Edge-Cut: %7d, Balance: ", nparts, edgecut); for (i=0; i<graph.ncon; i++) printf("%5.2f ", lbvec[i]); printf("\n"); starttimer(IOTmr); WritePartition(filename, part, graph.nvtxs, nparts); stoptimer(IOTmr); stoptimer(TOTALTmr); printf("\nTiming Information --------------------------------------------------\n"); printf(" I/O: \t\t %7.3f\n", gettimer(IOTmr)); printf(" Partitioning: \t\t %7.3f (KMETIS time)\n", gettimer(METISTmr)); printf(" Total: \t\t %7.3f\n", gettimer(TOTALTmr)); printf("**********************************************************************\n"); GKfree(&graph.xadj, &graph.adjncy, &graph.vwgt, &graph.adjwgt, &part, LTERM); }
int main (int argc, char **argv) { // see Metis documentation (p.20 ff for Metis 5.x) /*----------------------------------------------------------------------------*/ // example graph // 0 -- 3 // | | // 1 4 // | / | // 2 5 // number of vertices idx_t n_vertex = 6; // size of xadj is n_vertex+1. // This array stores for every vertex i in xadj[i] the index // where the adjacent vertices are found in adjncy[]. // Looking at the example graph: // Neighbors of vertex 0 are found starting at adjncy[0] and ending in adjncy[2-1], // i.e. one less than the starting point of the next vertex. // Neighbors of vertex 1 are found starting at adjncy[2] and ending in adjncy[4-1], // i.e. one less than the starting point of the next vertex. // etc. idx_t xadj[] = { 0, 2, 4, 6, 8, 11, 12 }; // number of edges (see below: adjcny stores every edge twice as (u,v) and (v,u)) idx_t n_edge = 6; // size of adjncy is 2 * n_edge (every edge appears twice in undirected graph) idx_t adjncy[] = { 1, 3, // neighbors of vertex 0 0, 2, // neighbors of vertex 1 1, 4, // neighbors of vertex 2 0, 4, // neighbors of vertex 3 2, 3, 5, // neighbors of vertex 4 4 // neighbors of vertex 5 }; // vertex weights (size of vwgt is n_vertex) idx_t vwgt[] = { 20, 30, 80, 60, 100, 90 }; // edge weights (size of adjwgt is n_edge*2) idx_t adjwgt[] = { 10, 20, 10, 10, 10, 30, 20, 10, 30, 10, 10, 10 }; // the resulting partition: // for every vertex i we find in part[i] afterwards the partition number for that vertex idx_t part[n_vertex]; // Metis options (initialized to default values) idx_t options[METIS_NOPTIONS]; METIS_SetDefaultOptions (options); // partitioning method: k-way partitioning options[METIS_OPTION_PTYPE] = METIS_PTYPE_KWAY; // edge cut minimization options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT; // C-style numbering options[METIS_OPTION_NUMBERING] = 0; // number of balancing constraints idx_t ncon = 1; // edge cut after partitioning idx_t edgecut; // !!!!!!!!!! you have to change this !!!!!!!!!!!! // number of partitions we want to partition to idx_t nparts = 2; // !!!!!!!!!! you have to change this !!!!!!!!!!!! /*--------------------------------------------------------------------------*/ // print header and input graph printf ("partitioning graph with %ld vertices and %ld edges for %ld partitions\n", (long)n_vertex, (long)n_edge, (long)nparts); print_graph (n_vertex, vwgt, xadj, adjncy, adjwgt); // call graph partitioning double t0 = gettime (); int rc = METIS_PartGraphKway (&n_vertex, // number of vertices &ncon, // number of balancing constraints xadj, // adjacency structure of graph adjncy, // adjacency structure of graph vwgt, // vertex weights NULL, // only for total communication volume adjwgt, // edge weights &nparts, // number of partitions wanted NULL, // tpwgts: no desired partition weights NULL, // ubvec: allowed load imbalance options, // special options &edgecut, // objective value (edge cut) part // vector with partition information for each vertex ); t0 = gettime () - t0; // check Metis return value switch (rc) { case METIS_OK: break; case METIS_ERROR_INPUT: printf ("error in Metis input\n"); exit (1); case METIS_ERROR_MEMORY: printf ("no more memory in Metis\n"); exit (1); case METIS_ERROR: printf ("some error in Metis\n"); exit (1); default: printf ("unknown return code ffrom Metis\n"); exit (1); } /*--------------------------------------------------------------------------*/ /* print results */ // time for partitioning printf ("partitioning time: %.9f s\n", t0); // print parition print_partition (n_vertex, vwgt, xadj, adjncy, adjwgt, part, nparts, edgecut); return 0; }
idx_t* gpmetis( int argc, char **argv ) /*************************************************************************/ /*! Let the game begin! */ /*************************************************************************/ //int main(int argc, char *argv[]) { idx_t i; char *curptr, *newptr; idx_t options[METIS_NOPTIONS]; graph_t *graph; idx_t *part; idx_t objval; params_t *params; int status=0; gk_optind = 0; //printf( "argc: %d\n", argc ); //printf( "gk_optind %d\n", gk_optind ); fflush( stdout ); for( i = 0; i < argc; i++ ) { //printf( "%s*\n", argv[ i ] ); } params = parse_cmdline(argc, argv); //printf( "gk_optind %d\n", gk_optind ); //fflush( stdout ); //return NULL; gk_startcputimer(params->iotimer); graph = ReadGraph(params); ReadTPwgts(params, graph->ncon); gk_stopcputimer(params->iotimer); /* Check if the graph is contiguous */ if (params->contig && !IsConnected(graph, 0)) { printf("***The input graph is not contiguous.\n" "***The specified -contig option will be ignored.\n"); params->contig = 0; } /* Get ubvec if supplied */ if (params->ubvecstr) { params->ubvec = rmalloc(graph->ncon, "main"); curptr = params->ubvecstr; for (i=0; i<graph->ncon; i++) { params->ubvec[i] = strtoreal(curptr, &newptr); if (curptr == newptr) errexit("Error parsing entry #%"PRIDX" of ubvec [%s] (possibly missing).\n", i, params->ubvecstr); curptr = newptr; } } /* Setup iptype */ if (params->iptype == -1) { if (params->ptype == METIS_PTYPE_RB) { if (graph->ncon == 1) params->iptype = METIS_IPTYPE_GROW; else params->iptype = METIS_IPTYPE_RANDOM; } } GPPrintInfo(params, graph); part = imalloc(graph->nvtxs, "main: part"); METIS_SetDefaultOptions(options); options[METIS_OPTION_OBJTYPE] = params->objtype; options[METIS_OPTION_CTYPE] = params->ctype; options[METIS_OPTION_IPTYPE] = params->iptype; options[METIS_OPTION_RTYPE] = params->rtype; options[METIS_OPTION_MINCONN] = params->minconn; options[METIS_OPTION_CONTIG] = params->contig; options[METIS_OPTION_SEED] = params->seed; options[METIS_OPTION_NITER] = params->niter; options[METIS_OPTION_NCUTS] = params->ncuts; options[METIS_OPTION_UFACTOR] = params->ufactor; options[METIS_OPTION_DBGLVL] = params->dbglvl; gk_malloc_init(); gk_startcputimer(params->parttimer); switch (params->ptype) { case METIS_PTYPE_RB: status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, ¶ms->nparts, params->tpwgts, params->ubvec, options, &objval, part); break; case METIS_PTYPE_KWAY: status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, ¶ms->nparts, params->tpwgts, params->ubvec, options, &objval, part); break; } gk_stopcputimer(params->parttimer); if (gk_GetCurMemoryUsed() != 0) printf("***It seems that Metis did not free all of its memory! Report this.\n"); params->maxmemory = gk_GetMaxMemoryUsed(); gk_malloc_cleanup(0); if (status != METIS_OK) { printf("\n***Metis returned with an error.\n"); } else { if (!params->nooutput) { /* Write the solution */ gk_startcputimer(params->iotimer); WritePartition(params->filename, part, graph->nvtxs, params->nparts); gk_stopcputimer(params->iotimer); } GPReportResults(params, graph, part, objval); } idx_t *r_part = ( idx_t* ) calloc( graph->nvtxs, sizeof( idx_t ) ); for( i = 0; i < graph->nvtxs; i++ ) { r_part[ i ] = part[ i ]; } FreeGraph(&graph); gk_free((void **)&part, LTERM); gk_free((void **)¶ms->filename, ¶ms->tpwgtsfile, ¶ms->tpwgts, ¶ms->ubvecstr, ¶ms->ubvec, ¶ms, LTERM); return r_part; }
int Zoltan_ParMetis( ZZ *zz, /* Zoltan structure */ float *part_sizes, /* Input: Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ int *num_imp, /* number of objects to be imported */ ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */ ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */ int **imp_procs, /* list of processors to import from */ int **imp_to_part, /* list of partitions to which imported objects are assigned. */ int *num_exp, /* number of objects to be exported */ ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */ ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */ int **exp_procs, /* list of processors to export to */ int **exp_to_part /* list of partitions to which exported objects are assigned. */ ) { char *yo = "Zoltan_ParMetis"; int ierr; ZOLTAN_Third_Graph gr; ZOLTAN_Third_Geom *geo = NULL; ZOLTAN_Third_Vsize vsp; ZOLTAN_Third_Part prt; ZOLTAN_Output_Part part; ZOLTAN_ID_PTR global_ids = NULL; ZOLTAN_ID_PTR local_ids = NULL; int use_timers = 0; int timer_p = -1; int get_times = 0; double times[5]; double pmv3_itr = 0.0; realtype itr = 0.0; indextype options[MAX_PARMETIS_OPTIONS]; char alg[MAX_PARAM_STRING_LEN]; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */ /* change our zz struct. */ #endif indextype i; realtype *imb_tols; indextype ncon; indextype edgecut; indextype wgtflag; indextype numflag = 0; indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */ ZOLTAN_TRACE_ENTER(zz, yo); Zoltan_Third_Init(&gr, &prt, &vsp, &part, imp_gids, imp_lids, imp_procs, imp_to_part, exp_gids, exp_lids, exp_procs, exp_to_part); if (sizeof(realtype) != sizeof(float)) { int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1); prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype)); for (i = 0; i < tmp; i++) prt.input_part_sizes[i] = (realtype) part_sizes[i]; /* KDD 2/2014: removed re-scaling part sizes so they sum to one. * part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes. * plus, the code here was wrong for multiple object weights. * similar scaling code did not exist in the Scotch interface. */ prt.part_sizes = prt.input_part_sizes; } else prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes; ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } gr.graph_type = 0; #ifdef ZOLTAN_PARMETIS SET_GLOBAL_GRAPH(&gr.graph_type); /* Select type of graph, negative because we impose them */ /* TODO: add a parameter to select the type, shared with Scotch */ /* if (strcmp (graph_type, "GLOBAL") != 0) { */ /* gr.graph_type = - LOCAL_GRAPH; */ /* if (zz->Num_Proc > 1) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */ /* gr.graph_type = - GLOBAL_GRAPH; */ /* retval = ZOLTAN_WARN; */ /* } */ /* } */ #else /* graph is local */ SET_LOCAL_GRAPH(&gr.graph_type); #endif /* ZOLTAN_PARMETIS */ /* Some algorithms use geometry data */ if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */ geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom)); memset (geo, 0, sizeof(ZOLTAN_Third_Geom)); /* ParMETIS will crash if geometric method and some procs have no nodes. */ /* Avoid fatal crash by setting scatter to level 2 or higher. */ gr.scatter_min = 2; if (geo == NULL) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory."); return (ZOLTAN_MEMERR); } if (strcmp(alg, "PARTGEOM") == 0) { gr.get_data = 0; } } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } vsp.vsize_malloc = 0; #ifdef PARMETIS31_ALWAYS_FREES_VSIZE if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) { /* ParMETIS will free this memory; use malloc to allocate so ZOLTAN_MALLOC counters don't show an error. */ vsp.vsize_malloc = 1 ; } #endif /* PARMETIS31_ALWAYS_FREES_VSIZE */ ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr, geo, &prt, &vsp); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } /* Get object sizes if requested */ if (options[PMV3_OPT_USE_OBJ_SIZE] && (zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) && (!strcmp(alg, "ADAPTIVEREPART") || gr.final_output)) gr.showMoveVol = 1; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); /* Get ready to call ParMETIS */ edgecut = -1; wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0); numflag = 0; ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1); if (!prt.part_sizes){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL."); } if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) { for (i=0; i<num_part; i++){ indextype j; printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i); for (j=0; j<ncon; j++) printf("%f ", prt.part_sizes[i*ncon+j]); printf("\n"); } } /* if (strcmp(alg, "ADAPTIVEREPART") == 0) */ for (i = 0; i < num_part*ncon; i++) if (prt.part_sizes[i] == 0) ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! " "ParMETIS 3.x will likely fail. Please use a " "different method, or remove the zero-sized " "parts from the problem."); /* Set Imbalance Tolerance for each weight component. */ imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype)); if (!imb_tols){ /* Not enough memory */ ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } for (i=0; i<ncon; i++) imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]); /* Now we can call ParMetis */ /* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */ #ifdef ZOLTAN_PARMETIS if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */ /* First check for ParMetis 3 routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartKway"); ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOMKWAY") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeomKway"); ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts, &wgtflag, &numflag, &ndims, geo->xyz, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOM") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeom"); ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "ADAPTIVEREPART") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_AdaptiveRepart"); ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, &itr, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "REFINEKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_RefineKway"); ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown ParMetis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_METIS /* TODO: I don't know how to set balance ! */ if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Check for Metis routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library "); /* Use default options for METIS */ #if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5 options[0] = 0; METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &num_part, prt.part_sizes, options, &edgecut, prt.part); #else METIS_SetDefaultOptions(options); METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown Metis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (gr.final_output) { /* Do final output now because after the data will not be coherent: unscatter only unscatter part data, not graph */ ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr); } /* Ignore the timings of Final Ouput */ if (get_times) times[3] = Zoltan_Time(zz->Timer); ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr, geo, &prt, &vsp, NULL, &part); Zoltan_Third_Export_User(&part, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part); /* Get a time here */ if (get_times) times[4] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers && timer_p >= 0) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL); if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols); if (geo != NULL) ZOLTAN_FREE(&geo); ZOLTAN_FREE(&global_ids); ZOLTAN_FREE(&local_ids); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
void metis_partgraphkway__(int *nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part) { METIS_PartGraphKway(nvtxs, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, part); }
/************************************************************************* * This function partitions a finite element mesh by partitioning its nodal * graph using KMETIS and then assigning elements in a load balanced fashion. **************************************************************************/ int METIS_PartMeshNodal(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind, idx_t *vwgt, idx_t *vsize, idx_t *nparts, real_t *tpwgts, idx_t *options, idx_t *objval, idx_t *epart, idx_t *npart) { int sigrval=0, renumber=0, ptype; idx_t *xadj=NULL, *adjncy=NULL; idx_t ncon=1, pnumflag=0; int rstatus=METIS_OK; /* set up malloc cleaning code and signal catchers */ if (!gk_malloc_init()) return METIS_ERROR_MEMORY; gk_sigtrap(); if ((sigrval = gk_sigcatch()) != 0) goto SIGTHROW; renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0); ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY); /* renumber the mesh */ if (renumber) { ChangeMesh2CNumbering(*ne, eptr, eind); options[METIS_OPTION_NUMBERING] = 0; } /* get the nodal graph */ rstatus = METIS_MeshToNodal(ne, nn, eptr, eind, &pnumflag, &xadj, &adjncy); if (rstatus != METIS_OK) raise(SIGERR); /* partition the graph */ if (ptype == METIS_PTYPE_KWAY) rstatus = METIS_PartGraphKway(nn, &ncon, xadj, adjncy, vwgt, vsize, NULL, nparts, tpwgts, NULL, options, objval, npart); else rstatus = METIS_PartGraphRecursive(nn, &ncon, xadj, adjncy, vwgt, vsize, NULL, nparts, tpwgts, NULL, options, objval, npart); if (rstatus != METIS_OK) raise(SIGERR); /* partition the other side of the mesh */ InduceRowPartFromColumnPart(*ne, eptr, eind, epart, npart, *nparts, tpwgts); SIGTHROW: if (renumber) { ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart); options[METIS_OPTION_NUMBERING] = 1; } METIS_Free(xadj); METIS_Free(adjncy); gk_siguntrap(); gk_malloc_cleanup(0); return metis_rcode(sigrval); }
/************************************************************************* * This function partitions a finite element mesh by partitioning its nodal * graph using KMETIS and then assigning elements in a load balanced fashion. **************************************************************************/ void METIS_PartMeshNodal(int *ne, int *nn, idxtype *elmnts, int *etype, int *numflag, int *nparts, int *edgecut, idxtype *epart, idxtype *npart) { int i, j, k, me; idxtype *xadj, *adjncy, *pwgts; int options[10], pnumflag=0, wgtflag=0; int nnbrs, nbrind[200], nbrwgt[200], maxpwgt; int esize, esizes[] = {-1, 3, 4, 8, 4}; esize = esizes[*etype]; if (*numflag == 1) ChangeMesh2CNumbering((*ne)*esize, elmnts); xadj = idxmalloc(*nn+1, "METIS_MESHPARTNODAL: xadj"); adjncy = idxmalloc(20*(*nn), "METIS_MESHPARTNODAL: adjncy"); METIS_MeshToNodal(ne, nn, elmnts, etype, &pnumflag, xadj, adjncy); adjncy = realloc(adjncy, xadj[*nn]*sizeof(idxtype)); options[0] = 0; METIS_PartGraphKway(nn, xadj, adjncy, NULL, NULL, &wgtflag, &pnumflag, nparts, options, edgecut, npart); /* OK, now compute an element partition based on the nodal partition npart */ idxset(*ne, -1, epart); pwgts = idxsmalloc(*nparts, 0, "METIS_MESHPARTNODAL: pwgts"); for (i=0; i<*ne; i++) { me = npart[elmnts[i*esize]]; for (j=1; j<esize; j++) { if (npart[elmnts[i*esize+j]] != me) break; } if (j == esize) { epart[i] = me; pwgts[me]++; } } maxpwgt = 1.03*(*ne)/(*nparts); for (i=0; i<*ne; i++) { if (epart[i] == -1) { /* Assign the boundary element */ nnbrs = 0; for (j=0; j<esize; j++) { me = npart[elmnts[i*esize+j]]; for (k=0; k<nnbrs; k++) { if (nbrind[k] == me) { nbrwgt[k]++; break; } } if (k == nnbrs) { nbrind[nnbrs] = me; nbrwgt[nnbrs++] = 1; } } /* Try to assign it first to the domain with most things in common */ j = iamax(nnbrs, nbrwgt); if (pwgts[nbrind[j]] < maxpwgt) { epart[i] = nbrind[j]; } else { /* If that fails, assign it to a light domain */ for (j=0; j<nnbrs; j++) { if (pwgts[nbrind[j]] < maxpwgt) { epart[i] = nbrind[j]; break; } } if (j == nnbrs) epart[i] = nbrind[iamax(nnbrs, nbrwgt)]; } pwgts[epart[i]]++; } } if (*numflag == 1) ChangeMesh2FNumbering2((*ne)*esize, elmnts, *ne, *nn, epart, npart); GKfree(&xadj, &adjncy, &pwgts, LTERM); }
sparse_matrix* graph_partition(LIST *list , partition_t* partition_info) { graph_t* graph; params_t *params; idx_t options[METIS_NOPTIONS], status; idx_t objval; sparse_matrix* matrix = (sparse_matrix*)malloc(sizeof(sparse_matrix)); graph = ReadGraph(list); params = bmalloc(sizeof(*params), "Allocating memory for params"); METIS_SetDefaultOptions(options); METIS_init_params(params, graph->ncon); options[METIS_OPTION_OBJTYPE] = params->objtype; options[METIS_OPTION_CTYPE] = params->ctype; options[METIS_OPTION_IPTYPE] = params->iptype; options[METIS_OPTION_RTYPE] = params->rtype; options[METIS_OPTION_NO2HOP] = params->no2hop; options[METIS_OPTION_MINCONN] = params->minconn; options[METIS_OPTION_CONTIG] = params->contig; options[METIS_OPTION_SEED] = params->seed; options[METIS_OPTION_NITER] = params->niter; options[METIS_OPTION_NCUTS] = params->ncuts; options[METIS_OPTION_UFACTOR] = params->ufactor; options[METIS_OPTION_DBGLVL] = params->dbglvl; //print_input_data(graph,params); params->tpwgts = NULL; params->ubvec = NULL; partition_info->partion_table = imalloc(graph->nvtxs, "Allocate memory for part"); switch (params->ptype) { case METIS_PTYPE_RB: status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, ¶ms->nparts, params->tpwgts, params->ubvec, options, &objval, partition_info->partion_table); break; case METIS_PTYPE_KWAY: status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, ¶ms->nparts, params->tpwgts, params->ubvec, options, &objval, partition_info->partion_table); break; } if (status != METIS_OK) { fprintf(stderr,"\n***Metis returned with an error.***\n"); return NULL; } partition_info->size = graph->nvtxs; partition_info->noOfParts = params->nparts; WritePartition(params->filename, partition_info->partion_table, graph->nvtxs, params->nparts); printf("Objval: %d\n",objval); matrix->nz = -1; matrix->n = graph->nvtxs; matrix->p = graph->xadj; matrix->i = graph->adjncy; matrix->x = NULL; return matrix; }
// Call Metis with options from dictionary. Foam::label Foam::metisDecomp::decompose ( const List<int>& adjncy, const List<int>& xadj, const scalarField& cWeights, List<int>& finalDecomp ) { // C style numbering int numFlag = 0; // Method of decomposition // recursive: multi-level recursive bisection (default) // k-way: multi-level k-way word method("k-way"); int numCells = xadj.size()-1; // decomposition options. 0 = use defaults List<int> options(5, 0); // processor weights initialised with no size, only used if specified in // a file Field<floatScalar> processorWeights; // cell weights (so on the vertices of the dual) List<int> cellWeights; // face weights (so on the edges of the dual) List<int> faceWeights; // Check for externally provided cellweights and if so initialise weights scalar minWeights = gMin(cWeights); if (cWeights.size() > 0) { if (minWeights <= 0) { WarningIn ( "metisDecomp::decompose" "(const pointField&, const scalarField&)" ) << "Illegal minimum weight " << minWeights << endl; } if (cWeights.size() != numCells) { FatalErrorIn ( "metisDecomp::decompose" "(const pointField&, const scalarField&)" ) << "Number of cell weights " << cWeights.size() << " does not equal number of cells " << numCells << exit(FatalError); } // Convert to integers. cellWeights.setSize(cWeights.size()); forAll(cellWeights, i) { cellWeights[i] = int(cWeights[i]/minWeights); } } // Check for user supplied weights and decomp options if (decompositionDict_.found("metisCoeffs")) { const dictionary& metisCoeffs = decompositionDict_.subDict("metisCoeffs"); word weightsFile; if (metisCoeffs.readIfPresent("method", method)) { if (method != "recursive" && method != "k-way") { FatalErrorIn("metisDecomp::decompose()") << "Method " << method << " in metisCoeffs in dictionary : " << decompositionDict_.name() << " should be 'recursive' or 'k-way'" << exit(FatalError); } Info<< "metisDecomp : Using Metis method " << method << nl << endl; } if (metisCoeffs.readIfPresent("options", options)) { if (options.size() != 5) { FatalErrorIn("metisDecomp::decompose()") << "Number of options in metisCoeffs in dictionary : " << decompositionDict_.name() << " should be 5" << exit(FatalError); } Info<< "metisDecomp : Using Metis options " << options << nl << endl; } if (metisCoeffs.readIfPresent("processorWeights", processorWeights)) { processorWeights /= sum(processorWeights); if (processorWeights.size() != nProcessors_) { FatalErrorIn("metisDecomp::decompose(const pointField&)") << "Number of processor weights " << processorWeights.size() << " does not equal number of domains " << nProcessors_ << exit(FatalError); } } //if (metisCoeffs.readIfPresent("cellWeightsFile", weightsFile)) //{ // Info<< "metisDecomp : Using cell-based weights." << endl; // // IOList<int> cellIOWeights // ( // IOobject // ( // weightsFile, // mesh_.time().timeName(), // mesh_, // IOobject::MUST_READ, // IOobject::AUTO_WRITE // ) // ); // cellWeights.transfer(cellIOWeights); // // if (cellWeights.size() != xadj.size()-1) // { // FatalErrorIn("metisDecomp::decompose(const pointField&)") // << "Number of cell weights " << cellWeights.size() // << " does not equal number of cells " << xadj.size()-1 // << exit(FatalError); // } //} } int nProcs = nProcessors_; // output: cell -> processor addressing finalDecomp.setSize(numCells); // output: number of cut edges int edgeCut = 0; // Vertex weight info int wgtFlag = 0; int* vwgtPtr = NULL; int* adjwgtPtr = NULL; if (cellWeights.size()) { vwgtPtr = cellWeights.begin(); wgtFlag += 2; // Weights on vertices } if (faceWeights.size()) { adjwgtPtr = faceWeights.begin(); wgtFlag += 1; // Weights on edges } if (method == "recursive") { if (processorWeights.size()) { METIS_WPartGraphRecursive ( &numCells, // num vertices in graph const_cast<List<int>&>(xadj).begin(), // indexing into adjncy const_cast<List<int>&>(adjncy).begin(), // neighbour info vwgtPtr, // vertexweights adjwgtPtr, // no edgeweights &wgtFlag, &numFlag, &nProcs, processorWeights.begin(), options.begin(), &edgeCut, finalDecomp.begin() ); } else { METIS_PartGraphRecursive ( &numCells, // num vertices in graph const_cast<List<int>&>(xadj).begin(), // indexing into adjncy const_cast<List<int>&>(adjncy).begin(), // neighbour info vwgtPtr, // vertexweights adjwgtPtr, // no edgeweights &wgtFlag, &numFlag, &nProcs, options.begin(), &edgeCut, finalDecomp.begin() ); } } else { if (processorWeights.size()) { METIS_WPartGraphKway ( &numCells, // num vertices in graph const_cast<List<int>&>(xadj).begin(), // indexing into adjncy const_cast<List<int>&>(adjncy).begin(), // neighbour info vwgtPtr, // vertexweights adjwgtPtr, // no edgeweights &wgtFlag, &numFlag, &nProcs, processorWeights.begin(), options.begin(), &edgeCut, finalDecomp.begin() ); } else { METIS_PartGraphKway ( &numCells, // num vertices in graph const_cast<List<int>&>(xadj).begin(), // indexing into adjncy const_cast<List<int>&>(adjncy).begin(), // neighbour info vwgtPtr, // vertexweights adjwgtPtr, // no edgeweights &wgtFlag, &numFlag, &nProcs, options.begin(), &edgeCut, finalDecomp.begin() ); } } return edgeCut; }
int generateLevels(std::vector< std::vector<sbfSElement *> > & selements, std::vector<int> & numTargetByLayers, std::vector<double> &maxImbalance, bool verbouse){ sbfMesh * mesh = selements[0][0]->mesh(); if(!mesh) return 1; int numRegElems = selements[0].size(); std::vector <double> facesWeigth; std::vector <int> facesOwners; std::vector <double> randoms; randoms.resize(50); for(size_t ct = 0; ct < randoms.size(); ct++) randoms[ct] = ((double)rand())/RAND_MAX; facesWeigth.reserve(numRegElems*50); facesOwners.reserve(numRegElems*50); for(int elemID = 0; elemID < numRegElems; elemID++){//Loop on elements sbfElement *elem = mesh->elemPtr(elemID); double faceWeigth; int facesOwner = elemID; std::list<int> inds; int count; std::vector< std::vector<int> > facesNodesIndexes = elem->facesNodesIndexes(); for(auto itFace = facesNodesIndexes.begin(); itFace != facesNodesIndexes.end(); itFace++){//Loop on faces inds.clear(); for(auto itIndex = (*itFace).begin(); itIndex != (*itFace).end(); itIndex++) inds.push_back(*itIndex); inds.sort(); count = 0; faceWeigth = 0; for(auto it = inds.begin(); it != inds.end(); it++) {faceWeigth += *it*randoms[count++];} facesWeigth.push_back(faceWeigth); facesOwners.push_back(facesOwner); }//Loop on faces }//Loop on elements quickAssociatedSort<double, int>(&facesWeigth[0], &facesOwners[0], 0, facesWeigth.size()-1); if ( verbouse ) std::cout << "sort done" << std::endl; for(size_t ctLevel = 0; ctLevel < numTargetByLayers.size(); ctLevel++){//Loop on levels int numTargetSE = numTargetByLayers[ctLevel]; int numSElems = selements[ctLevel].size(); int vertnbr, edgenbr; vertnbr = numSElems; edgenbr = 0; std::vector< std::list <int> > elemNeibour; elemNeibour.resize(vertnbr); int founded = 0, unfounded = 0; std::vector <double> facesWeigthNextLevel; std::vector <int> facesOwnersNextLevel; facesWeigthNextLevel.reserve(facesWeigth.size()); facesOwnersNextLevel.reserve(facesOwners.size()); auto facesWeigthIt = facesWeigth.begin(); auto facesOwnersIt = facesOwners.begin(); auto facesWeigthEndM1 = facesWeigth.end() - 1; for(; facesWeigthIt < facesWeigthEndM1; facesWeigthIt++, facesOwnersIt++){ if(*facesWeigthIt == *(facesWeigthIt+1)){ //Check if *facesOwnersIt and *(facesOwnersIt+1) are in one SE int owner0, owner1; owner0 = *facesOwnersIt; owner1 = *(facesOwnersIt+1); int ownerSE0 = -1, ownerSE1 = -1; bool inSame = false; if(ctLevel == 0){ownerSE0 = owner0; ownerSE1 = owner1;} else{ sbfSElement * se = selements[0][owner0]; for(size_t ct = 0; ct < ctLevel; ct++) se = se->parent(); ownerSE0 = se->index(); se = selements[0][owner1]; for(size_t ct = 0; ct < ctLevel; ct++) se = se->parent(); ownerSE1 = se->index(); if(ownerSE0 == ownerSE1) inSame = true; } if (inSame){ facesWeigthIt++; facesOwnersIt++; continue; } elemNeibour[ownerSE1].push_back(ownerSE0); elemNeibour[ownerSE0].push_back(ownerSE1); facesWeigthNextLevel.push_back(*facesWeigthIt); facesWeigthNextLevel.push_back(*facesWeigthIt); facesOwnersNextLevel.push_back(*facesOwnersIt); facesOwnersNextLevel.push_back(*(facesOwnersIt+1)); facesWeigthIt++; facesOwnersIt++; founded++; } else unfounded++; } std::vector<idx_t> verttab, edgetab, edlotab, parttab; verttab.resize(vertnbr+1); parttab.resize(vertnbr); int count = 0; verttab[0] = count; for(size_t ct = 0; ct < elemNeibour.size(); ct++){ std::list<int> elemNeibourAll, elemNeibourUnique; for(auto it = elemNeibour[ct].begin(); it != elemNeibour[ct].end(); it++) elemNeibourAll.push_back(*it); elemNeibourAll.sort(); elemNeibourUnique = elemNeibourAll; elemNeibourUnique.unique(); auto elemNeibourUniqueBegin = elemNeibourUnique.begin(); auto elemNeibourUniqueEnd = elemNeibourUnique.end(); auto elemNeibourAllBegin = elemNeibourAll.begin(); auto elemNeibourAllEnd = elemNeibourAll.end(); auto itA = elemNeibourAllBegin; for(auto itU = elemNeibourUniqueBegin; itU != elemNeibourUniqueEnd; itU++) { if(static_cast<int>(ct) != *itU){ int numAcuarence = 0; while(*itA == *itU && itA != elemNeibourAllEnd){ numAcuarence++; itA++; } edgetab.push_back(*itU); edlotab.push_back(numAcuarence); count++; edgenbr++; } } verttab[ct+1] = count; } idx_t nvtxs = vertnbr, ncon = 1, nparts = numTargetSE, objval; idx_t options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_VOL; options[METIS_OPTION_NUMBERING] = 0; options[METIS_OPTION_NITER] = 600; double maxLayerImbalance = maxImbalance.back(); if( ctLevel < maxImbalance.size() ) maxLayerImbalance = maxImbalance.at(ctLevel); options[METIS_OPTION_UFACTOR] = static_cast<int>((maxLayerImbalance-1)*1000); options[METIS_OPTION_CONTIG] = 1; //Force contiguous partitions // options[METIS_OPTION_DBGLVL] = 1; int rez = METIS_PartGraphKway(&nvtxs, &ncon, verttab.data(), edgetab.data(), /*idx_t *vwgt*/nullptr, /*idx_t *vsize*/nullptr, /*idx_t *adjwgt*/nullptr, &nparts, /*real_t *tpwgts*/nullptr, /*real_t *ubvec*/nullptr, options, &objval, parttab.data() ); if ( rez != METIS_OK ) throw std::runtime_error("Metis runtime failed :("); for(int ct = 0; ct < numSElems; ct++){ selements[ctLevel][ct]->setParent(selements[ctLevel+1][parttab[ct]]); selements[ctLevel+1][parttab[ct]]->addChildren(selements[ctLevel][ct]); } //Metis solves following problem, but still it should be checked //There are SElements with some disconnected clusters of elements. //For such SE split them to several SEs for(auto seIT = selements[ctLevel+1].begin(); seIT != selements[ctLevel+1].end(); seIT++){//Loop on SElements including new ones std::set<int> allElemIndexes;//Indexes of all elements in this SE for(int ct = 0; ct < (*seIT)->numSElements(); ct++) allElemIndexes.insert((*seIT)->children(ct)->index()); std::set<int> inOneSE; inOneSE.insert(*(allElemIndexes.begin())); bool flagChanges = true; while(flagChanges){ flagChanges = false; for(auto it = inOneSE.begin(); it != inOneSE.end(); it++){ for(auto itN = elemNeibour[*it].begin(); itN != elemNeibour[*it].end(); itN++){ if(allElemIndexes.count(*itN) && !inOneSE.count(*itN)){ inOneSE.insert(*itN); flagChanges = true; } } } } if(allElemIndexes.size() != inOneSE.size()){ std::set<int> inOtherSE; (*seIT)->setChildrens(std::vector<sbfSElement *>{}); (*seIT)->numSElements(); for(auto it = inOneSE.begin(); it != inOneSE.end(); it++){ (*seIT)->addChildren(selements[ctLevel][*it]); selements[ctLevel][*it]->setParent(*seIT); } for(auto it = allElemIndexes.begin(); it != allElemIndexes.end(); it++){ if(!inOneSE.count(*it)) inOtherSE.insert(*it); } sbfSElement *additionalSE = new sbfSElement((*seIT)->mesh(), selements[ctLevel+1].size()); for(auto it = inOtherSE.begin(); it != inOtherSE.end(); it++){ additionalSE->addChildren(selements[ctLevel][*it]); selements[ctLevel][*it]->setParent(additionalSE); } selements[ctLevel+1].push_back(additionalSE); if ( verbouse ) std::cout << "Split disconnected SE to two of sizes: " << inOneSE.size() << ", " << inOtherSE.size() << std::endl; } }//Loop on SElements including new ones facesWeigth = facesWeigthNextLevel; facesOwners = facesOwnersNextLevel; std::cout << "level " << ctLevel << " done" << std::endl; //get statistics std::list<int> selems; int numAll = 0; for(auto se : selements[ctLevel+1]){ selems.push_back(se->numSElements()); numAll += se->numSElements(); } if(numAll != numSElems) throw std::runtime_error("SElements contain not all elements of previous layer"); selems.sort(); selems.reverse(); if ( verbouse ) { for (auto se : selems ) std::cout << se << "\t"; std::cout << "Imbalance is " << (1.0*selems.front())/selems.back() << std::endl; } }//Loop on levels return 0; }
int MESH_PartitionWithMetis(Mesh_ptr mesh, int nparts, int **part) { MEdge_ptr fedge; MFace_ptr mf, oppf, rface; MRegion_ptr mr, oppr; List_ptr fedges, efaces, rfaces, fregions; int i, ncells, ipos; int nv, ne, nf, nr, nfe, nef, nfr, nrf, idx, idx2; #ifdef METIS_5 idx_t ngraphvtx, numflag, nedgecut, numparts, ncons; idx_t wtflag, metisopts[METIS_NOPTIONS]; idx_t *vsize, *idxpart; idx_t *xadj, *adjncy, *vwgt, *adjwgt; real_t *tpwgts, *ubvec; #else idxtype ngraphvtx, numflag, nedgecut, numparts; idxtype wtflag, metisopts[5] = {0,0,0,0,0}; idxtype *xadj, *adjncy, *vwgt, *adjwgt, *idxpart; #endif /* First build a nodal graph of the mesh in the format required by metis */ nv = MESH_Num_Vertices(mesh); ne = MESH_Num_Edges(mesh); nf = MESH_Num_Faces(mesh); nr = MESH_Num_Regions(mesh); ipos = 0; if (nr == 0) { if (nf == 0) { fprintf(stderr,"Cannot partition wire meshes\n"); exit(-1); } #ifdef METIS_5 xadj = (idx_t *) malloc((nf+1)*sizeof(idx_t)); adjncy = (idx_t *) malloc(2*ne*sizeof(idx_t)); #else xadj = (idxtype *) malloc((nf+1)*sizeof(idxtype)); adjncy = (idxtype *) malloc(2*ne*sizeof(idxtype)); #endif ncells = nf; /* Surface mesh */ idx = 0; i = 0; xadj[i] = ipos; while ((mf = MESH_Next_Face(mesh,&idx))) { fedges = MF_Edges(mf,1,0); nfe = List_Num_Entries(fedges); idx2 = 0; while ((fedge = List_Next_Entry(fedges,&idx2))) { efaces = ME_Faces(fedge); nef = List_Num_Entries(efaces); if (nef == 1) { continue; /* boundary edge; nothing to do */ } else { int j; for (j = 0; j < nef; j++) { oppf = List_Entry(efaces,j); if (oppf != mf) { adjncy[ipos] = MF_ID(oppf)-1; ipos++; } } } List_Delete(efaces); } List_Delete(fedges); i++; xadj[i] = ipos; } } else { #ifdef METIS_5 xadj = (idx_t *) malloc((nr+1)*sizeof(idx_t)); adjncy = (idx_t *) malloc(2*nf*sizeof(idx_t)); #else xadj = (idxtype *) malloc((nr+1)*sizeof(idxtype)); adjncy = (idxtype *) malloc(2*nf*sizeof(idxtype)); #endif ncells = nr; /* Volume mesh */ idx = 0; i = 0; xadj[i] = ipos; while ((mr = MESH_Next_Region(mesh,&idx))) { rfaces = MR_Faces(mr); nrf = List_Num_Entries(rfaces); idx2 = 0; while ((rface = List_Next_Entry(rfaces,&idx2))) { fregions = MF_Regions(rface); nfr = List_Num_Entries(fregions); if (nfr > 1) { oppr = List_Entry(fregions,0); if (oppr == mr) oppr = List_Entry(fregions,1); adjncy[ipos] = MR_ID(oppr)-1; ipos++; } List_Delete(fregions); } List_Delete(rfaces); i++; xadj[i] = ipos; } } /* Partition the graph */ wtflag = 0; /* No weights are specified */ vwgt = adjwgt = NULL; numflag = 0; /* C style numbering of elements (nodes of the dual graph) */ ngraphvtx = ncells; /* we want the variable to be of type idxtype or idx_t */ numparts = nparts; /* we want the variable to be of type idxtype or idx_t */ #ifdef METIS_5 idxpart = (idx_t *) malloc(ncells*sizeof(idx_t)); ncons = 1; /* Number of constraints */ vsize = NULL; tpwgts = NULL; ubvec = NULL; METIS_SetDefaultOptions(metisopts); metisopts[METIS_OPTION_NUMBERING] = 0; if (nparts <= 8) METIS_PartGraphRecursive(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt, &numparts,tpwgts,ubvec,metisopts,&nedgecut, idxpart); else METIS_PartGraphKway(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt, &numparts,tpwgts,ubvec,metisopts,&nedgecut,idxpart); #else idxpart = (idxtype *) malloc(ncells*sizeof(idxtype)); if (nparts <= 8) METIS_PartGraphRecursive(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag, &numflag,&numparts,metisopts,&nedgecut,idxpart); else METIS_PartGraphKway(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,&numflag, &numparts,metisopts,&nedgecut,idxpart); #endif free(xadj); free(adjncy); *part = (int *) malloc(ncells*sizeof(int)); for (i = 0; i < ncells; i++) (*part)[i] = (int) idxpart[i]; free(idxpart); return 1; }
void METIS_PARTGRAPHKWAY(int *nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part) { METIS_PartGraphKway(nvtxs, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, part); }
//============================================================================== // NOTE: // - matrix is supposed to be localized, and passes through the // singleton filter. This means that I do not have to look // for Dirichlet nodes (singletons). Also, all rows and columns are // local. int Ifpack_METISPartitioner::ComputePartitions() { int ierr; #ifdef HAVE_IFPACK_METIS int nbytes = 0; int edgecut; #endif Teuchos::RefCountPtr<Epetra_CrsGraph> SymGraph ; Teuchos::RefCountPtr<Epetra_Map> SymMap; Teuchos::RefCountPtr<Ifpack_Graph_Epetra_CrsGraph> SymIFPACKGraph; Teuchos::RefCountPtr<Ifpack_Graph> IFPACKGraph = Teuchos::rcp( (Ifpack_Graph*)Graph_, false ); int Length = 2 * MaxNumEntries(); int NumIndices; std::vector<int> Indices; Indices.resize(Length); /* construct the CSR graph information of the LOCAL matrix using the get_row function */ std::vector<idxtype> wgtflag; wgtflag.resize(4); std::vector<int> options; options.resize(4); int numflag; if (UseSymmetricGraph_) { #if !defined(EPETRA_NO_32BIT_GLOBAL_INDICES) || !defined(EPETRA_NO_64BIT_GLOBAL_INDICES) // need to build a symmetric graph. // I do this in two stages: // 1.- construct an Epetra_CrsMatrix, symmetric // 2.- convert the Epetra_CrsMatrix into METIS format SymMap = Teuchos::rcp( new Epetra_Map(NumMyRows(),0,Graph_->Comm()) ); SymGraph = Teuchos::rcp( new Epetra_CrsGraph(Copy,*SymMap,0) ); #endif #ifndef EPETRA_NO_32BIT_GLOBAL_INDICES if(SymGraph->RowMap().GlobalIndicesInt()) { for (int i = 0; i < NumMyRows() ; ++i) { ierr = Graph_->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]); IFPACK_CHK_ERR(ierr); for (int j = 0 ; j < NumIndices ; ++j) { int jj = Indices[j]; if (jj != i) { SymGraph->InsertGlobalIndices(i,1,&jj); SymGraph->InsertGlobalIndices(jj,1,&i); } } } } else #endif #ifndef EPETRA_NO_64BIT_GLOBAL_INDICES if(SymGraph->RowMap().GlobalIndicesLongLong()) { for (int i = 0; i < NumMyRows() ; ++i) { long long i_LL = i; ierr = Graph_->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]); IFPACK_CHK_ERR(ierr); for (int j = 0 ; j < NumIndices ; ++j) { long long jj = Indices[j]; if (jj != i_LL) { SymGraph->InsertGlobalIndices(i_LL,1,&jj); SymGraph->InsertGlobalIndices(jj,1,&i_LL); } } } } else #endif throw "Ifpack_METISPartitioner::ComputePartitions: GlobalIndices type unknown"; IFPACK_CHK_ERR(SymGraph->FillComplete()); SymIFPACKGraph = Teuchos::rcp( new Ifpack_Graph_Epetra_CrsGraph(SymGraph) ); IFPACKGraph = SymIFPACKGraph; } // now work on IFPACKGraph, that can be the symmetric or // the non-symmetric one /* set parameters */ wgtflag[0] = 0; /* no weights */ numflag = 0; /* C style */ options[0] = 0; /* default options */ std::vector<idxtype> xadj; xadj.resize(NumMyRows() + 1); std::vector<idxtype> adjncy; adjncy.resize(NumMyNonzeros()); int count = 0; int count2 = 0; xadj[0] = 0; for (int i = 0; i < NumMyRows() ; ++i) { xadj[count2+1] = xadj[count2]; /* nonzeros in row i-1 */ ierr = IFPACKGraph->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]); IFPACK_CHK_ERR(ierr); for (int j = 0 ; j < NumIndices ; ++j) { int jj = Indices[j]; if (jj != i) { adjncy[count++] = jj; xadj[count2+1]++; } } count2++; } std::vector<idxtype> NodesInSubgraph; NodesInSubgraph.resize(NumLocalParts_); // some cases can be handled separately int ok; if (NumLocalParts() == 1) { for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = 0; } else if (NumLocalParts() == NumMyRows()) { for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = i; } else { ok = 0; // sometimes METIS creates less partitions than specified. // ok will check this problem, and recall metis, asking // for NumLocalParts_/2 partitions while (ok == 0) { for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = -1; #ifdef HAVE_IFPACK_METIS int j = NumMyRows(); if (NumLocalParts_ < 8) { int i = 1; /* optype in the METIS manual */ numflag = 0; METIS_EstimateMemory(&j, &xadj[0], &adjncy[0], &numflag, &i, &nbytes ); METIS_PartGraphRecursive(&j, &xadj[0], &adjncy[0], NULL, NULL, &wgtflag[0], &numflag, &NumLocalParts_, &options[0], &edgecut, &Partition_[0]); } else { numflag = 0; METIS_PartGraphKway (&j, &xadj[0], &adjncy[0], NULL, NULL, &wgtflag[0], &numflag, &NumLocalParts_, &options[0], &edgecut, &Partition_[0]); } #else numflag = numflag * 2; // avoid warning for unused variable if (Graph_->Comm().MyPID() == 0) { cerr << "METIS was not linked; now I put all" << endl; cerr << "the local nodes in the same partition." << endl; } for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = 0; NumLocalParts_ = 1; #endif ok = 1; for (int i = 0 ; i < NumLocalParts() ; ++i) NodesInSubgraph[i] = 0; for (int i = 0 ; i < NumMyRows() ; ++i) { int j = Partition_[i]; if ((j < 0) || (j>= NumLocalParts())) { ok = 0; break; } else NodesInSubgraph[j]++; } for (int i = 0 ; i < NumLocalParts() ; ++i) { if( NodesInSubgraph[i] == 0 ) { ok = 0; break; } } if (ok == 0) { cerr << "Specified number of subgraphs (" << NumLocalParts_ << ") generates empty subgraphs." << endl; cerr << "Now I recall METIS with NumLocalParts_ = " << NumLocalParts_ / 2 << "..." << endl; NumLocalParts_ = NumLocalParts_/2; } if (NumLocalParts() == 0) { IFPACK_CHK_ERR(-10); // something went wrong } if (NumLocalParts() == 1) { for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = 0; ok = 1; } } /* while( ok == 0 ) */ } /* if( NumLocalParts_ == 1 ) */ return(0); }
/************************************************************************* * This function is the entry point of the initial partition algorithm * that does recursive bissection. * This algorithm assembles the graph to all the processors and preceeds * by parallelizing the recursive bisection step. **************************************************************************/ void InitPartition(ctrl_t *ctrl, graph_t *graph) { idx_t i, j, ncon, mype, npes, gnvtxs, ngroups; idx_t *xadj, *adjncy, *adjwgt, *vwgt; idx_t *part, *gwhere0, *gwhere1; idx_t *tmpwhere, *tmpvwgt, *tmpxadj, *tmpadjncy, *tmpadjwgt; graph_t *agraph; idx_t lnparts, fpart, fpe, lnpes; idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut; real_t *tpwgts, *tpwgts2, *lbvec, lbsum, min_lbsum, wsum; MPI_Comm ipcomm; struct { double sum; int rank; } lpesum, gpesum; WCOREPUSH; ncon = graph->ncon; ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, ctrl->npes), 1); IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm)); IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr)); lbvec = rwspacemalloc(ctrl, ncon); /* assemble the graph to all the processors */ agraph = AssembleAdaptiveGraph(ctrl, graph); gnvtxs = agraph->nvtxs; /* make a copy of the graph's structure for later */ xadj = icopy(gnvtxs+1, agraph->xadj, iwspacemalloc(ctrl, gnvtxs+1)); vwgt = icopy(gnvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, gnvtxs*ncon)); adjncy = icopy(agraph->nedges, agraph->adjncy, iwspacemalloc(ctrl, agraph->nedges)); adjwgt = icopy(agraph->nedges, agraph->adjwgt, iwspacemalloc(ctrl, agraph->nedges)); part = iwspacemalloc(ctrl, gnvtxs); /* create different processor groups */ gkMPI_Comm_split(ctrl->gcomm, ctrl->mype % ngroups, 0, &ipcomm); gkMPI_Comm_rank(ipcomm, &mype); gkMPI_Comm_size(ipcomm, &npes); /* Go into the recursive bisection */ METIS_SetDefaultOptions(moptions); moptions[METIS_OPTION_SEED] = ctrl->sync + (ctrl->mype % ngroups) + 1; tpwgts = ctrl->tpwgts; tpwgts2 = rwspacemalloc(ctrl, 2*ncon); lnparts = ctrl->nparts; fpart = fpe = 0; lnpes = npes; while (lnpes > 1 && lnparts > 1) { /* determine the weights of the two partitions as a function of the weight of the target partition weights */ for (j=(lnparts>>1), i=0; i<ncon; i++) { tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon); tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon); wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]); tpwgts2[i] *= wsum; tpwgts2[ncon+i] *= wsum; } METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt, &twoparts, tpwgts2, NULL, moptions, &edgecut, part); /* pick one of the branches */ if (mype < fpe+lnpes/2) { KeepPart(ctrl, agraph, part, 0); lnpes = lnpes/2; lnparts = lnparts/2; } else { KeepPart(ctrl, agraph, part, 1); fpart = fpart + lnparts/2; fpe = fpe + lnpes/2; lnpes = lnpes - lnpes/2; lnparts = lnparts - lnparts/2; } } gwhere0 = iset(gnvtxs, 0, iwspacemalloc(ctrl, gnvtxs)); gwhere1 = iwspacemalloc(ctrl, gnvtxs); if (lnparts == 1) { /* Case npes is greater than or equal to nparts */ /* Only the first process will assign labels (for the reduction to work) */ if (mype == fpe) { for (i=0; i<agraph->nvtxs; i++) gwhere0[agraph->label[i]] = fpart; } } else { /* Case in which npes is smaller than nparts */ /* create the normalized tpwgts for the lnparts from ctrl->tpwgts */ tpwgts = rwspacemalloc(ctrl, lnparts*ncon); for (j=0; j<ncon; j++) { for (wsum=0.0, i=0; i<lnparts; i++) { tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j]; wsum += tpwgts[i*ncon+j]; } for (wsum=1.0/wsum, i=0; i<lnparts; i++) tpwgts[i*ncon+j] *= wsum; } METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions, &edgecut, part); for (i=0; i<agraph->nvtxs; i++) gwhere0[agraph->label[i]] = fpart + part[i]; } gkMPI_Allreduce((void *)gwhere0, (void *)gwhere1, gnvtxs, IDX_T, MPI_SUM, ipcomm); if (ngroups > 1) { tmpxadj = agraph->xadj; tmpadjncy = agraph->adjncy; tmpadjwgt = agraph->adjwgt; tmpvwgt = agraph->vwgt; tmpwhere = agraph->where; agraph->xadj = xadj; agraph->adjncy = adjncy; agraph->adjwgt = adjwgt; agraph->vwgt = vwgt; agraph->where = gwhere1; agraph->vwgt = vwgt; agraph->nvtxs = gnvtxs; edgecut = ComputeSerialEdgeCut(agraph); ComputeSerialBalance(ctrl, agraph, gwhere1, lbvec); lbsum = rsum(ncon, lbvec, 1); gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ctrl->gcomm); gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ctrl->gcomm); lpesum.sum = lbsum; if (min_lbsum < UNBALANCE_FRACTION*ncon) { if (lbsum < UNBALANCE_FRACTION*ncon) lpesum.sum = edgecut; else lpesum.sum = max_cut; } lpesum.rank = ctrl->mype; gkMPI_Allreduce((void *)&lpesum, (void *)&gpesum, 1, MPI_DOUBLE_INT, MPI_MINLOC, ctrl->gcomm); gkMPI_Bcast((void *)gwhere1, gnvtxs, IDX_T, gpesum.rank, ctrl->gcomm); agraph->xadj = tmpxadj; agraph->adjncy = tmpadjncy; agraph->adjwgt = tmpadjwgt; agraph->vwgt = tmpvwgt; agraph->where = tmpwhere; } icopy(graph->nvtxs, gwhere1+graph->vtxdist[ctrl->mype], graph->where); FreeGraph(agraph); gkMPI_Comm_free(&ipcomm); IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm)); IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->InitPartTmr)); WCOREPOP; }
/*********************************************************************************** * This function is the entry point of the parallel kmetis algorithm that uses * coordinates to compute an initial graph distribution. ************************************************************************************/ int ParMETIS_V3_PartGeomKway(idx_t *vtxdist, idx_t *xadj, idx_t *adjncy, idx_t *vwgt, idx_t *adjwgt, idx_t *wgtflag, idx_t *numflag, idx_t *ndims, real_t *xyz, idx_t *ncon, idx_t *nparts, real_t *tpwgts, real_t *ubvec, idx_t *options, idx_t *edgecut, idx_t *part, MPI_Comm *comm) { idx_t h, i, j, npes, mype, status, nvtxs, seed, dbglvl; idx_t cut, gcut, maxnvtxs; idx_t moptions[METIS_NOPTIONS]; ctrl_t *ctrl; graph_t *graph, *mgraph; real_t balance; size_t curmem; /* Check the input parameters and return if an error */ status = CheckInputsPartGeomKway(vtxdist, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, ndims, xyz, ncon, nparts, tpwgts, ubvec, options, edgecut, part, comm); if (GlobalSEMinComm(*comm, status) == 0) return METIS_ERROR; status = METIS_OK; gk_malloc_init(); curmem = gk_GetCurMemoryUsed(); /* Setup the ctrl */ ctrl = SetupCtrl(PARMETIS_OP_GKMETIS, options, *ncon, *nparts, tpwgts, ubvec, *comm); npes = ctrl->npes; mype = ctrl->mype; /* Take care the nparts == 1 case */ if (*nparts == 1) { iset(vtxdist[mype+1]-vtxdist[mype], (*numflag == 0 ? 0 : 1), part); *edgecut = 0; goto DONE; } /* Take care of npes == 1 case */ if (npes == 1) { nvtxs = vtxdist[1] - vtxdist[0]; /* subtraction is required when numflag==1 */ METIS_SetDefaultOptions(moptions); moptions[METIS_OPTION_NUMBERING] = *numflag; status = METIS_PartGraphKway(&nvtxs, ncon, xadj, adjncy, vwgt, NULL, adjwgt, nparts, tpwgts, ubvec, moptions, edgecut, part); goto DONE; } /* Setup the graph */ if (*numflag > 0) ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 1); graph = SetupGraph(ctrl, *ncon, vtxdist, xadj, vwgt, NULL, adjncy, adjwgt, *wgtflag); gk_free((void **)&graph->nvwgt, LTERM); /* Allocate the workspace */ AllocateWSpace(ctrl, 10*graph->nvtxs); /* Compute the initial npes-way partitioning geometric partitioning */ STARTTIMER(ctrl, ctrl->TotalTmr); Coordinate_Partition(ctrl, graph, *ndims, xyz, 1); STOPTIMER(ctrl, ctrl->TotalTmr); /* Move the graph according to the partitioning */ STARTTIMER(ctrl, ctrl->MoveTmr); ctrl->nparts = npes; mgraph = MoveGraph(ctrl, graph); ctrl->nparts = *nparts; SetupGraph_nvwgts(ctrl, mgraph); /* compute nvwgts for the moved graph */ if (ctrl->dbglvl&DBG_INFO) { CommInterfaceData(ctrl, graph, graph->where, graph->where+graph->nvtxs); for (cut=0, i=0; i<graph->nvtxs; i++) { for (j=graph->xadj[i]; j<graph->xadj[i+1]; j++) { if (graph->where[i] != graph->where[graph->adjncy[j]]) cut += graph->adjwgt[j]; } } gcut = GlobalSESum(ctrl, cut)/2; maxnvtxs = GlobalSEMax(ctrl, mgraph->nvtxs); balance = (real_t)(maxnvtxs)/((real_t)(graph->gnvtxs)/(real_t)(npes)); rprintf(ctrl, "XYZ Cut: %6"PRIDX" \tBalance: %6.3"PRREAL" [%"PRIDX" %"PRIDX" %"PRIDX"]\n", gcut, balance, maxnvtxs, graph->gnvtxs, npes); } STOPTIMER(ctrl, ctrl->MoveTmr); /* Compute the partition of the moved graph */ STARTTIMER(ctrl, ctrl->TotalTmr); ctrl->CoarsenTo = gk_min(vtxdist[npes]+1, 25*(*ncon)*gk_max(npes, *nparts)); if (vtxdist[npes] < SMALLGRAPH || vtxdist[npes] < npes*20 || GlobalSESum(ctrl, mgraph->nedges) == 0) { /* serially */ IFSET(ctrl->dbglvl, DBG_INFO, rprintf(ctrl, "Partitioning a graph of size %"PRIDX" serially\n", vtxdist[npes])); PartitionSmallGraph(ctrl, mgraph); } else { /* in parallel */ Global_Partition(ctrl, mgraph); } ParallelReMapGraph(ctrl, mgraph); /* Invert the ordering back to the original graph */ ctrl->nparts = npes; ProjectInfoBack(ctrl, graph, part, mgraph->where); ctrl->nparts = *nparts; *edgecut = mgraph->mincut; STOPTIMER(ctrl, ctrl->TotalTmr); /* Print some stats */ IFSET(ctrl->dbglvl, DBG_TIME, PrintTimingInfo(ctrl)); IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->gcomm)); IFSET(ctrl->dbglvl, DBG_INFO, PrintPostPartInfo(ctrl, mgraph, 0)); FreeGraph(mgraph); FreeInitialGraphAndRemap(graph); if (*numflag > 0) ChangeNumbering(vtxdist, xadj, adjncy, part, npes, mype, 0); DONE: FreeCtrl(&ctrl); if (gk_GetCurMemoryUsed() - curmem > 0) { printf("ParMETIS appears to have a memory leak of %zdbytes. Report this.\n", (ssize_t)(gk_GetCurMemoryUsed() - curmem)); } gk_malloc_cleanup(0); return (int)status; }
/************************************************************************* * This function partitions a finite element mesh by partitioning its dual * graph using KMETIS and then assigning nodes in a load balanced fashion. **************************************************************************/ int METIS_PartMeshDual(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind, idx_t *vwgt, idx_t *vsize, idx_t *ncommon, idx_t *nparts, real_t *tpwgts, idx_t *options, idx_t *objval, idx_t *epart, idx_t *npart) { int sigrval=0, renumber=0, ptype; idx_t i, j; idx_t *xadj=NULL, *adjncy=NULL, *nptr=NULL, *nind=NULL; idx_t ncon=1, pnumflag=0; int rstatus = METIS_OK; /* set up malloc cleaning code and signal catchers */ if (!gk_malloc_init()) return METIS_ERROR_MEMORY; gk_sigtrap(); if ((sigrval = gk_sigcatch()) != 0) goto SIGTHROW; renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0); ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY); /* renumber the mesh */ if (renumber) { ChangeMesh2CNumbering(*ne, eptr, eind); options[METIS_OPTION_NUMBERING] = 0; } /* get the dual graph */ rstatus = METIS_MeshToDual(ne, nn, eptr, eind, ncommon, &pnumflag, &xadj, &adjncy); if (rstatus != METIS_OK) raise(SIGERR); /* partition the graph */ if (ptype == METIS_PTYPE_KWAY) rstatus = METIS_PartGraphKway(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL, nparts, tpwgts, NULL, options, objval, epart); else rstatus = METIS_PartGraphRecursive(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL, nparts, tpwgts, NULL, options, objval, epart); if (rstatus != METIS_OK) raise(SIGERR); /* construct the node-element list */ nptr = ismalloc(*nn+1, 0, "METIS_PartMeshDual: nptr"); nind = imalloc(eptr[*ne], "METIS_PartMeshDual: nind"); for (i=0; i<*ne; i++) { for (j=eptr[i]; j<eptr[i+1]; j++) nptr[eind[j]]++; } MAKECSR(i, *nn, nptr); for (i=0; i<*ne; i++) { for (j=eptr[i]; j<eptr[i+1]; j++) nind[nptr[eind[j]]++] = i; } SHIFTCSR(i, *nn, nptr); /* partition the other side of the mesh */ InduceRowPartFromColumnPart(*nn, nptr, nind, npart, epart, *nparts, tpwgts); gk_free((void **)&nptr, &nind, LTERM); SIGTHROW: if (renumber) { ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart); options[METIS_OPTION_NUMBERING] = 1; } METIS_Free(xadj); METIS_Free(adjncy); gk_siguntrap(); gk_malloc_cleanup(0); return metis_rcode(sigrval); }
/************************************************************************* * This function is the entry point of the initial balancing algorithm. * This algorithm assembles the graph to all the processors and preceeds * with the balancing step. **************************************************************************/ void Balance_Partition(ctrl_t *ctrl, graph_t *graph) { idx_t i, j, nvtxs, nedges, ncon; idx_t mype, npes, srnpes, srmype; idx_t *vtxdist, *xadj, *adjncy, *adjwgt, *vwgt, *vsize; idx_t *part, *lwhere, *home; idx_t lnparts, fpart, fpe, lnpes, ngroups; idx_t *rcounts, *rdispls; idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut; idx_t sr_pe, gd_pe, sr, gd, who_wins; real_t my_cut, my_totalv, my_cost = -1.0, my_balance = -1.0, wsum; real_t rating, max_rating, your_cost = -1.0, your_balance = -1.0; real_t lbsum, min_lbsum, *lbvec, *tpwgts, *tpwgts2, buffer[2]; graph_t *agraph, cgraph; ctrl_t *myctrl; MPI_Status status; MPI_Comm ipcomm, srcomm; struct { double cost; int rank; } lpecost, gpecost; IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr)); WCOREPUSH; vtxdist = graph->vtxdist; agraph = AssembleAdaptiveGraph(ctrl, graph); nvtxs = cgraph.nvtxs = agraph->nvtxs; nedges = cgraph.nedges = agraph->nedges; ncon = cgraph.ncon = agraph->ncon; xadj = cgraph.xadj = icopy(nvtxs+1, agraph->xadj, iwspacemalloc(ctrl, nvtxs+1)); vwgt = cgraph.vwgt = icopy(nvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, nvtxs*ncon)); vsize = cgraph.vsize = icopy(nvtxs, agraph->vsize, iwspacemalloc(ctrl, nvtxs)); adjncy = cgraph.adjncy = icopy(nedges, agraph->adjncy, iwspacemalloc(ctrl, nedges)); adjwgt = cgraph.adjwgt = icopy(nedges, agraph->adjwgt, iwspacemalloc(ctrl, nedges)); part = cgraph.where = agraph->where = iwspacemalloc(ctrl, nvtxs); lwhere = iwspacemalloc(ctrl, nvtxs); home = iwspacemalloc(ctrl, nvtxs); lbvec = rwspacemalloc(ctrl, graph->ncon); /****************************************/ /****************************************/ if (ctrl->ps_relation == PARMETIS_PSR_UNCOUPLED) { WCOREPUSH; rcounts = iwspacemalloc(ctrl, ctrl->npes); rdispls = iwspacemalloc(ctrl, ctrl->npes+1); for (i=0; i<ctrl->npes; i++) rdispls[i] = rcounts[i] = vtxdist[i+1]-vtxdist[i]; MAKECSR(i, ctrl->npes, rdispls); gkMPI_Allgatherv((void *)graph->home, graph->nvtxs, IDX_T, (void *)part, rcounts, rdispls, IDX_T, ctrl->comm); for (i=0; i<agraph->nvtxs; i++) home[i] = part[i]; WCOREPOP; /* local frees */ } else { for (i=0; i<ctrl->npes; i++) { for (j=vtxdist[i]; j<vtxdist[i+1]; j++) part[j] = home[j] = i; } } /* Ensure that the initial partitioning is legal */ for (i=0; i<agraph->nvtxs; i++) { if (part[i] >= ctrl->nparts) part[i] = home[i] = part[i] % ctrl->nparts; if (part[i] < 0) part[i] = home[i] = (-1*part[i]) % ctrl->nparts; } /****************************************/ /****************************************/ IFSET(ctrl->dbglvl, DBG_REFINEINFO, ComputeSerialBalance(ctrl, agraph, agraph->where, lbvec)); IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "input cut: %"PRIDX", balance: ", ComputeSerialEdgeCut(agraph))); for (i=0; i<agraph->ncon; i++) IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "%.3"PRREAL" ", lbvec[i])); IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "\n")); /****************************************/ /* Split the processors into two groups */ /****************************************/ sr = (ctrl->mype % 2 == 0) ? 1 : 0; gd = (ctrl->mype % 2 == 1) ? 1 : 0; if (graph->ncon > MAX_NCON_FOR_DIFFUSION || ctrl->npes == 1) { sr = 1; gd = 0; } sr_pe = 0; gd_pe = 1; gkMPI_Comm_split(ctrl->gcomm, sr, 0, &ipcomm); gkMPI_Comm_rank(ipcomm, &mype); gkMPI_Comm_size(ipcomm, &npes); if (sr == 1) { /* Half of the processors do scratch-remap */ ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, npes), 1); gkMPI_Comm_split(ipcomm, mype % ngroups, 0, &srcomm); gkMPI_Comm_rank(srcomm, &srmype); gkMPI_Comm_size(srcomm, &srnpes); METIS_SetDefaultOptions(moptions); moptions[METIS_OPTION_SEED] = ctrl->sync + (mype % ngroups) + 1; tpwgts = ctrl->tpwgts; tpwgts2 = rwspacemalloc(ctrl, 2*ncon); iset(nvtxs, 0, lwhere); lnparts = ctrl->nparts; fpart = fpe = 0; lnpes = srnpes; while (lnpes > 1 && lnparts > 1) { PASSERT(ctrl, agraph->nvtxs > 1); /* determine the weights of the two partitions as a function of the weight of the target partition weights */ for (j=(lnparts>>1), i=0; i<ncon; i++) { tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon); tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon); wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]); tpwgts2[i] *= wsum; tpwgts2[ncon+i] *= wsum; } METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt, &twoparts, tpwgts2, NULL, moptions, &edgecut, part); /* pick one of the branches */ if (srmype < fpe+lnpes/2) { KeepPart(ctrl, agraph, part, 0); lnpes = lnpes/2; lnparts = lnparts/2; } else { KeepPart(ctrl, agraph, part, 1); fpart = fpart + lnparts/2; fpe = fpe + lnpes/2; lnpes = lnpes - lnpes/2; lnparts = lnparts - lnparts/2; } } if (lnparts == 1) { /* Case in which srnpes is greater or equal to nparts */ /* Only the first process will assign labels (for the reduction to work) */ if (srmype == fpe) { for (i=0; i<agraph->nvtxs; i++) lwhere[agraph->label[i]] = fpart; } } else { /* Case in which srnpes is smaller than nparts */ /* create the normalized tpwgts for the lnparts from ctrl->tpwgts */ tpwgts = rwspacemalloc(ctrl, lnparts*ncon); for (j=0; j<ncon; j++) { for (wsum=0.0, i=0; i<lnparts; i++) { tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j]; wsum += tpwgts[i*ncon+j]; } for (wsum=1.0/wsum, i=0; i<lnparts; i++) tpwgts[i*ncon+j] *= wsum; } METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions, &edgecut, part); for (i=0; i<agraph->nvtxs; i++) lwhere[agraph->label[i]] = fpart + part[i]; } gkMPI_Allreduce((void *)lwhere, (void *)part, nvtxs, IDX_T, MPI_SUM, srcomm); edgecut = ComputeSerialEdgeCut(&cgraph); ComputeSerialBalance(ctrl, &cgraph, part, lbvec); lbsum = rsum(ncon, lbvec, 1); gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ipcomm); gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ipcomm); lpecost.rank = ctrl->mype; lpecost.cost = lbsum; if (min_lbsum < UNBALANCE_FRACTION * (real_t)(ncon)) { if (lbsum < UNBALANCE_FRACTION * (real_t)(ncon)) lpecost.cost = (double)edgecut; else lpecost.cost = (double)max_cut + lbsum; } gkMPI_Allreduce((void *)&lpecost, (void *)&gpecost, 1, MPI_DOUBLE_INT, MPI_MINLOC, ipcomm); if (ctrl->mype == gpecost.rank && ctrl->mype != sr_pe) gkMPI_Send((void *)part, nvtxs, IDX_T, sr_pe, 1, ctrl->comm); if (ctrl->mype != gpecost.rank && ctrl->mype == sr_pe) gkMPI_Recv((void *)part, nvtxs, IDX_T, gpecost.rank, 1, ctrl->comm, &status); if (ctrl->mype == sr_pe) { icopy(nvtxs, part, lwhere); SerialRemap(ctrl, &cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts); } gkMPI_Comm_free(&srcomm); }