int gpartition(const vector< set<int> > &graph, int npartitions, int partition_method, vector<int> &decomp){ // If no partitioning method is set, choose a default. if(partition_method<0){ if(npartitions<=8) partition_method = 0; // METIS PartGraphRecursive else partition_method = 1; // METIS PartGraphKway } int nnodes = graph.size(); // Compress graph vector<idxtype> xadj(nnodes+1), adjncy; int pos=0; xadj[0]=1; for(int i=0;i<nnodes;i++){ for(set<int>::iterator jt=graph[i].begin();jt!=graph[i].end();jt++){ adjncy.push_back(*jt); pos++; } xadj[i+1] = pos+1; } // Partition graph decomp.resize(nnodes); int wgtflag=0, numflag=1, edgecut=0; #ifdef PARMETIS_V3 int options[] = {0}; #else idx_t nbc=1; idx_t options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); options[METIS_OPTION_NUMBERING]=1; #endif if(partition_method){ #ifdef PARMETIS_V3 METIS_PartGraphKway(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag, &numflag, &npartitions, options, &edgecut, &(decomp[0])); #else METIS_PartGraphKway(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL, &npartitions, NULL, NULL,options,&edgecut, &(decomp[0])); #endif }else{ #ifdef PARMETIS_V3 METIS_PartGraphRecursive(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag, &numflag, &npartitions, options, &edgecut, &(decomp[0])); #else METIS_PartGraphRecursive(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL, &npartitions, NULL, NULL,options,&edgecut, &(decomp[0])); #endif } // number from zero for(int i=0;i<nnodes;i++) decomp[i]--; return edgecut; }
void metis_wrapper(int *n, int *xadj, int *adjncy, int *nparts, int *part) { #ifndef METIS_VER_MAJOR /* Metis 4.X (or METISLib in ParaMetis 3.X) */ int edgecut; int wgtflag = 0; int numflag = 1; /* fortran-stype numbering */ int options[5] = { 0, 3, 1, 1, 0 }; /* default values */ METIS_PartGraphRecursive(n, xadj, adjncy, NULL, NULL, &wgtflag, &numflag, nparts, options, &edgecut, part); #elif METIS_VER_MAJOR == 5 /* Metis 5.X */ int edgecut; int ncon = 1; int options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); options[METIS_OPTION_NUMBERING] = 1; /* fortran-stype numbering */ METIS_PartGraphRecursive(n, &ncon, xadj, adjncy, NULL, NULL, NULL, nparts, NULL, NULL, options, &edgecut, part); #else #error unknown Metis version #endif }
/*Partition this mesh's elements into n chunks, writing each element's 0-based chunk number to elem2chunk. */ void FEM_Mesh_partition(const FEM_Mesh *mesh,int nchunks,int *elem2chunk, bool faceGraph) { CkThresholdTimer time("FEM Split> Building graph for metis partitioner",1.0); int nelems=mesh->nElems(); if (nchunks==1) { //Metis doesn't handle this case (!) for (int i=0;i<nelems;i++) elem2chunk[i]=0; return; } Graph g(nelems); if (!faceGraph) { mesh2graph(mesh,&g); } else { mesh2graph_face(mesh,&g); } int *adjStart; /*Maps elem # -> start index in adjacency list*/ int *adjList; /*Lists adjacent vertices for each element*/ g.toAdjList(adjStart,adjList); int ecut,ncon=1; time.start("FEM Split> Calling metis partitioner"); if (nchunks<8) /*Metis manual says recursive version is higher-quality here*/ METIS_PartGraphRecursive(&nelems, &ncon, adjStart, adjList, NULL, NULL, NULL, &nchunks, NULL, NULL, NULL, &ecut, elem2chunk); else /*For many chunks, Kway is supposedly faster */ METIS_PartGraphKway(&nelems, &ncon, adjStart, adjList, NULL, NULL, NULL, &nchunks, NULL, NULL, NULL, &ecut, elem2chunk); delete[] adjStart; delete[] adjList; }
void FC_FUNC_(oct_metis_partgraphrecursive, OCT_METIS_PARTGRAPHRECURSIVE) (idx_t *nvtxs, idx_t *ncon, idx_t *xadj, idx_t *adjncy, idx_t *nparts, real_t *tpwgts, real_t *ubvec, idx_t *options, idx_t *objval, idx_t *part) { METIS_PartGraphRecursive(nvtxs, ncon, xadj, adjncy, NULL, NULL, NULL, nparts, tpwgts, ubvec, options, objval, part); }
void decompose(Graph_part & gp) { // Decompose METIS_PartGraphRecursive(Mg.nvtxs, Mg.ncon, Mg.xadj, Mg.adjncy, Mg.vwgt, Mg.vsize, Mg.adjwgt, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.options, Mg.objval, Mg.part); // vertex id size_t id = 0; // For each vertex store the processor that contain the data auto it = gp.getVertexIterator(); while (it.isNext()) { gp.vertex(it).template get<i>() = Mg.part[id]; ++id; ++it; } }
int do_metis_recursive_partition(network_t *network, options_t *opt, idx_t *part) { int ret = ORCC_OK; idx_t ncon = 1; idx_t metis_opt[METIS_NOPTIONS]; idx_t objval; adjacency_list *graph, *metis_graph; assert(network != NULL); assert(opt != NULL); assert(part != NULL); print_orcc_trace(ORCC_VL_VERBOSE_1, "Applying METIS Recursive partition for mapping"); METIS_SetDefaultOptions(metis_opt); graph = set_graph_from_network(network); metis_graph = fix_graph_for_metis(graph); ret = METIS_PartGraphRecursive(&metis_graph->nb_vertices, /* idx_t *nvtxs */ &ncon, /*idx_t *ncon*/ metis_graph->xadj, /*idx_t *xadj*/ metis_graph->adjncy, /*idx_t *adjncy*/ metis_graph->vwgt, /*idx_t *vwgt*/ NULL, /*idx_t *vsize*/ metis_graph->adjwgt, /*idx_t *adjwgt*/ &opt->nb_processors, /*idx_t *nparts*/ NULL, /*real t *tpwgts*/ NULL, /*real t ubvec*/ metis_opt, /*idx_t *options*/ &objval, /*idx_t *objval*/ part); /*idx_t *part*/ check_metis_error(ret); print_orcc_trace(ORCC_VL_VERBOSE_2, "METIS Edgecut : %d", objval); delete_graph(graph); delete_graph(metis_graph); return ret; }
void decompose() { if (Mg.nparts[0] != 1) { // Decompose METIS_PartGraphRecursive(Mg.nvtxs, Mg.ncon, Mg.xadj, Mg.adjncy, Mg.vwgt, Mg.vsize, Mg.adjwgt, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.options, Mg.objval, Mg.part); // vertex id size_t id = 0; // For each vertex store the processor that contain the data auto it = g.getVertexIterator(); while (it.isNext()) { g.vertex(it).template get<i>() = Mg.part[id]; ++id; ++it; } } else { // Trivially assign all the domains to the processor 0 auto it = g.getVertexIterator(); while (it.isNext()) { g.vertex(it).template get<i>() = 0; ++it; } } }
void METIS_PARTGRAPHRECURSIVE(int *nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part) { METIS_PartGraphRecursive(nvtxs, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, part); }
void metis_partgraphrecursive__(int *nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part) { METIS_PartGraphRecursive(nvtxs, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, part); }
int main() { /* We aim to partition the following mesh into 4 parts: * * 0---1---2---3---4 * | | | | | * 5---6---7---8---9 * | | | | | * 10--11--12--13--14 * */ const int nVertices = 15; const int nNeighs = 44; idx_t xadj [nVertices+1] = { 0, 2, 5, 8, 11, 13, 16, 20, 24, 28, 31, 33, 36, 39, 42, 44 }; idx_t adjncy [nNeighs] = { 1, 5, 0, 2, 6, 1, 3, 7, 2, 4, 8, 3, 9, 0, 6, 10, 1, 5, 7, 11, 2, 6, 8, 12, 3, 7, 9, 13, 4, 8, 14, 5, 11, 6, 10, 12, 7, 11, 13, 8, 12, 14, 9, 13 }; std::cout << "Before Partitioning:" << std::endl; std::cout << std::endl; std::cout << "0---0---0---0---0" << std::endl; std::cout << "| | | | |" << std::endl; std::cout << "0---0---0---0---0" << std::endl; std::cout << "| | | | |" << std::endl; std::cout << "0---0---0---0---0" << std::endl; std::cout << std::endl; // partition the graph using metis idx_t nvtxs = nVertices; // number of vertices idx_t ncon = 1; // number of balancing constraints idx_t *vwgt = NULL; // vertex weights idx_t *vsize = NULL; // vertex sizes idx_t *adjwgt = NULL; // edge weights idx_t nparts = 4; // number of partitions real_t *tpwgts = NULL; // target partition weight real_t *ubvec = NULL; // load balance tolerance for each constraint idx_t options[METIS_NOPTIONS]; // array of options METIS_SetDefaultOptions( options ); // use defaults idx_t objval; // edge-cut on return std::vector< idx_t > colors( nVertices, 0 ); // ranks int ierr = METIS_PartGraphRecursive( &nvtxs, &ncon, &xadj[0], &adjncy[0], vwgt, vsize, adjwgt, &nparts, tpwgts, ubvec, options, &objval, &colors[0] ); if ( ierr != METIS_OK ) throw std::runtime_error( "METIS graph partitioning failed!" ); std::cout << "After Partitioning:" << std::endl; std::cout << std::endl; std::cout << colors[0] << "---" << colors[1] << "---" << colors[2] << "---" << colors[3] << "---" << colors[4] << std::endl; std::cout << "| | | | |" << std::endl; std::cout << colors[5] << "---" << colors[6] << "---" << colors[7] << "---" << colors[8] << "---" << colors[9] << std::endl; std::cout << "| | | | |" << std::endl; std::cout << colors[10] << "---" << colors[11] << "---" << colors[12] << "---" << colors[13] << "---" << colors[14] << std::endl; std::cout << std::endl; return 0; }
JNIEXPORT jint JNICALL Java_jprime_JMetis_PartitionGraph( JNIEnv * env, jobject obj, jboolean j_force_PartGraphRecursive, jboolean j_force_PartGraphKway, jint j_num_vertices, jintArray j_xadj, jintArray j_adjncy, jintArray j_vwgt, jintArray j_adjwgt, jint j_wgtflag, jint j_nparts, jintArray j_options, jintArray j_partioned_nodes) { mysrand(7654321L); //delcare vars jint * xadj=NULL, * adjncy=NULL,* vwgt=NULL, * adjwgt=NULL, * options=NULL, * partioned_nodes=NULL; int idx, edges_cut=-1, num_vertices=0, nparts=0, wgtflag=0, numflag=0; jsize xadj_len, adjncy_len, vwgt_len, adjwgt_len, options_len, partioned_nodes_len; //copy the jsize vars num_vertices=j_num_vertices; nparts=j_nparts; wgtflag=j_wgtflag; //get array lengths xadj_len = (*env)->GetArrayLength(env,j_xadj); adjncy_len = (*env)->GetArrayLength(env,j_adjncy); vwgt_len = (*env)->GetArrayLength(env,j_vwgt); adjwgt_len = (*env)->GetArrayLength(env,j_adjwgt); options_len = (*env)->GetArrayLength(env,j_options); partioned_nodes_len = (*env)->GetArrayLength(env,j_partioned_nodes); //printf("xadj_len=%i, adjncy_len=%i, vwgt_len=%i, adjwgt_len=%i, options_len=%i, partioned_nodes_len=%i\n",xadj_len, adjncy_len, vwgt_len, adjwgt_len, options_len, partioned_nodes_len); //create/get local copies xadj = (*env)->GetIntArrayElements(env,j_xadj,0); adjncy = (*env)->GetIntArrayElements(env,j_adjncy,0); if(vwgt_len>0) vwgt = (*env)->GetIntArrayElements(env,j_vwgt,0); if(adjwgt_len>0) adjwgt = (*env)->GetIntArrayElements(env,j_adjwgt,0); if(options_len>0) options = (*env)->GetIntArrayElements(env,j_options,0); partioned_nodes = (int*)malloc(sizeof(int)*partioned_nodes_len); #if 0 printf("num_vertices=%i,wgtflag=%i, numflag=%i, nparts=%i\n", num_vertices, wgtflag, numflag, nparts); printf("xadj=%p, adjncy=%p, vwgt=%p, adjwgt=%p, options=%p, partioned_nodes=%p\n", xadj, adjncy, vwgt, adjwgt, options, partioned_nodes); printf("xadj:"); for(idx=0;idx<xadj_len;idx++) { printf("%i ",xadj[idx]); } printf("\n"); printf("adjncy:"); for(idx=0;idx<adjncy_len;idx++) { printf("%i ",adjncy[idx]); } printf("\n"); #endif //call func if((j_nparts<8 || j_force_PartGraphRecursive) && !j_force_PartGraphKway) { METIS_PartGraphRecursive( &num_vertices, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag, &nparts, options, &edges_cut, partioned_nodes); } else { METIS_PartGraphKway( &num_vertices, xadj, adjncy, vwgt, adjwgt, &wgtflag, &numflag, &nparts, options, &edges_cut, partioned_nodes); } //pop partioned_nodes (*env)->SetIntArrayRegion(env,j_partioned_nodes,0,partioned_nodes_len,partioned_nodes); //free local copies free(partioned_nodes); (*env)->ReleaseIntArrayElements(env, j_xadj, xadj, 0); (*env)->ReleaseIntArrayElements(env, j_adjncy, adjncy, 0); if(vwgt_len>0) (*env)->ReleaseIntArrayElements(env, j_vwgt, vwgt, 0); if(adjwgt_len>0) (*env)->ReleaseIntArrayElements(env, j_adjwgt, adjwgt, 0); if(options_len>0) (*env)->ReleaseIntArrayElements(env, j_options, options, 0); return edges_cut; }
/************************************************************************* * Let the game begin **************************************************************************/ main(int argc, char *argv[]) { int i, nparts, options[10]; idxtype *part; float lbvec[MAXNCON]; GraphType graph; char filename[256]; int numflag = 0, wgtflag = 0, edgecut; timer TOTALTmr, METISTmr, IOTmr; if (argc != 3) { printf("Usage: %s <GraphFile> <Nparts>\n",argv[0]); exit(0); } strcpy(filename, argv[1]); nparts = atoi(argv[2]); if (nparts < 2) { printf("The number of partitions should be greater than 1!\n"); exit(0); } cleartimer(TOTALTmr); cleartimer(METISTmr); cleartimer(IOTmr); starttimer(TOTALTmr); starttimer(IOTmr); ReadGraph(&graph, filename, &wgtflag); if (graph.nvtxs <= 0) { printf("Empty graph. Nothing to do.\n"); exit(0); } stoptimer(IOTmr); printf("**********************************************************************\n"); printf("%s", METISTITLE); printf("Graph Information ---------------------------------------------------\n"); printf(" Name: %s, #Vertices: %d, #Edges: %d, #Parts: %d\n", filename, graph.nvtxs, graph.nedges/2, nparts); if (graph.ncon > 1) printf(" Balancing Constraints: %d\n", graph.ncon); printf("\nRecursive Partitioning... -------------------------------------------\n"); part = idxmalloc(graph.nvtxs, "main: part"); options[0] = 0; starttimer(METISTmr); if (graph.ncon == 1) { METIS_PartGraphRecursive(&graph.nvtxs, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &nparts, options, &edgecut, part); } else { METIS_mCPartGraphRecursive(&graph.nvtxs, &graph.ncon, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt, &wgtflag, &numflag, &nparts, options, &edgecut, part); } stoptimer(METISTmr); ComputePartitionBalance(&graph, nparts, part, lbvec); printf(" %d-way Edge-Cut: %7d, Balance: ", nparts, edgecut); for (i=0; i<graph.ncon; i++) printf("%5.2f ", lbvec[i]); printf("\n"); starttimer(IOTmr); WritePartition(filename, part, graph.nvtxs, nparts); stoptimer(IOTmr); stoptimer(TOTALTmr); printf("\nTiming Information --------------------------------------------------\n"); printf(" I/O: \t\t %7.3f\n", gettimer(IOTmr)); printf(" Partitioning: \t\t %7.3f (PMETIS time)\n", gettimer(METISTmr)); printf(" Total: \t\t %7.3f\n", gettimer(TOTALTmr)); printf("**********************************************************************\n"); GKfree(&graph.xadj, &graph.adjncy, &graph.vwgt, &graph.adjwgt, &part, LTERM); }
idx_t* gpmetis( int argc, char **argv ) /*************************************************************************/ /*! Let the game begin! */ /*************************************************************************/ //int main(int argc, char *argv[]) { idx_t i; char *curptr, *newptr; idx_t options[METIS_NOPTIONS]; graph_t *graph; idx_t *part; idx_t objval; params_t *params; int status=0; gk_optind = 0; //printf( "argc: %d\n", argc ); //printf( "gk_optind %d\n", gk_optind ); fflush( stdout ); for( i = 0; i < argc; i++ ) { //printf( "%s*\n", argv[ i ] ); } params = parse_cmdline(argc, argv); //printf( "gk_optind %d\n", gk_optind ); //fflush( stdout ); //return NULL; gk_startcputimer(params->iotimer); graph = ReadGraph(params); ReadTPwgts(params, graph->ncon); gk_stopcputimer(params->iotimer); /* Check if the graph is contiguous */ if (params->contig && !IsConnected(graph, 0)) { printf("***The input graph is not contiguous.\n" "***The specified -contig option will be ignored.\n"); params->contig = 0; } /* Get ubvec if supplied */ if (params->ubvecstr) { params->ubvec = rmalloc(graph->ncon, "main"); curptr = params->ubvecstr; for (i=0; i<graph->ncon; i++) { params->ubvec[i] = strtoreal(curptr, &newptr); if (curptr == newptr) errexit("Error parsing entry #%"PRIDX" of ubvec [%s] (possibly missing).\n", i, params->ubvecstr); curptr = newptr; } } /* Setup iptype */ if (params->iptype == -1) { if (params->ptype == METIS_PTYPE_RB) { if (graph->ncon == 1) params->iptype = METIS_IPTYPE_GROW; else params->iptype = METIS_IPTYPE_RANDOM; } } GPPrintInfo(params, graph); part = imalloc(graph->nvtxs, "main: part"); METIS_SetDefaultOptions(options); options[METIS_OPTION_OBJTYPE] = params->objtype; options[METIS_OPTION_CTYPE] = params->ctype; options[METIS_OPTION_IPTYPE] = params->iptype; options[METIS_OPTION_RTYPE] = params->rtype; options[METIS_OPTION_MINCONN] = params->minconn; options[METIS_OPTION_CONTIG] = params->contig; options[METIS_OPTION_SEED] = params->seed; options[METIS_OPTION_NITER] = params->niter; options[METIS_OPTION_NCUTS] = params->ncuts; options[METIS_OPTION_UFACTOR] = params->ufactor; options[METIS_OPTION_DBGLVL] = params->dbglvl; gk_malloc_init(); gk_startcputimer(params->parttimer); switch (params->ptype) { case METIS_PTYPE_RB: status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, ¶ms->nparts, params->tpwgts, params->ubvec, options, &objval, part); break; case METIS_PTYPE_KWAY: status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, ¶ms->nparts, params->tpwgts, params->ubvec, options, &objval, part); break; } gk_stopcputimer(params->parttimer); if (gk_GetCurMemoryUsed() != 0) printf("***It seems that Metis did not free all of its memory! Report this.\n"); params->maxmemory = gk_GetMaxMemoryUsed(); gk_malloc_cleanup(0); if (status != METIS_OK) { printf("\n***Metis returned with an error.\n"); } else { if (!params->nooutput) { /* Write the solution */ gk_startcputimer(params->iotimer); WritePartition(params->filename, part, graph->nvtxs, params->nparts); gk_stopcputimer(params->iotimer); } GPReportResults(params, graph, part, objval); } idx_t *r_part = ( idx_t* ) calloc( graph->nvtxs, sizeof( idx_t ) ); for( i = 0; i < graph->nvtxs; i++ ) { r_part[ i ] = part[ i ]; } FreeGraph(&graph); gk_free((void **)&part, LTERM); gk_free((void **)¶ms->filename, ¶ms->tpwgtsfile, ¶ms->tpwgts, ¶ms->ubvecstr, ¶ms->ubvec, ¶ms, LTERM); return r_part; }
void MetisLB::work(LDStats* stats) { /** ========================== INITIALIZATION ============================= */ ProcArray *parr = new ProcArray(stats); ObjGraph *ogr = new ObjGraph(stats); /** ============================= STRATEGY ================================ */ if (_lb_args.debug() >= 2) { CkPrintf("[%d] In MetisLB Strategy...\n", CkMyPe()); } // convert ObjGraph to the adjacency structure int numVertices = ogr->vertices.size(); // number of vertices int numEdges = 0; // number of edges double maxLoad = 0.0; int i, j, k, vert; /** remove duplicate edges from recvFrom */ for(i = 0; i < numVertices; i++) { for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) { vert = ogr->vertices[i].sendToList[j].getNeighborId(); for(k = 0; k < ogr->vertices[i].recvFromList.size(); k++) { if(ogr->vertices[i].recvFromList[k].getNeighborId() == vert) { ogr->vertices[i].sendToList[j].setNumBytes(ogr->vertices[i].sendToList[j].getNumBytes() + ogr->vertices[i].recvFromList[k].getNumBytes()); ogr->vertices[i].recvFromList.erase(ogr->vertices[i].recvFromList.begin() + k); } } } } /** the object load is normalized to an integer between 0 and 256 */ for(i = 0; i < numVertices; i++) { if(ogr->vertices[i].getVertexLoad() > maxLoad) maxLoad = ogr->vertices[i].getVertexLoad(); numEdges = numEdges + ogr->vertices[i].sendToList.size() + ogr->vertices[i].recvFromList.size(); } /* adjacency list */ idx_t *xadj = new idx_t[numVertices + 1]; /* id of the neighbors */ idx_t *adjncy = new idx_t[numEdges]; /* weights of the vertices */ idx_t *vwgt = new idx_t[numVertices]; /* weights of the edges */ idx_t *adjwgt = new idx_t[numEdges]; int edgeNum = 0; double ratio = 256.0/maxLoad; for(i = 0; i < numVertices; i++) { xadj[i] = edgeNum; vwgt[i] = (int)ceil(ogr->vertices[i].getVertexLoad() * ratio); for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) { adjncy[edgeNum] = ogr->vertices[i].sendToList[j].getNeighborId(); adjwgt[edgeNum] = ogr->vertices[i].sendToList[j].getNumBytes(); edgeNum++; } for(j = 0; j < ogr->vertices[i].recvFromList.size(); j++) { adjncy[edgeNum] = ogr->vertices[i].recvFromList[j].getNeighborId(); adjwgt[edgeNum] = ogr->vertices[i].recvFromList[j].getNumBytes(); edgeNum++; } } xadj[i] = edgeNum; CkAssert(edgeNum == numEdges); idx_t edgecut; // number of edges cut by the partitioning idx_t *pemap; idx_t options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); //options[METIS_OPTION_PTYPE] = METIS_PTYPE_RB; // C style numbering options[METIS_OPTION_NUMBERING] = 0; // number of constrains idx_t ncon = 1; // number of partitions idx_t numPes = parr->procs.size(); real_t ubvec[ncon]; // allow 10% imbalance ubvec[0] = 1.1; // mapping of objs to partitions pemap = new idx_t[numVertices]; // Specifies size of vertices for computing the total communication volume idx_t *vsize = NULL; // This array of size nparts specifies the desired weight for each partition // and setting it to NULL indicates graph should be equally divided among // partitions real_t *tpwgts = NULL; int option = 0; if (WEIGHTED == option) { // set up the different weights between 0 and 1 tpwgts = new real_t[numPes]; for (i = 0; i < numPes; i++) { tpwgts[i] = 1.0/(real_t)numPes; } } else if (MULTI_CONSTRAINT == option) { CkAbort("Multiple constraints not implemented.\n"); } // numVertices: num vertices in the graph; ncon: num balancing constrains // xadj, adjncy: of size n+1 and adjncy of 2m, adjncy[xadj[i]] through and // including adjncy[xadj[i+1]-1]; // vwgt: weight of the vertices; vsize: amt of data that needs to be sent // for ith vertex is vsize[i] // adjwght: the weight of edges; numPes: total parts // tpwghts: target partition weight, can pass NULL to equally divide // ubvec: of size ncon to indicate allowed load imbalance tolerance (> 1.0) // options: array of options; edgecut: stores the edgecut; pemap: mapping METIS_PartGraphRecursive(&numVertices, &ncon, xadj, adjncy, vwgt, vsize, adjwgt, &numPes, tpwgts, ubvec, options, &edgecut, pemap); delete[] xadj; delete[] adjncy; delete[] vwgt; delete[] adjwgt; delete[] vsize; delete[] tpwgts; if (_lb_args.debug() >= 1) { CkPrintf("[%d] MetisLB done! \n", CkMyPe()); } for(i = 0; i < numVertices; i++) { if(pemap[i] != ogr->vertices[i].getCurrentPe()) ogr->vertices[i].setNewPe(pemap[i]); } delete[] pemap; /** ============================== CLEANUP ================================ */ ogr->convertDecisions(stats); delete parr; delete ogr; }
/************************************************************************** ** This takes objects and partitions them into clusters. */ void GridMetisLB::Partition_Objects_Into_Clusters (CentralLB::LDStats *stats) { int num_migratable_objects; int *migratable_objects; int index; int num_partitions; int *partition_to_cluster_map; int cluster; int partition; int partition_count; int *vertex_weights; int vertex; int **communication_matrix; LDCommData *com_data; int send_object; int recv_object; int send_index; int recv_index; LDObjKey *recv_objects; int num_objects; int *xadj; int num_edges; int *adjncy; int *edge_weights; int count; int weight_flag; int numbering_flag; int options[5]; int edgecut; int *newmap; int i; int j; if (Num_Clusters == 1) { for (i = 0; i < Num_Objects; i++) { (&Object_Data[i])->cluster = 0; } return; } for (i = 0; i < Num_Objects; i++) { (&Object_Data[i])->secondary_index = -1; } // Count the number of migratable objects, which are the only candidates to give to Metis. // (The non-migratable objects have been placed onto the correct destination PEs earlier.) // After getting the count, create a migratable_objects[] array to keep track of them. num_migratable_objects = 0; for (i = 0; i < Num_Objects; i++) { if ((&Object_Data[i])->migratable) { num_migratable_objects += 1; } } migratable_objects = new int[num_migratable_objects]; index = 0; for (i = 0; i < Num_Objects; i++) { if ((&Object_Data[i])->migratable) { (&Object_Data[i])->secondary_index = index; migratable_objects[index] = i; index += 1; } } // Compute the number of partitions for Metis, based on the scaled CPU power for each cluster. // Also create a partition-to-cluster mapping so the output of Metis can be mapped back to clusters. num_partitions = 0; for (i = 0; i < Num_Clusters; i++) { num_partitions += (int) ceil ((&Cluster_Data[i])->scaled_cpu_power); } partition_to_cluster_map = new int[num_partitions]; cluster = 0; partition = 0; while (partition < num_partitions) { partition_count = (int) ceil ((&Cluster_Data[cluster])->scaled_cpu_power); for (i = partition; i < (partition + partition_count); i++) { partition_to_cluster_map[i] = cluster; } partition += partition_count; cluster += 1; } if (CK_LDB_GridMetisLB_Mode == 1) { vertex_weights = new int[num_migratable_objects]; vertex = 0; for (i = 0; i < Num_Objects; i++) { if ((&Object_Data[i])->migratable) { vertex_weights[vertex] = (int) ceil ((&Object_Data[i])->load * 10000); vertex += 1; } } } // Create communication_matrix[] to hold all object-to-object message counts. communication_matrix = new int *[num_migratable_objects]; for (i = 0; i < num_migratable_objects; i++) { communication_matrix[i] = new int[num_migratable_objects]; for (j = 0; j < num_migratable_objects; j++) { communication_matrix[i][j] = 0; } } for (i = 0; i < stats->n_comm; i++) { com_data = &(stats->commData[i]); if ((!com_data->from_proc()) && (com_data->recv_type() == LD_OBJ_MSG)) { send_object = stats->getHash (com_data->sender); recv_object = stats->getHash (com_data->receiver.get_destObj()); //if ((recv_object == -1) && (stats->complete_flag == 0)) { if ((send_object < 0) || (send_object > Num_Objects) || (recv_object < 0) || (recv_object > Num_Objects)) { continue; } if ((!(&Object_Data[send_object])->migratable) || (!(&Object_Data[recv_object])->migratable)) { continue; } send_index = (&Object_Data[send_object])->secondary_index; recv_index = (&Object_Data[recv_object])->secondary_index; communication_matrix[send_index][recv_index] += com_data->messages; communication_matrix[recv_index][send_index] += com_data->messages; } else if (com_data->receiver.get_type() == LD_OBJLIST_MSG) { send_object = stats->getHash (com_data->sender); if ((send_object < 0) || (send_object > Num_Objects)) { continue; } if (!(&Object_Data[send_object])->migratable) { continue; } recv_objects = com_data->receiver.get_destObjs (num_objects); // (num_objects is passed by reference) for (j = 0; j < num_objects; j++) { recv_object = stats->getHash (recv_objects[j]); //if (recv_object == -1) { if ((recv_object < 0) || (recv_object > Num_Objects)) { continue; } if (!(&Object_Data[recv_object])->migratable) { continue; } send_index = (&Object_Data[send_object])->secondary_index; recv_index = (&Object_Data[recv_object])->secondary_index; communication_matrix[send_index][recv_index] += com_data->messages; communication_matrix[recv_index][send_index] += com_data->messages; } } } for (i = 0; i < num_migratable_objects; i++) { communication_matrix[i][i] = 0; } // Construct a graph in CSR format for input to Metis. xadj = new int[num_migratable_objects + 1]; num_edges = 0; for (i = 0; i < num_migratable_objects; i++) { for (j = 0; j < num_migratable_objects; j++) { if (communication_matrix[i][j] > 0) { num_edges += 1; } } } adjncy = new int[num_edges]; edge_weights = new int[num_edges]; count = 0; xadj[0] = 0; for (i = 0; i < num_migratable_objects; i++) { for (j = 0; j < num_migratable_objects; j++) { if (communication_matrix[i][j] > 0) { adjncy[count] = j; edge_weights[count] = communication_matrix[i][j]; count += 1; } } xadj[i+1] = count; } if (CK_LDB_GridMetisLB_Mode == 0) { // Call Metis to partition the communication graph. weight_flag = 1; // weights on edges only numbering_flag = 0; // C style numbering (base 0) options[0] = 0; newmap = new int[num_migratable_objects]; METIS_PartGraphRecursive (&num_migratable_objects, xadj, adjncy, NULL, edge_weights, &weight_flag, &numbering_flag, &num_partitions, options, &edgecut, newmap); } else if (CK_LDB_GridMetisLB_Mode == 1) { // Call Metis to partition the communication graph. weight_flag = 3; // weights on both vertices and edges numbering_flag = 0; // C style numbering (base 0) options[0] = 0; newmap = new int[num_migratable_objects]; METIS_PartGraphRecursive (&num_migratable_objects, xadj, adjncy, vertex_weights, edge_weights, &weight_flag, &numbering_flag, &num_partitions, options, &edgecut, newmap); } else { if (_lb_args.debug() > 0) { CkPrintf ("[%d] GridMetisLB was told to use bad mode (%d).\n", CkMyPe(), CK_LDB_GridMetisLB_Mode); } } // Place the partitioned objects into their correct clusters. for (i = 0; i < num_migratable_objects; i++) { partition = newmap[i]; cluster = partition_to_cluster_map[partition]; index = migratable_objects[i]; (&Object_Data[index])->cluster = cluster; } // Free memory. delete [] newmap; delete [] edge_weights; delete [] adjncy; delete [] xadj; for (i = 0; i < num_migratable_objects; i++) { delete [] communication_matrix[i]; } delete [] communication_matrix; if (CK_LDB_GridMetisLB_Mode == 1) { delete [] vertex_weights; } delete [] partition_to_cluster_map; delete [] migratable_objects; }
/************************************************************************* * This function partitions a finite element mesh by partitioning its nodal * graph using KMETIS and then assigning elements in a load balanced fashion. **************************************************************************/ int METIS_PartMeshNodal(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind, idx_t *vwgt, idx_t *vsize, idx_t *nparts, real_t *tpwgts, idx_t *options, idx_t *objval, idx_t *epart, idx_t *npart) { int sigrval=0, renumber=0, ptype; idx_t *xadj=NULL, *adjncy=NULL; idx_t ncon=1, pnumflag=0; int rstatus=METIS_OK; /* set up malloc cleaning code and signal catchers */ if (!gk_malloc_init()) return METIS_ERROR_MEMORY; gk_sigtrap(); if ((sigrval = gk_sigcatch()) != 0) goto SIGTHROW; renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0); ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY); /* renumber the mesh */ if (renumber) { ChangeMesh2CNumbering(*ne, eptr, eind); options[METIS_OPTION_NUMBERING] = 0; } /* get the nodal graph */ rstatus = METIS_MeshToNodal(ne, nn, eptr, eind, &pnumflag, &xadj, &adjncy); if (rstatus != METIS_OK) raise(SIGERR); /* partition the graph */ if (ptype == METIS_PTYPE_KWAY) rstatus = METIS_PartGraphKway(nn, &ncon, xadj, adjncy, vwgt, vsize, NULL, nparts, tpwgts, NULL, options, objval, npart); else rstatus = METIS_PartGraphRecursive(nn, &ncon, xadj, adjncy, vwgt, vsize, NULL, nparts, tpwgts, NULL, options, objval, npart); if (rstatus != METIS_OK) raise(SIGERR); /* partition the other side of the mesh */ InduceRowPartFromColumnPart(*ne, eptr, eind, epart, npart, *nparts, tpwgts); SIGTHROW: if (renumber) { ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart); options[METIS_OPTION_NUMBERING] = 1; } METIS_Free(xadj); METIS_Free(adjncy); gk_siguntrap(); gk_malloc_cleanup(0); return metis_rcode(sigrval); }
int MESH_PartitionWithMetis(Mesh_ptr mesh, int nparts, int **part) { MEdge_ptr fedge; MFace_ptr mf, oppf, rface; MRegion_ptr mr, oppr; List_ptr fedges, efaces, rfaces, fregions; int i, ncells, ipos; int nv, ne, nf, nr, nfe, nef, nfr, nrf, idx, idx2; #ifdef METIS_5 idx_t ngraphvtx, numflag, nedgecut, numparts, ncons; idx_t wtflag, metisopts[METIS_NOPTIONS]; idx_t *vsize, *idxpart; idx_t *xadj, *adjncy, *vwgt, *adjwgt; real_t *tpwgts, *ubvec; #else idxtype ngraphvtx, numflag, nedgecut, numparts; idxtype wtflag, metisopts[5] = {0,0,0,0,0}; idxtype *xadj, *adjncy, *vwgt, *adjwgt, *idxpart; #endif /* First build a nodal graph of the mesh in the format required by metis */ nv = MESH_Num_Vertices(mesh); ne = MESH_Num_Edges(mesh); nf = MESH_Num_Faces(mesh); nr = MESH_Num_Regions(mesh); ipos = 0; if (nr == 0) { if (nf == 0) { fprintf(stderr,"Cannot partition wire meshes\n"); exit(-1); } #ifdef METIS_5 xadj = (idx_t *) malloc((nf+1)*sizeof(idx_t)); adjncy = (idx_t *) malloc(2*ne*sizeof(idx_t)); #else xadj = (idxtype *) malloc((nf+1)*sizeof(idxtype)); adjncy = (idxtype *) malloc(2*ne*sizeof(idxtype)); #endif ncells = nf; /* Surface mesh */ idx = 0; i = 0; xadj[i] = ipos; while ((mf = MESH_Next_Face(mesh,&idx))) { fedges = MF_Edges(mf,1,0); nfe = List_Num_Entries(fedges); idx2 = 0; while ((fedge = List_Next_Entry(fedges,&idx2))) { efaces = ME_Faces(fedge); nef = List_Num_Entries(efaces); if (nef == 1) { continue; /* boundary edge; nothing to do */ } else { int j; for (j = 0; j < nef; j++) { oppf = List_Entry(efaces,j); if (oppf != mf) { adjncy[ipos] = MF_ID(oppf)-1; ipos++; } } } List_Delete(efaces); } List_Delete(fedges); i++; xadj[i] = ipos; } } else { #ifdef METIS_5 xadj = (idx_t *) malloc((nr+1)*sizeof(idx_t)); adjncy = (idx_t *) malloc(2*nf*sizeof(idx_t)); #else xadj = (idxtype *) malloc((nr+1)*sizeof(idxtype)); adjncy = (idxtype *) malloc(2*nf*sizeof(idxtype)); #endif ncells = nr; /* Volume mesh */ idx = 0; i = 0; xadj[i] = ipos; while ((mr = MESH_Next_Region(mesh,&idx))) { rfaces = MR_Faces(mr); nrf = List_Num_Entries(rfaces); idx2 = 0; while ((rface = List_Next_Entry(rfaces,&idx2))) { fregions = MF_Regions(rface); nfr = List_Num_Entries(fregions); if (nfr > 1) { oppr = List_Entry(fregions,0); if (oppr == mr) oppr = List_Entry(fregions,1); adjncy[ipos] = MR_ID(oppr)-1; ipos++; } List_Delete(fregions); } List_Delete(rfaces); i++; xadj[i] = ipos; } } /* Partition the graph */ wtflag = 0; /* No weights are specified */ vwgt = adjwgt = NULL; numflag = 0; /* C style numbering of elements (nodes of the dual graph) */ ngraphvtx = ncells; /* we want the variable to be of type idxtype or idx_t */ numparts = nparts; /* we want the variable to be of type idxtype or idx_t */ #ifdef METIS_5 idxpart = (idx_t *) malloc(ncells*sizeof(idx_t)); ncons = 1; /* Number of constraints */ vsize = NULL; tpwgts = NULL; ubvec = NULL; METIS_SetDefaultOptions(metisopts); metisopts[METIS_OPTION_NUMBERING] = 0; if (nparts <= 8) METIS_PartGraphRecursive(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt, &numparts,tpwgts,ubvec,metisopts,&nedgecut, idxpart); else METIS_PartGraphKway(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt, &numparts,tpwgts,ubvec,metisopts,&nedgecut,idxpart); #else idxpart = (idxtype *) malloc(ncells*sizeof(idxtype)); if (nparts <= 8) METIS_PartGraphRecursive(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag, &numflag,&numparts,metisopts,&nedgecut,idxpart); else METIS_PartGraphKway(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,&numflag, &numparts,metisopts,&nedgecut,idxpart); #endif free(xadj); free(adjncy); *part = (int *) malloc(ncells*sizeof(int)); for (i = 0; i < ncells; i++) (*part)[i] = (int) idxpart[i]; free(idxpart); return 1; }
//============================================================================== // NOTE: // - matrix is supposed to be localized, and passes through the // singleton filter. This means that I do not have to look // for Dirichlet nodes (singletons). Also, all rows and columns are // local. int Ifpack_METISPartitioner::ComputePartitions() { int ierr; #ifdef HAVE_IFPACK_METIS int nbytes = 0; int edgecut; #endif Teuchos::RefCountPtr<Epetra_CrsGraph> SymGraph ; Teuchos::RefCountPtr<Epetra_Map> SymMap; Teuchos::RefCountPtr<Ifpack_Graph_Epetra_CrsGraph> SymIFPACKGraph; Teuchos::RefCountPtr<Ifpack_Graph> IFPACKGraph = Teuchos::rcp( (Ifpack_Graph*)Graph_, false ); int Length = 2 * MaxNumEntries(); int NumIndices; std::vector<int> Indices; Indices.resize(Length); /* construct the CSR graph information of the LOCAL matrix using the get_row function */ std::vector<idxtype> wgtflag; wgtflag.resize(4); std::vector<int> options; options.resize(4); int numflag; if (UseSymmetricGraph_) { #if !defined(EPETRA_NO_32BIT_GLOBAL_INDICES) || !defined(EPETRA_NO_64BIT_GLOBAL_INDICES) // need to build a symmetric graph. // I do this in two stages: // 1.- construct an Epetra_CrsMatrix, symmetric // 2.- convert the Epetra_CrsMatrix into METIS format SymMap = Teuchos::rcp( new Epetra_Map(NumMyRows(),0,Graph_->Comm()) ); SymGraph = Teuchos::rcp( new Epetra_CrsGraph(Copy,*SymMap,0) ); #endif #ifndef EPETRA_NO_32BIT_GLOBAL_INDICES if(SymGraph->RowMap().GlobalIndicesInt()) { for (int i = 0; i < NumMyRows() ; ++i) { ierr = Graph_->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]); IFPACK_CHK_ERR(ierr); for (int j = 0 ; j < NumIndices ; ++j) { int jj = Indices[j]; if (jj != i) { SymGraph->InsertGlobalIndices(i,1,&jj); SymGraph->InsertGlobalIndices(jj,1,&i); } } } } else #endif #ifndef EPETRA_NO_64BIT_GLOBAL_INDICES if(SymGraph->RowMap().GlobalIndicesLongLong()) { for (int i = 0; i < NumMyRows() ; ++i) { long long i_LL = i; ierr = Graph_->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]); IFPACK_CHK_ERR(ierr); for (int j = 0 ; j < NumIndices ; ++j) { long long jj = Indices[j]; if (jj != i_LL) { SymGraph->InsertGlobalIndices(i_LL,1,&jj); SymGraph->InsertGlobalIndices(jj,1,&i_LL); } } } } else #endif throw "Ifpack_METISPartitioner::ComputePartitions: GlobalIndices type unknown"; IFPACK_CHK_ERR(SymGraph->FillComplete()); SymIFPACKGraph = Teuchos::rcp( new Ifpack_Graph_Epetra_CrsGraph(SymGraph) ); IFPACKGraph = SymIFPACKGraph; } // now work on IFPACKGraph, that can be the symmetric or // the non-symmetric one /* set parameters */ wgtflag[0] = 0; /* no weights */ numflag = 0; /* C style */ options[0] = 0; /* default options */ std::vector<idxtype> xadj; xadj.resize(NumMyRows() + 1); std::vector<idxtype> adjncy; adjncy.resize(NumMyNonzeros()); int count = 0; int count2 = 0; xadj[0] = 0; for (int i = 0; i < NumMyRows() ; ++i) { xadj[count2+1] = xadj[count2]; /* nonzeros in row i-1 */ ierr = IFPACKGraph->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]); IFPACK_CHK_ERR(ierr); for (int j = 0 ; j < NumIndices ; ++j) { int jj = Indices[j]; if (jj != i) { adjncy[count++] = jj; xadj[count2+1]++; } } count2++; } std::vector<idxtype> NodesInSubgraph; NodesInSubgraph.resize(NumLocalParts_); // some cases can be handled separately int ok; if (NumLocalParts() == 1) { for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = 0; } else if (NumLocalParts() == NumMyRows()) { for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = i; } else { ok = 0; // sometimes METIS creates less partitions than specified. // ok will check this problem, and recall metis, asking // for NumLocalParts_/2 partitions while (ok == 0) { for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = -1; #ifdef HAVE_IFPACK_METIS int j = NumMyRows(); if (NumLocalParts_ < 8) { int i = 1; /* optype in the METIS manual */ numflag = 0; METIS_EstimateMemory(&j, &xadj[0], &adjncy[0], &numflag, &i, &nbytes ); METIS_PartGraphRecursive(&j, &xadj[0], &adjncy[0], NULL, NULL, &wgtflag[0], &numflag, &NumLocalParts_, &options[0], &edgecut, &Partition_[0]); } else { numflag = 0; METIS_PartGraphKway (&j, &xadj[0], &adjncy[0], NULL, NULL, &wgtflag[0], &numflag, &NumLocalParts_, &options[0], &edgecut, &Partition_[0]); } #else numflag = numflag * 2; // avoid warning for unused variable if (Graph_->Comm().MyPID() == 0) { cerr << "METIS was not linked; now I put all" << endl; cerr << "the local nodes in the same partition." << endl; } for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = 0; NumLocalParts_ = 1; #endif ok = 1; for (int i = 0 ; i < NumLocalParts() ; ++i) NodesInSubgraph[i] = 0; for (int i = 0 ; i < NumMyRows() ; ++i) { int j = Partition_[i]; if ((j < 0) || (j>= NumLocalParts())) { ok = 0; break; } else NodesInSubgraph[j]++; } for (int i = 0 ; i < NumLocalParts() ; ++i) { if( NodesInSubgraph[i] == 0 ) { ok = 0; break; } } if (ok == 0) { cerr << "Specified number of subgraphs (" << NumLocalParts_ << ") generates empty subgraphs." << endl; cerr << "Now I recall METIS with NumLocalParts_ = " << NumLocalParts_ / 2 << "..." << endl; NumLocalParts_ = NumLocalParts_/2; } if (NumLocalParts() == 0) { IFPACK_CHK_ERR(-10); // something went wrong } if (NumLocalParts() == 1) { for (int i = 0 ; i < NumMyRows() ; ++i) Partition_[i] = 0; ok = 1; } } /* while( ok == 0 ) */ } /* if( NumLocalParts_ == 1 ) */ return(0); }
sparse_matrix* graph_partition(LIST *list , partition_t* partition_info) { graph_t* graph; params_t *params; idx_t options[METIS_NOPTIONS], status; idx_t objval; sparse_matrix* matrix = (sparse_matrix*)malloc(sizeof(sparse_matrix)); graph = ReadGraph(list); params = bmalloc(sizeof(*params), "Allocating memory for params"); METIS_SetDefaultOptions(options); METIS_init_params(params, graph->ncon); options[METIS_OPTION_OBJTYPE] = params->objtype; options[METIS_OPTION_CTYPE] = params->ctype; options[METIS_OPTION_IPTYPE] = params->iptype; options[METIS_OPTION_RTYPE] = params->rtype; options[METIS_OPTION_NO2HOP] = params->no2hop; options[METIS_OPTION_MINCONN] = params->minconn; options[METIS_OPTION_CONTIG] = params->contig; options[METIS_OPTION_SEED] = params->seed; options[METIS_OPTION_NITER] = params->niter; options[METIS_OPTION_NCUTS] = params->ncuts; options[METIS_OPTION_UFACTOR] = params->ufactor; options[METIS_OPTION_DBGLVL] = params->dbglvl; //print_input_data(graph,params); params->tpwgts = NULL; params->ubvec = NULL; partition_info->partion_table = imalloc(graph->nvtxs, "Allocate memory for part"); switch (params->ptype) { case METIS_PTYPE_RB: status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, ¶ms->nparts, params->tpwgts, params->ubvec, options, &objval, partition_info->partion_table); break; case METIS_PTYPE_KWAY: status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, ¶ms->nparts, params->tpwgts, params->ubvec, options, &objval, partition_info->partion_table); break; } if (status != METIS_OK) { fprintf(stderr,"\n***Metis returned with an error.***\n"); return NULL; } partition_info->size = graph->nvtxs; partition_info->noOfParts = params->nparts; WritePartition(params->filename, partition_info->partion_table, graph->nvtxs, params->nparts); printf("Objval: %d\n",objval); matrix->nz = -1; matrix->n = graph->nvtxs; matrix->p = graph->xadj; matrix->i = graph->adjncy; matrix->x = NULL; return matrix; }
/************************************************************************** ** This takes objects in a cluster and partitions them onto PEs. */ void GridMetisLB::Partition_ClusterObjects_Into_PEs (CentralLB::LDStats *stats, int cluster) { int num_migratable_cluster_objects; int *migratable_cluster_objects; int index; int num_available_cluster_pes; int num_partitions; int *partition_to_pe_map; int pe; int partition; int partition_count; int *vertex_weights; int vertex; int **communication_matrix; LDCommData *com_data; int send_object; int recv_object; int send_index; int recv_index; LDObjKey *recv_objects; int num_objects; int *xadj; int num_edges; int *adjncy; int *edge_weights; int count; int weight_flag; int numbering_flag; int options[5]; int edgecut; int *newmap; int i; int j; for (i = 0; i < Num_Objects; i++) { (&Object_Data[i])->secondary_index = -1; } // Count the number of migratable objects within this cluster, which are the only candidates to give to Metis. // (The non-migratable objects have been placed onto the correct destination PEs earlier.) // After getting the count, create a migratable_cluster_objects[] array to keep track of them. num_migratable_cluster_objects = 0; for (i = 0; i < Num_Objects; i++) { if (((&Object_Data[i])->migratable) && ((&Object_Data[i])->cluster == cluster)) { num_migratable_cluster_objects += 1; } } migratable_cluster_objects = new int[num_migratable_cluster_objects]; index = 0; for (i = 0; i < Num_Objects; i++) { if (((&Object_Data[i])->migratable) && ((&Object_Data[i])->cluster == cluster)) { (&Object_Data[i])->secondary_index = index; migratable_cluster_objects[index] = i; index += 1; } } // Count the number of available PEs in the cluster. num_available_cluster_pes = 0; for (i = 0; i < Num_PEs; i++) { if (((&PE_Data[i])->available) && ((&PE_Data[i])->cluster == cluster)) { num_available_cluster_pes += 1; } } // Compute the number of partitions for Metis, based on the relative speed of each PE. // Also create the partition-to-PE mapping so the output of Metis can be mapped back to PEs. num_partitions = 0; for (i = 0; i < Num_PEs; i++) { if (((&PE_Data[i])->available) && ((&PE_Data[i])->cluster == cluster)) { num_partitions += (int) ceil ((&PE_Data[i])->relative_speed); } } partition_to_pe_map = new int[num_partitions]; pe = 0; while (((!(&PE_Data[pe])->available) || ((&PE_Data[pe])->cluster != cluster)) && (pe < Num_PEs)) { pe += 1; } if (pe >= Num_PEs) { CmiAbort ("GridMetisLB: Error computing partition to PE map!\n"); } partition = 0; while (partition < num_partitions) { partition_count = (int) ceil ((&PE_Data[pe])->relative_speed); for (i = partition; i < (partition + partition_count); i++) { partition_to_pe_map[i] = pe; } partition += partition_count; pe += 1; while (((!(&PE_Data[pe])->available) || ((&PE_Data[pe])->cluster != cluster)) && (pe < Num_PEs)) { pe += 1; } if (pe > Num_PEs) { CmiAbort ("GridMetisLB: Error computing partition to PE map!\n"); } } // Compute vertex weights for the objects. vertex_weights = new int[num_migratable_cluster_objects]; vertex = 0; for (i = 0; i < Num_Objects; i++) { if ((&Object_Data[i])->migratable && ((&Object_Data[i])->cluster == cluster)) { vertex_weights[vertex] = (int) ceil ((&Object_Data[i])->load * 10000); vertex += 1; } } // Create communication_matrix[] to hold all object-to-object message counts; communication_matrix = new int *[num_migratable_cluster_objects]; for (i = 0; i < num_migratable_cluster_objects; i++) { communication_matrix[i] = new int[num_migratable_cluster_objects]; for (j = 0; j < num_migratable_cluster_objects; j++) { communication_matrix[i][j] = 0; } } for (i = 0; i < stats->n_comm; i++) { com_data = &(stats->commData[i]); if ((!com_data->from_proc()) && (com_data->recv_type() == LD_OBJ_MSG)) { send_object = stats->getHash (com_data->sender); recv_object = stats->getHash (com_data->receiver.get_destObj()); //if ((recv_object == -1) && (stats->complete_flag == 0)) { if ((send_object < 0) || (send_object > Num_Objects) || (recv_object < 0) || (recv_object > Num_Objects)) { continue; } if ((!(&Object_Data[send_object])->migratable) || (!(&Object_Data[recv_object])->migratable)) { continue; } if (((&Object_Data[send_object])->cluster != cluster) || ((&Object_Data[recv_object])->cluster != cluster)) { continue; } send_index = (&Object_Data[send_object])->secondary_index; recv_index = (&Object_Data[recv_object])->secondary_index; communication_matrix[send_index][recv_index] += com_data->messages; communication_matrix[recv_index][send_index] += com_data->messages; } else if (com_data->receiver.get_type() == LD_OBJLIST_MSG) { send_object = stats->getHash (com_data->sender); if ((send_object < 0) || (send_object > Num_Objects)) { continue; } if (!(&Object_Data[send_object])->migratable) { continue; } if ((&Object_Data[send_object])->cluster != cluster) { continue; } recv_objects = com_data->receiver.get_destObjs (num_objects); // (num_objects is passed by reference) for (j = 0; j < num_objects; j++) { recv_object = stats->getHash (recv_objects[j]); //if (recv_object == -1) { if ((recv_object < 0) || (recv_object > Num_Objects)) { continue; } if (!(&Object_Data[recv_object])->migratable) { continue; } if ((&Object_Data[recv_object])->cluster != cluster) { continue; } send_index = (&Object_Data[send_object])->secondary_index; recv_index = (&Object_Data[recv_object])->secondary_index; communication_matrix[send_index][recv_index] += com_data->messages; communication_matrix[recv_index][send_index] += com_data->messages; } } } for (i = 0; i < num_migratable_cluster_objects; i++) { communication_matrix[i][i] = 0; } // Construct a graph in CSR format for input to Metis. xadj = new int[num_migratable_cluster_objects + 1]; num_edges = 0; for (i = 0; i < num_migratable_cluster_objects; i++) { for (j = 0; j < num_migratable_cluster_objects; j++) { if (communication_matrix[i][j] > 0) { num_edges += 1; } } } adjncy = new int[num_edges]; edge_weights = new int[num_edges]; count = 0; xadj[0] = 0; for (i = 0; i < num_migratable_cluster_objects; i++) { for (j = 0; j < num_migratable_cluster_objects; j++) { if (communication_matrix[i][j] > 0) { adjncy[count] = j; edge_weights[count] = communication_matrix[i][j]; count += 1; } } xadj[i+1] = count; } // Call Metis to partition the communication graph. weight_flag = 3; // weights on both vertices and edges numbering_flag = 0; // C style numbering (base 0) options[0] = 0; newmap = new int[num_migratable_cluster_objects]; CmiPrintf ("[%d] GridMetisLB is partitioning %d objects in cluster %d into %d partitions.\n", CmiMyPe(), num_migratable_cluster_objects, cluster, num_partitions); METIS_PartGraphRecursive (&num_migratable_cluster_objects, xadj, adjncy, vertex_weights, edge_weights, &weight_flag, &numbering_flag, &num_partitions, options, &edgecut, newmap); // Place the partitioned objects onto their correct PEs. for (i = 0; i < num_migratable_cluster_objects; i++) { partition = newmap[i]; pe = partition_to_pe_map[partition]; index = migratable_cluster_objects[i]; /* WRONG! for (j = 0; j < Num_Objects; j++) { if ((&Object_Data[j])->secondary_index == index) { (&Object_Data[j])->to_pe = pe; break; } } */ (&Object_Data[index])->to_pe = pe; } // Free memory. delete [] newmap; delete [] edge_weights; delete [] adjncy; delete [] xadj; for (i = 0; i < num_migratable_cluster_objects; i++) { delete [] communication_matrix[i]; } delete [] communication_matrix; delete [] vertex_weights; delete [] partition_to_pe_map; delete [] migratable_cluster_objects; }
void InitKWayPartitioningRB(ctrl_t *ctrl, graph_t *graph) { idx_t i, options[METIS_NOPTIONS], curobj=0; idx_t *bestwhere=NULL; real_t *ubvec=NULL; int status; METIS_SetDefaultOptions(options); options[METIS_OPTION_NITER] = 10; options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT; options[METIS_OPTION_NO2HOP] = ctrl->no2hop; options[METIS_OPTION_ONDISK] = ctrl->ondisk; ubvec = rmalloc(graph->ncon, "InitKWayPartitioning: ubvec"); for (i=0; i<graph->ncon; i++) ubvec[i] = (real_t)pow(ctrl->ubfactors[i], 1.0/log(ctrl->nparts)); switch (ctrl->objtype) { case METIS_OBJTYPE_CUT: case METIS_OBJTYPE_VOL: options[METIS_OPTION_NCUTS] = ctrl->nIparts; status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, &ctrl->nparts, ctrl->tpwgts, ubvec, options, &curobj, graph->where); if (status != METIS_OK) gk_errexit(SIGERR, "Failed during initial partitioning\n"); break; #ifdef XXX /* This does not seem to help */ case METIS_OBJTYPE_VOL: bestwhere = imalloc(graph->nvtxs, "InitKWayPartitioning: bestwhere"); options[METIS_OPTION_NCUTS] = 2; ntrials = (ctrl->nIparts+1)/2; for (i=0; i<ntrials; i++) { status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj, graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt, &ctrl->nparts, ctrl->tpwgts, ubvec, options, &curobj, graph->where); if (status != METIS_OK) gk_errexit(SIGERR, "Failed during initial partitioning\n"); curobj = ComputeVolume(graph, graph->where); if (i == 0 || bestobj > curobj) { bestobj = curobj; if (i < ntrials-1) icopy(graph->nvtxs, graph->where, bestwhere); } if (bestobj == 0) break; } if (bestobj != curobj) icopy(graph->nvtxs, bestwhere, graph->where); break; #endif default: gk_errexit(SIGERR, "Unknown objtype: %d\n", ctrl->objtype); } gk_free((void **)&ubvec, &bestwhere, LTERM); }
/*It uses METIS library to accomplish that*/ void TopoCentLB::computePartitions(CentralLB::LDStats *stats,int count,int *newmap) { int numobjs = stats->n_objs; int i, j, m; // allocate space for the computing data double *objtime = new double[numobjs]; int *objwt = new int[numobjs]; int *origmap = new int[numobjs]; LDObjHandle *handles = new LDObjHandle[numobjs]; for(i=0;i<numobjs;i++) { objtime[i] = 0.0; objwt[i] = 0; origmap[i] = 0; } //Prepare compute loads for METIS library for (i=0; i<stats->n_objs; i++) { LDObjData &odata = stats->objData[i]; if (!odata.migratable) CmiAbort("MetisLB doesnot dupport nonmigratable object.\n"); int frompe = stats->from_proc[i]; origmap[i] = frompe; objtime[i] = odata.wallTime*stats->procs[frompe].pe_speed; handles[i] = odata.handle; } // to convert the weights on vertices to integers double max_objtime = objtime[0]; for(i=0; i<numobjs; i++) { if(max_objtime < objtime[i]) max_objtime = objtime[i]; } int maxobj=0; int totalwt=0; double ratio = 1000.0/max_objtime; for(i=0; i<numobjs; i++) { objwt[i] = (int)(objtime[i]*ratio); if(maxobj<objwt[i]) maxobj=objwt[i]; totalwt+=objwt[i]; } int **comm = new int*[numobjs]; for (i=0; i<numobjs; i++) { comm[i] = new int[numobjs]; for (j=0; j<numobjs; j++) { comm[i][j] = 0; } } //Prepare communication for METIS library const int csz = stats->n_comm; for(i=0; i<csz; i++) { LDCommData &cdata = stats->commData[i]; //if(cdata.from_proc() || cdata.receiver.get_type() != LD_OBJ_MSG) //continue; if(!cdata.from_proc() && cdata.receiver.get_type() == LD_OBJ_MSG){ int senderID = stats->getHash(cdata.sender); int recverID = stats->getHash(cdata.receiver.get_destObj()); CmiAssert(senderID < numobjs); CmiAssert(recverID < numobjs); comm[senderID][recverID] += cdata.messages; comm[recverID][senderID] += cdata.messages; //Use bytes or messages -- do i include messages for objlist too...?? } else if (cdata.receiver.get_type() == LD_OBJLIST_MSG) { //CkPrintf("in objlist..\n"); int nobjs; LDObjKey *objs = cdata.receiver.get_destObjs(nobjs); int senderID = stats->getHash(cdata.sender); for (j=0; j<nobjs; j++) { int recverID = stats->getHash(objs[j]); if((senderID == -1)||(recverID == -1)) if (_lb_args.migObjOnly()) continue; else CkAbort("Error in search\n"); comm[senderID][recverID] += cdata.messages; comm[recverID][senderID] += cdata.messages; } } } // ignore messages sent from an object to itself for (i=0; i<numobjs; i++) comm[i][i] = 0; // construct the graph in CSR format int *xadj = new int[numobjs+1]; int numedges = 0; for(i=0;i<numobjs;i++) { for(j=0;j<numobjs;j++) { if(comm[i][j] != 0) numedges++; } } int *adjncy = new int[numedges]; int *edgewt = new int[numedges]; int factor = 10; xadj[0] = 0; int count4all = 0; for (i=0; i<numobjs; i++) { for (j=0; j<numobjs; j++) { if (comm[i][j] != 0) { adjncy[count4all] = j; edgewt[count4all++] = comm[i][j]/factor; } } xadj[i+1] = count4all; } //Call METIS routine int wgtflag = 3; // Weights both on vertices and edges int numflag = 0; // C Style numbering int options[5]; int edgecut; options[0] = 0; if (count < 1) { CkPrintf("error: Number of Pe less than 1!"); } else if (count == 1) { for(m=0;m<numobjs;m++) newmap[i] = origmap[i]; } else { /* if (count > 8) METIS_PartGraphKway(&numobjs, xadj, adjncy, objwt, edgewt, &wgtflag, &numflag, &count, options, &edgecut, newmap); else METIS_PartGraphRecursive(&numobjs, xadj, adjncy, objwt, edgewt, &wgtflag, &numflag, &count, options, &edgecut, newmap); */ METIS_PartGraphRecursive(&numobjs, xadj, adjncy, objwt, edgewt, &wgtflag, &numflag, &count, options, &edgecut, newmap); } //Debugging code: Checking load on each partition if(_lb_args.debug() >=2){ int total=0; int *chkwt = new int[count]; for(i=0;i<count;i++) chkwt[i]=0; for(i=0;i<numobjs;i++){ chkwt[newmap[i]] += objwt[i]; total += objwt[i]; } for(i=0;i<count;i++) CkPrintf("%d -- %d\n",i,chkwt[i]); CkPrintf("Totalwt of all partitions after call to METIS:%d, Avg is %d\n",total,total/count); } //Clean up all the variables allocated in this routine for(i=0;i<numobjs;i++) delete[] comm[i]; delete[] comm; delete[] objtime; delete[] xadj; delete[] adjncy; delete[] objwt; delete[] edgewt; delete[] handles; delete[] origmap; }
/************************************************************************* * This function is the entry point of the initial partition algorithm * that does recursive bissection. * This algorithm assembles the graph to all the processors and preceeds * by parallelizing the recursive bisection step. **************************************************************************/ void InitPartition(ctrl_t *ctrl, graph_t *graph) { idx_t i, j, ncon, mype, npes, gnvtxs, ngroups; idx_t *xadj, *adjncy, *adjwgt, *vwgt; idx_t *part, *gwhere0, *gwhere1; idx_t *tmpwhere, *tmpvwgt, *tmpxadj, *tmpadjncy, *tmpadjwgt; graph_t *agraph; idx_t lnparts, fpart, fpe, lnpes; idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut; real_t *tpwgts, *tpwgts2, *lbvec, lbsum, min_lbsum, wsum; MPI_Comm ipcomm; struct { double sum; int rank; } lpesum, gpesum; WCOREPUSH; ncon = graph->ncon; ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, ctrl->npes), 1); IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm)); IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr)); lbvec = rwspacemalloc(ctrl, ncon); /* assemble the graph to all the processors */ agraph = AssembleAdaptiveGraph(ctrl, graph); gnvtxs = agraph->nvtxs; /* make a copy of the graph's structure for later */ xadj = icopy(gnvtxs+1, agraph->xadj, iwspacemalloc(ctrl, gnvtxs+1)); vwgt = icopy(gnvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, gnvtxs*ncon)); adjncy = icopy(agraph->nedges, agraph->adjncy, iwspacemalloc(ctrl, agraph->nedges)); adjwgt = icopy(agraph->nedges, agraph->adjwgt, iwspacemalloc(ctrl, agraph->nedges)); part = iwspacemalloc(ctrl, gnvtxs); /* create different processor groups */ gkMPI_Comm_split(ctrl->gcomm, ctrl->mype % ngroups, 0, &ipcomm); gkMPI_Comm_rank(ipcomm, &mype); gkMPI_Comm_size(ipcomm, &npes); /* Go into the recursive bisection */ METIS_SetDefaultOptions(moptions); moptions[METIS_OPTION_SEED] = ctrl->sync + (ctrl->mype % ngroups) + 1; tpwgts = ctrl->tpwgts; tpwgts2 = rwspacemalloc(ctrl, 2*ncon); lnparts = ctrl->nparts; fpart = fpe = 0; lnpes = npes; while (lnpes > 1 && lnparts > 1) { /* determine the weights of the two partitions as a function of the weight of the target partition weights */ for (j=(lnparts>>1), i=0; i<ncon; i++) { tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon); tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon); wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]); tpwgts2[i] *= wsum; tpwgts2[ncon+i] *= wsum; } METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt, &twoparts, tpwgts2, NULL, moptions, &edgecut, part); /* pick one of the branches */ if (mype < fpe+lnpes/2) { KeepPart(ctrl, agraph, part, 0); lnpes = lnpes/2; lnparts = lnparts/2; } else { KeepPart(ctrl, agraph, part, 1); fpart = fpart + lnparts/2; fpe = fpe + lnpes/2; lnpes = lnpes - lnpes/2; lnparts = lnparts - lnparts/2; } } gwhere0 = iset(gnvtxs, 0, iwspacemalloc(ctrl, gnvtxs)); gwhere1 = iwspacemalloc(ctrl, gnvtxs); if (lnparts == 1) { /* Case npes is greater than or equal to nparts */ /* Only the first process will assign labels (for the reduction to work) */ if (mype == fpe) { for (i=0; i<agraph->nvtxs; i++) gwhere0[agraph->label[i]] = fpart; } } else { /* Case in which npes is smaller than nparts */ /* create the normalized tpwgts for the lnparts from ctrl->tpwgts */ tpwgts = rwspacemalloc(ctrl, lnparts*ncon); for (j=0; j<ncon; j++) { for (wsum=0.0, i=0; i<lnparts; i++) { tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j]; wsum += tpwgts[i*ncon+j]; } for (wsum=1.0/wsum, i=0; i<lnparts; i++) tpwgts[i*ncon+j] *= wsum; } METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions, &edgecut, part); for (i=0; i<agraph->nvtxs; i++) gwhere0[agraph->label[i]] = fpart + part[i]; } gkMPI_Allreduce((void *)gwhere0, (void *)gwhere1, gnvtxs, IDX_T, MPI_SUM, ipcomm); if (ngroups > 1) { tmpxadj = agraph->xadj; tmpadjncy = agraph->adjncy; tmpadjwgt = agraph->adjwgt; tmpvwgt = agraph->vwgt; tmpwhere = agraph->where; agraph->xadj = xadj; agraph->adjncy = adjncy; agraph->adjwgt = adjwgt; agraph->vwgt = vwgt; agraph->where = gwhere1; agraph->vwgt = vwgt; agraph->nvtxs = gnvtxs; edgecut = ComputeSerialEdgeCut(agraph); ComputeSerialBalance(ctrl, agraph, gwhere1, lbvec); lbsum = rsum(ncon, lbvec, 1); gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ctrl->gcomm); gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ctrl->gcomm); lpesum.sum = lbsum; if (min_lbsum < UNBALANCE_FRACTION*ncon) { if (lbsum < UNBALANCE_FRACTION*ncon) lpesum.sum = edgecut; else lpesum.sum = max_cut; } lpesum.rank = ctrl->mype; gkMPI_Allreduce((void *)&lpesum, (void *)&gpesum, 1, MPI_DOUBLE_INT, MPI_MINLOC, ctrl->gcomm); gkMPI_Bcast((void *)gwhere1, gnvtxs, IDX_T, gpesum.rank, ctrl->gcomm); agraph->xadj = tmpxadj; agraph->adjncy = tmpadjncy; agraph->adjwgt = tmpadjwgt; agraph->vwgt = tmpvwgt; agraph->where = tmpwhere; } icopy(graph->nvtxs, gwhere1+graph->vtxdist[ctrl->mype], graph->where); FreeGraph(agraph); gkMPI_Comm_free(&ipcomm); IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm)); IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->InitPartTmr)); WCOREPOP; }
// Call Metis with options from dictionary. Foam::label Foam::metisDecomp::decompose ( const List<int>& adjncy, const List<int>& xadj, const scalarField& cWeights, List<int>& finalDecomp ) { // C style numbering int numFlag = 0; // Method of decomposition // recursive: multi-level recursive bisection (default) // k-way: multi-level k-way word method("k-way"); int numCells = xadj.size()-1; // decomposition options. 0 = use defaults List<int> options(5, 0); // processor weights initialised with no size, only used if specified in // a file Field<floatScalar> processorWeights; // cell weights (so on the vertices of the dual) List<int> cellWeights; // face weights (so on the edges of the dual) List<int> faceWeights; // Check for externally provided cellweights and if so initialise weights scalar minWeights = gMin(cWeights); if (cWeights.size() > 0) { if (minWeights <= 0) { WarningIn ( "metisDecomp::decompose" "(const pointField&, const scalarField&)" ) << "Illegal minimum weight " << minWeights << endl; } if (cWeights.size() != numCells) { FatalErrorIn ( "metisDecomp::decompose" "(const pointField&, const scalarField&)" ) << "Number of cell weights " << cWeights.size() << " does not equal number of cells " << numCells << exit(FatalError); } // Convert to integers. cellWeights.setSize(cWeights.size()); forAll(cellWeights, i) { cellWeights[i] = int(cWeights[i]/minWeights); } } // Check for user supplied weights and decomp options if (decompositionDict_.found("metisCoeffs")) { const dictionary& metisCoeffs = decompositionDict_.subDict("metisCoeffs"); word weightsFile; if (metisCoeffs.readIfPresent("method", method)) { if (method != "recursive" && method != "k-way") { FatalErrorIn("metisDecomp::decompose()") << "Method " << method << " in metisCoeffs in dictionary : " << decompositionDict_.name() << " should be 'recursive' or 'k-way'" << exit(FatalError); } Info<< "metisDecomp : Using Metis method " << method << nl << endl; } if (metisCoeffs.readIfPresent("options", options)) { if (options.size() != 5) { FatalErrorIn("metisDecomp::decompose()") << "Number of options in metisCoeffs in dictionary : " << decompositionDict_.name() << " should be 5" << exit(FatalError); } Info<< "metisDecomp : Using Metis options " << options << nl << endl; } if (metisCoeffs.readIfPresent("processorWeights", processorWeights)) { processorWeights /= sum(processorWeights); if (processorWeights.size() != nProcessors_) { FatalErrorIn("metisDecomp::decompose(const pointField&)") << "Number of processor weights " << processorWeights.size() << " does not equal number of domains " << nProcessors_ << exit(FatalError); } } //if (metisCoeffs.readIfPresent("cellWeightsFile", weightsFile)) //{ // Info<< "metisDecomp : Using cell-based weights." << endl; // // IOList<int> cellIOWeights // ( // IOobject // ( // weightsFile, // mesh_.time().timeName(), // mesh_, // IOobject::MUST_READ, // IOobject::AUTO_WRITE // ) // ); // cellWeights.transfer(cellIOWeights); // // if (cellWeights.size() != xadj.size()-1) // { // FatalErrorIn("metisDecomp::decompose(const pointField&)") // << "Number of cell weights " << cellWeights.size() // << " does not equal number of cells " << xadj.size()-1 // << exit(FatalError); // } //} } int nProcs = nProcessors_; // output: cell -> processor addressing finalDecomp.setSize(numCells); // output: number of cut edges int edgeCut = 0; // Vertex weight info int wgtFlag = 0; int* vwgtPtr = NULL; int* adjwgtPtr = NULL; if (cellWeights.size()) { vwgtPtr = cellWeights.begin(); wgtFlag += 2; // Weights on vertices } if (faceWeights.size()) { adjwgtPtr = faceWeights.begin(); wgtFlag += 1; // Weights on edges } if (method == "recursive") { if (processorWeights.size()) { METIS_WPartGraphRecursive ( &numCells, // num vertices in graph const_cast<List<int>&>(xadj).begin(), // indexing into adjncy const_cast<List<int>&>(adjncy).begin(), // neighbour info vwgtPtr, // vertexweights adjwgtPtr, // no edgeweights &wgtFlag, &numFlag, &nProcs, processorWeights.begin(), options.begin(), &edgeCut, finalDecomp.begin() ); } else { METIS_PartGraphRecursive ( &numCells, // num vertices in graph const_cast<List<int>&>(xadj).begin(), // indexing into adjncy const_cast<List<int>&>(adjncy).begin(), // neighbour info vwgtPtr, // vertexweights adjwgtPtr, // no edgeweights &wgtFlag, &numFlag, &nProcs, options.begin(), &edgeCut, finalDecomp.begin() ); } } else { if (processorWeights.size()) { METIS_WPartGraphKway ( &numCells, // num vertices in graph const_cast<List<int>&>(xadj).begin(), // indexing into adjncy const_cast<List<int>&>(adjncy).begin(), // neighbour info vwgtPtr, // vertexweights adjwgtPtr, // no edgeweights &wgtFlag, &numFlag, &nProcs, processorWeights.begin(), options.begin(), &edgeCut, finalDecomp.begin() ); } else { METIS_PartGraphKway ( &numCells, // num vertices in graph const_cast<List<int>&>(xadj).begin(), // indexing into adjncy const_cast<List<int>&>(adjncy).begin(), // neighbour info vwgtPtr, // vertexweights adjwgtPtr, // no edgeweights &wgtFlag, &numFlag, &nProcs, options.begin(), &edgeCut, finalDecomp.begin() ); } } return edgeCut; }
/************************************************************************* * This function partitions a finite element mesh by partitioning its dual * graph using KMETIS and then assigning nodes in a load balanced fashion. **************************************************************************/ int METIS_PartMeshDual(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind, idx_t *vwgt, idx_t *vsize, idx_t *ncommon, idx_t *nparts, real_t *tpwgts, idx_t *options, idx_t *objval, idx_t *epart, idx_t *npart) { int sigrval=0, renumber=0, ptype; idx_t i, j; idx_t *xadj=NULL, *adjncy=NULL, *nptr=NULL, *nind=NULL; idx_t ncon=1, pnumflag=0; int rstatus = METIS_OK; /* set up malloc cleaning code and signal catchers */ if (!gk_malloc_init()) return METIS_ERROR_MEMORY; gk_sigtrap(); if ((sigrval = gk_sigcatch()) != 0) goto SIGTHROW; renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0); ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY); /* renumber the mesh */ if (renumber) { ChangeMesh2CNumbering(*ne, eptr, eind); options[METIS_OPTION_NUMBERING] = 0; } /* get the dual graph */ rstatus = METIS_MeshToDual(ne, nn, eptr, eind, ncommon, &pnumflag, &xadj, &adjncy); if (rstatus != METIS_OK) raise(SIGERR); /* partition the graph */ if (ptype == METIS_PTYPE_KWAY) rstatus = METIS_PartGraphKway(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL, nparts, tpwgts, NULL, options, objval, epart); else rstatus = METIS_PartGraphRecursive(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL, nparts, tpwgts, NULL, options, objval, epart); if (rstatus != METIS_OK) raise(SIGERR); /* construct the node-element list */ nptr = ismalloc(*nn+1, 0, "METIS_PartMeshDual: nptr"); nind = imalloc(eptr[*ne], "METIS_PartMeshDual: nind"); for (i=0; i<*ne; i++) { for (j=eptr[i]; j<eptr[i+1]; j++) nptr[eind[j]]++; } MAKECSR(i, *nn, nptr); for (i=0; i<*ne; i++) { for (j=eptr[i]; j<eptr[i+1]; j++) nind[nptr[eind[j]]++] = i; } SHIFTCSR(i, *nn, nptr); /* partition the other side of the mesh */ InduceRowPartFromColumnPart(*nn, nptr, nind, npart, epart, *nparts, tpwgts); gk_free((void **)&nptr, &nind, LTERM); SIGTHROW: if (renumber) { ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart); options[METIS_OPTION_NUMBERING] = 1; } METIS_Free(xadj); METIS_Free(adjncy); gk_siguntrap(); gk_malloc_cleanup(0); return metis_rcode(sigrval); }
/************************************************************************* * This function is the entry point of the initial balancing algorithm. * This algorithm assembles the graph to all the processors and preceeds * with the balancing step. **************************************************************************/ void Balance_Partition(ctrl_t *ctrl, graph_t *graph) { idx_t i, j, nvtxs, nedges, ncon; idx_t mype, npes, srnpes, srmype; idx_t *vtxdist, *xadj, *adjncy, *adjwgt, *vwgt, *vsize; idx_t *part, *lwhere, *home; idx_t lnparts, fpart, fpe, lnpes, ngroups; idx_t *rcounts, *rdispls; idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut; idx_t sr_pe, gd_pe, sr, gd, who_wins; real_t my_cut, my_totalv, my_cost = -1.0, my_balance = -1.0, wsum; real_t rating, max_rating, your_cost = -1.0, your_balance = -1.0; real_t lbsum, min_lbsum, *lbvec, *tpwgts, *tpwgts2, buffer[2]; graph_t *agraph, cgraph; ctrl_t *myctrl; MPI_Status status; MPI_Comm ipcomm, srcomm; struct { double cost; int rank; } lpecost, gpecost; IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr)); WCOREPUSH; vtxdist = graph->vtxdist; agraph = AssembleAdaptiveGraph(ctrl, graph); nvtxs = cgraph.nvtxs = agraph->nvtxs; nedges = cgraph.nedges = agraph->nedges; ncon = cgraph.ncon = agraph->ncon; xadj = cgraph.xadj = icopy(nvtxs+1, agraph->xadj, iwspacemalloc(ctrl, nvtxs+1)); vwgt = cgraph.vwgt = icopy(nvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, nvtxs*ncon)); vsize = cgraph.vsize = icopy(nvtxs, agraph->vsize, iwspacemalloc(ctrl, nvtxs)); adjncy = cgraph.adjncy = icopy(nedges, agraph->adjncy, iwspacemalloc(ctrl, nedges)); adjwgt = cgraph.adjwgt = icopy(nedges, agraph->adjwgt, iwspacemalloc(ctrl, nedges)); part = cgraph.where = agraph->where = iwspacemalloc(ctrl, nvtxs); lwhere = iwspacemalloc(ctrl, nvtxs); home = iwspacemalloc(ctrl, nvtxs); lbvec = rwspacemalloc(ctrl, graph->ncon); /****************************************/ /****************************************/ if (ctrl->ps_relation == PARMETIS_PSR_UNCOUPLED) { WCOREPUSH; rcounts = iwspacemalloc(ctrl, ctrl->npes); rdispls = iwspacemalloc(ctrl, ctrl->npes+1); for (i=0; i<ctrl->npes; i++) rdispls[i] = rcounts[i] = vtxdist[i+1]-vtxdist[i]; MAKECSR(i, ctrl->npes, rdispls); gkMPI_Allgatherv((void *)graph->home, graph->nvtxs, IDX_T, (void *)part, rcounts, rdispls, IDX_T, ctrl->comm); for (i=0; i<agraph->nvtxs; i++) home[i] = part[i]; WCOREPOP; /* local frees */ } else { for (i=0; i<ctrl->npes; i++) { for (j=vtxdist[i]; j<vtxdist[i+1]; j++) part[j] = home[j] = i; } } /* Ensure that the initial partitioning is legal */ for (i=0; i<agraph->nvtxs; i++) { if (part[i] >= ctrl->nparts) part[i] = home[i] = part[i] % ctrl->nparts; if (part[i] < 0) part[i] = home[i] = (-1*part[i]) % ctrl->nparts; } /****************************************/ /****************************************/ IFSET(ctrl->dbglvl, DBG_REFINEINFO, ComputeSerialBalance(ctrl, agraph, agraph->where, lbvec)); IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "input cut: %"PRIDX", balance: ", ComputeSerialEdgeCut(agraph))); for (i=0; i<agraph->ncon; i++) IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "%.3"PRREAL" ", lbvec[i])); IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "\n")); /****************************************/ /* Split the processors into two groups */ /****************************************/ sr = (ctrl->mype % 2 == 0) ? 1 : 0; gd = (ctrl->mype % 2 == 1) ? 1 : 0; if (graph->ncon > MAX_NCON_FOR_DIFFUSION || ctrl->npes == 1) { sr = 1; gd = 0; } sr_pe = 0; gd_pe = 1; gkMPI_Comm_split(ctrl->gcomm, sr, 0, &ipcomm); gkMPI_Comm_rank(ipcomm, &mype); gkMPI_Comm_size(ipcomm, &npes); if (sr == 1) { /* Half of the processors do scratch-remap */ ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, npes), 1); gkMPI_Comm_split(ipcomm, mype % ngroups, 0, &srcomm); gkMPI_Comm_rank(srcomm, &srmype); gkMPI_Comm_size(srcomm, &srnpes); METIS_SetDefaultOptions(moptions); moptions[METIS_OPTION_SEED] = ctrl->sync + (mype % ngroups) + 1; tpwgts = ctrl->tpwgts; tpwgts2 = rwspacemalloc(ctrl, 2*ncon); iset(nvtxs, 0, lwhere); lnparts = ctrl->nparts; fpart = fpe = 0; lnpes = srnpes; while (lnpes > 1 && lnparts > 1) { PASSERT(ctrl, agraph->nvtxs > 1); /* determine the weights of the two partitions as a function of the weight of the target partition weights */ for (j=(lnparts>>1), i=0; i<ncon; i++) { tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon); tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon); wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]); tpwgts2[i] *= wsum; tpwgts2[ncon+i] *= wsum; } METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt, &twoparts, tpwgts2, NULL, moptions, &edgecut, part); /* pick one of the branches */ if (srmype < fpe+lnpes/2) { KeepPart(ctrl, agraph, part, 0); lnpes = lnpes/2; lnparts = lnparts/2; } else { KeepPart(ctrl, agraph, part, 1); fpart = fpart + lnparts/2; fpe = fpe + lnpes/2; lnpes = lnpes - lnpes/2; lnparts = lnparts - lnparts/2; } } if (lnparts == 1) { /* Case in which srnpes is greater or equal to nparts */ /* Only the first process will assign labels (for the reduction to work) */ if (srmype == fpe) { for (i=0; i<agraph->nvtxs; i++) lwhere[agraph->label[i]] = fpart; } } else { /* Case in which srnpes is smaller than nparts */ /* create the normalized tpwgts for the lnparts from ctrl->tpwgts */ tpwgts = rwspacemalloc(ctrl, lnparts*ncon); for (j=0; j<ncon; j++) { for (wsum=0.0, i=0; i<lnparts; i++) { tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j]; wsum += tpwgts[i*ncon+j]; } for (wsum=1.0/wsum, i=0; i<lnparts; i++) tpwgts[i*ncon+j] *= wsum; } METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions, &edgecut, part); for (i=0; i<agraph->nvtxs; i++) lwhere[agraph->label[i]] = fpart + part[i]; } gkMPI_Allreduce((void *)lwhere, (void *)part, nvtxs, IDX_T, MPI_SUM, srcomm); edgecut = ComputeSerialEdgeCut(&cgraph); ComputeSerialBalance(ctrl, &cgraph, part, lbvec); lbsum = rsum(ncon, lbvec, 1); gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ipcomm); gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ipcomm); lpecost.rank = ctrl->mype; lpecost.cost = lbsum; if (min_lbsum < UNBALANCE_FRACTION * (real_t)(ncon)) { if (lbsum < UNBALANCE_FRACTION * (real_t)(ncon)) lpecost.cost = (double)edgecut; else lpecost.cost = (double)max_cut + lbsum; } gkMPI_Allreduce((void *)&lpecost, (void *)&gpecost, 1, MPI_DOUBLE_INT, MPI_MINLOC, ipcomm); if (ctrl->mype == gpecost.rank && ctrl->mype != sr_pe) gkMPI_Send((void *)part, nvtxs, IDX_T, sr_pe, 1, ctrl->comm); if (ctrl->mype != gpecost.rank && ctrl->mype == sr_pe) gkMPI_Recv((void *)part, nvtxs, IDX_T, gpecost.rank, 1, ctrl->comm, &status); if (ctrl->mype == sr_pe) { icopy(nvtxs, part, lwhere); SerialRemap(ctrl, &cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts); } gkMPI_Comm_free(&srcomm); }