int gpartition(const vector< set<int> > &graph, int npartitions, int partition_method, vector<int> &decomp){
// If no partitioning method is set, choose a default.
if(partition_method<0){
if(npartitions<=8)
partition_method = 0; // METIS PartGraphRecursive
else
partition_method = 1; // METIS PartGraphKway
}
int nnodes = graph.size();
// Compress graph
vector<idxtype> xadj(nnodes+1), adjncy;
int pos=0;
xadj[0]=1;
for(int i=0;i<nnodes;i++){
for(set<int>::iterator jt=graph[i].begin();jt!=graph[i].end();jt++){
adjncy.push_back(*jt);
pos++;
}
xadj[i+1] = pos+1;
}
// Partition graph
decomp.resize(nnodes);
int wgtflag=0, numflag=1, edgecut=0;
#ifdef PARMETIS_V3
int options[] = {0};
#else
idx_t nbc=1;
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING]=1;
#endif
if(partition_method){
#ifdef PARMETIS_V3
METIS_PartGraphKway(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag,
&numflag, &npartitions, options, &edgecut, &(decomp[0]));
#else
METIS_PartGraphKway(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL,
&npartitions, NULL, NULL,options,&edgecut,
&(decomp[0]));
#endif
}else{
#ifdef PARMETIS_V3
METIS_PartGraphRecursive(&nnodes, &(xadj[0]), &(adjncy[0]), NULL, NULL, &wgtflag,
&numflag, &npartitions, options, &edgecut, &(decomp[0]));
#else
METIS_PartGraphRecursive(&nnodes,&nbc,&(xadj[0]),&(adjncy[0]), NULL, NULL,NULL,
&npartitions, NULL, NULL,options,&edgecut,
&(decomp[0]));
#endif
}
// number from zero
for(int i=0;i<nnodes;i++)
decomp[i]--;
return edgecut;
}
void metis_wrapper(int *n, int *xadj, int *adjncy, int *nparts, int *part)
{
#ifndef METIS_VER_MAJOR
/* Metis 4.X (or METISLib in ParaMetis 3.X) */
int edgecut;
int wgtflag = 0;
int numflag = 1; /* fortran-stype numbering */
int options[5] = { 0, 3, 1, 1, 0 }; /* default values */
METIS_PartGraphRecursive(n, xadj, adjncy, NULL, NULL, &wgtflag, &numflag,
nparts, options, &edgecut, part);
#elif METIS_VER_MAJOR == 5
/* Metis 5.X */
int edgecut;
int ncon = 1;
int options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NUMBERING] = 1; /* fortran-stype numbering */
METIS_PartGraphRecursive(n, &ncon, xadj, adjncy, NULL, NULL, NULL,
nparts, NULL, NULL, options, &edgecut, part);
#else
#error unknown Metis version
#endif
}
/*Partition this mesh's elements into n chunks,
writing each element's 0-based chunk number to elem2chunk.
*/
void FEM_Mesh_partition(const FEM_Mesh *mesh,int nchunks,int *elem2chunk, bool faceGraph)
{
CkThresholdTimer time("FEM Split> Building graph for metis partitioner",1.0);
int nelems=mesh->nElems();
if (nchunks==1) { //Metis doesn't handle this case (!)
for (int i=0;i<nelems;i++) elem2chunk[i]=0;
return;
}
Graph g(nelems);
if (!faceGraph) {
mesh2graph(mesh,&g);
} else {
mesh2graph_face(mesh,&g);
}
int *adjStart; /*Maps elem # -> start index in adjacency list*/
int *adjList; /*Lists adjacent vertices for each element*/
g.toAdjList(adjStart,adjList);
int ecut,ncon=1;
time.start("FEM Split> Calling metis partitioner");
if (nchunks<8) /*Metis manual says recursive version is higher-quality here*/
METIS_PartGraphRecursive(&nelems, &ncon, adjStart, adjList, NULL, NULL, NULL,
&nchunks, NULL, NULL, NULL, &ecut, elem2chunk);
else /*For many chunks, Kway is supposedly faster */
METIS_PartGraphKway(&nelems, &ncon, adjStart, adjList, NULL, NULL, NULL,
&nchunks, NULL, NULL, NULL, &ecut, elem2chunk);
delete[] adjStart;
delete[] adjList;
}
void FC_FUNC_(oct_metis_partgraphrecursive, OCT_METIS_PARTGRAPHRECURSIVE)
(idx_t *nvtxs, idx_t *ncon, idx_t *xadj, idx_t *adjncy, idx_t *nparts,
real_t *tpwgts, real_t *ubvec, idx_t *options, idx_t *objval, idx_t *part)
{
METIS_PartGraphRecursive(nvtxs, ncon, xadj, adjncy, NULL, NULL, NULL, nparts,
tpwgts, ubvec, options, objval, part);
}
void decompose(Graph_part & gp)
{
// Decompose
METIS_PartGraphRecursive(Mg.nvtxs, Mg.ncon, Mg.xadj, Mg.adjncy, Mg.vwgt, Mg.vsize, Mg.adjwgt, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.options, Mg.objval, Mg.part);
// vertex id
size_t id = 0;
// For each vertex store the processor that contain the data
auto it = gp.getVertexIterator();
while (it.isNext())
{
gp.vertex(it).template get<i>() = Mg.part[id];
++id;
++it;
}
}
int do_metis_recursive_partition(network_t *network, options_t *opt, idx_t *part) {
int ret = ORCC_OK;
idx_t ncon = 1;
idx_t metis_opt[METIS_NOPTIONS];
idx_t objval;
adjacency_list *graph, *metis_graph;
assert(network != NULL);
assert(opt != NULL);
assert(part != NULL);
print_orcc_trace(ORCC_VL_VERBOSE_1, "Applying METIS Recursive partition for mapping");
METIS_SetDefaultOptions(metis_opt);
graph = set_graph_from_network(network);
metis_graph = fix_graph_for_metis(graph);
ret = METIS_PartGraphRecursive(&metis_graph->nb_vertices, /* idx_t *nvtxs */
&ncon, /*idx_t *ncon*/
metis_graph->xadj, /*idx_t *xadj*/
metis_graph->adjncy, /*idx_t *adjncy*/
metis_graph->vwgt, /*idx_t *vwgt*/
NULL, /*idx_t *vsize*/
metis_graph->adjwgt, /*idx_t *adjwgt*/
&opt->nb_processors, /*idx_t *nparts*/
NULL, /*real t *tpwgts*/
NULL, /*real t ubvec*/
metis_opt, /*idx_t *options*/
&objval, /*idx_t *objval*/
part); /*idx_t *part*/
check_metis_error(ret);
print_orcc_trace(ORCC_VL_VERBOSE_2, "METIS Edgecut : %d", objval);
delete_graph(graph);
delete_graph(metis_graph);
return ret;
}
void decompose()
{
if (Mg.nparts[0] != 1)
{
// Decompose
METIS_PartGraphRecursive(Mg.nvtxs, Mg.ncon, Mg.xadj, Mg.adjncy, Mg.vwgt, Mg.vsize, Mg.adjwgt, Mg.nparts, Mg.tpwgts, Mg.ubvec, Mg.options, Mg.objval, Mg.part);
// vertex id
size_t id = 0;
// For each vertex store the processor that contain the data
auto it = g.getVertexIterator();
while (it.isNext())
{
g.vertex(it).template get<i>() = Mg.part[id];
++id;
++it;
}
}
else
{
// Trivially assign all the domains to the processor 0
auto it = g.getVertexIterator();
while (it.isNext())
{
g.vertex(it).template get<i>() = 0;
++it;
}
}
}
void METIS_PARTGRAPHRECURSIVE(int *nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part)
{
METIS_PartGraphRecursive(nvtxs, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, part);
}
void metis_partgraphrecursive__(int *nvtxs, idxtype *xadj, idxtype *adjncy, idxtype *vwgt, idxtype *adjwgt, int *wgtflag, int *numflag, int *nparts, int *options, int *edgecut, idxtype *part)
{
METIS_PartGraphRecursive(nvtxs, xadj, adjncy, vwgt, adjwgt, wgtflag, numflag, nparts, options, edgecut, part);
}
int main()
{
/* We aim to partition the following mesh into 4 parts:
*
* 0---1---2---3---4
* | | | | |
* 5---6---7---8---9
* | | | | |
* 10--11--12--13--14
*
*/
const int nVertices = 15;
const int nNeighs = 44;
idx_t xadj [nVertices+1] = { 0, 2, 5, 8, 11, 13, 16, 20, 24, 28, 31, 33, 36, 39, 42, 44 };
idx_t adjncy [nNeighs] = { 1, 5, 0, 2, 6, 1, 3, 7, 2, 4, 8, 3, 9, 0, 6, 10, 1, 5, 7, 11,
2, 6, 8, 12, 3, 7, 9, 13, 4, 8, 14, 5, 11, 6, 10, 12, 7, 11,
13, 8, 12, 14, 9, 13 };
std::cout << "Before Partitioning:" << std::endl;
std::cout << std::endl;
std::cout << "0---0---0---0---0" << std::endl;
std::cout << "| | | | |" << std::endl;
std::cout << "0---0---0---0---0" << std::endl;
std::cout << "| | | | |" << std::endl;
std::cout << "0---0---0---0---0" << std::endl;
std::cout << std::endl;
// partition the graph using metis
idx_t nvtxs = nVertices; // number of vertices
idx_t ncon = 1; // number of balancing constraints
idx_t *vwgt = NULL; // vertex weights
idx_t *vsize = NULL; // vertex sizes
idx_t *adjwgt = NULL; // edge weights
idx_t nparts = 4; // number of partitions
real_t *tpwgts = NULL; // target partition weight
real_t *ubvec = NULL; // load balance tolerance for each constraint
idx_t options[METIS_NOPTIONS]; // array of options
METIS_SetDefaultOptions( options ); // use defaults
idx_t objval; // edge-cut on return
std::vector< idx_t > colors( nVertices, 0 ); // ranks
int ierr = METIS_PartGraphRecursive( &nvtxs, &ncon, &xadj[0], &adjncy[0], vwgt, vsize,
adjwgt, &nparts, tpwgts, ubvec, options, &objval, &colors[0] );
if ( ierr != METIS_OK )
throw std::runtime_error( "METIS graph partitioning failed!" );
std::cout << "After Partitioning:" << std::endl;
std::cout << std::endl;
std::cout << colors[0] << "---" << colors[1] << "---" << colors[2] <<
"---" << colors[3] << "---" << colors[4] << std::endl;
std::cout << "| | | | |" << std::endl;
std::cout << colors[5] << "---" << colors[6] << "---" << colors[7] <<
"---" << colors[8] << "---" << colors[9] << std::endl;
std::cout << "| | | | |" << std::endl;
std::cout << colors[10] << "---" << colors[11] << "---" << colors[12] <<
"---" << colors[13] << "---" << colors[14] << std::endl;
std::cout << std::endl;
return 0;
}
JNIEXPORT jint JNICALL Java_jprime_JMetis_PartitionGraph(
JNIEnv * env,
jobject obj,
jboolean j_force_PartGraphRecursive,
jboolean j_force_PartGraphKway,
jint j_num_vertices,
jintArray j_xadj,
jintArray j_adjncy,
jintArray j_vwgt,
jintArray j_adjwgt,
jint j_wgtflag,
jint j_nparts,
jintArray j_options,
jintArray j_partioned_nodes) {
mysrand(7654321L);
//delcare vars
jint * xadj=NULL, * adjncy=NULL,* vwgt=NULL, * adjwgt=NULL, * options=NULL, * partioned_nodes=NULL;
int idx, edges_cut=-1, num_vertices=0, nparts=0, wgtflag=0, numflag=0;
jsize xadj_len, adjncy_len, vwgt_len, adjwgt_len, options_len, partioned_nodes_len;
//copy the jsize vars
num_vertices=j_num_vertices;
nparts=j_nparts;
wgtflag=j_wgtflag;
//get array lengths
xadj_len = (*env)->GetArrayLength(env,j_xadj);
adjncy_len = (*env)->GetArrayLength(env,j_adjncy);
vwgt_len = (*env)->GetArrayLength(env,j_vwgt);
adjwgt_len = (*env)->GetArrayLength(env,j_adjwgt);
options_len = (*env)->GetArrayLength(env,j_options);
partioned_nodes_len = (*env)->GetArrayLength(env,j_partioned_nodes);
//printf("xadj_len=%i, adjncy_len=%i, vwgt_len=%i, adjwgt_len=%i, options_len=%i, partioned_nodes_len=%i\n",xadj_len, adjncy_len, vwgt_len, adjwgt_len, options_len, partioned_nodes_len);
//create/get local copies
xadj = (*env)->GetIntArrayElements(env,j_xadj,0);
adjncy = (*env)->GetIntArrayElements(env,j_adjncy,0);
if(vwgt_len>0) vwgt = (*env)->GetIntArrayElements(env,j_vwgt,0);
if(adjwgt_len>0) adjwgt = (*env)->GetIntArrayElements(env,j_adjwgt,0);
if(options_len>0) options = (*env)->GetIntArrayElements(env,j_options,0);
partioned_nodes = (int*)malloc(sizeof(int)*partioned_nodes_len);
#if 0
printf("num_vertices=%i,wgtflag=%i, numflag=%i, nparts=%i\n", num_vertices, wgtflag, numflag, nparts);
printf("xadj=%p, adjncy=%p, vwgt=%p, adjwgt=%p, options=%p, partioned_nodes=%p\n",
xadj,
adjncy,
vwgt,
adjwgt,
options,
partioned_nodes);
printf("xadj:");
for(idx=0;idx<xadj_len;idx++) {
printf("%i ",xadj[idx]);
}
printf("\n");
printf("adjncy:");
for(idx=0;idx<adjncy_len;idx++) {
printf("%i ",adjncy[idx]);
}
printf("\n");
#endif
//call func
if((j_nparts<8 || j_force_PartGraphRecursive) && !j_force_PartGraphKway) {
METIS_PartGraphRecursive(
&num_vertices,
xadj,
adjncy,
vwgt,
adjwgt,
&wgtflag,
&numflag,
&nparts,
options,
&edges_cut,
partioned_nodes);
}
else {
METIS_PartGraphKway(
&num_vertices,
xadj,
adjncy,
vwgt,
adjwgt,
&wgtflag,
&numflag,
&nparts,
options,
&edges_cut,
partioned_nodes);
}
//pop partioned_nodes
(*env)->SetIntArrayRegion(env,j_partioned_nodes,0,partioned_nodes_len,partioned_nodes);
//free local copies
free(partioned_nodes);
(*env)->ReleaseIntArrayElements(env, j_xadj, xadj, 0);
(*env)->ReleaseIntArrayElements(env, j_adjncy, adjncy, 0);
if(vwgt_len>0) (*env)->ReleaseIntArrayElements(env, j_vwgt, vwgt, 0);
if(adjwgt_len>0) (*env)->ReleaseIntArrayElements(env, j_adjwgt, adjwgt, 0);
if(options_len>0) (*env)->ReleaseIntArrayElements(env, j_options, options, 0);
return edges_cut;
}
/*************************************************************************
* Let the game begin
**************************************************************************/
main(int argc, char *argv[])
{
int i, nparts, options[10];
idxtype *part;
float lbvec[MAXNCON];
GraphType graph;
char filename[256];
int numflag = 0, wgtflag = 0, edgecut;
timer TOTALTmr, METISTmr, IOTmr;
if (argc != 3) {
printf("Usage: %s <GraphFile> <Nparts>\n",argv[0]);
exit(0);
}
strcpy(filename, argv[1]);
nparts = atoi(argv[2]);
if (nparts < 2) {
printf("The number of partitions should be greater than 1!\n");
exit(0);
}
cleartimer(TOTALTmr);
cleartimer(METISTmr);
cleartimer(IOTmr);
starttimer(TOTALTmr);
starttimer(IOTmr);
ReadGraph(&graph, filename, &wgtflag);
if (graph.nvtxs <= 0) {
printf("Empty graph. Nothing to do.\n");
exit(0);
}
stoptimer(IOTmr);
printf("**********************************************************************\n");
printf("%s", METISTITLE);
printf("Graph Information ---------------------------------------------------\n");
printf(" Name: %s, #Vertices: %d, #Edges: %d, #Parts: %d\n", filename, graph.nvtxs, graph.nedges/2, nparts);
if (graph.ncon > 1)
printf(" Balancing Constraints: %d\n", graph.ncon);
printf("\nRecursive Partitioning... -------------------------------------------\n");
part = idxmalloc(graph.nvtxs, "main: part");
options[0] = 0;
starttimer(METISTmr);
if (graph.ncon == 1) {
METIS_PartGraphRecursive(&graph.nvtxs, graph.xadj, graph.adjncy, graph.vwgt, graph.adjwgt,
&wgtflag, &numflag, &nparts, options, &edgecut, part);
}
else {
METIS_mCPartGraphRecursive(&graph.nvtxs, &graph.ncon, graph.xadj, graph.adjncy, graph.vwgt,
graph.adjwgt, &wgtflag, &numflag, &nparts, options, &edgecut, part);
}
stoptimer(METISTmr);
ComputePartitionBalance(&graph, nparts, part, lbvec);
printf(" %d-way Edge-Cut: %7d, Balance: ", nparts, edgecut);
for (i=0; i<graph.ncon; i++)
printf("%5.2f ", lbvec[i]);
printf("\n");
starttimer(IOTmr);
WritePartition(filename, part, graph.nvtxs, nparts);
stoptimer(IOTmr);
stoptimer(TOTALTmr);
printf("\nTiming Information --------------------------------------------------\n");
printf(" I/O: \t\t %7.3f\n", gettimer(IOTmr));
printf(" Partitioning: \t\t %7.3f (PMETIS time)\n", gettimer(METISTmr));
printf(" Total: \t\t %7.3f\n", gettimer(TOTALTmr));
printf("**********************************************************************\n");
GKfree(&graph.xadj, &graph.adjncy, &graph.vwgt, &graph.adjwgt, &part, LTERM);
}
idx_t* gpmetis( int argc, char **argv )
/*************************************************************************/
/*! Let the game begin! */
/*************************************************************************/
//int main(int argc, char *argv[])
{
idx_t i;
char *curptr, *newptr;
idx_t options[METIS_NOPTIONS];
graph_t *graph;
idx_t *part;
idx_t objval;
params_t *params;
int status=0;
gk_optind = 0;
//printf( "argc: %d\n", argc );
//printf( "gk_optind %d\n", gk_optind );
fflush( stdout );
for( i = 0; i < argc; i++ )
{
//printf( "%s*\n", argv[ i ] );
}
params = parse_cmdline(argc, argv);
//printf( "gk_optind %d\n", gk_optind );
//fflush( stdout );
//return NULL;
gk_startcputimer(params->iotimer);
graph = ReadGraph(params);
ReadTPwgts(params, graph->ncon);
gk_stopcputimer(params->iotimer);
/* Check if the graph is contiguous */
if (params->contig && !IsConnected(graph, 0)) {
printf("***The input graph is not contiguous.\n"
"***The specified -contig option will be ignored.\n");
params->contig = 0;
}
/* Get ubvec if supplied */
if (params->ubvecstr) {
params->ubvec = rmalloc(graph->ncon, "main");
curptr = params->ubvecstr;
for (i=0; i<graph->ncon; i++) {
params->ubvec[i] = strtoreal(curptr, &newptr);
if (curptr == newptr)
errexit("Error parsing entry #%"PRIDX" of ubvec [%s] (possibly missing).\n",
i, params->ubvecstr);
curptr = newptr;
}
}
/* Setup iptype */
if (params->iptype == -1) {
if (params->ptype == METIS_PTYPE_RB) {
if (graph->ncon == 1)
params->iptype = METIS_IPTYPE_GROW;
else
params->iptype = METIS_IPTYPE_RANDOM;
}
}
GPPrintInfo(params, graph);
part = imalloc(graph->nvtxs, "main: part");
METIS_SetDefaultOptions(options);
options[METIS_OPTION_OBJTYPE] = params->objtype;
options[METIS_OPTION_CTYPE] = params->ctype;
options[METIS_OPTION_IPTYPE] = params->iptype;
options[METIS_OPTION_RTYPE] = params->rtype;
options[METIS_OPTION_MINCONN] = params->minconn;
options[METIS_OPTION_CONTIG] = params->contig;
options[METIS_OPTION_SEED] = params->seed;
options[METIS_OPTION_NITER] = params->niter;
options[METIS_OPTION_NCUTS] = params->ncuts;
options[METIS_OPTION_UFACTOR] = params->ufactor;
options[METIS_OPTION_DBGLVL] = params->dbglvl;
gk_malloc_init();
gk_startcputimer(params->parttimer);
switch (params->ptype) {
case METIS_PTYPE_RB:
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, part);
break;
case METIS_PTYPE_KWAY:
status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, part);
break;
}
gk_stopcputimer(params->parttimer);
if (gk_GetCurMemoryUsed() != 0)
printf("***It seems that Metis did not free all of its memory! Report this.\n");
params->maxmemory = gk_GetMaxMemoryUsed();
gk_malloc_cleanup(0);
if (status != METIS_OK) {
printf("\n***Metis returned with an error.\n");
}
else {
if (!params->nooutput) {
/* Write the solution */
gk_startcputimer(params->iotimer);
WritePartition(params->filename, part, graph->nvtxs, params->nparts);
gk_stopcputimer(params->iotimer);
}
GPReportResults(params, graph, part, objval);
}
idx_t *r_part = ( idx_t* ) calloc( graph->nvtxs, sizeof( idx_t ) );
for( i = 0; i < graph->nvtxs; i++ )
{
r_part[ i ] = part[ i ];
}
FreeGraph(&graph);
gk_free((void **)&part, LTERM);
gk_free((void **)¶ms->filename, ¶ms->tpwgtsfile, ¶ms->tpwgts,
¶ms->ubvecstr, ¶ms->ubvec, ¶ms, LTERM);
return r_part;
}
void MetisLB::work(LDStats* stats)
{
/** ========================== INITIALIZATION ============================= */
ProcArray *parr = new ProcArray(stats);
ObjGraph *ogr = new ObjGraph(stats);
/** ============================= STRATEGY ================================ */
if (_lb_args.debug() >= 2) {
CkPrintf("[%d] In MetisLB Strategy...\n", CkMyPe());
}
// convert ObjGraph to the adjacency structure
int numVertices = ogr->vertices.size(); // number of vertices
int numEdges = 0; // number of edges
double maxLoad = 0.0;
int i, j, k, vert;
/** remove duplicate edges from recvFrom */
for(i = 0; i < numVertices; i++) {
for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
vert = ogr->vertices[i].sendToList[j].getNeighborId();
for(k = 0; k < ogr->vertices[i].recvFromList.size(); k++) {
if(ogr->vertices[i].recvFromList[k].getNeighborId() == vert) {
ogr->vertices[i].sendToList[j].setNumBytes(ogr->vertices[i].sendToList[j].getNumBytes() + ogr->vertices[i].recvFromList[k].getNumBytes());
ogr->vertices[i].recvFromList.erase(ogr->vertices[i].recvFromList.begin() + k);
}
}
}
}
/** the object load is normalized to an integer between 0 and 256 */
for(i = 0; i < numVertices; i++) {
if(ogr->vertices[i].getVertexLoad() > maxLoad)
maxLoad = ogr->vertices[i].getVertexLoad();
numEdges = numEdges + ogr->vertices[i].sendToList.size() + ogr->vertices[i].recvFromList.size();
}
/* adjacency list */
idx_t *xadj = new idx_t[numVertices + 1];
/* id of the neighbors */
idx_t *adjncy = new idx_t[numEdges];
/* weights of the vertices */
idx_t *vwgt = new idx_t[numVertices];
/* weights of the edges */
idx_t *adjwgt = new idx_t[numEdges];
int edgeNum = 0;
double ratio = 256.0/maxLoad;
for(i = 0; i < numVertices; i++) {
xadj[i] = edgeNum;
vwgt[i] = (int)ceil(ogr->vertices[i].getVertexLoad() * ratio);
for(j = 0; j < ogr->vertices[i].sendToList.size(); j++) {
adjncy[edgeNum] = ogr->vertices[i].sendToList[j].getNeighborId();
adjwgt[edgeNum] = ogr->vertices[i].sendToList[j].getNumBytes();
edgeNum++;
}
for(j = 0; j < ogr->vertices[i].recvFromList.size(); j++) {
adjncy[edgeNum] = ogr->vertices[i].recvFromList[j].getNeighborId();
adjwgt[edgeNum] = ogr->vertices[i].recvFromList[j].getNumBytes();
edgeNum++;
}
}
xadj[i] = edgeNum;
CkAssert(edgeNum == numEdges);
idx_t edgecut; // number of edges cut by the partitioning
idx_t *pemap;
idx_t options[METIS_NOPTIONS];
METIS_SetDefaultOptions(options);
//options[METIS_OPTION_PTYPE] = METIS_PTYPE_RB;
// C style numbering
options[METIS_OPTION_NUMBERING] = 0;
// number of constrains
idx_t ncon = 1;
// number of partitions
idx_t numPes = parr->procs.size();
real_t ubvec[ncon];
// allow 10% imbalance
ubvec[0] = 1.1;
// mapping of objs to partitions
pemap = new idx_t[numVertices];
// Specifies size of vertices for computing the total communication volume
idx_t *vsize = NULL;
// This array of size nparts specifies the desired weight for each partition
// and setting it to NULL indicates graph should be equally divided among
// partitions
real_t *tpwgts = NULL;
int option = 0;
if (WEIGHTED == option) {
// set up the different weights between 0 and 1
tpwgts = new real_t[numPes];
for (i = 0; i < numPes; i++) {
tpwgts[i] = 1.0/(real_t)numPes;
}
} else if (MULTI_CONSTRAINT == option) {
CkAbort("Multiple constraints not implemented.\n");
}
// numVertices: num vertices in the graph; ncon: num balancing constrains
// xadj, adjncy: of size n+1 and adjncy of 2m, adjncy[xadj[i]] through and
// including adjncy[xadj[i+1]-1];
// vwgt: weight of the vertices; vsize: amt of data that needs to be sent
// for ith vertex is vsize[i]
// adjwght: the weight of edges; numPes: total parts
// tpwghts: target partition weight, can pass NULL to equally divide
// ubvec: of size ncon to indicate allowed load imbalance tolerance (> 1.0)
// options: array of options; edgecut: stores the edgecut; pemap: mapping
METIS_PartGraphRecursive(&numVertices, &ncon, xadj, adjncy, vwgt, vsize, adjwgt,
&numPes, tpwgts, ubvec, options, &edgecut, pemap);
delete[] xadj;
delete[] adjncy;
delete[] vwgt;
delete[] adjwgt;
delete[] vsize;
delete[] tpwgts;
if (_lb_args.debug() >= 1) {
CkPrintf("[%d] MetisLB done! \n", CkMyPe());
}
for(i = 0; i < numVertices; i++) {
if(pemap[i] != ogr->vertices[i].getCurrentPe())
ogr->vertices[i].setNewPe(pemap[i]);
}
delete[] pemap;
/** ============================== CLEANUP ================================ */
ogr->convertDecisions(stats);
delete parr;
delete ogr;
}
/**************************************************************************
** This takes objects and partitions them into clusters.
*/
void GridMetisLB::Partition_Objects_Into_Clusters (CentralLB::LDStats *stats)
{
int num_migratable_objects;
int *migratable_objects;
int index;
int num_partitions;
int *partition_to_cluster_map;
int cluster;
int partition;
int partition_count;
int *vertex_weights;
int vertex;
int **communication_matrix;
LDCommData *com_data;
int send_object;
int recv_object;
int send_index;
int recv_index;
LDObjKey *recv_objects;
int num_objects;
int *xadj;
int num_edges;
int *adjncy;
int *edge_weights;
int count;
int weight_flag;
int numbering_flag;
int options[5];
int edgecut;
int *newmap;
int i;
int j;
if (Num_Clusters == 1) {
for (i = 0; i < Num_Objects; i++) {
(&Object_Data[i])->cluster = 0;
}
return;
}
for (i = 0; i < Num_Objects; i++) {
(&Object_Data[i])->secondary_index = -1;
}
// Count the number of migratable objects, which are the only candidates to give to Metis.
// (The non-migratable objects have been placed onto the correct destination PEs earlier.)
// After getting the count, create a migratable_objects[] array to keep track of them.
num_migratable_objects = 0;
for (i = 0; i < Num_Objects; i++) {
if ((&Object_Data[i])->migratable) {
num_migratable_objects += 1;
}
}
migratable_objects = new int[num_migratable_objects];
index = 0;
for (i = 0; i < Num_Objects; i++) {
if ((&Object_Data[i])->migratable) {
(&Object_Data[i])->secondary_index = index;
migratable_objects[index] = i;
index += 1;
}
}
// Compute the number of partitions for Metis, based on the scaled CPU power for each cluster.
// Also create a partition-to-cluster mapping so the output of Metis can be mapped back to clusters.
num_partitions = 0;
for (i = 0; i < Num_Clusters; i++) {
num_partitions += (int) ceil ((&Cluster_Data[i])->scaled_cpu_power);
}
partition_to_cluster_map = new int[num_partitions];
cluster = 0;
partition = 0;
while (partition < num_partitions) {
partition_count = (int) ceil ((&Cluster_Data[cluster])->scaled_cpu_power);
for (i = partition; i < (partition + partition_count); i++) {
partition_to_cluster_map[i] = cluster;
}
partition += partition_count;
cluster += 1;
}
if (CK_LDB_GridMetisLB_Mode == 1) {
vertex_weights = new int[num_migratable_objects];
vertex = 0;
for (i = 0; i < Num_Objects; i++) {
if ((&Object_Data[i])->migratable) {
vertex_weights[vertex] = (int) ceil ((&Object_Data[i])->load * 10000);
vertex += 1;
}
}
}
// Create communication_matrix[] to hold all object-to-object message counts.
communication_matrix = new int *[num_migratable_objects];
for (i = 0; i < num_migratable_objects; i++) {
communication_matrix[i] = new int[num_migratable_objects];
for (j = 0; j < num_migratable_objects; j++) {
communication_matrix[i][j] = 0;
}
}
for (i = 0; i < stats->n_comm; i++) {
com_data = &(stats->commData[i]);
if ((!com_data->from_proc()) && (com_data->recv_type() == LD_OBJ_MSG)) {
send_object = stats->getHash (com_data->sender);
recv_object = stats->getHash (com_data->receiver.get_destObj());
//if ((recv_object == -1) && (stats->complete_flag == 0)) {
if ((send_object < 0) || (send_object > Num_Objects) || (recv_object < 0) || (recv_object > Num_Objects)) {
continue;
}
if ((!(&Object_Data[send_object])->migratable) || (!(&Object_Data[recv_object])->migratable)) {
continue;
}
send_index = (&Object_Data[send_object])->secondary_index;
recv_index = (&Object_Data[recv_object])->secondary_index;
communication_matrix[send_index][recv_index] += com_data->messages;
communication_matrix[recv_index][send_index] += com_data->messages;
} else if (com_data->receiver.get_type() == LD_OBJLIST_MSG) {
send_object = stats->getHash (com_data->sender);
if ((send_object < 0) || (send_object > Num_Objects)) {
continue;
}
if (!(&Object_Data[send_object])->migratable) {
continue;
}
recv_objects = com_data->receiver.get_destObjs (num_objects); // (num_objects is passed by reference)
for (j = 0; j < num_objects; j++) {
recv_object = stats->getHash (recv_objects[j]);
//if (recv_object == -1) {
if ((recv_object < 0) || (recv_object > Num_Objects)) {
continue;
}
if (!(&Object_Data[recv_object])->migratable) {
continue;
}
send_index = (&Object_Data[send_object])->secondary_index;
recv_index = (&Object_Data[recv_object])->secondary_index;
communication_matrix[send_index][recv_index] += com_data->messages;
communication_matrix[recv_index][send_index] += com_data->messages;
}
}
}
for (i = 0; i < num_migratable_objects; i++) {
communication_matrix[i][i] = 0;
}
// Construct a graph in CSR format for input to Metis.
xadj = new int[num_migratable_objects + 1];
num_edges = 0;
for (i = 0; i < num_migratable_objects; i++) {
for (j = 0; j < num_migratable_objects; j++) {
if (communication_matrix[i][j] > 0) {
num_edges += 1;
}
}
}
adjncy = new int[num_edges];
edge_weights = new int[num_edges];
count = 0;
xadj[0] = 0;
for (i = 0; i < num_migratable_objects; i++) {
for (j = 0; j < num_migratable_objects; j++) {
if (communication_matrix[i][j] > 0) {
adjncy[count] = j;
edge_weights[count] = communication_matrix[i][j];
count += 1;
}
}
xadj[i+1] = count;
}
if (CK_LDB_GridMetisLB_Mode == 0) {
// Call Metis to partition the communication graph.
weight_flag = 1; // weights on edges only
numbering_flag = 0; // C style numbering (base 0)
options[0] = 0;
newmap = new int[num_migratable_objects];
METIS_PartGraphRecursive (&num_migratable_objects, xadj, adjncy, NULL, edge_weights, &weight_flag, &numbering_flag, &num_partitions, options, &edgecut, newmap);
} else if (CK_LDB_GridMetisLB_Mode == 1) {
// Call Metis to partition the communication graph.
weight_flag = 3; // weights on both vertices and edges
numbering_flag = 0; // C style numbering (base 0)
options[0] = 0;
newmap = new int[num_migratable_objects];
METIS_PartGraphRecursive (&num_migratable_objects, xadj, adjncy, vertex_weights, edge_weights, &weight_flag, &numbering_flag, &num_partitions, options, &edgecut, newmap);
} else {
if (_lb_args.debug() > 0) {
CkPrintf ("[%d] GridMetisLB was told to use bad mode (%d).\n", CkMyPe(), CK_LDB_GridMetisLB_Mode);
}
}
// Place the partitioned objects into their correct clusters.
for (i = 0; i < num_migratable_objects; i++) {
partition = newmap[i];
cluster = partition_to_cluster_map[partition];
index = migratable_objects[i];
(&Object_Data[index])->cluster = cluster;
}
// Free memory.
delete [] newmap;
delete [] edge_weights;
delete [] adjncy;
delete [] xadj;
for (i = 0; i < num_migratable_objects; i++) {
delete [] communication_matrix[i];
}
delete [] communication_matrix;
if (CK_LDB_GridMetisLB_Mode == 1) {
delete [] vertex_weights;
}
delete [] partition_to_cluster_map;
delete [] migratable_objects;
}
/*************************************************************************
* This function partitions a finite element mesh by partitioning its nodal
* graph using KMETIS and then assigning elements in a load balanced fashion.
**************************************************************************/
int METIS_PartMeshNodal(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind,
idx_t *vwgt, idx_t *vsize, idx_t *nparts, real_t *tpwgts,
idx_t *options, idx_t *objval, idx_t *epart, idx_t *npart)
{
int sigrval=0, renumber=0, ptype;
idx_t *xadj=NULL, *adjncy=NULL;
idx_t ncon=1, pnumflag=0;
int rstatus=METIS_OK;
/* set up malloc cleaning code and signal catchers */
if (!gk_malloc_init())
return METIS_ERROR_MEMORY;
gk_sigtrap();
if ((sigrval = gk_sigcatch()) != 0)
goto SIGTHROW;
renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0);
ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY);
/* renumber the mesh */
if (renumber) {
ChangeMesh2CNumbering(*ne, eptr, eind);
options[METIS_OPTION_NUMBERING] = 0;
}
/* get the nodal graph */
rstatus = METIS_MeshToNodal(ne, nn, eptr, eind, &pnumflag, &xadj, &adjncy);
if (rstatus != METIS_OK)
raise(SIGERR);
/* partition the graph */
if (ptype == METIS_PTYPE_KWAY)
rstatus = METIS_PartGraphKway(nn, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, npart);
else
rstatus = METIS_PartGraphRecursive(nn, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, npart);
if (rstatus != METIS_OK)
raise(SIGERR);
/* partition the other side of the mesh */
InduceRowPartFromColumnPart(*ne, eptr, eind, epart, npart, *nparts, tpwgts);
SIGTHROW:
if (renumber) {
ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart);
options[METIS_OPTION_NUMBERING] = 1;
}
METIS_Free(xadj);
METIS_Free(adjncy);
gk_siguntrap();
gk_malloc_cleanup(0);
return metis_rcode(sigrval);
}
int MESH_PartitionWithMetis(Mesh_ptr mesh, int nparts, int **part) {
MEdge_ptr fedge;
MFace_ptr mf, oppf, rface;
MRegion_ptr mr, oppr;
List_ptr fedges, efaces, rfaces, fregions;
int i, ncells, ipos;
int nv, ne, nf, nr, nfe, nef, nfr, nrf, idx, idx2;
#ifdef METIS_5
idx_t ngraphvtx, numflag, nedgecut, numparts, ncons;
idx_t wtflag, metisopts[METIS_NOPTIONS];
idx_t *vsize, *idxpart;
idx_t *xadj, *adjncy, *vwgt, *adjwgt;
real_t *tpwgts, *ubvec;
#else
idxtype ngraphvtx, numflag, nedgecut, numparts;
idxtype wtflag, metisopts[5] = {0,0,0,0,0};
idxtype *xadj, *adjncy, *vwgt, *adjwgt, *idxpart;
#endif
/* First build a nodal graph of the mesh in the format required by
metis */
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
ipos = 0;
if (nr == 0) {
if (nf == 0) {
fprintf(stderr,"Cannot partition wire meshes\n");
exit(-1);
}
#ifdef METIS_5
xadj = (idx_t *) malloc((nf+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*ne*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nf+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*ne*sizeof(idxtype));
#endif
ncells = nf;
/* Surface mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mf = MESH_Next_Face(mesh,&idx))) {
fedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(fedges);
idx2 = 0;
while ((fedge = List_Next_Entry(fedges,&idx2))) {
efaces = ME_Faces(fedge);
nef = List_Num_Entries(efaces);
if (nef == 1) {
continue; /* boundary edge; nothing to do */
}
else {
int j;
for (j = 0; j < nef; j++) {
oppf = List_Entry(efaces,j);
if (oppf != mf) {
adjncy[ipos] = MF_ID(oppf)-1;
ipos++;
}
}
}
List_Delete(efaces);
}
List_Delete(fedges);
i++;
xadj[i] = ipos;
}
}
else {
#ifdef METIS_5
xadj = (idx_t *) malloc((nr+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*nf*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nr+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*nf*sizeof(idxtype));
#endif
ncells = nr;
/* Volume mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mr = MESH_Next_Region(mesh,&idx))) {
rfaces = MR_Faces(mr);
nrf = List_Num_Entries(rfaces);
idx2 = 0;
while ((rface = List_Next_Entry(rfaces,&idx2))) {
fregions = MF_Regions(rface);
nfr = List_Num_Entries(fregions);
if (nfr > 1) {
oppr = List_Entry(fregions,0);
if (oppr == mr)
oppr = List_Entry(fregions,1);
adjncy[ipos] = MR_ID(oppr)-1;
ipos++;
}
List_Delete(fregions);
}
List_Delete(rfaces);
i++;
xadj[i] = ipos;
}
}
/* Partition the graph */
wtflag = 0; /* No weights are specified */
vwgt = adjwgt = NULL;
numflag = 0; /* C style numbering of elements (nodes of the dual graph) */
ngraphvtx = ncells; /* we want the variable to be of type idxtype or idx_t */
numparts = nparts; /* we want the variable to be of type idxtype or idx_t */
#ifdef METIS_5
idxpart = (idx_t *) malloc(ncells*sizeof(idx_t));
ncons = 1; /* Number of constraints */
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
METIS_SetDefaultOptions(metisopts);
metisopts[METIS_OPTION_NUMBERING] = 0;
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,
idxpart);
else
METIS_PartGraphKway(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,idxpart);
#else
idxpart = (idxtype *) malloc(ncells*sizeof(idxtype));
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,
&numflag,&numparts,metisopts,&nedgecut,idxpart);
else
METIS_PartGraphKway(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,&numflag,
&numparts,metisopts,&nedgecut,idxpart);
#endif
free(xadj);
free(adjncy);
*part = (int *) malloc(ncells*sizeof(int));
for (i = 0; i < ncells; i++)
(*part)[i] = (int) idxpart[i];
free(idxpart);
return 1;
}
//==============================================================================
// NOTE:
// - matrix is supposed to be localized, and passes through the
// singleton filter. This means that I do not have to look
// for Dirichlet nodes (singletons). Also, all rows and columns are
// local.
int Ifpack_METISPartitioner::ComputePartitions()
{
int ierr;
#ifdef HAVE_IFPACK_METIS
int nbytes = 0;
int edgecut;
#endif
Teuchos::RefCountPtr<Epetra_CrsGraph> SymGraph ;
Teuchos::RefCountPtr<Epetra_Map> SymMap;
Teuchos::RefCountPtr<Ifpack_Graph_Epetra_CrsGraph> SymIFPACKGraph;
Teuchos::RefCountPtr<Ifpack_Graph> IFPACKGraph = Teuchos::rcp( (Ifpack_Graph*)Graph_, false );
int Length = 2 * MaxNumEntries();
int NumIndices;
std::vector<int> Indices;
Indices.resize(Length);
/* construct the CSR graph information of the LOCAL matrix
using the get_row function */
std::vector<idxtype> wgtflag;
wgtflag.resize(4);
std::vector<int> options;
options.resize(4);
int numflag;
if (UseSymmetricGraph_) {
#if !defined(EPETRA_NO_32BIT_GLOBAL_INDICES) || !defined(EPETRA_NO_64BIT_GLOBAL_INDICES)
// need to build a symmetric graph.
// I do this in two stages:
// 1.- construct an Epetra_CrsMatrix, symmetric
// 2.- convert the Epetra_CrsMatrix into METIS format
SymMap = Teuchos::rcp( new Epetra_Map(NumMyRows(),0,Graph_->Comm()) );
SymGraph = Teuchos::rcp( new Epetra_CrsGraph(Copy,*SymMap,0) );
#endif
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
if(SymGraph->RowMap().GlobalIndicesInt()) {
for (int i = 0; i < NumMyRows() ; ++i) {
ierr = Graph_->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]);
IFPACK_CHK_ERR(ierr);
for (int j = 0 ; j < NumIndices ; ++j) {
int jj = Indices[j];
if (jj != i) {
SymGraph->InsertGlobalIndices(i,1,&jj);
SymGraph->InsertGlobalIndices(jj,1,&i);
}
}
}
}
else
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
if(SymGraph->RowMap().GlobalIndicesLongLong()) {
for (int i = 0; i < NumMyRows() ; ++i) {
long long i_LL = i;
ierr = Graph_->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]);
IFPACK_CHK_ERR(ierr);
for (int j = 0 ; j < NumIndices ; ++j) {
long long jj = Indices[j];
if (jj != i_LL) {
SymGraph->InsertGlobalIndices(i_LL,1,&jj);
SymGraph->InsertGlobalIndices(jj,1,&i_LL);
}
}
}
}
else
#endif
throw "Ifpack_METISPartitioner::ComputePartitions: GlobalIndices type unknown";
IFPACK_CHK_ERR(SymGraph->FillComplete());
SymIFPACKGraph = Teuchos::rcp( new Ifpack_Graph_Epetra_CrsGraph(SymGraph) );
IFPACKGraph = SymIFPACKGraph;
}
// now work on IFPACKGraph, that can be the symmetric or
// the non-symmetric one
/* set parameters */
wgtflag[0] = 0; /* no weights */
numflag = 0; /* C style */
options[0] = 0; /* default options */
std::vector<idxtype> xadj;
xadj.resize(NumMyRows() + 1);
std::vector<idxtype> adjncy;
adjncy.resize(NumMyNonzeros());
int count = 0;
int count2 = 0;
xadj[0] = 0;
for (int i = 0; i < NumMyRows() ; ++i) {
xadj[count2+1] = xadj[count2]; /* nonzeros in row i-1 */
ierr = IFPACKGraph->ExtractMyRowCopy(i, Length, NumIndices, &Indices[0]);
IFPACK_CHK_ERR(ierr);
for (int j = 0 ; j < NumIndices ; ++j) {
int jj = Indices[j];
if (jj != i) {
adjncy[count++] = jj;
xadj[count2+1]++;
}
}
count2++;
}
std::vector<idxtype> NodesInSubgraph;
NodesInSubgraph.resize(NumLocalParts_);
// some cases can be handled separately
int ok;
if (NumLocalParts() == 1) {
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = 0;
} else if (NumLocalParts() == NumMyRows()) {
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = i;
} else {
ok = 0;
// sometimes METIS creates less partitions than specified.
// ok will check this problem, and recall metis, asking
// for NumLocalParts_/2 partitions
while (ok == 0) {
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = -1;
#ifdef HAVE_IFPACK_METIS
int j = NumMyRows();
if (NumLocalParts_ < 8) {
int i = 1; /* optype in the METIS manual */
numflag = 0;
METIS_EstimateMemory(&j, &xadj[0], &adjncy[0],
&numflag, &i, &nbytes );
METIS_PartGraphRecursive(&j, &xadj[0], &adjncy[0],
NULL, NULL,
&wgtflag[0], &numflag, &NumLocalParts_,
&options[0], &edgecut, &Partition_[0]);
} else {
numflag = 0;
METIS_PartGraphKway (&j, &xadj[0], &adjncy[0],
NULL,
NULL, &wgtflag[0], &numflag,
&NumLocalParts_, &options[0],
&edgecut, &Partition_[0]);
}
#else
numflag = numflag * 2; // avoid warning for unused variable
if (Graph_->Comm().MyPID() == 0) {
cerr << "METIS was not linked; now I put all" << endl;
cerr << "the local nodes in the same partition." << endl;
}
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = 0;
NumLocalParts_ = 1;
#endif
ok = 1;
for (int i = 0 ; i < NumLocalParts() ; ++i)
NodesInSubgraph[i] = 0;
for (int i = 0 ; i < NumMyRows() ; ++i) {
int j = Partition_[i];
if ((j < 0) || (j>= NumLocalParts())) {
ok = 0;
break;
}
else NodesInSubgraph[j]++;
}
for (int i = 0 ; i < NumLocalParts() ; ++i) {
if( NodesInSubgraph[i] == 0 ) {
ok = 0;
break;
}
}
if (ok == 0) {
cerr << "Specified number of subgraphs ("
<< NumLocalParts_ << ") generates empty subgraphs." << endl;
cerr << "Now I recall METIS with NumLocalParts_ = "
<< NumLocalParts_ / 2 << "..." << endl;
NumLocalParts_ = NumLocalParts_/2;
}
if (NumLocalParts() == 0) {
IFPACK_CHK_ERR(-10); // something went wrong
}
if (NumLocalParts() == 1) {
for (int i = 0 ; i < NumMyRows() ; ++i)
Partition_[i] = 0;
ok = 1;
}
} /* while( ok == 0 ) */
} /* if( NumLocalParts_ == 1 ) */
return(0);
}
sparse_matrix* graph_partition(LIST *list , partition_t* partition_info)
{
graph_t* graph;
params_t *params;
idx_t options[METIS_NOPTIONS], status;
idx_t objval;
sparse_matrix* matrix = (sparse_matrix*)malloc(sizeof(sparse_matrix));
graph = ReadGraph(list);
params = bmalloc(sizeof(*params), "Allocating memory for params");
METIS_SetDefaultOptions(options);
METIS_init_params(params, graph->ncon);
options[METIS_OPTION_OBJTYPE] = params->objtype;
options[METIS_OPTION_CTYPE] = params->ctype;
options[METIS_OPTION_IPTYPE] = params->iptype;
options[METIS_OPTION_RTYPE] = params->rtype;
options[METIS_OPTION_NO2HOP] = params->no2hop;
options[METIS_OPTION_MINCONN] = params->minconn;
options[METIS_OPTION_CONTIG] = params->contig;
options[METIS_OPTION_SEED] = params->seed;
options[METIS_OPTION_NITER] = params->niter;
options[METIS_OPTION_NCUTS] = params->ncuts;
options[METIS_OPTION_UFACTOR] = params->ufactor;
options[METIS_OPTION_DBGLVL] = params->dbglvl;
//print_input_data(graph,params);
params->tpwgts = NULL;
params->ubvec = NULL;
partition_info->partion_table = imalloc(graph->nvtxs, "Allocate memory for part");
switch (params->ptype) {
case METIS_PTYPE_RB:
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, partition_info->partion_table);
break;
case METIS_PTYPE_KWAY:
status = METIS_PartGraphKway(&graph->nvtxs, &graph->ncon, graph->xadj,
graph->adjncy, graph->vwgt, graph->vsize, graph->adjwgt,
¶ms->nparts, params->tpwgts, params->ubvec, options,
&objval, partition_info->partion_table);
break;
}
if (status != METIS_OK) {
fprintf(stderr,"\n***Metis returned with an error.***\n");
return NULL;
}
partition_info->size = graph->nvtxs;
partition_info->noOfParts = params->nparts;
WritePartition(params->filename, partition_info->partion_table, graph->nvtxs, params->nparts);
printf("Objval: %d\n",objval);
matrix->nz = -1;
matrix->n = graph->nvtxs;
matrix->p = graph->xadj;
matrix->i = graph->adjncy;
matrix->x = NULL;
return matrix;
}
/**************************************************************************
** This takes objects in a cluster and partitions them onto PEs.
*/
void GridMetisLB::Partition_ClusterObjects_Into_PEs (CentralLB::LDStats *stats, int cluster)
{
int num_migratable_cluster_objects;
int *migratable_cluster_objects;
int index;
int num_available_cluster_pes;
int num_partitions;
int *partition_to_pe_map;
int pe;
int partition;
int partition_count;
int *vertex_weights;
int vertex;
int **communication_matrix;
LDCommData *com_data;
int send_object;
int recv_object;
int send_index;
int recv_index;
LDObjKey *recv_objects;
int num_objects;
int *xadj;
int num_edges;
int *adjncy;
int *edge_weights;
int count;
int weight_flag;
int numbering_flag;
int options[5];
int edgecut;
int *newmap;
int i;
int j;
for (i = 0; i < Num_Objects; i++) {
(&Object_Data[i])->secondary_index = -1;
}
// Count the number of migratable objects within this cluster, which are the only candidates to give to Metis.
// (The non-migratable objects have been placed onto the correct destination PEs earlier.)
// After getting the count, create a migratable_cluster_objects[] array to keep track of them.
num_migratable_cluster_objects = 0;
for (i = 0; i < Num_Objects; i++) {
if (((&Object_Data[i])->migratable) && ((&Object_Data[i])->cluster == cluster)) {
num_migratable_cluster_objects += 1;
}
}
migratable_cluster_objects = new int[num_migratable_cluster_objects];
index = 0;
for (i = 0; i < Num_Objects; i++) {
if (((&Object_Data[i])->migratable) && ((&Object_Data[i])->cluster == cluster)) {
(&Object_Data[i])->secondary_index = index;
migratable_cluster_objects[index] = i;
index += 1;
}
}
// Count the number of available PEs in the cluster.
num_available_cluster_pes = 0;
for (i = 0; i < Num_PEs; i++) {
if (((&PE_Data[i])->available) && ((&PE_Data[i])->cluster == cluster)) {
num_available_cluster_pes += 1;
}
}
// Compute the number of partitions for Metis, based on the relative speed of each PE.
// Also create the partition-to-PE mapping so the output of Metis can be mapped back to PEs.
num_partitions = 0;
for (i = 0; i < Num_PEs; i++) {
if (((&PE_Data[i])->available) && ((&PE_Data[i])->cluster == cluster)) {
num_partitions += (int) ceil ((&PE_Data[i])->relative_speed);
}
}
partition_to_pe_map = new int[num_partitions];
pe = 0;
while (((!(&PE_Data[pe])->available) || ((&PE_Data[pe])->cluster != cluster)) && (pe < Num_PEs)) {
pe += 1;
}
if (pe >= Num_PEs) {
CmiAbort ("GridMetisLB: Error computing partition to PE map!\n");
}
partition = 0;
while (partition < num_partitions) {
partition_count = (int) ceil ((&PE_Data[pe])->relative_speed);
for (i = partition; i < (partition + partition_count); i++) {
partition_to_pe_map[i] = pe;
}
partition += partition_count;
pe += 1;
while (((!(&PE_Data[pe])->available) || ((&PE_Data[pe])->cluster != cluster)) && (pe < Num_PEs)) {
pe += 1;
}
if (pe > Num_PEs) {
CmiAbort ("GridMetisLB: Error computing partition to PE map!\n");
}
}
// Compute vertex weights for the objects.
vertex_weights = new int[num_migratable_cluster_objects];
vertex = 0;
for (i = 0; i < Num_Objects; i++) {
if ((&Object_Data[i])->migratable && ((&Object_Data[i])->cluster == cluster)) {
vertex_weights[vertex] = (int) ceil ((&Object_Data[i])->load * 10000);
vertex += 1;
}
}
// Create communication_matrix[] to hold all object-to-object message counts;
communication_matrix = new int *[num_migratable_cluster_objects];
for (i = 0; i < num_migratable_cluster_objects; i++) {
communication_matrix[i] = new int[num_migratable_cluster_objects];
for (j = 0; j < num_migratable_cluster_objects; j++) {
communication_matrix[i][j] = 0;
}
}
for (i = 0; i < stats->n_comm; i++) {
com_data = &(stats->commData[i]);
if ((!com_data->from_proc()) && (com_data->recv_type() == LD_OBJ_MSG)) {
send_object = stats->getHash (com_data->sender);
recv_object = stats->getHash (com_data->receiver.get_destObj());
//if ((recv_object == -1) && (stats->complete_flag == 0)) {
if ((send_object < 0) || (send_object > Num_Objects) || (recv_object < 0) || (recv_object > Num_Objects)) {
continue;
}
if ((!(&Object_Data[send_object])->migratable) || (!(&Object_Data[recv_object])->migratable)) {
continue;
}
if (((&Object_Data[send_object])->cluster != cluster) || ((&Object_Data[recv_object])->cluster != cluster)) {
continue;
}
send_index = (&Object_Data[send_object])->secondary_index;
recv_index = (&Object_Data[recv_object])->secondary_index;
communication_matrix[send_index][recv_index] += com_data->messages;
communication_matrix[recv_index][send_index] += com_data->messages;
} else if (com_data->receiver.get_type() == LD_OBJLIST_MSG) {
send_object = stats->getHash (com_data->sender);
if ((send_object < 0) || (send_object > Num_Objects)) {
continue;
}
if (!(&Object_Data[send_object])->migratable) {
continue;
}
if ((&Object_Data[send_object])->cluster != cluster) {
continue;
}
recv_objects = com_data->receiver.get_destObjs (num_objects); // (num_objects is passed by reference)
for (j = 0; j < num_objects; j++) {
recv_object = stats->getHash (recv_objects[j]);
//if (recv_object == -1) {
if ((recv_object < 0) || (recv_object > Num_Objects)) {
continue;
}
if (!(&Object_Data[recv_object])->migratable) {
continue;
}
if ((&Object_Data[recv_object])->cluster != cluster) {
continue;
}
send_index = (&Object_Data[send_object])->secondary_index;
recv_index = (&Object_Data[recv_object])->secondary_index;
communication_matrix[send_index][recv_index] += com_data->messages;
communication_matrix[recv_index][send_index] += com_data->messages;
}
}
}
for (i = 0; i < num_migratable_cluster_objects; i++) {
communication_matrix[i][i] = 0;
}
// Construct a graph in CSR format for input to Metis.
xadj = new int[num_migratable_cluster_objects + 1];
num_edges = 0;
for (i = 0; i < num_migratable_cluster_objects; i++) {
for (j = 0; j < num_migratable_cluster_objects; j++) {
if (communication_matrix[i][j] > 0) {
num_edges += 1;
}
}
}
adjncy = new int[num_edges];
edge_weights = new int[num_edges];
count = 0;
xadj[0] = 0;
for (i = 0; i < num_migratable_cluster_objects; i++) {
for (j = 0; j < num_migratable_cluster_objects; j++) {
if (communication_matrix[i][j] > 0) {
adjncy[count] = j;
edge_weights[count] = communication_matrix[i][j];
count += 1;
}
}
xadj[i+1] = count;
}
// Call Metis to partition the communication graph.
weight_flag = 3; // weights on both vertices and edges
numbering_flag = 0; // C style numbering (base 0)
options[0] = 0;
newmap = new int[num_migratable_cluster_objects];
CmiPrintf ("[%d] GridMetisLB is partitioning %d objects in cluster %d into %d partitions.\n", CmiMyPe(), num_migratable_cluster_objects, cluster, num_partitions);
METIS_PartGraphRecursive (&num_migratable_cluster_objects, xadj, adjncy, vertex_weights, edge_weights, &weight_flag, &numbering_flag, &num_partitions, options, &edgecut, newmap);
// Place the partitioned objects onto their correct PEs.
for (i = 0; i < num_migratable_cluster_objects; i++) {
partition = newmap[i];
pe = partition_to_pe_map[partition];
index = migratable_cluster_objects[i];
/* WRONG!
for (j = 0; j < Num_Objects; j++) {
if ((&Object_Data[j])->secondary_index == index) {
(&Object_Data[j])->to_pe = pe;
break;
}
}
*/
(&Object_Data[index])->to_pe = pe;
}
// Free memory.
delete [] newmap;
delete [] edge_weights;
delete [] adjncy;
delete [] xadj;
for (i = 0; i < num_migratable_cluster_objects; i++) {
delete [] communication_matrix[i];
}
delete [] communication_matrix;
delete [] vertex_weights;
delete [] partition_to_pe_map;
delete [] migratable_cluster_objects;
}
void InitKWayPartitioningRB(ctrl_t *ctrl, graph_t *graph)
{
idx_t i, options[METIS_NOPTIONS], curobj=0;
idx_t *bestwhere=NULL;
real_t *ubvec=NULL;
int status;
METIS_SetDefaultOptions(options);
options[METIS_OPTION_NITER] = 10;
options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT;
options[METIS_OPTION_NO2HOP] = ctrl->no2hop;
options[METIS_OPTION_ONDISK] = ctrl->ondisk;
ubvec = rmalloc(graph->ncon, "InitKWayPartitioning: ubvec");
for (i=0; i<graph->ncon; i++)
ubvec[i] = (real_t)pow(ctrl->ubfactors[i], 1.0/log(ctrl->nparts));
switch (ctrl->objtype) {
case METIS_OBJTYPE_CUT:
case METIS_OBJTYPE_VOL:
options[METIS_OPTION_NCUTS] = ctrl->nIparts;
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon,
graph->xadj, graph->adjncy, graph->vwgt, graph->vsize,
graph->adjwgt, &ctrl->nparts, ctrl->tpwgts, ubvec,
options, &curobj, graph->where);
if (status != METIS_OK)
gk_errexit(SIGERR, "Failed during initial partitioning\n");
break;
#ifdef XXX /* This does not seem to help */
case METIS_OBJTYPE_VOL:
bestwhere = imalloc(graph->nvtxs, "InitKWayPartitioning: bestwhere");
options[METIS_OPTION_NCUTS] = 2;
ntrials = (ctrl->nIparts+1)/2;
for (i=0; i<ntrials; i++) {
status = METIS_PartGraphRecursive(&graph->nvtxs, &graph->ncon,
graph->xadj, graph->adjncy, graph->vwgt, graph->vsize,
graph->adjwgt, &ctrl->nparts, ctrl->tpwgts, ubvec,
options, &curobj, graph->where);
if (status != METIS_OK)
gk_errexit(SIGERR, "Failed during initial partitioning\n");
curobj = ComputeVolume(graph, graph->where);
if (i == 0 || bestobj > curobj) {
bestobj = curobj;
if (i < ntrials-1)
icopy(graph->nvtxs, graph->where, bestwhere);
}
if (bestobj == 0)
break;
}
if (bestobj != curobj)
icopy(graph->nvtxs, bestwhere, graph->where);
break;
#endif
default:
gk_errexit(SIGERR, "Unknown objtype: %d\n", ctrl->objtype);
}
gk_free((void **)&ubvec, &bestwhere, LTERM);
}
/*It uses METIS library to accomplish that*/
void TopoCentLB::computePartitions(CentralLB::LDStats *stats,int count,int *newmap)
{
int numobjs = stats->n_objs;
int i, j, m;
// allocate space for the computing data
double *objtime = new double[numobjs];
int *objwt = new int[numobjs];
int *origmap = new int[numobjs];
LDObjHandle *handles = new LDObjHandle[numobjs];
for(i=0;i<numobjs;i++) {
objtime[i] = 0.0;
objwt[i] = 0;
origmap[i] = 0;
}
//Prepare compute loads for METIS library
for (i=0; i<stats->n_objs; i++) {
LDObjData &odata = stats->objData[i];
if (!odata.migratable)
CmiAbort("MetisLB doesnot dupport nonmigratable object.\n");
int frompe = stats->from_proc[i];
origmap[i] = frompe;
objtime[i] = odata.wallTime*stats->procs[frompe].pe_speed;
handles[i] = odata.handle;
}
// to convert the weights on vertices to integers
double max_objtime = objtime[0];
for(i=0; i<numobjs; i++) {
if(max_objtime < objtime[i])
max_objtime = objtime[i];
}
int maxobj=0;
int totalwt=0;
double ratio = 1000.0/max_objtime;
for(i=0; i<numobjs; i++) {
objwt[i] = (int)(objtime[i]*ratio);
if(maxobj<objwt[i])
maxobj=objwt[i];
totalwt+=objwt[i];
}
int **comm = new int*[numobjs];
for (i=0; i<numobjs; i++) {
comm[i] = new int[numobjs];
for (j=0; j<numobjs; j++) {
comm[i][j] = 0;
}
}
//Prepare communication for METIS library
const int csz = stats->n_comm;
for(i=0; i<csz; i++) {
LDCommData &cdata = stats->commData[i];
//if(cdata.from_proc() || cdata.receiver.get_type() != LD_OBJ_MSG)
//continue;
if(!cdata.from_proc() && cdata.receiver.get_type() == LD_OBJ_MSG){
int senderID = stats->getHash(cdata.sender);
int recverID = stats->getHash(cdata.receiver.get_destObj());
CmiAssert(senderID < numobjs);
CmiAssert(recverID < numobjs);
comm[senderID][recverID] += cdata.messages;
comm[recverID][senderID] += cdata.messages;
//Use bytes or messages -- do i include messages for objlist too...??
}
else if (cdata.receiver.get_type() == LD_OBJLIST_MSG) {
//CkPrintf("in objlist..\n");
int nobjs;
LDObjKey *objs = cdata.receiver.get_destObjs(nobjs);
int senderID = stats->getHash(cdata.sender);
for (j=0; j<nobjs; j++) {
int recverID = stats->getHash(objs[j]);
if((senderID == -1)||(recverID == -1))
if (_lb_args.migObjOnly()) continue;
else CkAbort("Error in search\n");
comm[senderID][recverID] += cdata.messages;
comm[recverID][senderID] += cdata.messages;
}
}
}
// ignore messages sent from an object to itself
for (i=0; i<numobjs; i++)
comm[i][i] = 0;
// construct the graph in CSR format
int *xadj = new int[numobjs+1];
int numedges = 0;
for(i=0;i<numobjs;i++) {
for(j=0;j<numobjs;j++) {
if(comm[i][j] != 0)
numedges++;
}
}
int *adjncy = new int[numedges];
int *edgewt = new int[numedges];
int factor = 10;
xadj[0] = 0;
int count4all = 0;
for (i=0; i<numobjs; i++) {
for (j=0; j<numobjs; j++) {
if (comm[i][j] != 0) {
adjncy[count4all] = j;
edgewt[count4all++] = comm[i][j]/factor;
}
}
xadj[i+1] = count4all;
}
//Call METIS routine
int wgtflag = 3; // Weights both on vertices and edges
int numflag = 0; // C Style numbering
int options[5];
int edgecut;
options[0] = 0;
if (count < 1) {
CkPrintf("error: Number of Pe less than 1!");
}
else if (count == 1) {
for(m=0;m<numobjs;m++)
newmap[i] = origmap[i];
}
else {
/*
if (count > 8)
METIS_PartGraphKway(&numobjs, xadj, adjncy, objwt, edgewt,
&wgtflag, &numflag, &count, options,
&edgecut, newmap);
else
METIS_PartGraphRecursive(&numobjs, xadj, adjncy, objwt, edgewt,
&wgtflag, &numflag, &count, options,
&edgecut, newmap);
*/
METIS_PartGraphRecursive(&numobjs, xadj, adjncy, objwt, edgewt,
&wgtflag, &numflag, &count, options,
&edgecut, newmap);
}
//Debugging code: Checking load on each partition
if(_lb_args.debug() >=2){
int total=0;
int *chkwt = new int[count];
for(i=0;i<count;i++)
chkwt[i]=0;
for(i=0;i<numobjs;i++){
chkwt[newmap[i]] += objwt[i];
total += objwt[i];
}
for(i=0;i<count;i++)
CkPrintf("%d -- %d\n",i,chkwt[i]);
CkPrintf("Totalwt of all partitions after call to METIS:%d, Avg is %d\n",total,total/count);
}
//Clean up all the variables allocated in this routine
for(i=0;i<numobjs;i++)
delete[] comm[i];
delete[] comm;
delete[] objtime;
delete[] xadj;
delete[] adjncy;
delete[] objwt;
delete[] edgewt;
delete[] handles;
delete[] origmap;
}
/*************************************************************************
* This function is the entry point of the initial partition algorithm
* that does recursive bissection.
* This algorithm assembles the graph to all the processors and preceeds
* by parallelizing the recursive bisection step.
**************************************************************************/
void InitPartition(ctrl_t *ctrl, graph_t *graph)
{
idx_t i, j, ncon, mype, npes, gnvtxs, ngroups;
idx_t *xadj, *adjncy, *adjwgt, *vwgt;
idx_t *part, *gwhere0, *gwhere1;
idx_t *tmpwhere, *tmpvwgt, *tmpxadj, *tmpadjncy, *tmpadjwgt;
graph_t *agraph;
idx_t lnparts, fpart, fpe, lnpes;
idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut;
real_t *tpwgts, *tpwgts2, *lbvec, lbsum, min_lbsum, wsum;
MPI_Comm ipcomm;
struct {
double sum;
int rank;
} lpesum, gpesum;
WCOREPUSH;
ncon = graph->ncon;
ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, ctrl->npes), 1);
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
lbvec = rwspacemalloc(ctrl, ncon);
/* assemble the graph to all the processors */
agraph = AssembleAdaptiveGraph(ctrl, graph);
gnvtxs = agraph->nvtxs;
/* make a copy of the graph's structure for later */
xadj = icopy(gnvtxs+1, agraph->xadj, iwspacemalloc(ctrl, gnvtxs+1));
vwgt = icopy(gnvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, gnvtxs*ncon));
adjncy = icopy(agraph->nedges, agraph->adjncy, iwspacemalloc(ctrl, agraph->nedges));
adjwgt = icopy(agraph->nedges, agraph->adjwgt, iwspacemalloc(ctrl, agraph->nedges));
part = iwspacemalloc(ctrl, gnvtxs);
/* create different processor groups */
gkMPI_Comm_split(ctrl->gcomm, ctrl->mype % ngroups, 0, &ipcomm);
gkMPI_Comm_rank(ipcomm, &mype);
gkMPI_Comm_size(ipcomm, &npes);
/* Go into the recursive bisection */
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_SEED] = ctrl->sync + (ctrl->mype % ngroups) + 1;
tpwgts = ctrl->tpwgts;
tpwgts2 = rwspacemalloc(ctrl, 2*ncon);
lnparts = ctrl->nparts;
fpart = fpe = 0;
lnpes = npes;
while (lnpes > 1 && lnparts > 1) {
/* determine the weights of the two partitions as a function of the
weight of the target partition weights */
for (j=(lnparts>>1), i=0; i<ncon; i++) {
tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon);
tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon);
wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]);
tpwgts2[i] *= wsum;
tpwgts2[ncon+i] *= wsum;
}
METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &twoparts, tpwgts2, NULL, moptions,
&edgecut, part);
/* pick one of the branches */
if (mype < fpe+lnpes/2) {
KeepPart(ctrl, agraph, part, 0);
lnpes = lnpes/2;
lnparts = lnparts/2;
}
else {
KeepPart(ctrl, agraph, part, 1);
fpart = fpart + lnparts/2;
fpe = fpe + lnpes/2;
lnpes = lnpes - lnpes/2;
lnparts = lnparts - lnparts/2;
}
}
gwhere0 = iset(gnvtxs, 0, iwspacemalloc(ctrl, gnvtxs));
gwhere1 = iwspacemalloc(ctrl, gnvtxs);
if (lnparts == 1) { /* Case npes is greater than or equal to nparts */
/* Only the first process will assign labels (for the reduction to work) */
if (mype == fpe) {
for (i=0; i<agraph->nvtxs; i++)
gwhere0[agraph->label[i]] = fpart;
}
}
else { /* Case in which npes is smaller than nparts */
/* create the normalized tpwgts for the lnparts from ctrl->tpwgts */
tpwgts = rwspacemalloc(ctrl, lnparts*ncon);
for (j=0; j<ncon; j++) {
for (wsum=0.0, i=0; i<lnparts; i++) {
tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j];
wsum += tpwgts[i*ncon+j];
}
for (wsum=1.0/wsum, i=0; i<lnparts; i++)
tpwgts[i*ncon+j] *= wsum;
}
METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions,
&edgecut, part);
for (i=0; i<agraph->nvtxs; i++)
gwhere0[agraph->label[i]] = fpart + part[i];
}
gkMPI_Allreduce((void *)gwhere0, (void *)gwhere1, gnvtxs, IDX_T, MPI_SUM, ipcomm);
if (ngroups > 1) {
tmpxadj = agraph->xadj;
tmpadjncy = agraph->adjncy;
tmpadjwgt = agraph->adjwgt;
tmpvwgt = agraph->vwgt;
tmpwhere = agraph->where;
agraph->xadj = xadj;
agraph->adjncy = adjncy;
agraph->adjwgt = adjwgt;
agraph->vwgt = vwgt;
agraph->where = gwhere1;
agraph->vwgt = vwgt;
agraph->nvtxs = gnvtxs;
edgecut = ComputeSerialEdgeCut(agraph);
ComputeSerialBalance(ctrl, agraph, gwhere1, lbvec);
lbsum = rsum(ncon, lbvec, 1);
gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ctrl->gcomm);
gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ctrl->gcomm);
lpesum.sum = lbsum;
if (min_lbsum < UNBALANCE_FRACTION*ncon) {
if (lbsum < UNBALANCE_FRACTION*ncon)
lpesum.sum = edgecut;
else
lpesum.sum = max_cut;
}
lpesum.rank = ctrl->mype;
gkMPI_Allreduce((void *)&lpesum, (void *)&gpesum, 1, MPI_DOUBLE_INT,
MPI_MINLOC, ctrl->gcomm);
gkMPI_Bcast((void *)gwhere1, gnvtxs, IDX_T, gpesum.rank, ctrl->gcomm);
agraph->xadj = tmpxadj;
agraph->adjncy = tmpadjncy;
agraph->adjwgt = tmpadjwgt;
agraph->vwgt = tmpvwgt;
agraph->where = tmpwhere;
}
icopy(graph->nvtxs, gwhere1+graph->vtxdist[ctrl->mype], graph->where);
FreeGraph(agraph);
gkMPI_Comm_free(&ipcomm);
IFSET(ctrl->dbglvl, DBG_TIME, gkMPI_Barrier(ctrl->comm));
IFSET(ctrl->dbglvl, DBG_TIME, stoptimer(ctrl->InitPartTmr));
WCOREPOP;
}
// Call Metis with options from dictionary.
Foam::label Foam::metisDecomp::decompose
(
const List<int>& adjncy,
const List<int>& xadj,
const scalarField& cWeights,
List<int>& finalDecomp
)
{
// C style numbering
int numFlag = 0;
// Method of decomposition
// recursive: multi-level recursive bisection (default)
// k-way: multi-level k-way
word method("k-way");
int numCells = xadj.size()-1;
// decomposition options. 0 = use defaults
List<int> options(5, 0);
// processor weights initialised with no size, only used if specified in
// a file
Field<floatScalar> processorWeights;
// cell weights (so on the vertices of the dual)
List<int> cellWeights;
// face weights (so on the edges of the dual)
List<int> faceWeights;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningIn
(
"metisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != numCells)
{
FatalErrorIn
(
"metisDecomp::decompose"
"(const pointField&, const scalarField&)"
) << "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << numCells
<< exit(FatalError);
}
// Convert to integers.
cellWeights.setSize(cWeights.size());
forAll(cellWeights, i)
{
cellWeights[i] = int(cWeights[i]/minWeights);
}
}
// Check for user supplied weights and decomp options
if (decompositionDict_.found("metisCoeffs"))
{
const dictionary& metisCoeffs =
decompositionDict_.subDict("metisCoeffs");
word weightsFile;
if (metisCoeffs.readIfPresent("method", method))
{
if (method != "recursive" && method != "k-way")
{
FatalErrorIn("metisDecomp::decompose()")
<< "Method " << method << " in metisCoeffs in dictionary : "
<< decompositionDict_.name()
<< " should be 'recursive' or 'k-way'"
<< exit(FatalError);
}
Info<< "metisDecomp : Using Metis method " << method
<< nl << endl;
}
if (metisCoeffs.readIfPresent("options", options))
{
if (options.size() != 5)
{
FatalErrorIn("metisDecomp::decompose()")
<< "Number of options in metisCoeffs in dictionary : "
<< decompositionDict_.name()
<< " should be 5"
<< exit(FatalError);
}
Info<< "metisDecomp : Using Metis options " << options
<< nl << endl;
}
if (metisCoeffs.readIfPresent("processorWeights", processorWeights))
{
processorWeights /= sum(processorWeights);
if (processorWeights.size() != nProcessors_)
{
FatalErrorIn("metisDecomp::decompose(const pointField&)")
<< "Number of processor weights "
<< processorWeights.size()
<< " does not equal number of domains " << nProcessors_
<< exit(FatalError);
}
}
//if (metisCoeffs.readIfPresent("cellWeightsFile", weightsFile))
//{
// Info<< "metisDecomp : Using cell-based weights." << endl;
//
// IOList<int> cellIOWeights
// (
// IOobject
// (
// weightsFile,
// mesh_.time().timeName(),
// mesh_,
// IOobject::MUST_READ,
// IOobject::AUTO_WRITE
// )
// );
// cellWeights.transfer(cellIOWeights);
//
// if (cellWeights.size() != xadj.size()-1)
// {
// FatalErrorIn("metisDecomp::decompose(const pointField&)")
// << "Number of cell weights " << cellWeights.size()
// << " does not equal number of cells " << xadj.size()-1
// << exit(FatalError);
// }
//}
}
int nProcs = nProcessors_;
// output: cell -> processor addressing
finalDecomp.setSize(numCells);
// output: number of cut edges
int edgeCut = 0;
// Vertex weight info
int wgtFlag = 0;
int* vwgtPtr = NULL;
int* adjwgtPtr = NULL;
if (cellWeights.size())
{
vwgtPtr = cellWeights.begin();
wgtFlag += 2; // Weights on vertices
}
if (faceWeights.size())
{
adjwgtPtr = faceWeights.begin();
wgtFlag += 1; // Weights on edges
}
if (method == "recursive")
{
if (processorWeights.size())
{
METIS_WPartGraphRecursive
(
&numCells, // num vertices in graph
const_cast<List<int>&>(xadj).begin(), // indexing into adjncy
const_cast<List<int>&>(adjncy).begin(), // neighbour info
vwgtPtr, // vertexweights
adjwgtPtr, // no edgeweights
&wgtFlag,
&numFlag,
&nProcs,
processorWeights.begin(),
options.begin(),
&edgeCut,
finalDecomp.begin()
);
}
else
{
METIS_PartGraphRecursive
(
&numCells, // num vertices in graph
const_cast<List<int>&>(xadj).begin(), // indexing into adjncy
const_cast<List<int>&>(adjncy).begin(), // neighbour info
vwgtPtr, // vertexweights
adjwgtPtr, // no edgeweights
&wgtFlag,
&numFlag,
&nProcs,
options.begin(),
&edgeCut,
finalDecomp.begin()
);
}
}
else
{
if (processorWeights.size())
{
METIS_WPartGraphKway
(
&numCells, // num vertices in graph
const_cast<List<int>&>(xadj).begin(), // indexing into adjncy
const_cast<List<int>&>(adjncy).begin(), // neighbour info
vwgtPtr, // vertexweights
adjwgtPtr, // no edgeweights
&wgtFlag,
&numFlag,
&nProcs,
processorWeights.begin(),
options.begin(),
&edgeCut,
finalDecomp.begin()
);
}
else
{
METIS_PartGraphKway
(
&numCells, // num vertices in graph
const_cast<List<int>&>(xadj).begin(), // indexing into adjncy
const_cast<List<int>&>(adjncy).begin(), // neighbour info
vwgtPtr, // vertexweights
adjwgtPtr, // no edgeweights
&wgtFlag,
&numFlag,
&nProcs,
options.begin(),
&edgeCut,
finalDecomp.begin()
);
}
}
return edgeCut;
}
/*************************************************************************
* This function partitions a finite element mesh by partitioning its dual
* graph using KMETIS and then assigning nodes in a load balanced fashion.
**************************************************************************/
int METIS_PartMeshDual(idx_t *ne, idx_t *nn, idx_t *eptr, idx_t *eind,
idx_t *vwgt, idx_t *vsize, idx_t *ncommon, idx_t *nparts,
real_t *tpwgts, idx_t *options, idx_t *objval, idx_t *epart,
idx_t *npart)
{
int sigrval=0, renumber=0, ptype;
idx_t i, j;
idx_t *xadj=NULL, *adjncy=NULL, *nptr=NULL, *nind=NULL;
idx_t ncon=1, pnumflag=0;
int rstatus = METIS_OK;
/* set up malloc cleaning code and signal catchers */
if (!gk_malloc_init())
return METIS_ERROR_MEMORY;
gk_sigtrap();
if ((sigrval = gk_sigcatch()) != 0)
goto SIGTHROW;
renumber = GETOPTION(options, METIS_OPTION_NUMBERING, 0);
ptype = GETOPTION(options, METIS_OPTION_PTYPE, METIS_PTYPE_KWAY);
/* renumber the mesh */
if (renumber) {
ChangeMesh2CNumbering(*ne, eptr, eind);
options[METIS_OPTION_NUMBERING] = 0;
}
/* get the dual graph */
rstatus = METIS_MeshToDual(ne, nn, eptr, eind, ncommon, &pnumflag, &xadj, &adjncy);
if (rstatus != METIS_OK)
raise(SIGERR);
/* partition the graph */
if (ptype == METIS_PTYPE_KWAY)
rstatus = METIS_PartGraphKway(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, epart);
else
rstatus = METIS_PartGraphRecursive(ne, &ncon, xadj, adjncy, vwgt, vsize, NULL,
nparts, tpwgts, NULL, options, objval, epart);
if (rstatus != METIS_OK)
raise(SIGERR);
/* construct the node-element list */
nptr = ismalloc(*nn+1, 0, "METIS_PartMeshDual: nptr");
nind = imalloc(eptr[*ne], "METIS_PartMeshDual: nind");
for (i=0; i<*ne; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++)
nptr[eind[j]]++;
}
MAKECSR(i, *nn, nptr);
for (i=0; i<*ne; i++) {
for (j=eptr[i]; j<eptr[i+1]; j++)
nind[nptr[eind[j]]++] = i;
}
SHIFTCSR(i, *nn, nptr);
/* partition the other side of the mesh */
InduceRowPartFromColumnPart(*nn, nptr, nind, npart, epart, *nparts, tpwgts);
gk_free((void **)&nptr, &nind, LTERM);
SIGTHROW:
if (renumber) {
ChangeMesh2FNumbering2(*ne, *nn, eptr, eind, epart, npart);
options[METIS_OPTION_NUMBERING] = 1;
}
METIS_Free(xadj);
METIS_Free(adjncy);
gk_siguntrap();
gk_malloc_cleanup(0);
return metis_rcode(sigrval);
}
/*************************************************************************
* This function is the entry point of the initial balancing algorithm.
* This algorithm assembles the graph to all the processors and preceeds
* with the balancing step.
**************************************************************************/
void Balance_Partition(ctrl_t *ctrl, graph_t *graph)
{
idx_t i, j, nvtxs, nedges, ncon;
idx_t mype, npes, srnpes, srmype;
idx_t *vtxdist, *xadj, *adjncy, *adjwgt, *vwgt, *vsize;
idx_t *part, *lwhere, *home;
idx_t lnparts, fpart, fpe, lnpes, ngroups;
idx_t *rcounts, *rdispls;
idx_t twoparts=2, moptions[METIS_NOPTIONS], edgecut, max_cut;
idx_t sr_pe, gd_pe, sr, gd, who_wins;
real_t my_cut, my_totalv, my_cost = -1.0, my_balance = -1.0, wsum;
real_t rating, max_rating, your_cost = -1.0, your_balance = -1.0;
real_t lbsum, min_lbsum, *lbvec, *tpwgts, *tpwgts2, buffer[2];
graph_t *agraph, cgraph;
ctrl_t *myctrl;
MPI_Status status;
MPI_Comm ipcomm, srcomm;
struct {
double cost;
int rank;
} lpecost, gpecost;
IFSET(ctrl->dbglvl, DBG_TIME, starttimer(ctrl->InitPartTmr));
WCOREPUSH;
vtxdist = graph->vtxdist;
agraph = AssembleAdaptiveGraph(ctrl, graph);
nvtxs = cgraph.nvtxs = agraph->nvtxs;
nedges = cgraph.nedges = agraph->nedges;
ncon = cgraph.ncon = agraph->ncon;
xadj = cgraph.xadj = icopy(nvtxs+1, agraph->xadj, iwspacemalloc(ctrl, nvtxs+1));
vwgt = cgraph.vwgt = icopy(nvtxs*ncon, agraph->vwgt, iwspacemalloc(ctrl, nvtxs*ncon));
vsize = cgraph.vsize = icopy(nvtxs, agraph->vsize, iwspacemalloc(ctrl, nvtxs));
adjncy = cgraph.adjncy = icopy(nedges, agraph->adjncy, iwspacemalloc(ctrl, nedges));
adjwgt = cgraph.adjwgt = icopy(nedges, agraph->adjwgt, iwspacemalloc(ctrl, nedges));
part = cgraph.where = agraph->where = iwspacemalloc(ctrl, nvtxs);
lwhere = iwspacemalloc(ctrl, nvtxs);
home = iwspacemalloc(ctrl, nvtxs);
lbvec = rwspacemalloc(ctrl, graph->ncon);
/****************************************/
/****************************************/
if (ctrl->ps_relation == PARMETIS_PSR_UNCOUPLED) {
WCOREPUSH;
rcounts = iwspacemalloc(ctrl, ctrl->npes);
rdispls = iwspacemalloc(ctrl, ctrl->npes+1);
for (i=0; i<ctrl->npes; i++)
rdispls[i] = rcounts[i] = vtxdist[i+1]-vtxdist[i];
MAKECSR(i, ctrl->npes, rdispls);
gkMPI_Allgatherv((void *)graph->home, graph->nvtxs, IDX_T,
(void *)part, rcounts, rdispls, IDX_T, ctrl->comm);
for (i=0; i<agraph->nvtxs; i++)
home[i] = part[i];
WCOREPOP; /* local frees */
}
else {
for (i=0; i<ctrl->npes; i++) {
for (j=vtxdist[i]; j<vtxdist[i+1]; j++)
part[j] = home[j] = i;
}
}
/* Ensure that the initial partitioning is legal */
for (i=0; i<agraph->nvtxs; i++) {
if (part[i] >= ctrl->nparts)
part[i] = home[i] = part[i] % ctrl->nparts;
if (part[i] < 0)
part[i] = home[i] = (-1*part[i]) % ctrl->nparts;
}
/****************************************/
/****************************************/
IFSET(ctrl->dbglvl, DBG_REFINEINFO,
ComputeSerialBalance(ctrl, agraph, agraph->where, lbvec));
IFSET(ctrl->dbglvl, DBG_REFINEINFO,
rprintf(ctrl, "input cut: %"PRIDX", balance: ", ComputeSerialEdgeCut(agraph)));
for (i=0; i<agraph->ncon; i++)
IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "%.3"PRREAL" ", lbvec[i]));
IFSET(ctrl->dbglvl, DBG_REFINEINFO, rprintf(ctrl, "\n"));
/****************************************/
/* Split the processors into two groups */
/****************************************/
sr = (ctrl->mype % 2 == 0) ? 1 : 0;
gd = (ctrl->mype % 2 == 1) ? 1 : 0;
if (graph->ncon > MAX_NCON_FOR_DIFFUSION || ctrl->npes == 1) {
sr = 1;
gd = 0;
}
sr_pe = 0;
gd_pe = 1;
gkMPI_Comm_split(ctrl->gcomm, sr, 0, &ipcomm);
gkMPI_Comm_rank(ipcomm, &mype);
gkMPI_Comm_size(ipcomm, &npes);
if (sr == 1) { /* Half of the processors do scratch-remap */
ngroups = gk_max(gk_min(RIP_SPLIT_FACTOR, npes), 1);
gkMPI_Comm_split(ipcomm, mype % ngroups, 0, &srcomm);
gkMPI_Comm_rank(srcomm, &srmype);
gkMPI_Comm_size(srcomm, &srnpes);
METIS_SetDefaultOptions(moptions);
moptions[METIS_OPTION_SEED] = ctrl->sync + (mype % ngroups) + 1;
tpwgts = ctrl->tpwgts;
tpwgts2 = rwspacemalloc(ctrl, 2*ncon);
iset(nvtxs, 0, lwhere);
lnparts = ctrl->nparts;
fpart = fpe = 0;
lnpes = srnpes;
while (lnpes > 1 && lnparts > 1) {
PASSERT(ctrl, agraph->nvtxs > 1);
/* determine the weights of the two partitions as a function of the
weight of the target partition weights */
for (j=(lnparts>>1), i=0; i<ncon; i++) {
tpwgts2[i] = rsum(j, tpwgts+fpart*ncon+i, ncon);
tpwgts2[ncon+i] = rsum(lnparts-j, tpwgts+(fpart+j)*ncon+i, ncon);
wsum = 1.0/(tpwgts2[i] + tpwgts2[ncon+i]);
tpwgts2[i] *= wsum;
tpwgts2[ncon+i] *= wsum;
}
METIS_PartGraphRecursive(&agraph->nvtxs, &ncon, agraph->xadj,
agraph->adjncy, agraph->vwgt, NULL, agraph->adjwgt,
&twoparts, tpwgts2, NULL, moptions, &edgecut, part);
/* pick one of the branches */
if (srmype < fpe+lnpes/2) {
KeepPart(ctrl, agraph, part, 0);
lnpes = lnpes/2;
lnparts = lnparts/2;
}
else {
KeepPart(ctrl, agraph, part, 1);
fpart = fpart + lnparts/2;
fpe = fpe + lnpes/2;
lnpes = lnpes - lnpes/2;
lnparts = lnparts - lnparts/2;
}
}
if (lnparts == 1) { /* Case in which srnpes is greater or equal to nparts */
/* Only the first process will assign labels (for the reduction to work) */
if (srmype == fpe) {
for (i=0; i<agraph->nvtxs; i++)
lwhere[agraph->label[i]] = fpart;
}
}
else { /* Case in which srnpes is smaller than nparts */
/* create the normalized tpwgts for the lnparts from ctrl->tpwgts */
tpwgts = rwspacemalloc(ctrl, lnparts*ncon);
for (j=0; j<ncon; j++) {
for (wsum=0.0, i=0; i<lnparts; i++) {
tpwgts[i*ncon+j] = ctrl->tpwgts[(fpart+i)*ncon+j];
wsum += tpwgts[i*ncon+j];
}
for (wsum=1.0/wsum, i=0; i<lnparts; i++)
tpwgts[i*ncon+j] *= wsum;
}
METIS_PartGraphKway(&agraph->nvtxs, &ncon, agraph->xadj, agraph->adjncy,
agraph->vwgt, NULL, agraph->adjwgt, &lnparts, tpwgts, NULL, moptions,
&edgecut, part);
for (i=0; i<agraph->nvtxs; i++)
lwhere[agraph->label[i]] = fpart + part[i];
}
gkMPI_Allreduce((void *)lwhere, (void *)part, nvtxs, IDX_T, MPI_SUM, srcomm);
edgecut = ComputeSerialEdgeCut(&cgraph);
ComputeSerialBalance(ctrl, &cgraph, part, lbvec);
lbsum = rsum(ncon, lbvec, 1);
gkMPI_Allreduce((void *)&edgecut, (void *)&max_cut, 1, IDX_T, MPI_MAX, ipcomm);
gkMPI_Allreduce((void *)&lbsum, (void *)&min_lbsum, 1, REAL_T, MPI_MIN, ipcomm);
lpecost.rank = ctrl->mype;
lpecost.cost = lbsum;
if (min_lbsum < UNBALANCE_FRACTION * (real_t)(ncon)) {
if (lbsum < UNBALANCE_FRACTION * (real_t)(ncon))
lpecost.cost = (double)edgecut;
else
lpecost.cost = (double)max_cut + lbsum;
}
gkMPI_Allreduce((void *)&lpecost, (void *)&gpecost, 1, MPI_DOUBLE_INT,
MPI_MINLOC, ipcomm);
if (ctrl->mype == gpecost.rank && ctrl->mype != sr_pe)
gkMPI_Send((void *)part, nvtxs, IDX_T, sr_pe, 1, ctrl->comm);
if (ctrl->mype != gpecost.rank && ctrl->mype == sr_pe)
gkMPI_Recv((void *)part, nvtxs, IDX_T, gpecost.rank, 1, ctrl->comm, &status);
if (ctrl->mype == sr_pe) {
icopy(nvtxs, part, lwhere);
SerialRemap(ctrl, &cgraph, ctrl->nparts, home, lwhere, part, ctrl->tpwgts);
}
gkMPI_Comm_free(&srcomm);
}