int run_zoltan(struct Zoltan_Struct *zz, Matrix A)
{
/* Variables returned by Zoltan */
ZOLTAN_ID_PTR import_gids = NULL; /* Global nums of objs to be imported */
ZOLTAN_ID_PTR import_lids = NULL; /* Local indices to objs to be imported */
int *import_procs = NULL; /* Proc IDs of procs owning objs to be
imported. */
int *import_to_part = NULL; /* Partition #s to which imported objs
should be assigned. */
ZOLTAN_ID_PTR export_gids = NULL; /* Global nums of objs to be exported */
ZOLTAN_ID_PTR export_lids = NULL; /* local indices to objs to be exported */
int *export_procs = NULL; /* Proc IDs of destination procs for objs
to be exported. */
int *export_to_part = NULL; /* Partition #s for objs to be exported.*/
int num_imported; /* Number of objs to be imported. */
int num_exported; /* Number of objs to be exported. */
int new_decomp; /* Flag indicating whether the decomposition
has changed */
int num_gid_entries; /* Number of array entries in a global ID. */
int num_lid_entries; /* Number of array entries in a local ID. */
int *ptr;
int myrank;
int k;
/*
* Call Zoltan to compute a new decomposition.
* The result is given as export and/or import lists.
* If you need only the export lists, perform the
* following call before this function:
* Zoltan_Set_Param(zz, "RETURN_LISTS", "EXPORT");
*
* In this example, we ignore the import_to_part/export_to_part
* arguments, as they are only useful if we distinguish between
* partitions and processors.
*/
if (Zoltan_LB_Partition(zz, &new_decomp, &num_gid_entries, &num_lid_entries,
&num_imported, &import_gids,
&import_lids, &import_procs, &import_to_part,
&num_exported, &export_gids,
&export_lids, &export_procs, &export_to_part) == ZOLTAN_FATAL){
printf("fatal: error returned from Zoltan_LB_Partition()\n");
return 0;
}
/* Print results. */
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
if (new_decomp){
printf("[Proc %1d] My data to export are:\n", myrank);
for (k=0; k<num_exported; k++){
ptr = (int *) &export_gids[num_gid_entries*k];
printf("[Proc %1d] Export (%d,%d) to proc %d\n", myrank,
*ptr, *(ptr+1), export_procs[k]);
}
}
else
printf("[Proc %1d] No changes over here.\n", myrank);
/* Data migration goes here. */
/* Clean up */
Zoltan_LB_Free_Part(&import_gids, &import_lids,
&import_procs, &import_to_part);
Zoltan_LB_Free_Part(&export_gids, &export_lids,
&export_procs, &export_to_part);
return 1;
}
int main(int argc, char *argv[])
{
int rc, i;
ZOLTAN_GNO_TYPE numGlobalVertices;
float ver;
char dimstring[16];
double min, max, avg, local;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
#ifdef HOST_LINUX
signal(SIGSEGV, meminfo_signal_handler);
signal(SIGINT, meminfo_signal_handler);
signal(SIGTERM, meminfo_signal_handler);
signal(SIGABRT, meminfo_signal_handler);
signal(SIGFPE, meminfo_signal_handler);
#endif
/******************************************************************
** Problem size
******************************************************************/
numGlobalVertices = NUM_GLOBAL_VERTICES;
vertexWeightDim = VERTEX_WEIGHT_DIMENSION;
vertexDim = VERTEX_DIMENSION;
if (argc > 1){
sscanf(argv[1], "%zd", &numGlobalVertices);
if (argc > 2){
vertexWeightDim = atoi(argv[2]);
if (argc > 3){
vertexDim = atoi(argv[3]);
}
}
}
sprintf(dimstring,"%d",vertexWeightDim);
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
exit(1);
}
Zoltan_Memory_Debug(2);
/******************************************************************
** Create vertices
******************************************************************/
rc = create_vertices(numGlobalVertices, vertexDim, vertexWeightDim, numProcs, myRank);
if (rc){
fprintf(stderr,"Process rank %d: insufficient memory\n",myRank);
MPI_Finalize();
exit(1);
}
first_gid = vertex_gid[myRank];
/******************************************************************
** Create a Zoltan library structure for this instance of load
** balancing. Set the parameters and query functions that will
** govern the library's calculation. See the Zoltan User's
** Guide for the definition of these and many other parameters.
******************************************************************/
zz = Zoltan_Create(MPI_COMM_WORLD);
/* General parameters */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", "RIB");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", dimstring);
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
/* RIB parameters */
Zoltan_Set_Param(zz, "RIB_OUTPUT_LEVEL", "0");
/* Query functions, to provide geometry to Zoltan */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, NULL);
Zoltan_Set_Obj_List_Fn(zz, get_object_list, NULL);
Zoltan_Set_Num_Geom_Fn(zz, get_num_geometry, NULL);
Zoltan_Set_Geom_Multi_Fn(zz, get_geometry_list, NULL);
Zoltan_Set_Part_Multi_Fn(zz, get_partition_list, NULL);
/******************************************************************
** Zoltan can now partition the vertices in the simple mesh.
** In this simple example, we assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
if (myRank == 0){
printf("Run Zoltan\n");
}
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK){
if (myRank == 0)printf("sorry...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Check the balance of the partitions before running zoltan.
** The query function get_partition_list() will give the
** partitions of the vertices before we called Zoltan.
******************************************************************/
if (myRank == 0){
printf("\nBALANCE before running Zoltan\n");
}
rc = Zoltan_LB_Eval_Balance(zz, 1, NULL);
if (rc != ZOLTAN_OK){
printf("sorry first LB_Eval_Balance...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Print out the balance of the new partitions.
******************************************************************/
vertex_part = (int *)malloc(sizeof(int) * numLocalVertices);
if (!vertex_part){
printf("sorry memory error...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
for (i=0; i < numLocalVertices; i++){
vertex_part[i] = myRank;
}
if (numExport > 0){
for (i=0; i < numExport; i++){
vertex_part[exportLocalGids[i]] = exportToPart[i];
}
}
if (myRank == 0){
printf("\nBALANCE after running Zoltan\n");
}
rc = Zoltan_LB_Eval_Balance(zz, 1, NULL);
if (rc != ZOLTAN_OK){
printf("sorry second LB_Eval_Balance...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Free the arrays allocated by Zoltan_LB_Partition, and free
** the storage allocated for the Zoltan structure.
******************************************************************/
if (myRank == 0){
printf("Free structures\n");
}
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
Zoltan_Destroy(&zz);
if (vertex_part) free(vertex_part);
if (v_x) free(v_x);
if (v_y) free(v_y);
if (v_z) free(v_z);
if (vertex_weight) free(vertex_weight);
if (vertex_gid) free(vertex_gid);
/**********************
** all done ***********
**********************/
local= (double)Zoltan_Memory_Usage(ZOLTAN_MEM_STAT_MAXIMUM)/(1024.0*1024);
MPI_Reduce(&local, &avg, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
avg /= (double)numProcs;
MPI_Reduce(&local, &max, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&local, &min, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
if (myRank == 0){
printf("Total MBytes in use by test while Zoltan is running: %12.3lf\n",
mbytes/(1024.0*1024));
printf("Min/Avg/Max of maximum MBytes in use by Zoltan: %12.3lf / %12.3lf / %12.3lf\n",
min, avg, max);
}
MPI_Finalize();
return 0;
}
void lb_zoltan(PSTopology top, LBMethod method, unsigned int dimen, list_type& pl)
{
const par::communicator& comm = par::comm_world();
float ver;
Zoltan_Initialize(0, 0, &ver);
struct Zoltan_Struct *zz;
// Create ZData (moves pl into zd)
ZData zd(par::comm_world(), std::move(pl));
// Allocate the Zoltan data
zz = Zoltan_Create(zd.comm.raw());
// Set some default sane parameters
if (method == LBMethod::RCB)
Zoltan_Set_Param(zz, "LB_METHOD", "RCB");
else
throw std::runtime_error("Unknown load balancing method");
// Zoltan_Set_Param(zz, "KEEP_CUTS", "1");
// Zoltan_Set_Param(zz, "LB_APPROACH", "REPARTITION");
// Zoltan_Set_Param(zz, "MIGRATE_ONLY_PROC_CHANGES", "1");
Zoltan_Set_Param(zz, "AUTO_MIGRATE", "TRUE");
// Set higher for more debugging output
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
// Set partition query methods
Zoltan_Set_Num_Obj_Fn(zz, pl_num_obj,
static_cast<void *>(&zd));
Zoltan_Set_Obj_List_Fn(zz, pl_obj_list,
static_cast<void *>(&zd));
Zoltan_Set_Num_Geom_Fn(zz, pl_num_geom,
static_cast<void *>(&zd));
Zoltan_Set_Geom_Multi_Fn(zz, pl_geom_multi,
static_cast<void *>(&zd));
// Migration query methods
Zoltan_Set_Mid_Migrate_PP_Fn(zz, pl_mid_migrate_pp,
static_cast<void *>(&zd));
Zoltan_Set_Obj_Size_Multi_Fn(zz, pl_obj_size_multi,
static_cast<void *>(&zd));
Zoltan_Set_Pack_Obj_Multi_Fn(zz, pl_pack_obj_multi,
static_cast<void *>(&zd));
Zoltan_Set_Unpack_Obj_Multi_Fn(zz, pl_unpack_obj_multi,
static_cast<void *>(&zd));
int
zerr,
changes,
num_gid_entries, num_lid_entries,
num_import,
*import_procs,
*import_to_part,
num_export,
*export_procs,
*export_to_part;
unsigned int
*import_global_ids,
*import_local_ids,
*export_global_ids,
*export_local_ids;
zerr = Zoltan_LB_Partition(zz,
&changes,
&num_gid_entries,
&num_lid_entries,
&num_import,
&import_global_ids,
&import_local_ids,
&import_procs,
&import_to_part,
&num_export,
&export_global_ids,
&export_local_ids,
&export_procs,
&export_to_part);
if (zerr != ZOLTAN_OK) comm.abort("Zoltan error", 1);
// Move the data back again out of the struct
pl = std::move(zd.list);
Zoltan_Destroy(&zz);
}
int main(int argc, char *argv[])
{
int i, rc;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport;
int myRank, numProcs;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int *parts;
FILE *fp;
GRAPH_DATA myGraph;
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
exit(0);
}
/******************************************************************
** Read graph from input file and distribute it
******************************************************************/
fp = fopen(fname, "r");
if (!fp){
if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
MPI_Finalize();
exit(1);
}
fclose(fp);
read_input_file(myRank, numProcs, fname, &myGraph);
/******************************************************************
** Create a Zoltan library structure for this instance of load
** balancing. Set the parameters and query functions that will
** govern the library's calculation. See the Zoltan User's
** Guide for the definition of these and many other parameters.
******************************************************************/
zz = Zoltan_Create(MPI_COMM_WORLD);
/* General parameters */
/*
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
*/
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz,"LB_METHOD","GRAPH");
#ifdef HAVE_PARMETIS
Zoltan_Set_Param(zz,"GRAPH_PACKAGE","PARMETIS");
#else
#ifdef HAVE_SCOTCH
Zoltan_Set_Param(zz,"GRAPH_PACKAGE","SCOTCH");
#endif
#endif
Zoltan_Set_Param(zz,"EDGE_WEIGHT_DIM","1");
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");
Zoltan_Set_Param(zz,"LB_APPROACH","REPARTITION");
Zoltan_Set_Param(zz,"GRAPH_SYMMETRIZE","TRANSPOSE");
Zoltan_Set_Param(zz,"GRAPH_SYM_WEIGHT","ADD");
/* Graph parameters */
Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */
/* Query functions - defined in simpleQueries.h */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph);
Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph);
/******************************************************************
** Zoltan can now partition the simple graph.
** In this simple example, we assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Visualize the graph partitioning before and after calling Zoltan.
******************************************************************/
parts = (int *)malloc(sizeof(int) * myGraph.numMyVertices);
for (i=0; i < myGraph.numMyVertices; i++){
parts[i] = myRank;
}
if (myRank== 0){
printf("\nGraph partition before calling Zoltan\n");
}
showGraphPartitions(myRank, myGraph.numMyVertices, myGraph.vertexGID, parts, numProcs);
for (i=0; i < numExport; i++){
parts[exportLocalGids[i]] = exportToPart[i];
}
if (myRank == 0){
printf("Graph partition after calling Zoltan\n");
}
showGraphPartitions(myRank, myGraph.numMyVertices, myGraph.vertexGID, parts, numProcs);
free(parts);
/******************************************************************
** Free the arrays allocated by Zoltan_LB_Partition, and free
** the storage allocated for the Zoltan structure.
******************************************************************/
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
Zoltan_Destroy(&zz);
/**********************
** all done ***********
**********************/
MPI_Finalize();
if (myGraph.numMyVertices > 0){
free(myGraph.vertexGID);
free(myGraph.nborIndex);
if (myGraph.numAllNbors > 0){
free(myGraph.nborGID);
free(myGraph.nborProc);
}
}
return 0;
}
int main(int argc, char *argv[])
{
int rc, i, myRank, numProcs;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int *parts;
FILE *fp;
MESH_DATA myMesh;
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
exit(0);
}
/******************************************************************
** Read geometry from input file and distribute it unevenly
******************************************************************/
fp = fopen(fname, "r");
if (!fp){
if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
MPI_Finalize();
exit(1);
}
fclose(fp);
read_input_objects(myRank, numProcs, fname, &myMesh);
/******************************************************************
** Create a Zoltan library structure for this instance of load
** balancing. Set the parameters and query functions that will
** govern the library's calculation. See the Zoltan User's
** Guide for the definition of these and many other parameters.
******************************************************************/
zz = Zoltan_Create(MPI_COMM_WORLD);
/* General parameters */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", "RCB");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
/* RCB parameters */
Zoltan_Set_Param(zz, "RCB_OUTPUT_LEVEL", "0");
Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS", "1");
/*Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS", "0"); */
/* Query functions, to provide geometry to Zoltan */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, &myMesh);
Zoltan_Set_Obj_List_Fn(zz, get_object_list, &myMesh);
Zoltan_Set_Num_Geom_Fn(zz, get_num_geometry, &myMesh);
Zoltan_Set_Geom_Multi_Fn(zz, get_geometry_list, &myMesh);
/******************************************************************
** Zoltan can now partition the vertices in the simple mesh.
** In this simple example, we assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Visualize the mesh partitioning before and after calling Zoltan.
******************************************************************/
parts = (int *)malloc(sizeof(int) * myMesh.numMyPoints);
for (i=0; i < myMesh.numMyPoints; i++){
parts[i] = myRank;
}
if (myRank== 0){
printf("\nMesh partition assignments before calling Zoltan\n");
}
showSimpleMeshPartitions(myRank, myMesh.numMyPoints, myMesh.myGlobalIDs, parts);
for (i=0; i < numExport; i++){
parts[exportLocalGids[i]] = exportToPart[i];
}
if (myRank == 0){
printf("Mesh partition assignments after calling Zoltan\n");
}
showSimpleMeshPartitions(myRank, myMesh.numMyPoints, myMesh.myGlobalIDs, parts);
free(parts);
/******************************************************************
** Free the arrays allocated by Zoltan_LB_Partition, and free
** the storage allocated for the Zoltan structure.
******************************************************************/
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
Zoltan_Destroy(&zz);
/**********************
** all done ***********
**********************/
MPI_Finalize();
if (myMesh.numMyPoints > 0){
free(myMesh.myGlobalIDs);
free(myMesh.x);
free(myMesh.y);
}
return 0;
}
std::vector<int> zoltanGraphPartitionGridOnRoot(const CpGrid& cpgrid,
const CollectiveCommunication<MPI_Comm>& cc,
int root)
{
int rc;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int argc=0;
char** argv;
rc = Zoltan_Initialize(argc, argv, &ver);
zz = Zoltan_Create(cc);
if ( rc != ZOLTAN_OK )
{
OPM_THROW(std::runtime_error, "Could not initialize Zoltan!");
}
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "3");
Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */
bool pretendEmptyGrid = cc.rank()!=root;
Dune::cpgrid::setCpGridZoltanGraphFunctions(zz, cpgrid, pretendEmptyGrid);
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
int size = cpgrid.numCells();
int rank = cc.rank();
std::vector<int> parts=std::vector<int>(size, rank);
for ( int i=0; i < numExport; ++i )
{
parts[exportLocalGids[i]] = exportProcs[i];
}
cc.broadcast(&parts[0], parts.size(), root);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart);
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart);
Zoltan_Destroy(&zz);
return parts;
}
int main(int argc, char *argv[])
{
int i, rc;
int myRank, numProcs;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport, start_gid, num_nbors;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int *parts=NULL;
ZOLTAN_ID_PTR lids=NULL;
FILE *fp;
struct Zoltan_DD_Struct *dd;
GRAPH_DATA myGraph;
int gid_length = 1; /* our global IDs consist of 1 integer */
int lid_length = 1; /* our local IDs consist of 1 integer */
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
exit(0);
}
/******************************************************************
** Read graph from input file and distribute it
******************************************************************/
fp = fopen(fname, "r");
if (!fp){
if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
MPI_Finalize();
exit(1);
}
fclose(fp);
read_input_file(myRank, numProcs, fname, &myGraph);
/*fprintf(stderr,"%d have %d objects\n",myRank,myGraph.numMyVertices);*/
/******************************************************************
** Create a distributed data directory which maps vertex
** global IDs to their current partition number. We'll use this
** after migrating vertices, to update the partition in which
** our vertices neighbors are.
**
** Our local IDs (array "lids") are of type ZOLTAN_ID_TYPE because
** we are using Zoltan's distributed data directory. It assumes
** that a global ID is a sequence of "gid_length" ZOLTAN_ID_TYPEs.
** It assumes that a local ID is a sequence of "lid_length"
** ZOLTAN_ID_TYPEs.
******************************************************************/
rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD,
gid_length, /* length of a global ID */
lid_length, /* length of a local ID */
0, /* length of user data */
myGraph.numMyVertices, /* hash table size */
0); /* debug level */
parts = malloc(myGraph.numMyVertices * sizeof(int));
lids = malloc(myGraph.numMyVertices * sizeof(ZOLTAN_ID_TYPE));
for (i=0; i < myGraph.numMyVertices; i++){
parts[i] = myRank; /* part number of this vertex */
lids[i] = (ZOLTAN_ID_TYPE)i; /* local ID on my process for this vertex */
}
rc = Zoltan_DD_Update(dd,
myGraph.vertexGID,
lids,
NULL,
parts,
myGraph.numMyVertices);
myGraph.dd = dd;
/******************************************************************
** Create a Zoltan library structure for this instance of load
** balancing. Set the parameters and query functions that will
** govern the library's calculation. See the Zoltan User's
** Guide for the definition of these and many other parameters.
******************************************************************/
zz = Zoltan_Create(MPI_COMM_WORLD);
/* General parameters */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
/* Graph parameters */
Zoltan_Set_Param(zz, "CHECK_GRAPH", "2");
Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */
/* Query functions, defined in this source file */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph);
Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph);
Zoltan_Set_Obj_Size_Multi_Fn(zz, get_message_sizes,&myGraph);
Zoltan_Set_Pack_Obj_Multi_Fn(zz, pack_object_messages,&myGraph);
Zoltan_Set_Unpack_Obj_Multi_Fn(zz, unpack_object_messages,&myGraph);
Zoltan_Set_Mid_Migrate_PP_Fn(zz, mid_migrate,&myGraph);
/******************************************************************
** Visualize the graph partitioning before calling Zoltan.
******************************************************************/
if (myRank== 0){
printf("\nGraph partition before calling Zoltan\n");
}
showGraphPartitions(myRank, myGraph.dd);
/******************************************************************
** Zoltan can now partition the simple graph.
** In this simple example, we assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK){
printf("sorry...\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/*fprintf(stderr,"%d export %d import %d\n",myRank,numExport,numImport);*/
/******************************************************************
** Update the data directory with the new partition numbers
******************************************************************/
for (i=0; i < numExport; i++){
parts[exportLocalGids[i]] = exportToPart[i];
}
rc = Zoltan_DD_Update(dd,
myGraph.vertexGID,
lids,
NULL,
parts,
myGraph.numMyVertices);
/******************************************************************
** Migrate vertices to new partitions
******************************************************************/
rc = Zoltan_Migrate(zz,
numImport, importGlobalGids, importLocalGids,
importProcs, importToPart,
numExport, exportGlobalGids, exportLocalGids,
exportProcs, exportToPart);
/******************************************************************
** Use the data dictionary to find neighbors' partitions
******************************************************************/
start_gid = myGraph.numMyVertices - numImport;
num_nbors = myGraph.nborIndex[myGraph.numMyVertices] - myGraph.nborIndex[start_gid];
rc = Zoltan_DD_Find(dd,
(ZOLTAN_ID_PTR)(myGraph.nborGID + start_gid), NULL, NULL,
myGraph.nborPart + start_gid, num_nbors, NULL);
/******************************************************************
** Visualize the graph partitioning after calling Zoltan.
******************************************************************/
if (myRank == 0){
printf("Graph partition after calling Zoltan\n");
}
showGraphPartitions(myRank, myGraph.dd);
/******************************************************************
** Free the arrays allocated by Zoltan_LB_Partition, and free
** the storage allocated for the Zoltan structure.
******************************************************************/
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
Zoltan_Destroy(&zz);
/**********************
** all done ***********
**********************/
MPI_Finalize();
if (myGraph.vertex_capacity > 0){
free(myGraph.vertexGID);
free(myGraph.nborIndex);
if (myGraph.nbor_capacity > 0){
free(myGraph.nborGID);
free(myGraph.nborPart);
}
}
if (parts) free(parts);
if (lids) free(lids);
return 0;
}
int main(int argc, char *argv[])
{
int rc, i;
int myRank, numProcs;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids;
ZOLTAN_ID_PTR exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int *parts = NULL;
FILE *fp;
OBJECT_DATA myData;
/******************************************************************
** Initialize MPI and Zoltan
******************************************************************/
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
rc = Zoltan_Initialize(argc, argv, &ver);
if (rc != ZOLTAN_OK){
printf("Error initializing Zoltan\n");
MPI_Finalize();
exit(0);
}
/******************************************************************
** Read objects from input file and distribute them unevenly
******************************************************************/
fp = fopen(fname, "r");
if (!fp){
if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname);
MPI_Finalize();
exit(1);
}
fclose(fp);
read_input_objects(myRank, numProcs, fname, &myData);
/******************************************************************
** Create a Zoltan library structure for this instance of load
** balancing. Set the parameters and query functions.
******************************************************************/
zz = Zoltan_Create(MPI_COMM_WORLD);
/* General parameters */
Zoltan_Set_Param(zz, "LB_METHOD", "BLOCK"); /* Zoltan method: "BLOCK" */
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); /* global ID is 1 integer */
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); /* local ID is 1 integer */
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0"); /* we omit object weights */
/* Query functions */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, &myData);
Zoltan_Set_Obj_List_Fn(zz, get_object_list, &myData);
/******************************************************************
** Call Zoltan to partition the objects.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of objects to be sent to me */
&importGlobalGids, /* Global IDs of objects to be sent to me */
&importLocalGids, /* Local IDs of objects to be sent to me */
&importProcs, /* Process rank for source of each incoming object */
&importToPart, /* New partition for each incoming object */
&numExport, /* Number of objects I must send to other processes*/
&exportGlobalGids, /* Global IDs of the objects I must send */
&exportLocalGids, /* Local IDs of the objects I must send */
&exportProcs, /* Process to which I send each of the objects */
&exportToPart); /* Partition to which each object will belong */
if (rc != ZOLTAN_OK){
printf("Error in Zoltan library\n");
MPI_Finalize();
Zoltan_Destroy(&zz);
exit(0);
}
/******************************************************************
** Visualize the object partitioning before and after calling Zoltan.
**
** In this example, partition number equals process rank.
******************************************************************/
parts = (int *)malloc(sizeof(int) * myData.numMyObjects);
for (i=0; i < myData.numMyObjects; i++){
parts[i] = myRank;
}
if (myRank== 0){
printf("\nObject partition assignments before calling Zoltan\n");
}
showSimpleMeshPartitions(myRank, myData.numMyObjects, myData.myGlobalIDs, parts);
for (i=0; i < numExport; i++){
parts[exportLocalGids[i]] = exportToPart[i];
}
if (myRank == 0){
printf("Object partition assignments after calling Zoltan\n");
}
showSimpleMeshPartitions(myRank, myData.numMyObjects, myData.myGlobalIDs, parts);
/******************************************************************
** Free the arrays allocated by Zoltan_LB_Partition, and free
** the storage allocated for the Zoltan structure.
******************************************************************/
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
Zoltan_Destroy(&zz);
MPI_Finalize();
return 0;
}
int MESH_PartitionWithZoltan(Mesh_ptr mesh, int nparts, int **part, int noptions,
char **options, MSTK_Comm comm) {
MEdge_ptr fedge;
MFace_ptr mf, oppf, rface;
MRegion_ptr mr, oppr;
List_ptr fedges, efaces, rfaces, fregions;
int i, j, k, id;
int nv, ne, nf, nr=0, nfe, nef, nfr, nrf, idx, idx2;
int numflag, nedgecut, ipos;
int wtflag;
int rc;
float ver;
struct Zoltan_Struct *zz;
GRAPH_DATA graph;
int changes, numGidEntries, numLidEntries, numImport, numExport;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int rank;
MPI_Comm_rank(comm,&rank);
rc = Zoltan_Initialize(0, NULL, &ver);
if (rc != ZOLTAN_OK){
MSTK_Report("MESH_PartitionWithZoltan","Could not initialize Zoltan",MSTK_FATAL);
MPI_Finalize();
exit(0);
}
/******************************************************************
** Create a Zoltan library structure for this instance of partition
********************************************************************/
zz = Zoltan_Create(comm);
/*****************************************************************
** Figure out partitioning method
*****************************************************************/
char partition_method_str[32];
strcpy(partition_method_str,"RCB"); /* Default - Recursive Coordinate Bisection */
if (noptions) {
for (i = 0; i < noptions; i++) {
if (strncmp(options[i],"LB_PARTITION",12) == 0) {
char *result = NULL, instring[256];
strcpy(instring,options[i]);
result = strtok(instring,"=");
result = strtok(NULL," ");
strcpy(partition_method_str,result);
}
}
}
if (rank == 0) {
char mesg[256];
sprintf(mesg,"Using partitioning method %s for ZOLTAN\n",partition_method_str);
MSTK_Report("MESH_PartitionWithZoltan",mesg,MSTK_MESG);
}
/* General parameters for Zoltan */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", partition_method_str);
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
graph.numMyNodes = 0;
graph.numAllNbors = 0;
graph.nodeGID = NULL;
graph.nodeCoords = NULL;
graph.nborIndex = NULL;
graph.nborGID = NULL;
graph.nborProc = NULL;
if (strcmp(partition_method_str,"RCB") == 0) {
if (rank == 0) {
nr = MESH_Num_Regions(mesh);
nf = MESH_Num_Faces(mesh);
if (!nf && !nr)
MSTK_Report("MESH_PartitionWithZoltan","Cannot partition wire meshes",
MSTK_FATAL);
if (nr == 0) { /* Surface or planar mesh */
int ndim = 2; /* assume mesh is planar */
idx = 0;
MVertex_ptr mv;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
double vxyz[3];
MV_Coords(mv,vxyz);
if (vxyz[2] != 0.0) {
ndim = 3; /* non-planar or planar with non-zero z */
break;
}
}
NDIM_4_ZOLTAN = ndim-1; /* ignore last dimension to avoid partitioning in that dimension */
graph.numMyNodes = nf;
graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nf);
graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nf);
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
double fxyz[MAXPV2][3], cen[3];
int nfv;
MF_Coords(mf,&nfv,fxyz);
cen[0] = cen[1] = cen[2] = 0.0;
for (j = 0; j < nfv; j++)
for (k = 0; k < NDIM_4_ZOLTAN; k++)
cen[k] += fxyz[j][k];
for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nfv;
id = MF_ID(mf);
graph.nodeGID[id-1] = id;
memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double));
}
}
else { /* Volume mesh */
int ndim = 3;
NDIM_4_ZOLTAN = ndim-1; /* ignore last dimension to avoid partitioning in that dimension */
graph.numMyNodes = nr;
graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nr);
graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nr);
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
double rxyz[MAXPV3][3], cen[3];
int nrv;
MR_Coords(mr,&nrv,rxyz);
cen[0] = cen[1] = cen[2] = 0.0;
for (j = 0; j < nrv; j++)
for (k = 0; k < NDIM_4_ZOLTAN; k++)
cen[k] += rxyz[j][k];
for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nrv;
for (k = 0; k < NDIM_4_ZOLTAN; k++)
if (fabs(cen[k]) < 1.0e-10) cen[k] = 0.0;
id = MR_ID(mr);
graph.nodeGID[id-1] = id;
memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double));
}
}
}
MPI_Bcast(&NDIM_4_ZOLTAN,1,MPI_INT,0,comm);
/* Set some default values */
Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS","1");
// Zoltan_Set_Param(zz, "AVERAGE_CUTS", "1");
if (noptions > 1) {
for (i = 1; i < noptions; i++) {
char *paramstr = NULL, *valuestr = NULL, instring[256];
strcpy(instring,options[i]);
paramstr = strtok(instring,"=");
valuestr = strtok(NULL," ");
Zoltan_Set_Param(zz,paramstr,valuestr);
}
}
/* Query functions - defined in simpleQueries.h */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph);
Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph);
Zoltan_Set_Num_Geom_Fn(zz, get_num_dimensions_reduced, &graph); /* reduced dimensions */
Zoltan_Set_Geom_Multi_Fn(zz, get_element_centers_reduced, &graph); /* reduced dimension centers */
}
else if (strcmp(partition_method_str,"GRAPH") == 0) {
if(rank == 0) {
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
ipos = 0;
/* build nodes and neighbors list, similar as in partition with metis
Assign processor 0 the whole mesh, assign other processors a NULL mesh */
if (nr == 0) {
if (nf == 0) {
MSTK_Report("MESH_PartitionWithZoltan",
"Cannot partition wire meshes with Zoltan",MSTK_FATAL);
exit(-1);
}
graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nf);
graph.nborIndex = (int *)malloc(sizeof(int) * (nf + 1));
graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*ne);
graph.nborProc = (int *)malloc(sizeof(int) * 2*ne);
graph.nborIndex[0] = 0;
/* Surface mesh */
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
graph.nodeGID[i] = MF_ID(mf);
fedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(fedges);
idx2 = 0;
while ((fedge = List_Next_Entry(fedges,&idx2))) {
efaces = ME_Faces(fedge);
nef = List_Num_Entries(efaces);
if (nef == 1) {
continue; /* boundary edge; nothing to do */
} else {
int j;
for (j = 0; j < nef; j++) {
oppf = List_Entry(efaces,j);
if (oppf == mf) {
graph.nborGID[ipos] = MF_ID(oppf);
/* initially set all nodes on processor 0 */
graph.nborProc[ipos] = 0;
ipos++;
}
}
}
List_Delete(efaces);
}
List_Delete(fedges);
i++;
graph.nborIndex[i] = ipos;
}
graph.numMyNodes = i;
graph.numAllNbors = ipos;
}
else {
graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nr);
graph.nborIndex = (int *)malloc(sizeof(int) * (nr + 1));
graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*nf);
graph.nborProc = (int *)malloc(sizeof(int) * 2*nf);
graph.nborIndex[0] = 0;
/* Volume mesh */
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
graph.nodeGID[i] = MR_ID(mr);
rfaces = MR_Faces(mr);
nrf = List_Num_Entries(rfaces);
idx2 = 0;
while ((rface = List_Next_Entry(rfaces,&idx2))) {
fregions = MF_Regions(rface);
nfr = List_Num_Entries(fregions);
if (nfr > 1) {
oppr = List_Entry(fregions,0);
if (oppr == mr)
oppr = List_Entry(fregions,1);
graph.nborGID[ipos] = MR_ID(oppr);
/* initially set all nodes on processor 0 */
graph.nborProc[ipos] = 0;
ipos++;
}
List_Delete(fregions);
}
List_Delete(rfaces);
i++;
graph.nborIndex[i] = ipos;
}
graph.numMyNodes = i;
graph.numAllNbors = ipos;
}
}
/* Graph parameters */
/* Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); */
Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */
/* Query functions - defined in simpleQueries.h */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph);
Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &graph);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &graph);
}
/* Partition the graph */
/******************************************************************
** Zoltan can now partition the graph.
** We assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of nodes to be sent to me */
&importGlobalGids, /* Global IDs of nodes to be sent to me */
&importLocalGids, /* Local IDs of nodes to be sent to me */
&importProcs, /* Process rank for source of each incoming node */
&importToPart, /* New partition for each incoming node */
&numExport, /* Number of nodes I must send to other processes*/
&exportGlobalGids, /* Global IDs of the nodes I must send */
&exportLocalGids, /* Local IDs of the nodes I must send */
&exportProcs, /* Process to which I send each of the nodes */
&exportToPart); /* Partition to which each node will belong */
if (rc != ZOLTAN_OK){
if (rank == 0)
MSTK_Report("MESH_PartitionWithZoltan","Could not partition mesh with ZOLTAN",
MSTK_ERROR);
Zoltan_Destroy(&zz);
MPI_Finalize();
return 0;
}
if(rank == 0) {
*part = (int *) calloc(graph.numMyNodes,sizeof(int));
for ( i = 0; i < numExport; i++ ) {
(*part)[exportGlobalGids[i]-1] = exportToPart[i];
}
if (graph.nodeGID) free(graph.nodeGID);
if (graph.nodeCoords) free(graph.nodeCoords);
if (graph.nborIndex) free(graph.nborIndex);
if (graph.nborGID) free(graph.nborGID);
if (graph.nborProc) free(graph.nborProc);
}
else {
*part = NULL;
}
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart);
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart);
Zoltan_Destroy(&zz);
return 1;
}
PeridigmNS::ZoltanSearchTree::ZoltanSearchTree(int numPoints, double* coordinates) : SearchTree(numPoints, coordinates),
callbackdata(numPoints,coordinates),zoltan(0),
partToCoordIdx(new int[numPoints]),
searchParts(new int[numPoints])
{
zoltan = Zoltan_Create(MPI_COMM_SELF);
Zoltan_Set_Param(zoltan, "debug_level", "0");
Zoltan_Set_Param(zoltan, "rcb_output_level", "0");
/*
* query function returns the number of objects that are currently assigned to the processor
*/
Zoltan_Set_Num_Obj_Fn(zoltan, &get_num_points, &callbackdata);
/*
* query function fills two (three if weights are used) arrays with information about the objects
* currently assigned to the processor. Both arrays are allocated (and subsequently freed) by Zoltan;
* their size is determined by a call to a ZOLTAN_NUM_OBJ_FN query function to get the array size.
*/
Zoltan_Set_Obj_List_Fn(zoltan, &get_point_ids, &callbackdata);
/*
* query function returns the number of values needed to express the geometry of an object.
* For example, for a two-dimensional mesh-based application, (x,y) coordinates are needed
* to describe an object's geometry; thus the ZOLTAN_NUM_GEOM_FN query function should return
* the value of two. For a similar three-dimensional application, the return value should be three.
*/
Zoltan_Set_Num_Geom_Fn(zoltan, &get_dimension, NULL);
/*
* query function returns a vector of geometry values for a list of given objects. The geometry
* vector is allocated by Zoltan to be of size num_obj * num_dim.
*/
Zoltan_Set_Geom_Multi_Fn(zoltan, &get_point_coordinates, &callbackdata);
/*
* setting method
*/
Zoltan_Set_Param(zoltan,"lb_method","rcb");
/*
* keep cuts
*/
Zoltan_Set_Param(zoltan,"keep_cuts","1");
Zoltan_Set_Param(zoltan,"return_lists","part");
Zoltan_Set_Param(zoltan, "num_lid_entries", "0");
char s[11];
sprintf(s,"%d",numPoints);
Zoltan_Set_Param(zoltan, "num_global_parts", s);
int numimp = -1, numexp = -1;
int numgid = 1, numlid = 0;
ZOLTAN_ID_PTR impgid=NULL, implid=NULL;
ZOLTAN_ID_PTR gid=NULL, lid=NULL;
int *imppart = NULL, *impproc = NULL;
int *procs = NULL, *parts=NULL;
int changes;
int ierr = Zoltan_LB_Partition(zoltan, &changes, &numgid, &numlid,
&numimp, &impgid, &implid, &imppart, &impproc,
&numexp, &gid, &lid, &procs, &parts);
TEUCHOS_TEST_FOR_EXCEPT_MSG(ierr != 0, "Error in ZoltanSearchTree::ZoltanSearchTree(), call to Zoltan_LB_Partition() returned a nonzero error code.");
for (int I = 0; I < numPoints; I++)
partToCoordIdx[parts[I]] = I;
Zoltan_LB_Free_Part(&impgid, &implid, &impproc, &imppart);
Zoltan_LB_Free_Part(&gid, &lid, &procs, &parts);
}
int main(int argc, char *argv[])
{
int rc, do_hier, status;
float ver;
struct Zoltan_Struct *zz;
int changes, numGidEntries, numLidEntries, numImport, numExport;
int generate_files = 0;
char *platform=NULL, *topology=NULL;
char *graph_package=NULL;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
struct option opts[10];
double comm_time[10];
float cut_weight[3] = {0., 0., 0.};
long nvert=0;
char *debug_level=NULL;
status = 0;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
Zoltan_Initialize(argc, argv, &ver);
zz = Zoltan_Create(MPI_COMM_WORLD);
/******************************************************************
** Check that this test makes sense.
******************************************************************/
if (sizeof(long) < sizeof(ZOLTAN_ID_TYPE)){
if (myRank == 0){
printf("ERROR: This code assumes that a long is at least %d bytes\n",(int)sizeof(ZOLTAN_ID_TYPE));
}
status = 1;
}
check_error_status(status, "configuration error");
/******************************************************************
** Initialize zoltan
******************************************************************/
/* options */
opts[0].name = "platform";
opts[0].has_arg = 1;
opts[0].flag = NULL;
opts[0].val = 1;
opts[1].name = "topology";
opts[1].has_arg = 1;
opts[1].flag = NULL;
opts[1].val = 2;
opts[2].name = "size";
opts[2].has_arg = 1;
opts[2].flag = NULL;
opts[2].val = 4;
opts[3].name = "verbose";
opts[3].has_arg = 0;
opts[3].flag = NULL;
opts[3].val = 5;
opts[4].name = "help";
opts[4].has_arg = 0;
opts[4].flag = NULL;
opts[4].val = 6;
opts[5].name = "graph_package";
opts[5].has_arg = 1;
opts[5].flag = NULL;
opts[5].val = 7;
opts[6].name = "generate_files";
opts[6].has_arg = 0;
opts[6].flag = NULL;
opts[6].val = 8;
opts[7].name = "debug_level";
opts[7].has_arg = 1;
opts[7].flag = NULL;
opts[7].val = 9;
opts[8].name = 0;
opts[8].has_arg = 0;
opts[8].flag = NULL;
opts[8].val = 0;
status = 0;
while (1){
rc = getopt_long_only(argc, argv, "", opts, NULL);
if (rc == '?'){
MPI_Barrier(MPI_COMM_WORLD);
if (myRank == 0) usage();
MPI_Finalize();
exit(0);
}
else if (rc == 1){
platform = optarg;
if (myRank == 0)
printf( "For platform %s\n",optarg );
}
else if (rc == 2){
topology = optarg;
if (myRank == 0)
printf( "For topology %s\n",optarg);
}
else if (rc == 7){
graph_package = optarg;
if (myRank == 0)
printf( "Zoltan parameter GRAPH_PACKAGE = %s\n",graph_package);
}
else if (rc == 8){
generate_files = 1;
if (myRank == 0)
printf( "Zoltan_Generate_Files will be called for each level.\n");
}
else if (rc == 4){
nvert = atol(optarg);
if (nvert < 1) status = 1;
check_error_status(status, "--size={approximate number of vertices}");
if (myRank == 0){
printf( "Graph will have approximately %ld vertices.\n",nvert);
}
}
else if (rc == 5){
verbose = 1;
}
else if (rc == 6){
if (myRank == 0) usage();
MPI_Finalize();
exit(0);
}
else if (rc == 9){
debug_level = optarg;
}
else if (rc <= 0){
break;
}
}
if ((platform==NULL) && (topology==NULL)){
if (myRank == 0)
fprintf(stdout,"No platform or topology, so we'll skip hierarchical partitioning\n");
do_hier = 0;
}
else if (graph_package == NULL){
if (myRank == 0)
fprintf(stdout,"No graph package, so we'll skip hierarchical partitioning\n");
do_hier = 0;
}
else{
do_hier = 1;
}
/* start */
Zoltan_Memory_Debug(0);
if (nvert > 0)
numGlobalVertices = nvert;
else
numGlobalVertices = NUM_GLOBAL_VERTICES;
status = create_a_graph();
check_error_status(status, "creating the graph");
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "REMAP", "0");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* export AND import lists */
Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "1"); /* number of weights per vertex */
Zoltan_Set_Param(zz, "EDGE_WEIGHT_DIM", "1");/* number of weights per hyperedge */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, NULL);
Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, NULL);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, NULL);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, NULL);
/* GRAPH PARTITION */
Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH");
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
if (graph_package)
Zoltan_Set_Param(zz, "GRAPH_PACKAGE", graph_package);
if (verbose){
debug(zz, "Initial graph", 0);
}
if (generate_files){
rc = Zoltan_Generate_Files(zz, "flat", myRank, 0, 1, 0);
if (rc != ZOLTAN_OK) status = 1;
check_error_status(status, "Zoltan_Generate_Files");
}
/* Performance before partitioning */
time_communication(comm_time+0);
cut_weight[0] = get_edge_cut_weight(zz);
if (cut_weight[0] < 0.0) status = 1;
check_error_status(status, "First call to get_edge_cut_weight");
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK) status = 1;
check_error_status(status, "First call to LB_Partition");
status = migrate_graph(numExport, numImport, exportLocalGids, importGlobalGids);
check_error_status(status, "migration");
if (verbose){
debug(zz, "After flat partitioning and migration", 0);
}
time_communication(comm_time+1); /* With graph partitioning */
cut_weight[1] = get_edge_cut_weight(zz);
if (cut_weight[1] < 0.0) status = 1;
check_error_status(status, "Second call to get_edge_cut_weight");
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
if (do_hier){
/* HIERARCHICAL PARTITION */
free_graph();
status = create_a_graph();
check_error_status(status, "create graph for hierarchical partitioning");
Zoltan_Set_Param(zz, "LB_METHOD", "HIER");
Zoltan_Set_Param(zz, "HIER_ASSIST", "1");
if (generate_files){
Zoltan_Set_Param(zz, "HIER_GENERATE_FILES", "1");
}
if (debug_level) /* 1, 2 or 3 */
Zoltan_Set_Param(zz, "HIER_DEBUG_LEVEL", debug_level);
else
Zoltan_Set_Param(zz, "HIER_DEBUG_LEVEL", "0");
/* TODO: Suppose graph is not symmetric, and we request SYMMETRIZE. Do we still get
* a "good" answer when each sub-graph in the hierarchy is symmetrized?
*/
if (topology)
Zoltan_Set_Param(zz, "TOPOLOGY", topology);
else if (platform)
Zoltan_Set_Param(zz, "PLATFORM", platform);
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of vertices to be sent to me */
&importGlobalGids, /* Global IDs of vertices to be sent to me */
&importLocalGids, /* Local IDs of vertices to be sent to me */
&importProcs, /* Process rank for source of each incoming vertex */
&importToPart, /* New partition for each incoming vertex */
&numExport, /* Number of vertices I must send to other processes*/
&exportGlobalGids, /* Global IDs of the vertices I must send */
&exportLocalGids, /* Local IDs of the vertices I must send */
&exportProcs, /* Process to which I send each of the vertices */
&exportToPart); /* Partition to which each vertex will belong */
if (rc != ZOLTAN_OK) status = 1;
check_error_status(status, "Second call to LB_Partition");
status = migrate_graph(numExport, numImport, exportLocalGids, importGlobalGids);
check_error_status(status, "second migration");
if (verbose){
debug(zz, "After hierarchical partitioning and migration", 0);
}
time_communication(comm_time+2); /* With hierarchical graph partitioning */
cut_weight[2] = get_edge_cut_weight(zz);
if (cut_weight[2] < 0.0) status = 1;
check_error_status(status, "Third call to get_edge_cut_weight");
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids,
&importProcs, &importToPart);
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids,
&exportProcs, &exportToPart);
}
Zoltan_Destroy(&zz);
free_graph();
if (myRank == 0){
fprintf(stdout,"Graph cut weight before partitioning: %f\n",cut_weight[0]);
fprintf(stdout," after flat partitioning: %f\n",cut_weight[1]);
if (do_hier)
fprintf(stdout," after hierarchical partitioning: %f\n",cut_weight[2]);
fflush(stdout);
}
if (cut_weight[1] >= cut_weight[0]){
status = 1;
if (zz->Proc == 0){
fprintf(stderr,"FAILED: No improvement shown in flat partitioning");
}
}
if (do_hier && (cut_weight[2] > cut_weight[0])){
status = 1;
if (zz->Proc == 0){
fprintf(stderr,"FAILED: No improvement shown in hierarchical partitioning");
}
}
MPI_Finalize();
return status;
}