int Zoltan_Set_Fn(ZZ *zz, ZOLTAN_FN_TYPE fn_type, ZOLTAN_VOID_FN *fn, void *data) { /* * Function to initialize a given LB interface function. * Input: * zz -- Pointer to a Zoltan structure. * fn_type -- Enum type indicating the function to be set. * fn -- Pointer to the function to be used in the * assignment. * data -- Pointer to data that the LB library will * pass as an argument to fn(). May be NULL. * Output: * zz -- Appropriate field set to value in void *(). */ char *yo = "Zoltan_Set_Fn"; char msg[256]; int ierr; switch (fn_type) { case ZOLTAN_PART_FN_TYPE: ierr = Zoltan_Set_Part_Fn(zz, (ZOLTAN_PART_FN *) fn, data); break; case ZOLTAN_PART_MULTI_FN_TYPE: ierr = Zoltan_Set_Part_Multi_Fn(zz, (ZOLTAN_PART_MULTI_FN *) fn, data); break; case ZOLTAN_NUM_EDGES_FN_TYPE: ierr = Zoltan_Set_Num_Edges_Fn(zz, (ZOLTAN_NUM_EDGES_FN *) fn, data); break; case ZOLTAN_NUM_EDGES_MULTI_FN_TYPE: ierr = Zoltan_Set_Num_Edges_Multi_Fn(zz, (ZOLTAN_NUM_EDGES_MULTI_FN *) fn, data); break; case ZOLTAN_EDGE_LIST_FN_TYPE: ierr = Zoltan_Set_Edge_List_Fn(zz, (ZOLTAN_EDGE_LIST_FN *) fn, data); break; case ZOLTAN_EDGE_LIST_MULTI_FN_TYPE: ierr = Zoltan_Set_Edge_List_Multi_Fn(zz, (ZOLTAN_EDGE_LIST_MULTI_FN *) fn, data); break; case ZOLTAN_NUM_GEOM_FN_TYPE: ierr = Zoltan_Set_Num_Geom_Fn(zz, (ZOLTAN_NUM_GEOM_FN *) fn, data); break; case ZOLTAN_GEOM_MULTI_FN_TYPE: ierr = Zoltan_Set_Geom_Multi_Fn(zz, (ZOLTAN_GEOM_MULTI_FN *) fn, data); break; case ZOLTAN_GEOM_FN_TYPE: ierr = Zoltan_Set_Geom_Fn(zz, (ZOLTAN_GEOM_FN *) fn, data); break; case ZOLTAN_NUM_OBJ_FN_TYPE: ierr = Zoltan_Set_Num_Obj_Fn(zz, (ZOLTAN_NUM_OBJ_FN *) fn, data); break; case ZOLTAN_OBJ_LIST_FN_TYPE: ierr = Zoltan_Set_Obj_List_Fn(zz, (ZOLTAN_OBJ_LIST_FN *) fn, data); break; case ZOLTAN_FIRST_OBJ_FN_TYPE: ierr = Zoltan_Set_First_Obj_Fn(zz, (ZOLTAN_FIRST_OBJ_FN *) fn, data); break; case ZOLTAN_NEXT_OBJ_FN_TYPE: ierr = Zoltan_Set_Next_Obj_Fn(zz, (ZOLTAN_NEXT_OBJ_FN *) fn, data); break; case ZOLTAN_NUM_BORDER_OBJ_FN_TYPE: ierr = Zoltan_Set_Num_Border_Obj_Fn(zz, (ZOLTAN_NUM_BORDER_OBJ_FN *) fn, data); break; case ZOLTAN_BORDER_OBJ_LIST_FN_TYPE: ierr = Zoltan_Set_Border_Obj_List_Fn(zz, (ZOLTAN_BORDER_OBJ_LIST_FN *) fn, data); break; case ZOLTAN_FIRST_BORDER_OBJ_FN_TYPE: ierr = Zoltan_Set_First_Border_Obj_Fn(zz, (ZOLTAN_FIRST_BORDER_OBJ_FN *) fn, data); break; case ZOLTAN_NEXT_BORDER_OBJ_FN_TYPE: ierr = Zoltan_Set_Next_Border_Obj_Fn(zz, (ZOLTAN_NEXT_BORDER_OBJ_FN *) fn, data); break; case ZOLTAN_PRE_MIGRATE_PP_FN_TYPE: ierr = Zoltan_Set_Pre_Migrate_PP_Fn(zz, (ZOLTAN_PRE_MIGRATE_PP_FN *) fn, data); break; case ZOLTAN_MID_MIGRATE_PP_FN_TYPE: ierr = Zoltan_Set_Mid_Migrate_PP_Fn(zz, (ZOLTAN_MID_MIGRATE_PP_FN *) fn, data); break; case ZOLTAN_POST_MIGRATE_PP_FN_TYPE: ierr = Zoltan_Set_Post_Migrate_PP_Fn(zz, (ZOLTAN_POST_MIGRATE_PP_FN *) fn, data); break; case ZOLTAN_PRE_MIGRATE_FN_TYPE: ierr = Zoltan_Set_Pre_Migrate_Fn(zz, (ZOLTAN_PRE_MIGRATE_FN *) fn, data); break; case ZOLTAN_MID_MIGRATE_FN_TYPE: ierr = Zoltan_Set_Mid_Migrate_Fn(zz, (ZOLTAN_MID_MIGRATE_FN *) fn, data); break; case ZOLTAN_POST_MIGRATE_FN_TYPE: ierr = Zoltan_Set_Post_Migrate_Fn(zz, (ZOLTAN_POST_MIGRATE_FN *) fn, data); break; case ZOLTAN_OBJ_SIZE_FN_TYPE: ierr = Zoltan_Set_Obj_Size_Fn(zz, (ZOLTAN_OBJ_SIZE_FN *) fn, data); break; case ZOLTAN_OBJ_SIZE_MULTI_FN_TYPE: ierr = Zoltan_Set_Obj_Size_Multi_Fn(zz, (ZOLTAN_OBJ_SIZE_MULTI_FN *) fn, data); break; case ZOLTAN_PACK_OBJ_FN_TYPE: ierr = Zoltan_Set_Pack_Obj_Fn(zz, (ZOLTAN_PACK_OBJ_FN *) fn, data); break; case ZOLTAN_PACK_OBJ_MULTI_FN_TYPE: ierr = Zoltan_Set_Pack_Obj_Multi_Fn(zz, (ZOLTAN_PACK_OBJ_MULTI_FN *) fn, data); break; case ZOLTAN_UNPACK_OBJ_FN_TYPE: ierr = Zoltan_Set_Unpack_Obj_Fn(zz, (ZOLTAN_UNPACK_OBJ_FN *) fn, data); break; case ZOLTAN_UNPACK_OBJ_MULTI_FN_TYPE: ierr = Zoltan_Set_Unpack_Obj_Multi_Fn(zz, (ZOLTAN_UNPACK_OBJ_MULTI_FN *) fn, data); break; case ZOLTAN_NUM_COARSE_OBJ_FN_TYPE: ierr = Zoltan_Set_Num_Coarse_Obj_Fn(zz, (ZOLTAN_NUM_COARSE_OBJ_FN *) fn, data); break; case ZOLTAN_COARSE_OBJ_LIST_FN_TYPE: ierr = Zoltan_Set_Coarse_Obj_List_Fn(zz, (ZOLTAN_COARSE_OBJ_LIST_FN *) fn, data); break; case ZOLTAN_FIRST_COARSE_OBJ_FN_TYPE: ierr = Zoltan_Set_First_Coarse_Obj_Fn(zz, (ZOLTAN_FIRST_COARSE_OBJ_FN *) fn, data); break; case ZOLTAN_NEXT_COARSE_OBJ_FN_TYPE: ierr = Zoltan_Set_Next_Coarse_Obj_Fn(zz, (ZOLTAN_NEXT_COARSE_OBJ_FN *) fn, data); break; case ZOLTAN_NUM_CHILD_FN_TYPE: ierr = Zoltan_Set_Num_Child_Fn(zz, (ZOLTAN_NUM_CHILD_FN *) fn, data); break; case ZOLTAN_CHILD_LIST_FN_TYPE: ierr = Zoltan_Set_Child_List_Fn(zz, (ZOLTAN_CHILD_LIST_FN *) fn, data); break; case ZOLTAN_CHILD_WEIGHT_FN_TYPE: ierr = Zoltan_Set_Child_Weight_Fn(zz, (ZOLTAN_CHILD_WEIGHT_FN *) fn, data); break; case ZOLTAN_HG_SIZE_CS_FN_TYPE: ierr = Zoltan_Set_HG_Size_CS_Fn(zz, (ZOLTAN_HG_SIZE_CS_FN *) fn, data); break; case ZOLTAN_HG_CS_FN_TYPE: ierr = Zoltan_Set_HG_CS_Fn(zz, (ZOLTAN_HG_CS_FN *) fn, data); break; case ZOLTAN_HG_SIZE_EDGE_WTS_FN_TYPE: ierr = Zoltan_Set_HG_Size_Edge_Wts_Fn(zz, (ZOLTAN_HG_SIZE_EDGE_WTS_FN *) fn, data); break; case ZOLTAN_HG_EDGE_WTS_FN_TYPE: ierr = Zoltan_Set_HG_Edge_Wts_Fn(zz, (ZOLTAN_HG_EDGE_WTS_FN *) fn, data); break; case ZOLTAN_NUM_FIXED_OBJ_FN_TYPE: ierr = Zoltan_Set_Num_Fixed_Obj_Fn(zz, (ZOLTAN_NUM_FIXED_OBJ_FN *) fn, data); break; case ZOLTAN_FIXED_OBJ_LIST_FN_TYPE: ierr = Zoltan_Set_Fixed_Obj_List_Fn(zz, (ZOLTAN_FIXED_OBJ_LIST_FN *) fn, data); break; case ZOLTAN_HIER_NUM_LEVELS_FN_TYPE: ierr = Zoltan_Set_Hier_Num_Levels_Fn(zz, (ZOLTAN_HIER_NUM_LEVELS_FN *) fn, data); break; case ZOLTAN_HIER_PART_FN_TYPE: ierr = Zoltan_Set_Hier_Part_Fn(zz, (ZOLTAN_HIER_PART_FN *) fn, data); break; case ZOLTAN_HIER_METHOD_FN_TYPE: ierr = Zoltan_Set_Hier_Method_Fn(zz, (ZOLTAN_HIER_METHOD_FN *) fn, data); break; default: sprintf(msg, "ZOLTAN_FN_TYPE %d is invalid.\n" "Value must be in range 0 to %d.", fn_type, ZOLTAN_MAX_FN_TYPES); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ierr = ZOLTAN_WARN; } return (ierr); }
void lb_zoltan(PSTopology top, LBMethod method, unsigned int dimen, list_type& pl) { const par::communicator& comm = par::comm_world(); float ver; Zoltan_Initialize(0, 0, &ver); struct Zoltan_Struct *zz; // Create ZData (moves pl into zd) ZData zd(par::comm_world(), std::move(pl)); // Allocate the Zoltan data zz = Zoltan_Create(zd.comm.raw()); // Set some default sane parameters if (method == LBMethod::RCB) Zoltan_Set_Param(zz, "LB_METHOD", "RCB"); else throw std::runtime_error("Unknown load balancing method"); // Zoltan_Set_Param(zz, "KEEP_CUTS", "1"); // Zoltan_Set_Param(zz, "LB_APPROACH", "REPARTITION"); // Zoltan_Set_Param(zz, "MIGRATE_ONLY_PROC_CHANGES", "1"); Zoltan_Set_Param(zz, "AUTO_MIGRATE", "TRUE"); // Set higher for more debugging output Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); // Set partition query methods Zoltan_Set_Num_Obj_Fn(zz, pl_num_obj, static_cast<void *>(&zd)); Zoltan_Set_Obj_List_Fn(zz, pl_obj_list, static_cast<void *>(&zd)); Zoltan_Set_Num_Geom_Fn(zz, pl_num_geom, static_cast<void *>(&zd)); Zoltan_Set_Geom_Multi_Fn(zz, pl_geom_multi, static_cast<void *>(&zd)); // Migration query methods Zoltan_Set_Mid_Migrate_PP_Fn(zz, pl_mid_migrate_pp, static_cast<void *>(&zd)); Zoltan_Set_Obj_Size_Multi_Fn(zz, pl_obj_size_multi, static_cast<void *>(&zd)); Zoltan_Set_Pack_Obj_Multi_Fn(zz, pl_pack_obj_multi, static_cast<void *>(&zd)); Zoltan_Set_Unpack_Obj_Multi_Fn(zz, pl_unpack_obj_multi, static_cast<void *>(&zd)); int zerr, changes, num_gid_entries, num_lid_entries, num_import, *import_procs, *import_to_part, num_export, *export_procs, *export_to_part; unsigned int *import_global_ids, *import_local_ids, *export_global_ids, *export_local_ids; zerr = Zoltan_LB_Partition(zz, &changes, &num_gid_entries, &num_lid_entries, &num_import, &import_global_ids, &import_local_ids, &import_procs, &import_to_part, &num_export, &export_global_ids, &export_local_ids, &export_procs, &export_to_part); if (zerr != ZOLTAN_OK) comm.abort("Zoltan error", 1); // Move the data back again out of the struct pl = std::move(zd.list); Zoltan_Destroy(&zz); }
int main(int argc, char *argv[]) { int i, rc; int myRank, numProcs; float ver; struct Zoltan_Struct *zz; int changes, numGidEntries, numLidEntries, numImport, numExport, start_gid, num_nbors; ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; int *importProcs, *importToPart, *exportProcs, *exportToPart; int *parts=NULL; ZOLTAN_ID_PTR lids=NULL; FILE *fp; struct Zoltan_DD_Struct *dd; GRAPH_DATA myGraph; int gid_length = 1; /* our global IDs consist of 1 integer */ int lid_length = 1; /* our local IDs consist of 1 integer */ /****************************************************************** ** Initialize MPI and Zoltan ******************************************************************/ MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &myRank); MPI_Comm_size(MPI_COMM_WORLD, &numProcs); rc = Zoltan_Initialize(argc, argv, &ver); if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); exit(0); } /****************************************************************** ** Read graph from input file and distribute it ******************************************************************/ fp = fopen(fname, "r"); if (!fp){ if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname); MPI_Finalize(); exit(1); } fclose(fp); read_input_file(myRank, numProcs, fname, &myGraph); /*fprintf(stderr,"%d have %d objects\n",myRank,myGraph.numMyVertices);*/ /****************************************************************** ** Create a distributed data directory which maps vertex ** global IDs to their current partition number. We'll use this ** after migrating vertices, to update the partition in which ** our vertices neighbors are. ** ** Our local IDs (array "lids") are of type ZOLTAN_ID_TYPE because ** we are using Zoltan's distributed data directory. It assumes ** that a global ID is a sequence of "gid_length" ZOLTAN_ID_TYPEs. ** It assumes that a local ID is a sequence of "lid_length" ** ZOLTAN_ID_TYPEs. ******************************************************************/ rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD, gid_length, /* length of a global ID */ lid_length, /* length of a local ID */ 0, /* length of user data */ myGraph.numMyVertices, /* hash table size */ 0); /* debug level */ parts = malloc(myGraph.numMyVertices * sizeof(int)); lids = malloc(myGraph.numMyVertices * sizeof(ZOLTAN_ID_TYPE)); for (i=0; i < myGraph.numMyVertices; i++){ parts[i] = myRank; /* part number of this vertex */ lids[i] = (ZOLTAN_ID_TYPE)i; /* local ID on my process for this vertex */ } rc = Zoltan_DD_Update(dd, myGraph.vertexGID, lids, NULL, parts, myGraph.numMyVertices); myGraph.dd = dd; /****************************************************************** ** Create a Zoltan library structure for this instance of load ** balancing. Set the parameters and query functions that will ** govern the library's calculation. See the Zoltan User's ** Guide for the definition of these and many other parameters. ******************************************************************/ zz = Zoltan_Create(MPI_COMM_WORLD); /* General parameters */ Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION"); Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* Graph parameters */ Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */ /* Query functions, defined in this source file */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph); Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph); Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph); Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph); Zoltan_Set_Obj_Size_Multi_Fn(zz, get_message_sizes,&myGraph); Zoltan_Set_Pack_Obj_Multi_Fn(zz, pack_object_messages,&myGraph); Zoltan_Set_Unpack_Obj_Multi_Fn(zz, unpack_object_messages,&myGraph); Zoltan_Set_Mid_Migrate_PP_Fn(zz, mid_migrate,&myGraph); /****************************************************************** ** Visualize the graph partitioning before calling Zoltan. ******************************************************************/ if (myRank== 0){ printf("\nGraph partition before calling Zoltan\n"); } showGraphPartitions(myRank, myGraph.dd); /****************************************************************** ** Zoltan can now partition the simple graph. ** In this simple example, we assume the number of partitions is ** equal to the number of processes. Process rank 0 will own ** partition 0, process rank 1 will own partition 1, and so on. ******************************************************************/ rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partitioning was changed, 0 otherwise */ &numGidEntries, /* Number of integers used for a global ID */ &numLidEntries, /* Number of integers used for a local ID */ &numImport, /* Number of vertices to be sent to me */ &importGlobalGids, /* Global IDs of vertices to be sent to me */ &importLocalGids, /* Local IDs of vertices to be sent to me */ &importProcs, /* Process rank for source of each incoming vertex */ &importToPart, /* New partition for each incoming vertex */ &numExport, /* Number of vertices I must send to other processes*/ &exportGlobalGids, /* Global IDs of the vertices I must send */ &exportLocalGids, /* Local IDs of the vertices I must send */ &exportProcs, /* Process to which I send each of the vertices */ &exportToPart); /* Partition to which each vertex will belong */ if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } /*fprintf(stderr,"%d export %d import %d\n",myRank,numExport,numImport);*/ /****************************************************************** ** Update the data directory with the new partition numbers ******************************************************************/ for (i=0; i < numExport; i++){ parts[exportLocalGids[i]] = exportToPart[i]; } rc = Zoltan_DD_Update(dd, myGraph.vertexGID, lids, NULL, parts, myGraph.numMyVertices); /****************************************************************** ** Migrate vertices to new partitions ******************************************************************/ rc = Zoltan_Migrate(zz, numImport, importGlobalGids, importLocalGids, importProcs, importToPart, numExport, exportGlobalGids, exportLocalGids, exportProcs, exportToPart); /****************************************************************** ** Use the data dictionary to find neighbors' partitions ******************************************************************/ start_gid = myGraph.numMyVertices - numImport; num_nbors = myGraph.nborIndex[myGraph.numMyVertices] - myGraph.nborIndex[start_gid]; rc = Zoltan_DD_Find(dd, (ZOLTAN_ID_PTR)(myGraph.nborGID + start_gid), NULL, NULL, myGraph.nborPart + start_gid, num_nbors, NULL); /****************************************************************** ** Visualize the graph partitioning after calling Zoltan. ******************************************************************/ if (myRank == 0){ printf("Graph partition after calling Zoltan\n"); } showGraphPartitions(myRank, myGraph.dd); /****************************************************************** ** Free the arrays allocated by Zoltan_LB_Partition, and free ** the storage allocated for the Zoltan structure. ******************************************************************/ Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart); Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart); Zoltan_Destroy(&zz); /********************** ** all done *********** **********************/ MPI_Finalize(); if (myGraph.vertex_capacity > 0){ free(myGraph.vertexGID); free(myGraph.nborIndex); if (myGraph.nbor_capacity > 0){ free(myGraph.nborGID); free(myGraph.nborPart); } } if (parts) free(parts); if (lids) free(lids); return 0; }