int main(int argc, char *argv[]) { int rc, i; ZOLTAN_GNO_TYPE numGlobalVertices; float ver; char dimstring[16]; double min, max, avg, local; struct Zoltan_Struct *zz; int changes, numGidEntries, numLidEntries, numImport, numExport; ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; int *importProcs, *importToPart, *exportProcs, *exportToPart; #ifdef HOST_LINUX signal(SIGSEGV, meminfo_signal_handler); signal(SIGINT, meminfo_signal_handler); signal(SIGTERM, meminfo_signal_handler); signal(SIGABRT, meminfo_signal_handler); signal(SIGFPE, meminfo_signal_handler); #endif /****************************************************************** ** Problem size ******************************************************************/ numGlobalVertices = NUM_GLOBAL_VERTICES; vertexWeightDim = VERTEX_WEIGHT_DIMENSION; vertexDim = VERTEX_DIMENSION; if (argc > 1){ sscanf(argv[1], "%zd", &numGlobalVertices); if (argc > 2){ vertexWeightDim = atoi(argv[2]); if (argc > 3){ vertexDim = atoi(argv[3]); } } } sprintf(dimstring,"%d",vertexWeightDim); /****************************************************************** ** Initialize MPI and Zoltan ******************************************************************/ MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &myRank); MPI_Comm_size(MPI_COMM_WORLD, &numProcs); rc = Zoltan_Initialize(argc, argv, &ver); if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); exit(1); } Zoltan_Memory_Debug(2); /****************************************************************** ** Create vertices ******************************************************************/ rc = create_vertices(numGlobalVertices, vertexDim, vertexWeightDim, numProcs, myRank); if (rc){ fprintf(stderr,"Process rank %d: insufficient memory\n",myRank); MPI_Finalize(); exit(1); } first_gid = vertex_gid[myRank]; /****************************************************************** ** Create a Zoltan library structure for this instance of load ** balancing. Set the parameters and query functions that will ** govern the library's calculation. See the Zoltan User's ** Guide for the definition of these and many other parameters. ******************************************************************/ zz = Zoltan_Create(MPI_COMM_WORLD); /* General parameters */ Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); Zoltan_Set_Param(zz, "LB_METHOD", "RIB"); Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", dimstring); Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* RIB parameters */ Zoltan_Set_Param(zz, "RIB_OUTPUT_LEVEL", "0"); /* Query functions, to provide geometry to Zoltan */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, NULL); Zoltan_Set_Obj_List_Fn(zz, get_object_list, NULL); Zoltan_Set_Num_Geom_Fn(zz, get_num_geometry, NULL); Zoltan_Set_Geom_Multi_Fn(zz, get_geometry_list, NULL); Zoltan_Set_Part_Multi_Fn(zz, get_partition_list, NULL); /****************************************************************** ** Zoltan can now partition the vertices in the simple mesh. ** In this simple example, we assume the number of partitions is ** equal to the number of processes. Process rank 0 will own ** partition 0, process rank 1 will own partition 1, and so on. ******************************************************************/ if (myRank == 0){ printf("Run Zoltan\n"); } rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partitioning was changed, 0 otherwise */ &numGidEntries, /* Number of integers used for a global ID */ &numLidEntries, /* Number of integers used for a local ID */ &numImport, /* Number of vertices to be sent to me */ &importGlobalGids, /* Global IDs of vertices to be sent to me */ &importLocalGids, /* Local IDs of vertices to be sent to me */ &importProcs, /* Process rank for source of each incoming vertex */ &importToPart, /* New partition for each incoming vertex */ &numExport, /* Number of vertices I must send to other processes*/ &exportGlobalGids, /* Global IDs of the vertices I must send */ &exportLocalGids, /* Local IDs of the vertices I must send */ &exportProcs, /* Process to which I send each of the vertices */ &exportToPart); /* Partition to which each vertex will belong */ if (rc != ZOLTAN_OK){ if (myRank == 0)printf("sorry...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } /****************************************************************** ** Check the balance of the partitions before running zoltan. ** The query function get_partition_list() will give the ** partitions of the vertices before we called Zoltan. ******************************************************************/ if (myRank == 0){ printf("\nBALANCE before running Zoltan\n"); } rc = Zoltan_LB_Eval_Balance(zz, 1, NULL); if (rc != ZOLTAN_OK){ printf("sorry first LB_Eval_Balance...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } /****************************************************************** ** Print out the balance of the new partitions. ******************************************************************/ vertex_part = (int *)malloc(sizeof(int) * numLocalVertices); if (!vertex_part){ printf("sorry memory error...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } for (i=0; i < numLocalVertices; i++){ vertex_part[i] = myRank; } if (numExport > 0){ for (i=0; i < numExport; i++){ vertex_part[exportLocalGids[i]] = exportToPart[i]; } } if (myRank == 0){ printf("\nBALANCE after running Zoltan\n"); } rc = Zoltan_LB_Eval_Balance(zz, 1, NULL); if (rc != ZOLTAN_OK){ printf("sorry second LB_Eval_Balance...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } /****************************************************************** ** Free the arrays allocated by Zoltan_LB_Partition, and free ** the storage allocated for the Zoltan structure. ******************************************************************/ if (myRank == 0){ printf("Free structures\n"); } Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart); Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart); Zoltan_Destroy(&zz); if (vertex_part) free(vertex_part); if (v_x) free(v_x); if (v_y) free(v_y); if (v_z) free(v_z); if (vertex_weight) free(vertex_weight); if (vertex_gid) free(vertex_gid); /********************** ** all done *********** **********************/ local= (double)Zoltan_Memory_Usage(ZOLTAN_MEM_STAT_MAXIMUM)/(1024.0*1024); MPI_Reduce(&local, &avg, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); avg /= (double)numProcs; MPI_Reduce(&local, &max, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); MPI_Reduce(&local, &min, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); if (myRank == 0){ printf("Total MBytes in use by test while Zoltan is running: %12.3lf\n", mbytes/(1024.0*1024)); printf("Min/Avg/Max of maximum MBytes in use by Zoltan: %12.3lf / %12.3lf / %12.3lf\n", min, avg, max); } MPI_Finalize(); return 0; }
void lb_zoltan(PSTopology top, LBMethod method, unsigned int dimen, list_type& pl) { const par::communicator& comm = par::comm_world(); float ver; Zoltan_Initialize(0, 0, &ver); struct Zoltan_Struct *zz; // Create ZData (moves pl into zd) ZData zd(par::comm_world(), std::move(pl)); // Allocate the Zoltan data zz = Zoltan_Create(zd.comm.raw()); // Set some default sane parameters if (method == LBMethod::RCB) Zoltan_Set_Param(zz, "LB_METHOD", "RCB"); else throw std::runtime_error("Unknown load balancing method"); // Zoltan_Set_Param(zz, "KEEP_CUTS", "1"); // Zoltan_Set_Param(zz, "LB_APPROACH", "REPARTITION"); // Zoltan_Set_Param(zz, "MIGRATE_ONLY_PROC_CHANGES", "1"); Zoltan_Set_Param(zz, "AUTO_MIGRATE", "TRUE"); // Set higher for more debugging output Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); // Set partition query methods Zoltan_Set_Num_Obj_Fn(zz, pl_num_obj, static_cast<void *>(&zd)); Zoltan_Set_Obj_List_Fn(zz, pl_obj_list, static_cast<void *>(&zd)); Zoltan_Set_Num_Geom_Fn(zz, pl_num_geom, static_cast<void *>(&zd)); Zoltan_Set_Geom_Multi_Fn(zz, pl_geom_multi, static_cast<void *>(&zd)); // Migration query methods Zoltan_Set_Mid_Migrate_PP_Fn(zz, pl_mid_migrate_pp, static_cast<void *>(&zd)); Zoltan_Set_Obj_Size_Multi_Fn(zz, pl_obj_size_multi, static_cast<void *>(&zd)); Zoltan_Set_Pack_Obj_Multi_Fn(zz, pl_pack_obj_multi, static_cast<void *>(&zd)); Zoltan_Set_Unpack_Obj_Multi_Fn(zz, pl_unpack_obj_multi, static_cast<void *>(&zd)); int zerr, changes, num_gid_entries, num_lid_entries, num_import, *import_procs, *import_to_part, num_export, *export_procs, *export_to_part; unsigned int *import_global_ids, *import_local_ids, *export_global_ids, *export_local_ids; zerr = Zoltan_LB_Partition(zz, &changes, &num_gid_entries, &num_lid_entries, &num_import, &import_global_ids, &import_local_ids, &import_procs, &import_to_part, &num_export, &export_global_ids, &export_local_ids, &export_procs, &export_to_part); if (zerr != ZOLTAN_OK) comm.abort("Zoltan error", 1); // Move the data back again out of the struct pl = std::move(zd.list); Zoltan_Destroy(&zz); }
int main(int argc, char *argv[]) { int rc, i, myRank, numProcs; float ver; struct Zoltan_Struct *zz; int changes, numGidEntries, numLidEntries, numImport, numExport; ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; int *importProcs, *importToPart, *exportProcs, *exportToPart; int *parts; FILE *fp; MESH_DATA myMesh; /****************************************************************** ** Initialize MPI and Zoltan ******************************************************************/ MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &myRank); MPI_Comm_size(MPI_COMM_WORLD, &numProcs); rc = Zoltan_Initialize(argc, argv, &ver); if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); exit(0); } /****************************************************************** ** Read geometry from input file and distribute it unevenly ******************************************************************/ fp = fopen(fname, "r"); if (!fp){ if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname); MPI_Finalize(); exit(1); } fclose(fp); read_input_objects(myRank, numProcs, fname, &myMesh); /****************************************************************** ** Create a Zoltan library structure for this instance of load ** balancing. Set the parameters and query functions that will ** govern the library's calculation. See the Zoltan User's ** Guide for the definition of these and many other parameters. ******************************************************************/ zz = Zoltan_Create(MPI_COMM_WORLD); /* General parameters */ Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); Zoltan_Set_Param(zz, "LB_METHOD", "RCB"); Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "0"); Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* RCB parameters */ Zoltan_Set_Param(zz, "RCB_OUTPUT_LEVEL", "0"); Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS", "1"); /*Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS", "0"); */ /* Query functions, to provide geometry to Zoltan */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_objects, &myMesh); Zoltan_Set_Obj_List_Fn(zz, get_object_list, &myMesh); Zoltan_Set_Num_Geom_Fn(zz, get_num_geometry, &myMesh); Zoltan_Set_Geom_Multi_Fn(zz, get_geometry_list, &myMesh); /****************************************************************** ** Zoltan can now partition the vertices in the simple mesh. ** In this simple example, we assume the number of partitions is ** equal to the number of processes. Process rank 0 will own ** partition 0, process rank 1 will own partition 1, and so on. ******************************************************************/ rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partitioning was changed, 0 otherwise */ &numGidEntries, /* Number of integers used for a global ID */ &numLidEntries, /* Number of integers used for a local ID */ &numImport, /* Number of vertices to be sent to me */ &importGlobalGids, /* Global IDs of vertices to be sent to me */ &importLocalGids, /* Local IDs of vertices to be sent to me */ &importProcs, /* Process rank for source of each incoming vertex */ &importToPart, /* New partition for each incoming vertex */ &numExport, /* Number of vertices I must send to other processes*/ &exportGlobalGids, /* Global IDs of the vertices I must send */ &exportLocalGids, /* Local IDs of the vertices I must send */ &exportProcs, /* Process to which I send each of the vertices */ &exportToPart); /* Partition to which each vertex will belong */ if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } /****************************************************************** ** Visualize the mesh partitioning before and after calling Zoltan. ******************************************************************/ parts = (int *)malloc(sizeof(int) * myMesh.numMyPoints); for (i=0; i < myMesh.numMyPoints; i++){ parts[i] = myRank; } if (myRank== 0){ printf("\nMesh partition assignments before calling Zoltan\n"); } showSimpleMeshPartitions(myRank, myMesh.numMyPoints, myMesh.myGlobalIDs, parts); for (i=0; i < numExport; i++){ parts[exportLocalGids[i]] = exportToPart[i]; } if (myRank == 0){ printf("Mesh partition assignments after calling Zoltan\n"); } showSimpleMeshPartitions(myRank, myMesh.numMyPoints, myMesh.myGlobalIDs, parts); free(parts); /****************************************************************** ** Free the arrays allocated by Zoltan_LB_Partition, and free ** the storage allocated for the Zoltan structure. ******************************************************************/ Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart); Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart); Zoltan_Destroy(&zz); /********************** ** all done *********** **********************/ MPI_Finalize(); if (myMesh.numMyPoints > 0){ free(myMesh.myGlobalIDs); free(myMesh.x); free(myMesh.y); } return 0; }
int Zoltan_Set_Fn(ZZ *zz, ZOLTAN_FN_TYPE fn_type, ZOLTAN_VOID_FN *fn, void *data) { /* * Function to initialize a given LB interface function. * Input: * zz -- Pointer to a Zoltan structure. * fn_type -- Enum type indicating the function to be set. * fn -- Pointer to the function to be used in the * assignment. * data -- Pointer to data that the LB library will * pass as an argument to fn(). May be NULL. * Output: * zz -- Appropriate field set to value in void *(). */ char *yo = "Zoltan_Set_Fn"; char msg[256]; int ierr; switch (fn_type) { case ZOLTAN_PART_FN_TYPE: ierr = Zoltan_Set_Part_Fn(zz, (ZOLTAN_PART_FN *) fn, data); break; case ZOLTAN_PART_MULTI_FN_TYPE: ierr = Zoltan_Set_Part_Multi_Fn(zz, (ZOLTAN_PART_MULTI_FN *) fn, data); break; case ZOLTAN_NUM_EDGES_FN_TYPE: ierr = Zoltan_Set_Num_Edges_Fn(zz, (ZOLTAN_NUM_EDGES_FN *) fn, data); break; case ZOLTAN_NUM_EDGES_MULTI_FN_TYPE: ierr = Zoltan_Set_Num_Edges_Multi_Fn(zz, (ZOLTAN_NUM_EDGES_MULTI_FN *) fn, data); break; case ZOLTAN_EDGE_LIST_FN_TYPE: ierr = Zoltan_Set_Edge_List_Fn(zz, (ZOLTAN_EDGE_LIST_FN *) fn, data); break; case ZOLTAN_EDGE_LIST_MULTI_FN_TYPE: ierr = Zoltan_Set_Edge_List_Multi_Fn(zz, (ZOLTAN_EDGE_LIST_MULTI_FN *) fn, data); break; case ZOLTAN_NUM_GEOM_FN_TYPE: ierr = Zoltan_Set_Num_Geom_Fn(zz, (ZOLTAN_NUM_GEOM_FN *) fn, data); break; case ZOLTAN_GEOM_MULTI_FN_TYPE: ierr = Zoltan_Set_Geom_Multi_Fn(zz, (ZOLTAN_GEOM_MULTI_FN *) fn, data); break; case ZOLTAN_GEOM_FN_TYPE: ierr = Zoltan_Set_Geom_Fn(zz, (ZOLTAN_GEOM_FN *) fn, data); break; case ZOLTAN_NUM_OBJ_FN_TYPE: ierr = Zoltan_Set_Num_Obj_Fn(zz, (ZOLTAN_NUM_OBJ_FN *) fn, data); break; case ZOLTAN_OBJ_LIST_FN_TYPE: ierr = Zoltan_Set_Obj_List_Fn(zz, (ZOLTAN_OBJ_LIST_FN *) fn, data); break; case ZOLTAN_FIRST_OBJ_FN_TYPE: ierr = Zoltan_Set_First_Obj_Fn(zz, (ZOLTAN_FIRST_OBJ_FN *) fn, data); break; case ZOLTAN_NEXT_OBJ_FN_TYPE: ierr = Zoltan_Set_Next_Obj_Fn(zz, (ZOLTAN_NEXT_OBJ_FN *) fn, data); break; case ZOLTAN_NUM_BORDER_OBJ_FN_TYPE: ierr = Zoltan_Set_Num_Border_Obj_Fn(zz, (ZOLTAN_NUM_BORDER_OBJ_FN *) fn, data); break; case ZOLTAN_BORDER_OBJ_LIST_FN_TYPE: ierr = Zoltan_Set_Border_Obj_List_Fn(zz, (ZOLTAN_BORDER_OBJ_LIST_FN *) fn, data); break; case ZOLTAN_FIRST_BORDER_OBJ_FN_TYPE: ierr = Zoltan_Set_First_Border_Obj_Fn(zz, (ZOLTAN_FIRST_BORDER_OBJ_FN *) fn, data); break; case ZOLTAN_NEXT_BORDER_OBJ_FN_TYPE: ierr = Zoltan_Set_Next_Border_Obj_Fn(zz, (ZOLTAN_NEXT_BORDER_OBJ_FN *) fn, data); break; case ZOLTAN_PRE_MIGRATE_PP_FN_TYPE: ierr = Zoltan_Set_Pre_Migrate_PP_Fn(zz, (ZOLTAN_PRE_MIGRATE_PP_FN *) fn, data); break; case ZOLTAN_MID_MIGRATE_PP_FN_TYPE: ierr = Zoltan_Set_Mid_Migrate_PP_Fn(zz, (ZOLTAN_MID_MIGRATE_PP_FN *) fn, data); break; case ZOLTAN_POST_MIGRATE_PP_FN_TYPE: ierr = Zoltan_Set_Post_Migrate_PP_Fn(zz, (ZOLTAN_POST_MIGRATE_PP_FN *) fn, data); break; case ZOLTAN_PRE_MIGRATE_FN_TYPE: ierr = Zoltan_Set_Pre_Migrate_Fn(zz, (ZOLTAN_PRE_MIGRATE_FN *) fn, data); break; case ZOLTAN_MID_MIGRATE_FN_TYPE: ierr = Zoltan_Set_Mid_Migrate_Fn(zz, (ZOLTAN_MID_MIGRATE_FN *) fn, data); break; case ZOLTAN_POST_MIGRATE_FN_TYPE: ierr = Zoltan_Set_Post_Migrate_Fn(zz, (ZOLTAN_POST_MIGRATE_FN *) fn, data); break; case ZOLTAN_OBJ_SIZE_FN_TYPE: ierr = Zoltan_Set_Obj_Size_Fn(zz, (ZOLTAN_OBJ_SIZE_FN *) fn, data); break; case ZOLTAN_OBJ_SIZE_MULTI_FN_TYPE: ierr = Zoltan_Set_Obj_Size_Multi_Fn(zz, (ZOLTAN_OBJ_SIZE_MULTI_FN *) fn, data); break; case ZOLTAN_PACK_OBJ_FN_TYPE: ierr = Zoltan_Set_Pack_Obj_Fn(zz, (ZOLTAN_PACK_OBJ_FN *) fn, data); break; case ZOLTAN_PACK_OBJ_MULTI_FN_TYPE: ierr = Zoltan_Set_Pack_Obj_Multi_Fn(zz, (ZOLTAN_PACK_OBJ_MULTI_FN *) fn, data); break; case ZOLTAN_UNPACK_OBJ_FN_TYPE: ierr = Zoltan_Set_Unpack_Obj_Fn(zz, (ZOLTAN_UNPACK_OBJ_FN *) fn, data); break; case ZOLTAN_UNPACK_OBJ_MULTI_FN_TYPE: ierr = Zoltan_Set_Unpack_Obj_Multi_Fn(zz, (ZOLTAN_UNPACK_OBJ_MULTI_FN *) fn, data); break; case ZOLTAN_NUM_COARSE_OBJ_FN_TYPE: ierr = Zoltan_Set_Num_Coarse_Obj_Fn(zz, (ZOLTAN_NUM_COARSE_OBJ_FN *) fn, data); break; case ZOLTAN_COARSE_OBJ_LIST_FN_TYPE: ierr = Zoltan_Set_Coarse_Obj_List_Fn(zz, (ZOLTAN_COARSE_OBJ_LIST_FN *) fn, data); break; case ZOLTAN_FIRST_COARSE_OBJ_FN_TYPE: ierr = Zoltan_Set_First_Coarse_Obj_Fn(zz, (ZOLTAN_FIRST_COARSE_OBJ_FN *) fn, data); break; case ZOLTAN_NEXT_COARSE_OBJ_FN_TYPE: ierr = Zoltan_Set_Next_Coarse_Obj_Fn(zz, (ZOLTAN_NEXT_COARSE_OBJ_FN *) fn, data); break; case ZOLTAN_NUM_CHILD_FN_TYPE: ierr = Zoltan_Set_Num_Child_Fn(zz, (ZOLTAN_NUM_CHILD_FN *) fn, data); break; case ZOLTAN_CHILD_LIST_FN_TYPE: ierr = Zoltan_Set_Child_List_Fn(zz, (ZOLTAN_CHILD_LIST_FN *) fn, data); break; case ZOLTAN_CHILD_WEIGHT_FN_TYPE: ierr = Zoltan_Set_Child_Weight_Fn(zz, (ZOLTAN_CHILD_WEIGHT_FN *) fn, data); break; case ZOLTAN_HG_SIZE_CS_FN_TYPE: ierr = Zoltan_Set_HG_Size_CS_Fn(zz, (ZOLTAN_HG_SIZE_CS_FN *) fn, data); break; case ZOLTAN_HG_CS_FN_TYPE: ierr = Zoltan_Set_HG_CS_Fn(zz, (ZOLTAN_HG_CS_FN *) fn, data); break; case ZOLTAN_HG_SIZE_EDGE_WTS_FN_TYPE: ierr = Zoltan_Set_HG_Size_Edge_Wts_Fn(zz, (ZOLTAN_HG_SIZE_EDGE_WTS_FN *) fn, data); break; case ZOLTAN_HG_EDGE_WTS_FN_TYPE: ierr = Zoltan_Set_HG_Edge_Wts_Fn(zz, (ZOLTAN_HG_EDGE_WTS_FN *) fn, data); break; case ZOLTAN_NUM_FIXED_OBJ_FN_TYPE: ierr = Zoltan_Set_Num_Fixed_Obj_Fn(zz, (ZOLTAN_NUM_FIXED_OBJ_FN *) fn, data); break; case ZOLTAN_FIXED_OBJ_LIST_FN_TYPE: ierr = Zoltan_Set_Fixed_Obj_List_Fn(zz, (ZOLTAN_FIXED_OBJ_LIST_FN *) fn, data); break; case ZOLTAN_HIER_NUM_LEVELS_FN_TYPE: ierr = Zoltan_Set_Hier_Num_Levels_Fn(zz, (ZOLTAN_HIER_NUM_LEVELS_FN *) fn, data); break; case ZOLTAN_HIER_PART_FN_TYPE: ierr = Zoltan_Set_Hier_Part_Fn(zz, (ZOLTAN_HIER_PART_FN *) fn, data); break; case ZOLTAN_HIER_METHOD_FN_TYPE: ierr = Zoltan_Set_Hier_Method_Fn(zz, (ZOLTAN_HIER_METHOD_FN *) fn, data); break; default: sprintf(msg, "ZOLTAN_FN_TYPE %d is invalid.\n" "Value must be in range 0 to %d.", fn_type, ZOLTAN_MAX_FN_TYPES); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ierr = ZOLTAN_WARN; } return (ierr); }
int MESH_PartitionWithZoltan(Mesh_ptr mesh, int nparts, int **part, int noptions, char **options, MSTK_Comm comm) { MEdge_ptr fedge; MFace_ptr mf, oppf, rface; MRegion_ptr mr, oppr; List_ptr fedges, efaces, rfaces, fregions; int i, j, k, id; int nv, ne, nf, nr=0, nfe, nef, nfr, nrf, idx, idx2; int numflag, nedgecut, ipos; int wtflag; int rc; float ver; struct Zoltan_Struct *zz; GRAPH_DATA graph; int changes, numGidEntries, numLidEntries, numImport, numExport; ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; int *importProcs, *importToPart, *exportProcs, *exportToPart; int rank; MPI_Comm_rank(comm,&rank); rc = Zoltan_Initialize(0, NULL, &ver); if (rc != ZOLTAN_OK){ MSTK_Report("MESH_PartitionWithZoltan","Could not initialize Zoltan",MSTK_FATAL); MPI_Finalize(); exit(0); } /****************************************************************** ** Create a Zoltan library structure for this instance of partition ********************************************************************/ zz = Zoltan_Create(comm); /***************************************************************** ** Figure out partitioning method *****************************************************************/ char partition_method_str[32]; strcpy(partition_method_str,"RCB"); /* Default - Recursive Coordinate Bisection */ if (noptions) { for (i = 0; i < noptions; i++) { if (strncmp(options[i],"LB_PARTITION",12) == 0) { char *result = NULL, instring[256]; strcpy(instring,options[i]); result = strtok(instring,"="); result = strtok(NULL," "); strcpy(partition_method_str,result); } } } if (rank == 0) { char mesg[256]; sprintf(mesg,"Using partitioning method %s for ZOLTAN\n",partition_method_str); MSTK_Report("MESH_PartitionWithZoltan",mesg,MSTK_MESG); } /* General parameters for Zoltan */ Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); Zoltan_Set_Param(zz, "LB_METHOD", partition_method_str); Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION"); Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); graph.numMyNodes = 0; graph.numAllNbors = 0; graph.nodeGID = NULL; graph.nodeCoords = NULL; graph.nborIndex = NULL; graph.nborGID = NULL; graph.nborProc = NULL; if (strcmp(partition_method_str,"RCB") == 0) { if (rank == 0) { nr = MESH_Num_Regions(mesh); nf = MESH_Num_Faces(mesh); if (!nf && !nr) MSTK_Report("MESH_PartitionWithZoltan","Cannot partition wire meshes", MSTK_FATAL); if (nr == 0) { /* Surface or planar mesh */ int ndim = 2; /* assume mesh is planar */ idx = 0; MVertex_ptr mv; while ((mv = MESH_Next_Vertex(mesh,&idx))) { double vxyz[3]; MV_Coords(mv,vxyz); if (vxyz[2] != 0.0) { ndim = 3; /* non-planar or planar with non-zero z */ break; } } NDIM_4_ZOLTAN = ndim-1; /* ignore last dimension to avoid partitioning in that dimension */ graph.numMyNodes = nf; graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nf); graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nf); idx = 0; while ((mf = MESH_Next_Face(mesh,&idx))) { double fxyz[MAXPV2][3], cen[3]; int nfv; MF_Coords(mf,&nfv,fxyz); cen[0] = cen[1] = cen[2] = 0.0; for (j = 0; j < nfv; j++) for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] += fxyz[j][k]; for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nfv; id = MF_ID(mf); graph.nodeGID[id-1] = id; memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double)); } } else { /* Volume mesh */ int ndim = 3; NDIM_4_ZOLTAN = ndim-1; /* ignore last dimension to avoid partitioning in that dimension */ graph.numMyNodes = nr; graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nr); graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nr); idx = 0; while ((mr = MESH_Next_Region(mesh,&idx))) { double rxyz[MAXPV3][3], cen[3]; int nrv; MR_Coords(mr,&nrv,rxyz); cen[0] = cen[1] = cen[2] = 0.0; for (j = 0; j < nrv; j++) for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] += rxyz[j][k]; for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nrv; for (k = 0; k < NDIM_4_ZOLTAN; k++) if (fabs(cen[k]) < 1.0e-10) cen[k] = 0.0; id = MR_ID(mr); graph.nodeGID[id-1] = id; memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double)); } } } MPI_Bcast(&NDIM_4_ZOLTAN,1,MPI_INT,0,comm); /* Set some default values */ Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS","1"); // Zoltan_Set_Param(zz, "AVERAGE_CUTS", "1"); if (noptions > 1) { for (i = 1; i < noptions; i++) { char *paramstr = NULL, *valuestr = NULL, instring[256]; strcpy(instring,options[i]); paramstr = strtok(instring,"="); valuestr = strtok(NULL," "); Zoltan_Set_Param(zz,paramstr,valuestr); } } /* Query functions - defined in simpleQueries.h */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph); Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph); Zoltan_Set_Num_Geom_Fn(zz, get_num_dimensions_reduced, &graph); /* reduced dimensions */ Zoltan_Set_Geom_Multi_Fn(zz, get_element_centers_reduced, &graph); /* reduced dimension centers */ } else if (strcmp(partition_method_str,"GRAPH") == 0) { if(rank == 0) { nv = MESH_Num_Vertices(mesh); ne = MESH_Num_Edges(mesh); nf = MESH_Num_Faces(mesh); nr = MESH_Num_Regions(mesh); ipos = 0; /* build nodes and neighbors list, similar as in partition with metis Assign processor 0 the whole mesh, assign other processors a NULL mesh */ if (nr == 0) { if (nf == 0) { MSTK_Report("MESH_PartitionWithZoltan", "Cannot partition wire meshes with Zoltan",MSTK_FATAL); exit(-1); } graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nf); graph.nborIndex = (int *)malloc(sizeof(int) * (nf + 1)); graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*ne); graph.nborProc = (int *)malloc(sizeof(int) * 2*ne); graph.nborIndex[0] = 0; /* Surface mesh */ idx = 0; i = 0; while ((mf = MESH_Next_Face(mesh,&idx))) { graph.nodeGID[i] = MF_ID(mf); fedges = MF_Edges(mf,1,0); nfe = List_Num_Entries(fedges); idx2 = 0; while ((fedge = List_Next_Entry(fedges,&idx2))) { efaces = ME_Faces(fedge); nef = List_Num_Entries(efaces); if (nef == 1) { continue; /* boundary edge; nothing to do */ } else { int j; for (j = 0; j < nef; j++) { oppf = List_Entry(efaces,j); if (oppf == mf) { graph.nborGID[ipos] = MF_ID(oppf); /* initially set all nodes on processor 0 */ graph.nborProc[ipos] = 0; ipos++; } } } List_Delete(efaces); } List_Delete(fedges); i++; graph.nborIndex[i] = ipos; } graph.numMyNodes = i; graph.numAllNbors = ipos; } else { graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nr); graph.nborIndex = (int *)malloc(sizeof(int) * (nr + 1)); graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*nf); graph.nborProc = (int *)malloc(sizeof(int) * 2*nf); graph.nborIndex[0] = 0; /* Volume mesh */ idx = 0; i = 0; while ((mr = MESH_Next_Region(mesh,&idx))) { graph.nodeGID[i] = MR_ID(mr); rfaces = MR_Faces(mr); nrf = List_Num_Entries(rfaces); idx2 = 0; while ((rface = List_Next_Entry(rfaces,&idx2))) { fregions = MF_Regions(rface); nfr = List_Num_Entries(fregions); if (nfr > 1) { oppr = List_Entry(fregions,0); if (oppr == mr) oppr = List_Entry(fregions,1); graph.nborGID[ipos] = MR_ID(oppr); /* initially set all nodes on processor 0 */ graph.nborProc[ipos] = 0; ipos++; } List_Delete(fregions); } List_Delete(rfaces); i++; graph.nborIndex[i] = ipos; } graph.numMyNodes = i; graph.numAllNbors = ipos; } } /* Graph parameters */ /* Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); */ Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */ /* Query functions - defined in simpleQueries.h */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph); Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph); Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &graph); Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &graph); } /* Partition the graph */ /****************************************************************** ** Zoltan can now partition the graph. ** We assume the number of partitions is ** equal to the number of processes. Process rank 0 will own ** partition 0, process rank 1 will own partition 1, and so on. ******************************************************************/ rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partitioning was changed, 0 otherwise */ &numGidEntries, /* Number of integers used for a global ID */ &numLidEntries, /* Number of integers used for a local ID */ &numImport, /* Number of nodes to be sent to me */ &importGlobalGids, /* Global IDs of nodes to be sent to me */ &importLocalGids, /* Local IDs of nodes to be sent to me */ &importProcs, /* Process rank for source of each incoming node */ &importToPart, /* New partition for each incoming node */ &numExport, /* Number of nodes I must send to other processes*/ &exportGlobalGids, /* Global IDs of the nodes I must send */ &exportLocalGids, /* Local IDs of the nodes I must send */ &exportProcs, /* Process to which I send each of the nodes */ &exportToPart); /* Partition to which each node will belong */ if (rc != ZOLTAN_OK){ if (rank == 0) MSTK_Report("MESH_PartitionWithZoltan","Could not partition mesh with ZOLTAN", MSTK_ERROR); Zoltan_Destroy(&zz); MPI_Finalize(); return 0; } if(rank == 0) { *part = (int *) calloc(graph.numMyNodes,sizeof(int)); for ( i = 0; i < numExport; i++ ) { (*part)[exportGlobalGids[i]-1] = exportToPart[i]; } if (graph.nodeGID) free(graph.nodeGID); if (graph.nodeCoords) free(graph.nodeCoords); if (graph.nborIndex) free(graph.nborIndex); if (graph.nborGID) free(graph.nborGID); if (graph.nborProc) free(graph.nborProc); } else { *part = NULL; } Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart); Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart); Zoltan_Destroy(&zz); return 1; }
PeridigmNS::ZoltanSearchTree::ZoltanSearchTree(int numPoints, double* coordinates) : SearchTree(numPoints, coordinates), callbackdata(numPoints,coordinates),zoltan(0), partToCoordIdx(new int[numPoints]), searchParts(new int[numPoints]) { zoltan = Zoltan_Create(MPI_COMM_SELF); Zoltan_Set_Param(zoltan, "debug_level", "0"); Zoltan_Set_Param(zoltan, "rcb_output_level", "0"); /* * query function returns the number of objects that are currently assigned to the processor */ Zoltan_Set_Num_Obj_Fn(zoltan, &get_num_points, &callbackdata); /* * query function fills two (three if weights are used) arrays with information about the objects * currently assigned to the processor. Both arrays are allocated (and subsequently freed) by Zoltan; * their size is determined by a call to a ZOLTAN_NUM_OBJ_FN query function to get the array size. */ Zoltan_Set_Obj_List_Fn(zoltan, &get_point_ids, &callbackdata); /* * query function returns the number of values needed to express the geometry of an object. * For example, for a two-dimensional mesh-based application, (x,y) coordinates are needed * to describe an object's geometry; thus the ZOLTAN_NUM_GEOM_FN query function should return * the value of two. For a similar three-dimensional application, the return value should be three. */ Zoltan_Set_Num_Geom_Fn(zoltan, &get_dimension, NULL); /* * query function returns a vector of geometry values for a list of given objects. The geometry * vector is allocated by Zoltan to be of size num_obj * num_dim. */ Zoltan_Set_Geom_Multi_Fn(zoltan, &get_point_coordinates, &callbackdata); /* * setting method */ Zoltan_Set_Param(zoltan,"lb_method","rcb"); /* * keep cuts */ Zoltan_Set_Param(zoltan,"keep_cuts","1"); Zoltan_Set_Param(zoltan,"return_lists","part"); Zoltan_Set_Param(zoltan, "num_lid_entries", "0"); char s[11]; sprintf(s,"%d",numPoints); Zoltan_Set_Param(zoltan, "num_global_parts", s); int numimp = -1, numexp = -1; int numgid = 1, numlid = 0; ZOLTAN_ID_PTR impgid=NULL, implid=NULL; ZOLTAN_ID_PTR gid=NULL, lid=NULL; int *imppart = NULL, *impproc = NULL; int *procs = NULL, *parts=NULL; int changes; int ierr = Zoltan_LB_Partition(zoltan, &changes, &numgid, &numlid, &numimp, &impgid, &implid, &imppart, &impproc, &numexp, &gid, &lid, &procs, &parts); TEUCHOS_TEST_FOR_EXCEPT_MSG(ierr != 0, "Error in ZoltanSearchTree::ZoltanSearchTree(), call to Zoltan_LB_Partition() returned a nonzero error code."); for (int I = 0; I < numPoints; I++) partToCoordIdx[parts[I]] = I; Zoltan_LB_Free_Part(&impgid, &implid, &impproc, &imppart); Zoltan_LB_Free_Part(&gid, &lid, &procs, &parts); }