int Zoltan_Help_Migrate( ZZ *zz, int num_import, ZOLTAN_ID_PTR import_global_ids, ZOLTAN_ID_PTR import_local_ids, int *import_procs, int num_export, ZOLTAN_ID_PTR export_global_ids, ZOLTAN_ID_PTR export_local_ids, int *export_procs ) { /* * Wrapper around Zoltan_Migrate with NULL pointers for partition arrays. * Maintained for backward compatibility. * Arguments are same as for Zoltan_Migrate. */ char *yo = "Zoltan_Help_Migrate"; int ierr; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->LB.PartDist != NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Non-uniform distribution of partitions over processors is specified; " "use Zoltan_Migrate\n"); ierr = ZOLTAN_FATAL; goto End; } if (zz->Migrate.Pre_Migrate_PP || zz->Migrate.Mid_Migrate_PP || zz->Migrate.Post_Migrate_PP) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Partition information not available in Zoltan_Help_Migrate for " "ZOLTAN_*_MIGRATE_PP_FNs; use ZOLTAN_*_MIGRATE_FNs instead."); ierr = ZOLTAN_FATAL; goto End; } /* * Wrapper (for backward compatilibity) around Zoltan_Migrate. * Passes NULL for partition assignment arrays. */ ierr = Zoltan_Migrate(zz, num_import, import_global_ids, import_local_ids, import_procs, NULL, num_export, export_global_ids, export_local_ids, export_procs, NULL); End: ZOLTAN_TRACE_EXIT(zz, yo); return ierr; }
int migrate_elements( int Proc, MESH_INFO_PTR mesh, struct Zoltan_Struct *zz, int num_gid_entries, int num_lid_entries, int num_imp, ZOLTAN_ID_PTR imp_gids, ZOLTAN_ID_PTR imp_lids, int *imp_procs, int *imp_to_part, int num_exp, ZOLTAN_ID_PTR exp_gids, ZOLTAN_ID_PTR exp_lids, int *exp_procs, int *exp_to_part) { /* Local declarations. */ char *yo = "migrate_elements"; /***************************** BEGIN EXECUTION ******************************/ DEBUG_TRACE_START(Proc, yo); /* * register migration functions */ if (!Test.Null_Lists) { /* If not passing NULL lists, let Help_Migrate call the * pre-processing and post-processing routines. */ if (Zoltan_Set_Fn(zz, ZOLTAN_PRE_MIGRATE_PP_FN_TYPE, (void (*)()) migrate_pre_process, (void *) mesh) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n"); return 0; } if (Zoltan_Set_Fn(zz, ZOLTAN_POST_MIGRATE_PP_FN_TYPE, (void (*)()) migrate_post_process, (void *) mesh) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n"); return 0; } } if (Test.Multi_Callbacks) { if (Zoltan_Set_Fn(zz, ZOLTAN_OBJ_SIZE_MULTI_FN_TYPE, (void (*)()) migrate_elem_size_multi, (void *) mesh) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n"); return 0; } if (Zoltan_Set_Fn(zz, ZOLTAN_PACK_OBJ_MULTI_FN_TYPE, (void (*)()) migrate_pack_elem_multi, (void *) mesh) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n"); return 0; } if (Zoltan_Set_Fn(zz, ZOLTAN_UNPACK_OBJ_MULTI_FN_TYPE, (void (*)()) migrate_unpack_elem_multi, (void *) mesh) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n"); return 0; } } else { if (Zoltan_Set_Fn(zz, ZOLTAN_OBJ_SIZE_FN_TYPE, (void (*)()) migrate_elem_size, (void *) mesh) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n"); return 0; } if (Zoltan_Set_Fn(zz, ZOLTAN_PACK_OBJ_FN_TYPE, (void (*)()) migrate_pack_elem, (void *) mesh) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n"); return 0; } if (Zoltan_Set_Fn(zz, ZOLTAN_UNPACK_OBJ_FN_TYPE, (void (*)()) migrate_unpack_elem, (void *) mesh) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n"); return 0; } } if (Test.Null_Lists == NONE) { if (Zoltan_Migrate(zz, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Migrate()\n"); return 0; } } else { /* Call Zoltan_Help_Migrate with empty import lists. */ /* Have to "manually" call migrate_pre_process and migrate_post_process. */ int ierr = 0; migrate_pre_process((void *) mesh, 1, 1, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part, &ierr); if (Test.Null_Lists == IMPORT_LISTS) { if (Zoltan_Migrate(zz, -1, NULL, NULL, NULL, NULL, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Migrate()\n"); return 0; } } else { if (Zoltan_Migrate(zz, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, -1, NULL, NULL, NULL, NULL) == ZOLTAN_FATAL) { Gen_Error(0, "fatal: error returned from Zoltan_Migrate()\n"); return 0; } } migrate_post_process((void *) mesh, 1, 1, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part, &ierr); } DEBUG_TRACE_END(Proc, yo); return 1; }
static int Zoltan_LB( ZZ *zz, int include_parts, /* Flag indicating whether to generate part informtion; 0 if called by Zoltan_LB_Balance, 1 if called by Zoltan_LB_Partition. */ int *changes, /* Set to zero or one depending on if Zoltan determines a new decomposition or not: zero - No changes to the decomposition were made by the load-balancing algorithm; migration is not needed. one - A new decomposition is suggested by the load-balancer; migration is needed to establish the new decomposition. */ int *num_gid_entries, /* The number of array entries in a global ID; set to be the max over all processors in zz->Communicator of the parameter Num_Global_ID_Entries. */ int *num_lid_entries, /* The number of array entries in a local ID; set to be the max over all processors in zz->Communicator of the parameter Num_Local_ID_Entries. */ int *num_import_objs, /* The number of non-local objects in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ int **import_procs, /* Array of processor IDs for processors currently owning non-local objects (i.e., objs to be imported) in this processor's new decomposition. */ int **import_to_part, /* Partition to which the objects should be imported. */ int *num_export_objs, /* The number of local objects that need to be exported from the processor to establish the new decomposition. */ ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ int **export_procs, /* Array of destination processor IDs for objects that need to be exported to establish the new decomposition. */ int **export_to_part /* Partition to which objects should be exported. */ ) { /* * Main load-balancing routine. * Input: a Zoltan structure with appropriate function pointers set. * Output: * changes * num_import_objs * import_global_ids * import_local_ids * import_procs * import_to_part * num_export_objs * export_global_ids * export_local_ids * export_procs * export_to_part * Return values: * Zoltan error code. */ char *yo = "Zoltan_LB"; int gmax; /* Maximum number of imported/exported objects over all processors. */ int error = ZOLTAN_OK; /* Error code */ double start_time, end_time; double lb_time[2] = {0.0,0.0}; char msg[256]; int comm[3],gcomm[3]; float *part_sizes = NULL, *fdummy = NULL; int wgt_dim, part_dim; int all_num_obj, i, ts, idIdx; struct Hash_Node **ht; int *export_all_procs, *export_all_to_part, *parts=NULL; ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL; ZOLTAN_ID_PTR gid; #ifdef ZOLTAN_OVIS struct OVIS_parameters ovisParameters; #endif ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS){ printf("Build configuration:\n"); Zoltan_Print_Configuration(" "); printf("\n"); Zoltan_Print_Key_Params(zz); } start_time = Zoltan_Time(zz->Timer); #ifdef ZOLTAN_OVIS Zoltan_OVIS_Setup(zz, &ovisParameters); if (zz->Proc == 0) printf("OVIS PARAMETERS %s %s %d %f\n", ovisParameters.hello, ovisParameters.dll, ovisParameters.outputLevel, ovisParameters.minVersion); ovis_enabled(zz->Proc, ovisParameters.dll); #endif /* * Compute Max number of array entries per ID over all processors. * Compute Max number of return arguments for Zoltan_LB_Balance. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; comm[2] = zz->LB.Return_Lists; MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; zz->LB.Return_Lists = gcomm[2]; /* assume no changes */ *changes = 0; *num_import_objs = *num_export_objs = 0; *import_global_ids = NULL; *import_local_ids = NULL; *import_procs = NULL; *import_to_part = NULL; *export_global_ids = NULL; *export_local_ids = NULL; *export_procs = NULL; *export_to_part = NULL; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) goto End; if (zz->LB.Method == NONE) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s Balancing method selected == NONE; no balancing performed\n", yo); error = ZOLTAN_WARN; goto End; } /* * Sync the random number generator across processors. */ Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator); /* Since generating a new partition, need to free old mapping vector */ zz->LB.OldRemap = zz->LB.Remap; zz->LB.Remap = NULL; error = Zoltan_LB_Build_PartDist(zz); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) goto End; if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { int i, np, fp; for (i = 0; i < zz->Num_Proc; i++) { Zoltan_LB_Proc_To_Part(zz, i, &np, &fp); printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n", zz->Proc, i, np, fp); } } /* * Generate parts sizes. */ #ifdef ZOLTAN_OVIS /* set part sizes computed by OVIS, if requested. Processes set only their own value */ { float part_sizes[1]; int part_ids[1], wgt_idx[1]; wgt_idx[0] = 0; part_ids[0] = 0; ovis_getPartsize(&(part_sizes[0])); printf("Rank %d ps %f\n",zz->Proc, part_sizes[0]); /* clear out old part size info first */ Zoltan_LB_Set_Part_Sizes(zz, 0, -1, NULL, NULL, NULL); Zoltan_LB_Set_Part_Sizes(zz, 0, 1, part_ids, wgt_idx, part_sizes); } #endif wgt_dim = zz->Obj_Weight_Dim; part_dim = ((wgt_dim > 0) ? wgt_dim : 1); part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim * zz->LB.Num_Global_Parts); if (part_sizes == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); error = ZOLTAN_MEMERR; goto End; } /* Get part sizes. */ Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim, part_sizes); #ifdef ZOLTAN_OVIS /* if (ovisParameters.outputlevel > 3) */ { int myRank = zz->Proc; if (myRank == 0){ int i, j; for (i = 0; i < zz->LB.Num_Global_Parts; i++){ for (j = 0; j < part_dim; j++){ printf("Rank %d AG: part_sizes[%d] = %f (Num_Global_Parts = %d, part_dim = %d)\n",zz->Proc, (i*part_dim+j), part_sizes[i*part_dim+j],zz->LB.Num_Global_Parts, part_dim); } } } } #endif /* * Call the actual load-balancing function. */ error = zz->LB.LB_Fn(zz, part_sizes, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); ZOLTAN_FREE(&part_sizes); if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){ sprintf(msg, "Partitioning routine returned code %d.", error); #ifdef HOST_LINUX if ((error == ZOLTAN_MEMERR) && (Zoltan_Memory_Get_Debug() > 0)){ Zoltan_write_linux_meminfo(0, "State of /proc/meminfo after malloc failure\n", 0); } #endif ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } else if (error){ if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) { sprintf(msg, "Partitioning routine returned code %d.", error); ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning"); if (*num_import_objs >= 0) MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); else /* use export data */ MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); if (gmax == 0) { /* * Decomposition was not changed by the load balancing; no migration * is needed. */ if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s No changes to the decomposition due to partitioning; " "no migration is needed.\n", yo); /* * Reset num_import_objs and num_export_objs; don't want to return * -1 for the arrays that weren't returned by ZOLTAN_LB_FN. */ *num_import_objs = *num_export_objs = 0; if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){ /* * This parameter setting requires that all local objects * and their assignments appear in the export list. */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } goto End; } /* * Check whether we know the import data, export data, or both. * * If we were given the import data, * we know what the new decomposition should look like on the * processor, but we don't know which of our local objects we have * to export to other processors to establish the new decomposition. * Reverse the argument if we were given the export data. * * Unless we were given both maps, compute the inverse map. */ if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) { if (*num_import_objs >= 0) Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); if (*num_export_objs >= 0) Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); *num_import_objs = *num_export_objs = -1; } if (*num_import_objs >= 0){ if (*num_export_objs >= 0) { /* Both maps already available; nothing to do. */; } else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Export lists are requested; compute export map */ error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Method returned import lists, but only export lists were desired. */ /* Import lists not needed; free them. */ *num_import_objs = -1; Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); } } } else { /* (*num_import_objs < 0) */ if (*num_export_objs >= 0) { /* Only export lists have been returned. */ if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Compute import map */ error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Method returned export lists, but only import lists are desired. */ /* Export lists not needed; free them. */ *num_export_objs = -1; Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); } } } else { /* *num_export_objs < 0 && *num_import_objs < 0) */ if (zz->LB.Return_Lists) { /* No map at all available */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned " "neither import nor export data."); error = ZOLTAN_WARN; goto End; } } } if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* * Normally, Zoltan_LB returns in the export lists all local * objects that are moving off processor, or that are assigned * to a part on the local processor that is not the * default part. This setting of Return_Lists requests * that all local objects be included in the export list. */ if (*num_export_objs == 0){ /* all local objects are remaining on processor */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (*num_export_objs) { if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End; } else{ all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error); if (*num_export_objs < all_num_obj){ /* Create a lookup table for exported IDs */ ts = Zoltan_Recommended_Hash_Size(*num_export_objs); ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts); /* Create a list of all gids, lids and parts */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj, &all_global_ids, &all_local_ids, wgt_dim, &fdummy, &parts); if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){ ZOLTAN_FREE(&fdummy); if ((Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_procs, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) || (Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_to_part, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){ error = ZOLTAN_MEMERR; } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){ sprintf(msg, "Error building complete export list; " "%d returned by Zoltan_Get_Obj_List\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } gid = all_global_ids; for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){ idIdx = search_hash_table(zz, gid, ht, ts); if (idIdx >= 0){ export_all_procs[i] = (*export_procs)[idIdx]; export_all_to_part[i] = (*export_to_part)[idIdx]; } else{ export_all_procs[i] = zz->Proc; export_all_to_part[i] = parts[i]; } } free_hash_table(ht, ts); Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); Zoltan_Special_Free(zz, (void **)(void*)&parts, ZOLTAN_SPECIAL_MALLOC_INT); *export_global_ids = all_global_ids; *export_local_ids = all_local_ids; *export_procs = export_all_procs; *export_to_part = export_all_to_part; *num_export_objs = all_num_obj; } } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments"); end_time = Zoltan_Time(zz->Timer); lb_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc); for (i = 0; i < *num_import_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID])); printf(" To part: %4d", (*import_to_part != NULL ? (*import_to_part)[i] : zz->Proc)); printf(" From processor: %4d\n", (*import_procs)[i]); } printf("\n"); printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc); for (i = 0; i < *num_export_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID])); printf(" To part: %4d", (*export_to_part != NULL ? (*export_to_part)[i] : (*export_procs)[i])); printf(" To processor: %4d\n", (*export_procs)[i]); } Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* * If the Help_Migrate flag is set, perform migration for the application. */ if (zz->Migrate.Auto_Migrate) { ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration"); start_time = Zoltan_Time(zz->Timer); error = Zoltan_Migrate(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error in auto-migration; %d returned from " "Zoltan_Help_Migrate\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } end_time = Zoltan_Time(zz->Timer); lb_time[1] = end_time - start_time; ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration"); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0], "ZOLTAN Partition: "); if (zz->Migrate.Auto_Migrate) Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1], "ZOLTAN Migrate: "); } *changes = 1; End: ZOLTAN_TRACE_EXIT(zz, yo); return (error); }
int main(int argc, char *argv[]) { int i, rc; int myRank, numProcs; float ver; struct Zoltan_Struct *zz; int changes, numGidEntries, numLidEntries, numImport, numExport, start_gid, num_nbors; ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids; int *importProcs, *importToPart, *exportProcs, *exportToPart; int *parts=NULL; ZOLTAN_ID_PTR lids=NULL; FILE *fp; struct Zoltan_DD_Struct *dd; GRAPH_DATA myGraph; int gid_length = 1; /* our global IDs consist of 1 integer */ int lid_length = 1; /* our local IDs consist of 1 integer */ /****************************************************************** ** Initialize MPI and Zoltan ******************************************************************/ MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &myRank); MPI_Comm_size(MPI_COMM_WORLD, &numProcs); rc = Zoltan_Initialize(argc, argv, &ver); if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); exit(0); } /****************************************************************** ** Read graph from input file and distribute it ******************************************************************/ fp = fopen(fname, "r"); if (!fp){ if (myRank == 0) fprintf(stderr,"ERROR: Can not open %s\n",fname); MPI_Finalize(); exit(1); } fclose(fp); read_input_file(myRank, numProcs, fname, &myGraph); /*fprintf(stderr,"%d have %d objects\n",myRank,myGraph.numMyVertices);*/ /****************************************************************** ** Create a distributed data directory which maps vertex ** global IDs to their current partition number. We'll use this ** after migrating vertices, to update the partition in which ** our vertices neighbors are. ** ** Our local IDs (array "lids") are of type ZOLTAN_ID_TYPE because ** we are using Zoltan's distributed data directory. It assumes ** that a global ID is a sequence of "gid_length" ZOLTAN_ID_TYPEs. ** It assumes that a local ID is a sequence of "lid_length" ** ZOLTAN_ID_TYPEs. ******************************************************************/ rc = Zoltan_DD_Create(&dd, MPI_COMM_WORLD, gid_length, /* length of a global ID */ lid_length, /* length of a local ID */ 0, /* length of user data */ myGraph.numMyVertices, /* hash table size */ 0); /* debug level */ parts = malloc(myGraph.numMyVertices * sizeof(int)); lids = malloc(myGraph.numMyVertices * sizeof(ZOLTAN_ID_TYPE)); for (i=0; i < myGraph.numMyVertices; i++){ parts[i] = myRank; /* part number of this vertex */ lids[i] = (ZOLTAN_ID_TYPE)i; /* local ID on my process for this vertex */ } rc = Zoltan_DD_Update(dd, myGraph.vertexGID, lids, NULL, parts, myGraph.numMyVertices); myGraph.dd = dd; /****************************************************************** ** Create a Zoltan library structure for this instance of load ** balancing. Set the parameters and query functions that will ** govern the library's calculation. See the Zoltan User's ** Guide for the definition of these and many other parameters. ******************************************************************/ zz = Zoltan_Create(MPI_COMM_WORLD); /* General parameters */ Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"); Zoltan_Set_Param(zz, "LB_METHOD", "GRAPH"); Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION"); Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL"); /* Graph parameters */ Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */ /* Query functions, defined in this source file */ Zoltan_Set_Num_Obj_Fn(zz, get_number_of_vertices, &myGraph); Zoltan_Set_Obj_List_Fn(zz, get_vertex_list, &myGraph); Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &myGraph); Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &myGraph); Zoltan_Set_Obj_Size_Multi_Fn(zz, get_message_sizes,&myGraph); Zoltan_Set_Pack_Obj_Multi_Fn(zz, pack_object_messages,&myGraph); Zoltan_Set_Unpack_Obj_Multi_Fn(zz, unpack_object_messages,&myGraph); Zoltan_Set_Mid_Migrate_PP_Fn(zz, mid_migrate,&myGraph); /****************************************************************** ** Visualize the graph partitioning before calling Zoltan. ******************************************************************/ if (myRank== 0){ printf("\nGraph partition before calling Zoltan\n"); } showGraphPartitions(myRank, myGraph.dd); /****************************************************************** ** Zoltan can now partition the simple graph. ** In this simple example, we assume the number of partitions is ** equal to the number of processes. Process rank 0 will own ** partition 0, process rank 1 will own partition 1, and so on. ******************************************************************/ rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */ &changes, /* 1 if partitioning was changed, 0 otherwise */ &numGidEntries, /* Number of integers used for a global ID */ &numLidEntries, /* Number of integers used for a local ID */ &numImport, /* Number of vertices to be sent to me */ &importGlobalGids, /* Global IDs of vertices to be sent to me */ &importLocalGids, /* Local IDs of vertices to be sent to me */ &importProcs, /* Process rank for source of each incoming vertex */ &importToPart, /* New partition for each incoming vertex */ &numExport, /* Number of vertices I must send to other processes*/ &exportGlobalGids, /* Global IDs of the vertices I must send */ &exportLocalGids, /* Local IDs of the vertices I must send */ &exportProcs, /* Process to which I send each of the vertices */ &exportToPart); /* Partition to which each vertex will belong */ if (rc != ZOLTAN_OK){ printf("sorry...\n"); MPI_Finalize(); Zoltan_Destroy(&zz); exit(0); } /*fprintf(stderr,"%d export %d import %d\n",myRank,numExport,numImport);*/ /****************************************************************** ** Update the data directory with the new partition numbers ******************************************************************/ for (i=0; i < numExport; i++){ parts[exportLocalGids[i]] = exportToPart[i]; } rc = Zoltan_DD_Update(dd, myGraph.vertexGID, lids, NULL, parts, myGraph.numMyVertices); /****************************************************************** ** Migrate vertices to new partitions ******************************************************************/ rc = Zoltan_Migrate(zz, numImport, importGlobalGids, importLocalGids, importProcs, importToPart, numExport, exportGlobalGids, exportLocalGids, exportProcs, exportToPart); /****************************************************************** ** Use the data dictionary to find neighbors' partitions ******************************************************************/ start_gid = myGraph.numMyVertices - numImport; num_nbors = myGraph.nborIndex[myGraph.numMyVertices] - myGraph.nborIndex[start_gid]; rc = Zoltan_DD_Find(dd, (ZOLTAN_ID_PTR)(myGraph.nborGID + start_gid), NULL, NULL, myGraph.nborPart + start_gid, num_nbors, NULL); /****************************************************************** ** Visualize the graph partitioning after calling Zoltan. ******************************************************************/ if (myRank == 0){ printf("Graph partition after calling Zoltan\n"); } showGraphPartitions(myRank, myGraph.dd); /****************************************************************** ** Free the arrays allocated by Zoltan_LB_Partition, and free ** the storage allocated for the Zoltan structure. ******************************************************************/ Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart); Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart); Zoltan_Destroy(&zz); /********************** ** all done *********** **********************/ MPI_Finalize(); if (myGraph.vertex_capacity > 0){ free(myGraph.vertexGID); free(myGraph.nborIndex); if (myGraph.nbor_capacity > 0){ free(myGraph.nborGID); free(myGraph.nborPart); } } if (parts) free(parts); if (lids) free(lids); return 0; }