void Zoltan_Third_DisplayTime(ZZ* zz, double* times) { if (zz->Proc == zz->Debug_Proc) printf("\nZOLTAN timing statistics:\n"); Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, times[1]-times[0], " Partitioner Pre-processing time "); Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, times[2]-times[1], " Partitioner Library time "); Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, times[3]-times[2], " Partitioner Post-processing time "); Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, times[3]-times[0], " Partitioner Total time "); if (zz->Proc==zz->Debug_Proc) printf("\n"); }
int Zoltan_Reftree_Part( ZZ *zz, /* The Zoltan structure */ float *part_sizes, /* Input: Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ int *num_import, /* Not computed, set to -1 */ ZOLTAN_ID_PTR *import_global_ids, /* Not computed */ ZOLTAN_ID_PTR *import_local_ids, /* Not computed */ int **import_procs, /* Not computed */ int **import_to_part, /* Not computed */ int *num_export, /* Number of objects to be exported */ ZOLTAN_ID_PTR *export_global_ids, /* global ids of objects to be exported */ ZOLTAN_ID_PTR *export_local_ids, /* local ids of objects to be exported */ int **export_procs, /* list of processors to export to */ int **export_to_partition /* list of partitions to export to */ ) { char *yo = "Zoltan_Reftree_Part"; int ierr; /* error code returned by called routines */ int final_ierr; /* error code returned by this routine */ double time0 = 0, time1= 0, time2 = 0, time3 =0, time4 =0; ZOLTAN_TRACE_ENTER(zz, yo); /* Initializations in case of early exit. */ *num_export = -1; *num_import = -1; final_ierr = ZOLTAN_OK; /* * initialize the tree (first call only) */ if (zz->LB.Data_Structure == NULL) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time0 = Zoltan_Time(zz->Timer); ierr = Zoltan_Reftree_Init(zz); if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned by Zoltan_Reftree_Init."); ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); } if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN; if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time1 = Zoltan_Time(zz->Timer); } else { if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) { time1 = Zoltan_Time(zz->Timer); time0 = time1 + 1.0; } } /* * build the refinement tree */ ierr = Zoltan_Reftree_Build(zz); if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned by Zoltan_Reftree_Build."); ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); } if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN; if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time2 = Zoltan_Time(zz->Timer); /* * sum the weights in the tree */ ierr = Zoltan_Reftree_Sum_Weights(zz); if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned by Zoltan_Reftree_Sum_Weights."); ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); } if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN; if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time3 = Zoltan_Time(zz->Timer); /* * determine the new partition */ ierr = Zoltan_Reftree_Partition(zz, part_sizes, num_export, export_global_ids, export_local_ids, export_to_partition, export_procs); if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned by Zoltan_Reftree_Partition."); ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); } if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN; if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time4 = Zoltan_Time(zz->Timer); if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) { if (time0 <= time1) { Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time1-time0, "REFTREE Time to initialize :"); } Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time2-time1, "REFTREE Time to build tree :"); Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time3-time2, "REFTREE Time to sum weights:"); Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time4-time3, "REFTREE Time to partition :"); } ZOLTAN_TRACE_EXIT(zz, yo); return(final_ierr); }
static int Zoltan_LB( ZZ *zz, int include_parts, /* Flag indicating whether to generate part informtion; 0 if called by Zoltan_LB_Balance, 1 if called by Zoltan_LB_Partition. */ int *changes, /* Set to zero or one depending on if Zoltan determines a new decomposition or not: zero - No changes to the decomposition were made by the load-balancing algorithm; migration is not needed. one - A new decomposition is suggested by the load-balancer; migration is needed to establish the new decomposition. */ int *num_gid_entries, /* The number of array entries in a global ID; set to be the max over all processors in zz->Communicator of the parameter Num_Global_ID_Entries. */ int *num_lid_entries, /* The number of array entries in a local ID; set to be the max over all processors in zz->Communicator of the parameter Num_Local_ID_Entries. */ int *num_import_objs, /* The number of non-local objects in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ int **import_procs, /* Array of processor IDs for processors currently owning non-local objects (i.e., objs to be imported) in this processor's new decomposition. */ int **import_to_part, /* Partition to which the objects should be imported. */ int *num_export_objs, /* The number of local objects that need to be exported from the processor to establish the new decomposition. */ ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ int **export_procs, /* Array of destination processor IDs for objects that need to be exported to establish the new decomposition. */ int **export_to_part /* Partition to which objects should be exported. */ ) { /* * Main load-balancing routine. * Input: a Zoltan structure with appropriate function pointers set. * Output: * changes * num_import_objs * import_global_ids * import_local_ids * import_procs * import_to_part * num_export_objs * export_global_ids * export_local_ids * export_procs * export_to_part * Return values: * Zoltan error code. */ char *yo = "Zoltan_LB"; int gmax; /* Maximum number of imported/exported objects over all processors. */ int error = ZOLTAN_OK; /* Error code */ double start_time, end_time; double lb_time[2] = {0.0,0.0}; char msg[256]; int comm[3],gcomm[3]; float *part_sizes = NULL, *fdummy = NULL; int wgt_dim, part_dim; int all_num_obj, i, ts, idIdx; struct Hash_Node **ht; int *export_all_procs, *export_all_to_part, *parts=NULL; ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL; ZOLTAN_ID_PTR gid; #ifdef ZOLTAN_OVIS struct OVIS_parameters ovisParameters; #endif ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS){ printf("Build configuration:\n"); Zoltan_Print_Configuration(" "); printf("\n"); Zoltan_Print_Key_Params(zz); } start_time = Zoltan_Time(zz->Timer); #ifdef ZOLTAN_OVIS Zoltan_OVIS_Setup(zz, &ovisParameters); if (zz->Proc == 0) printf("OVIS PARAMETERS %s %s %d %f\n", ovisParameters.hello, ovisParameters.dll, ovisParameters.outputLevel, ovisParameters.minVersion); ovis_enabled(zz->Proc, ovisParameters.dll); #endif /* * Compute Max number of array entries per ID over all processors. * Compute Max number of return arguments for Zoltan_LB_Balance. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; comm[2] = zz->LB.Return_Lists; MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; zz->LB.Return_Lists = gcomm[2]; /* assume no changes */ *changes = 0; *num_import_objs = *num_export_objs = 0; *import_global_ids = NULL; *import_local_ids = NULL; *import_procs = NULL; *import_to_part = NULL; *export_global_ids = NULL; *export_local_ids = NULL; *export_procs = NULL; *export_to_part = NULL; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) goto End; if (zz->LB.Method == NONE) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s Balancing method selected == NONE; no balancing performed\n", yo); error = ZOLTAN_WARN; goto End; } /* * Sync the random number generator across processors. */ Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator); /* Since generating a new partition, need to free old mapping vector */ zz->LB.OldRemap = zz->LB.Remap; zz->LB.Remap = NULL; error = Zoltan_LB_Build_PartDist(zz); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) goto End; if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { int i, np, fp; for (i = 0; i < zz->Num_Proc; i++) { Zoltan_LB_Proc_To_Part(zz, i, &np, &fp); printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n", zz->Proc, i, np, fp); } } /* * Generate parts sizes. */ #ifdef ZOLTAN_OVIS /* set part sizes computed by OVIS, if requested. Processes set only their own value */ { float part_sizes[1]; int part_ids[1], wgt_idx[1]; wgt_idx[0] = 0; part_ids[0] = 0; ovis_getPartsize(&(part_sizes[0])); printf("Rank %d ps %f\n",zz->Proc, part_sizes[0]); /* clear out old part size info first */ Zoltan_LB_Set_Part_Sizes(zz, 0, -1, NULL, NULL, NULL); Zoltan_LB_Set_Part_Sizes(zz, 0, 1, part_ids, wgt_idx, part_sizes); } #endif wgt_dim = zz->Obj_Weight_Dim; part_dim = ((wgt_dim > 0) ? wgt_dim : 1); part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim * zz->LB.Num_Global_Parts); if (part_sizes == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); error = ZOLTAN_MEMERR; goto End; } /* Get part sizes. */ Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim, part_sizes); #ifdef ZOLTAN_OVIS /* if (ovisParameters.outputlevel > 3) */ { int myRank = zz->Proc; if (myRank == 0){ int i, j; for (i = 0; i < zz->LB.Num_Global_Parts; i++){ for (j = 0; j < part_dim; j++){ printf("Rank %d AG: part_sizes[%d] = %f (Num_Global_Parts = %d, part_dim = %d)\n",zz->Proc, (i*part_dim+j), part_sizes[i*part_dim+j],zz->LB.Num_Global_Parts, part_dim); } } } } #endif /* * Call the actual load-balancing function. */ error = zz->LB.LB_Fn(zz, part_sizes, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); ZOLTAN_FREE(&part_sizes); if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){ sprintf(msg, "Partitioning routine returned code %d.", error); #ifdef HOST_LINUX if ((error == ZOLTAN_MEMERR) && (Zoltan_Memory_Get_Debug() > 0)){ Zoltan_write_linux_meminfo(0, "State of /proc/meminfo after malloc failure\n", 0); } #endif ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } else if (error){ if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) { sprintf(msg, "Partitioning routine returned code %d.", error); ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning"); if (*num_import_objs >= 0) MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); else /* use export data */ MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); if (gmax == 0) { /* * Decomposition was not changed by the load balancing; no migration * is needed. */ if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s No changes to the decomposition due to partitioning; " "no migration is needed.\n", yo); /* * Reset num_import_objs and num_export_objs; don't want to return * -1 for the arrays that weren't returned by ZOLTAN_LB_FN. */ *num_import_objs = *num_export_objs = 0; if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){ /* * This parameter setting requires that all local objects * and their assignments appear in the export list. */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } goto End; } /* * Check whether we know the import data, export data, or both. * * If we were given the import data, * we know what the new decomposition should look like on the * processor, but we don't know which of our local objects we have * to export to other processors to establish the new decomposition. * Reverse the argument if we were given the export data. * * Unless we were given both maps, compute the inverse map. */ if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) { if (*num_import_objs >= 0) Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); if (*num_export_objs >= 0) Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); *num_import_objs = *num_export_objs = -1; } if (*num_import_objs >= 0){ if (*num_export_objs >= 0) { /* Both maps already available; nothing to do. */; } else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Export lists are requested; compute export map */ error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Method returned import lists, but only export lists were desired. */ /* Import lists not needed; free them. */ *num_import_objs = -1; Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); } } } else { /* (*num_import_objs < 0) */ if (*num_export_objs >= 0) { /* Only export lists have been returned. */ if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Compute import map */ error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Method returned export lists, but only import lists are desired. */ /* Export lists not needed; free them. */ *num_export_objs = -1; Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); } } } else { /* *num_export_objs < 0 && *num_import_objs < 0) */ if (zz->LB.Return_Lists) { /* No map at all available */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned " "neither import nor export data."); error = ZOLTAN_WARN; goto End; } } } if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* * Normally, Zoltan_LB returns in the export lists all local * objects that are moving off processor, or that are assigned * to a part on the local processor that is not the * default part. This setting of Return_Lists requests * that all local objects be included in the export list. */ if (*num_export_objs == 0){ /* all local objects are remaining on processor */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (*num_export_objs) { if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End; } else{ all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error); if (*num_export_objs < all_num_obj){ /* Create a lookup table for exported IDs */ ts = Zoltan_Recommended_Hash_Size(*num_export_objs); ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts); /* Create a list of all gids, lids and parts */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj, &all_global_ids, &all_local_ids, wgt_dim, &fdummy, &parts); if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){ ZOLTAN_FREE(&fdummy); if ((Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_procs, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) || (Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_to_part, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){ error = ZOLTAN_MEMERR; } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){ sprintf(msg, "Error building complete export list; " "%d returned by Zoltan_Get_Obj_List\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } gid = all_global_ids; for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){ idIdx = search_hash_table(zz, gid, ht, ts); if (idIdx >= 0){ export_all_procs[i] = (*export_procs)[idIdx]; export_all_to_part[i] = (*export_to_part)[idIdx]; } else{ export_all_procs[i] = zz->Proc; export_all_to_part[i] = parts[i]; } } free_hash_table(ht, ts); Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); Zoltan_Special_Free(zz, (void **)(void*)&parts, ZOLTAN_SPECIAL_MALLOC_INT); *export_global_ids = all_global_ids; *export_local_ids = all_local_ids; *export_procs = export_all_procs; *export_to_part = export_all_to_part; *num_export_objs = all_num_obj; } } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments"); end_time = Zoltan_Time(zz->Timer); lb_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc); for (i = 0; i < *num_import_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID])); printf(" To part: %4d", (*import_to_part != NULL ? (*import_to_part)[i] : zz->Proc)); printf(" From processor: %4d\n", (*import_procs)[i]); } printf("\n"); printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc); for (i = 0; i < *num_export_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID])); printf(" To part: %4d", (*export_to_part != NULL ? (*export_to_part)[i] : (*export_procs)[i])); printf(" To processor: %4d\n", (*export_procs)[i]); } Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* * If the Help_Migrate flag is set, perform migration for the application. */ if (zz->Migrate.Auto_Migrate) { ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration"); start_time = Zoltan_Time(zz->Timer); error = Zoltan_Migrate(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error in auto-migration; %d returned from " "Zoltan_Help_Migrate\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } end_time = Zoltan_Time(zz->Timer); lb_time[1] = end_time - start_time; ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration"); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0], "ZOLTAN Partition: "); if (zz->Migrate.Auto_Migrate) Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1], "ZOLTAN Migrate: "); } *changes = 1; End: ZOLTAN_TRACE_EXIT(zz, yo); return (error); }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int *num_gid_entries, /* # of entries for a global id */ int *num_lid_entries, /* # of entries for a local id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm, /* inverse permutation of rank */ ZOS *order_info /* Method-specific ordering info. Currently not used. */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * iperm, inverse permutation of rank * order_info, a Zoltan Ordering Struct with additional info. * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; int *vtxdist; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strncpy(opt.order_type, "GLOBAL", MAX_PARAM_STRING_LEN); opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); if (opt.use_order_info == 0) order_info = NULL; /* * Check that the user has allocated space for the return args. */ if (!(gids && lids && rank && iperm)){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL){ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. */ ierr = (*Order_fn)(zz, num_obj, gids, lids, rank, iperm, &opt, order_info); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN){ ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr){ /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK)){ /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, iperm, rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM)){ /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, rank, iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i, nobjs; nobjs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &i); Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < nobjs; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(*num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); printf("ZOLTAN: inverse permutation on Proc %d\n", zz->Proc); for (i = 0; i < nobjs; i++) { printf("iperm[%3d] = %3d\n", i, iperm[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
static int rib_fn( ZZ *zz, /* The Zoltan structure with info for the RIB balancer. */ int *num_import, /* Number of non-local objects assigned to this processor in the new decomposition. When LB.Return_Lists==CANDIDATE_LISTS, num_import returns the number of input objects as given by ZOLTAN_NUM_OBJ_FN. */ ZOLTAN_ID_PTR *import_global_ids, /* Returned value: array of global IDs for non-local objects in this processor's new decomposition. When LB.Return_Lists==CANDIDATE_LISTS, this array contains GIDs for all input objs as given by ZOLTAN_OBJ_LIST_FN.*/ ZOLTAN_ID_PTR *import_local_ids, /* Returned value: array of local IDs for non-local objects in this processor's new decomposition. When LB.Return_Lists==CANDIDATE_LISTS, this array contains LIDs for all input objs as given by ZOLTAN_OBJ_LIST_FN.*/ int **import_procs, /* Returned value: array of processor IDs for processors owning the non-local objects in this processor's new decomposition. When LB.Return_Lists==CANDIDATE_LISTS, the returned array is NULL. */ int **import_to_part, /* Returned value: array of parts to which objects are imported. When LB.Return_Lists==CANDIDATE_LISTS, the returned array is NULL. */ int *num_export, /* Returned value only when LB.Return_Lists==CANDIDATE_LISTS; number of input objs as given by ZOLTAN_NUM_OBJ_FN */ ZOLTAN_ID_PTR *export_global_ids, /* Returned value only when LB.Return_Lists==CANDIDATE_LISTS; for each input obj (from ZOLTAN_OBJ_LIST_FN), return a candidate obj from the part to which the obj is assigned; used in PHG matching */ double overalloc, /* amount to overallocate by when realloc of dot array must be done. 1.0 = no extra; 1.5 = 50% extra; etc. */ int wgtflag, /* No. of weights per dot supplied by user. */ int check_geom, /* Check input & output for consistency? */ int stats, /* Print timing & count summary? */ int gen_tree, /* (0) do not (1) do generate full treept */ int average_cuts, /* (0) don't (1) compute the cut to be the average of the closest dots. */ float *part_sizes /* Input: Array of size zz->Num_Global_Parts * max(zz->Obj_Weight_Dim, 1) containing the percentage of work to be assigned to each part. */ ) { char yo[] = "rib_fn"; int proc,nprocs; /* my proc id, total # of procs */ struct Dot_Struct *dotpt; /* temporary pointer to local dot arrays */ int pdotnum; /* # of dots - decomposition changes it */ int *dotmark = NULL; /* which side of median for each dot */ int dotnum; /* number of dots */ int dotmax = 0; /* max # of dots arrays can hold */ int dottop; /* dots >= this index are new */ int proclower; /* 1st proc in lower set */ int procmid; /* 1st proc in upper set */ int partlower; /* 1st part in lower set */ int partmid; /* 1st part in upper set */ int set; /* which set processor is in = 0/1 */ int old_set; /* set processor was in last cut = 0/1 */ int root; /* part that stores last cut */ int num_procs; /* number of procs in current set */ int num_parts; /* number of parts in current set */ int ierr = ZOLTAN_OK; /* error flag. */ double *value = NULL; /* temp array for median_find */ double *wgts = NULL; /* temp array for serial_rib */ double valuehalf; /* median cut position */ double cm[3]; /* Center of mass of objects */ double evec[3]; /* Eigenvector defining direction */ int first_guess = 0; /* flag if first guess for median search */ int allocflag; /* have to re-allocate space */ double time1=0,time2=0; /* timers */ double time3=0,time4=0; /* timers */ double timestart=0,timestop=0; /* timers */ double timers[4]={0.,0.,0.,0.}; /* diagnostic timers 0 = start-up time before recursion 1 = time before median iterations 2 = time in median iterations 3 = communication time */ ZOLTAN_GNO_TYPE counters[7]; /* diagnostic counts 0 = unused 1 = # of dots sent 2 = # of dots received 3 = most dots this proc ever owns 4 = most dot memory this proc ever allocs 5 = # of times a previous cut is re-used 6 = # of reallocs of dot array */ int i, j; /* local variables */ int use_ids; /* When true, global and local IDs will be stored along with dots in the RCB_STRUCT. When false, storage, manipulation, and communication of IDs is avoided. Set by call to Zoltan_RB_Use_IDs(). */ RIB_STRUCT *rib = NULL; /* Pointer to data structures for RIB */ struct rib_tree *treept = NULL; /* tree of cuts - single cut on exit*/ double start_time, end_time; double lb_time[2]={0,0}; int tfs[2], tmp_tfs[2]; /* added for Tflops_Special; max number of procs and parts over all processors in each iteration (while loop) of parallel partitioning. */ int old_nprocs; /* added for Tflops_Special */ int old_nparts; /* added for Tflops_Special */ double valuelo; /* smallest value of value[i] */ double valuehi; /* largest value of value[i] */ double weight[RB_MAX_WGTS]; /* weight for current set */ double weightlo[RB_MAX_WGTS]; /* weight of lower side of cut */ double weighthi[RB_MAX_WGTS]; /* weight of upper side of cut */ double fractionlo[RB_MAX_WGTS]; /* desired wt in lower half */ int *dotlist = NULL; /* list of dots for find_median. allocated above find_median for better efficiency (don't necessarily have to realloc for each find_median).*/ int rectilinear_blocks = 0; /* parameter for find_median (not used by rib) */ int fp=0; /* first part assigned to this proc. */ int np=0; /* number of parts assigned to this proc. */ int wgtdim; /* max(wgtflag,1) */ int *dindx = NULL, *tmpdindx = NULL; /* MPI data types and user functions */ MPI_Comm local_comm, tmp_comm; int free_comm = FALSE; /* Flag indicating whether MPI_Comm_free should be called on local_comm at end. */ ZOLTAN_TRACE_ENTER(zz, yo); if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) { MPI_Barrier(zz->Communicator); timestart = time1 = Zoltan_Time(zz->Timer); } /* setup for parallel */ proc = zz->Proc; nprocs = zz->Num_Proc; num_parts = zz->LB.Num_Global_Parts; /* * Determine whether to store, manipulate, and communicate global and * local IDs. */ use_ids = Zoltan_RB_Use_IDs(zz); /* * Build the RIB Data structure and * set pointers to information in it. */ start_time = Zoltan_Time(zz->Timer); ierr = Zoltan_RIB_Build_Structure(zz, &pdotnum, &dotmax, wgtflag, overalloc, use_ids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from Zoltan_RIB_Build_Structure."); goto End; } rib = (RIB_STRUCT *) (zz->LB.Data_Structure); treept = rib->Tree_Ptr; end_time = Zoltan_Time(zz->Timer); lb_time[0] = end_time - start_time; start_time = end_time; /* local copies of calling parameters */ dottop = dotnum = pdotnum; /* initialize timers and counters */ counters[0] = 0; counters[1] = 0; counters[2] = 0; counters[3] = dotnum; counters[4] = dotmax; counters[5] = 0; counters[6] = 0; /* Ensure there are dots */ MPI_Allreduce(&dotnum, &i, 1, MPI_INT, MPI_MAX, zz->Communicator); if (i == 0){ if (proc == 0){ ZOLTAN_PRINT_WARN(proc, yo, "RIB partitioning called with no objects"); } timestart = timestop = 0; goto EndReporting; } /* If using RIB for matching, need to generate candidate lists. * Candidate lists include input GIDs, LIDs as provided by the application. * We need to capture that input here before we move any dots! * We return it in the import lists. * Candidates will be computed after partitioning and returned in the * export lists. */ if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) { ierr = Zoltan_RB_Candidates_Copy_Input(zz, dotnum, rib->Global_IDs, rib->Local_IDs, &rib->Dots, num_import, import_global_ids, import_local_ids, import_procs, import_to_part); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc,yo, "Error returned from Zoltan_RB_Return_Arguments."); goto End; } } /* create mark and list arrays for dots */ allocflag = 0; if (dotmax > 0) { if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int))) || !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double))) || !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) { ierr = ZOLTAN_MEMERR; goto End; } } else { dotmark = NULL; value = NULL; dotlist = NULL; } /* set dot weights = 1.0 if user didn't and determine total weight */ dotpt = &rib->Dots; if (dotpt->nWeights == 0) { weightlo[0] = (double) dotnum; dotpt->uniformWeight = 1.0; wgtdim = 1; } else { double *wgt; for (j=0; j<dotpt->nWeights; j++){ weightlo[j] = 0.0; wgt = dotpt->Weight + j; for (i=0; i < dotnum; i++){ weightlo[j] += *wgt; wgt += dotpt->nWeights; } } wgtdim = dotpt->nWeights; } MPI_Allreduce(weightlo, weight, wgtdim, MPI_DOUBLE, MPI_SUM, zz->Communicator); if (check_geom) { ierr = Zoltan_RB_check_geom_input(zz, dotpt, dotnum); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from Zoltan_RB_check_geom_input"); goto End; } } /* create local communicator for use in recursion */ if (zz->Tflops_Special) local_comm = zz->Communicator; else { MPI_Comm_dup(zz->Communicator,&local_comm); free_comm = TRUE; } if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) { time2 = Zoltan_Time(zz->Timer); timers[0] = time2 - time1; } /* recursively halve until just one part or proc in set */ old_nprocs = num_procs = nprocs; old_nparts = num_parts; partlower = 0; root = 0; old_set = 1; ierr = Zoltan_LB_Proc_To_Part(zz, proc, &np, &fp); for (i = fp; i < (fp + np); i++) { treept[i].parent = 0; treept[i].left_leaf = 0; } if (zz->Tflops_Special) { proclower = 0; tfs[0] = nprocs; tfs[1] = num_parts; } while ((num_parts > 1 && num_procs > 1) || (zz->Tflops_Special && tfs[0] > 1 && tfs[1] > 1)) { ierr = Zoltan_Divide_Machine(zz, zz->Obj_Weight_Dim, part_sizes, proc, local_comm, &set, &proclower, &procmid, &num_procs, &partlower, &partmid, &num_parts, fractionlo); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc, yo, "Error in Zoltan_Divide_Machine."); goto End; } /* tfs[0] is max number of processors in all sets over all processors - * tfs[1] is max number of parts in all sets over all processors - * force all processors to go through all levels of parallel rib */ if (zz->Tflops_Special) { tmp_tfs[0] = num_procs; tmp_tfs[1] = num_parts; MPI_Allreduce(tmp_tfs, tfs, 2, MPI_INT, MPI_MAX, local_comm); } /* create mark array and active list for dots */ if (allocflag) { allocflag = 0; ZOLTAN_FREE(&dotmark); ZOLTAN_FREE(&value); ZOLTAN_FREE(&dotlist); if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int))) || !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double))) || !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) { ierr = ZOLTAN_MEMERR; goto End; } } dotpt = &rib->Dots; if (old_nparts > 1 && old_nprocs > 1) { /* test added for Tflops_Special; compute values only if looping to decompose, not if looping to keep Tflops_Special happy. */ ierr = compute_rib_direction(zz, zz->Tflops_Special, rib->Num_Geom, &valuelo, &valuehi, dotpt, NULL, dotnum, wgtflag, cm, evec, value, local_comm, proc, old_nprocs, proclower); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from compute_rib_direction"); goto End; } } else { /* For Tflops_Special: initialize value when looping only for Tflops_Special */ for (i = 0; i < dotmax; i++) value[i] = 0.0; valuelo = valuehi = 0.0; } if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) time2 = Zoltan_Time(zz->Timer); if (!Zoltan_RB_find_median( zz->Tflops_Special, value, dotpt->Weight, dotpt->uniformWeight, dotmark, dotnum, proc, fractionlo, local_comm, &valuehalf, first_guess, nprocs, old_nprocs, proclower, old_nparts, wgtflag, valuelo, valuehi, weight[0], weightlo, weighthi, dotlist, rectilinear_blocks, average_cuts)) { ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from Zoltan_RB_find_median."); ierr = ZOLTAN_FATAL; goto End; } if (set) /* set weight for current part */ for (j=0; j<wgtdim; j++) weight[j] = weighthi[j]; else for (j=0; j<wgtdim; j++) weight[j] = weightlo[j]; if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) time3 = Zoltan_Time(zz->Timer); /* store cut info in tree only if proc "owns" partmid */ /* test of partmid > 0 prevents treept[0] being set when this cut is only removing low-numbered processors (proclower to procmid-1) that have no parts in them from the processors remaining to be partitioned. */ if (partmid > 0 && partmid == fp) { treept[partmid].cm[0] = cm[0]; treept[partmid].cm[1] = cm[1]; treept[partmid].cm[2] = cm[2]; treept[partmid].ev[0] = evec[0]; treept[partmid].ev[1] = evec[1]; treept[partmid].ev[2] = evec[2]; treept[partmid].cut = valuehalf; treept[partmid].parent = old_set ? -(root+1) : root+1; /* The following two will get overwritten when the information is assembled if this is not a terminal cut */ treept[partmid].left_leaf = -partlower; treept[partmid].right_leaf = -partmid; } if (old_nprocs > 1 && partmid > 0 && partmid != partlower + old_nparts) { /* old_nprocs > 1 test: Don't reset these values if proc is in loop only * because of other procs for Tflops_Special. * partmid > 0 test: Don't reset these values if low-numbered processors * (proclower to procmid-1) have zero parts and this cut is removing * them from the processors remaining to be partitioned. * partmid != partlower + old_nparts test: Don't reset these values if * cut is removing high-numbered processors with zero parts from * the processors remaining to be partitioned. */ old_set = set; root = partmid; } ierr = Zoltan_RB_Send_Outgoing(zz, &(rib->Global_IDs), &(rib->Local_IDs), &(rib->Dots), &dotmark, &dottop, &dotnum, &dotmax, set, &allocflag, overalloc, stats, counters, use_ids, local_comm, proclower, old_nprocs, partlower, partmid); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from Zoltan_RB_Send_Outgoing."); goto End; } /* create new communicators */ if (zz->Tflops_Special) { if (set) { proclower = procmid; partlower = partmid; } old_nprocs = num_procs; old_nparts = num_parts; } else { if (set) partlower = partmid; MPI_Comm_split(local_comm,set,proc,&tmp_comm); MPI_Comm_free(&local_comm); local_comm = tmp_comm; old_nprocs = num_procs; old_nparts = num_parts; } if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) { time4 = Zoltan_Time(zz->Timer); timers[1] += time2 - time1; timers[2] += time3 - time2; timers[3] += time4 - time3; } } /* have recursed all the way to a single processor sub-domain */ /* Send dots to correct processors for their parts. This is needed most notably when a processor has zero parts on it, but still has some dots after the parallel partitioning. */ ierr = Zoltan_RB_Send_To_Part(zz, &(rib->Global_IDs), &(rib->Local_IDs), &(rib->Dots), &dotmark, &dottop, &dotnum, &dotmax, &allocflag, overalloc, stats, counters, use_ids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_RB_Send_To_Part"); goto End; } /* All dots are now on the processors they will end up on; now generate * more parts if needed. */ if (num_parts > 1) { if (dotpt->nWeights) wgts = (double *) ZOLTAN_MALLOC(dotpt->nWeights * dotnum * sizeof(double)); dindx = (int *) ZOLTAN_MALLOC(dotnum * 2 * sizeof(int)); tmpdindx = dindx + dotnum; if (allocflag) { ZOLTAN_FREE(&dotmark); ZOLTAN_FREE(&value); ZOLTAN_FREE(&dotlist); if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int))) || !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double))) || !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) { ZOLTAN_PRINT_ERROR(proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } for (i = 0; i < dotnum; i++) dindx[i] = i; ierr = serial_rib(zz, &rib->Dots, dotmark, dotlist, old_set, root, rib->Num_Geom, weight[0], dotnum, num_parts, &(dindx[0]), &(tmpdindx[0]), partlower, proc, wgtflag, stats, gen_tree, rectilinear_blocks, average_cuts, treept, value, wgts, part_sizes); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from serial_rib"); goto End; } ZOLTAN_FREE(&wgts); } end_time = Zoltan_Time(zz->Timer); lb_time[1] = end_time - start_time; if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) { MPI_Barrier(zz->Communicator); timestop = time1 = Zoltan_Time(zz->Timer); } /* error checking and statistics */ if (check_geom) { ierr = Zoltan_RB_check_geom_output(zz, &rib->Dots, part_sizes, np, fp, dotnum, pdotnum, NULL); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from Zoltan_RB_check_geom_output"); goto End; } } EndReporting: /* update calling routine parameters */ start_time = Zoltan_Time(zz->Timer); pdotnum = dotnum; /* Perform remapping (if requested) */ if (zz->LB.Remap_Flag) { ierr = Zoltan_RB_Remap(zz, &(rib->Global_IDs), &(rib->Local_IDs), &(rib->Dots), &dotnum, &dotmax, &allocflag, overalloc, stats, counters, use_ids); /* Note: dottop is no longer valid after remapping. Remapping might destroy the nice local-followed-by-non-local ordering of the dots array. Do not use dottop after remapping. */ if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_RB_Remap."); goto End; } } /* build return arguments */ if (zz->LB.Return_Lists != ZOLTAN_LB_NO_LISTS && zz->LB.Return_Lists != ZOLTAN_LB_CANDIDATE_LISTS) { /* zz->LB.Return_Lists is true ==> use_ids is true */ ierr = Zoltan_RB_Return_Arguments(zz, rib->Global_IDs, rib->Local_IDs, &rib->Dots, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, dotnum); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from Zoltan_RB_Return_Arguments."); goto End; } } else if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) { /* Select a candidate for each part and return it in the export_GIDs. */ ierr = Zoltan_RB_Candidates_Output(zz, dotnum, dindx, rib->Global_IDs, rib->Local_IDs, &rib->Dots, *num_import, *import_global_ids, num_export, export_global_ids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(proc,yo, "Error returned from Zoltan_RB_Return_Candidates."); goto End; } } ZOLTAN_FREE(&dindx); if (gen_tree) { int *displ, *recvcount; int sendcount; struct rib_tree *treetmp = NULL; /* temporary tree of cuts; used to keep valgrind from reporting overlapped memory in MPI_Allgatherv */ treetmp = (struct rib_tree *) ZOLTAN_MALLOC(zz->LB.Num_Global_Parts* sizeof(struct rib_tree)); displ = (int *) ZOLTAN_MALLOC(2 * zz->Num_Proc * sizeof(int)); if (!displ || !treetmp) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } recvcount = displ + zz->Num_Proc; ierr = Zoltan_RB_Tree_Gatherv(zz, sizeof(struct rib_tree), &sendcount, recvcount, displ); /* * Create copy of treept so that MPI_Allgatherv doesn't use same * memory for sending and receiving; removes valgrind warning. */ for (i = 0; i < zz->LB.Num_Global_Parts; i++) treetmp[i] = treept[i]; MPI_Allgatherv(&treetmp[fp], sendcount, MPI_BYTE, treept, recvcount, displ, MPI_BYTE, zz->Communicator); for (i = 1; i < zz->LB.Num_Global_Parts; i++){ if (treept[i].parent > 0) treept[treept[i].parent - 1].left_leaf = i; else if (treept[i].parent < 0) treept[-treept[i].parent - 1].right_leaf = i; } ZOLTAN_FREE(&displ); ZOLTAN_FREE(&treetmp); } else { treept[0].right_leaf = -1; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) print_rib_tree(zz, np, fp, &(treept[fp])); end_time = Zoltan_Time(zz->Timer); lb_time[0] += (end_time - start_time); if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) Zoltan_RB_stats(zz, timestop-timestart, &rib->Dots, dotnum, part_sizes, timers, counters, stats, NULL, NULL, FALSE); if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) { if (zz->Proc == zz->Debug_Proc) printf("ZOLTAN RIB Times: \n"); Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, lb_time[0], "ZOLTAN Build: "); Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, lb_time[1], "ZOLTAN RIB: "); } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { /* zz->Debug_Level >= ZOLTAN_DEBUG_ALL ==> use_ids is true */ Zoltan_RB_Print_All(zz, rib->Global_IDs, &rib->Dots, dotnum, *num_import, *import_global_ids, *import_procs); } End: /* Free memory allocated by the algorithm. */ if (free_comm) MPI_Comm_free(&local_comm); ZOLTAN_FREE(&dotmark); ZOLTAN_FREE(&value); ZOLTAN_FREE(&dotlist); if (!gen_tree && /* don't need parts */ rib && (rib->Tran.Target_Dim < 0)) { /* don't need transformation */ /* Free all memory used. */ Zoltan_RIB_Free_Structure(zz); } else if (rib != NULL) { /* Free only Dots and IDs; keep other structures. */ ZOLTAN_FREE(&(rib->Global_IDs)); ZOLTAN_FREE(&(rib->Local_IDs)); Zoltan_Free_And_Reset_Dot_Structure(&(rib->Dots)); } ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int num_gid_entries, /* # of entries for a global id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm /* iperm[rank[i]]=i, only for sequential ordering */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; int * vtxdist = NULL; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL, *local_iperm=NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value"); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.rank = rank; zz->Order.iperm = iperm; zz->Order.gids = gids; zz->Order.lids = lids; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); #ifdef HAVE_MPI strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN); #else strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); zz->Order.start_index = opt.start_index; /* * Check that the user has allocated space for the return args. */ if (!(gids && rank)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); /* strcpy(opt.order_type, "GLOBAL"); */ } #endif /* ZOLTAN_SCOTCH */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } if (!strcmp(opt.order_type, "GLOBAL")) strcpy (opt.order_type, "DIST"); if (!strcmp(opt.order_type, "LOCAL")) strcpy (opt.order_type, "SERIAL"); strcpy(zz->Order.order_type, opt.order_type); /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. * Compute gid according to the local graph. */ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 3, &local_gids, &local_rank, &local_iperm); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ if ((!(opt.return_args & RETURN_RANK) && (rank != NULL)) || (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) { ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr) { /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) { /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) { /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* I store : GNO, rank, iperm */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); ZOLTAN_FREE(&local_iperm); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
int Zoltan_Order ( struct Zoltan_Struct *zz, int num_gid_entries, int num_obj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR permuted_global_ids ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids. * num_gid_entries * Output: * permuted_global_ids * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { char msg[253]; sprintf(msg, "num_gid_entries=%d is not equal to parameter setting " "NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ #ifdef HAVE_MPI strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "GLOBAL"); #else strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "LOCAL"); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* * Check that the user has allocated space for the return args. */ if (num_obj && !(gids && permuted_global_ids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "LOCAL_HSFC")) { Order_fn = Zoltan_LocalHSFC_Order; strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/ } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PTSCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_SCOTCH */ #ifdef ZOLTAN_HUND else if (!strcasecmp(opt.method, "HUND")) { ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL); goto End; } #endif /* ZOLTAN_HUND */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* TODO : Ask why useful ! */ /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /************************************ * Check for required query function ************************************/ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } /* TODO allocate all this stuff with the graph */ local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); /* * Call the actual ordering function. * Compute gid according to the local graph. */ ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 2, &local_gids, &local_rank); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* TODO: At this time, I consider rank == permuted_global_ids */ /* I store : GNO, rank, permuted GID */ /* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0); /* Hope a linear assignment will help a little */ if (local_num_obj) Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", permuted_global_ids[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } #ifdef ZOLTAN_HUND End: #endif /*ZOLTAN_HUND*/ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }