static int Zoltan_LB( ZZ *zz, int include_parts, /* Flag indicating whether to generate part informtion; 0 if called by Zoltan_LB_Balance, 1 if called by Zoltan_LB_Partition. */ int *changes, /* Set to zero or one depending on if Zoltan determines a new decomposition or not: zero - No changes to the decomposition were made by the load-balancing algorithm; migration is not needed. one - A new decomposition is suggested by the load-balancer; migration is needed to establish the new decomposition. */ int *num_gid_entries, /* The number of array entries in a global ID; set to be the max over all processors in zz->Communicator of the parameter Num_Global_ID_Entries. */ int *num_lid_entries, /* The number of array entries in a local ID; set to be the max over all processors in zz->Communicator of the parameter Num_Local_ID_Entries. */ int *num_import_objs, /* The number of non-local objects in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ int **import_procs, /* Array of processor IDs for processors currently owning non-local objects (i.e., objs to be imported) in this processor's new decomposition. */ int **import_to_part, /* Partition to which the objects should be imported. */ int *num_export_objs, /* The number of local objects that need to be exported from the processor to establish the new decomposition. */ ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ int **export_procs, /* Array of destination processor IDs for objects that need to be exported to establish the new decomposition. */ int **export_to_part /* Partition to which objects should be exported. */ ) { /* * Main load-balancing routine. * Input: a Zoltan structure with appropriate function pointers set. * Output: * changes * num_import_objs * import_global_ids * import_local_ids * import_procs * import_to_part * num_export_objs * export_global_ids * export_local_ids * export_procs * export_to_part * Return values: * Zoltan error code. */ char *yo = "Zoltan_LB"; int gmax; /* Maximum number of imported/exported objects over all processors. */ int error = ZOLTAN_OK; /* Error code */ double start_time, end_time; double lb_time[2] = {0.0,0.0}; char msg[256]; int comm[3],gcomm[3]; float *part_sizes = NULL, *fdummy = NULL; int wgt_dim, part_dim; int all_num_obj, i, ts, idIdx; struct Hash_Node **ht; int *export_all_procs, *export_all_to_part, *parts=NULL; ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL; ZOLTAN_ID_PTR gid; #ifdef ZOLTAN_OVIS struct OVIS_parameters ovisParameters; #endif ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS){ printf("Build configuration:\n"); Zoltan_Print_Configuration(" "); printf("\n"); Zoltan_Print_Key_Params(zz); } start_time = Zoltan_Time(zz->Timer); #ifdef ZOLTAN_OVIS Zoltan_OVIS_Setup(zz, &ovisParameters); if (zz->Proc == 0) printf("OVIS PARAMETERS %s %s %d %f\n", ovisParameters.hello, ovisParameters.dll, ovisParameters.outputLevel, ovisParameters.minVersion); ovis_enabled(zz->Proc, ovisParameters.dll); #endif /* * Compute Max number of array entries per ID over all processors. * Compute Max number of return arguments for Zoltan_LB_Balance. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; comm[2] = zz->LB.Return_Lists; MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; zz->LB.Return_Lists = gcomm[2]; /* assume no changes */ *changes = 0; *num_import_objs = *num_export_objs = 0; *import_global_ids = NULL; *import_local_ids = NULL; *import_procs = NULL; *import_to_part = NULL; *export_global_ids = NULL; *export_local_ids = NULL; *export_procs = NULL; *export_to_part = NULL; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) goto End; if (zz->LB.Method == NONE) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s Balancing method selected == NONE; no balancing performed\n", yo); error = ZOLTAN_WARN; goto End; } /* * Sync the random number generator across processors. */ Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator); /* Since generating a new partition, need to free old mapping vector */ zz->LB.OldRemap = zz->LB.Remap; zz->LB.Remap = NULL; error = Zoltan_LB_Build_PartDist(zz); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) goto End; if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { int i, np, fp; for (i = 0; i < zz->Num_Proc; i++) { Zoltan_LB_Proc_To_Part(zz, i, &np, &fp); printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n", zz->Proc, i, np, fp); } } /* * Generate parts sizes. */ #ifdef ZOLTAN_OVIS /* set part sizes computed by OVIS, if requested. Processes set only their own value */ { float part_sizes[1]; int part_ids[1], wgt_idx[1]; wgt_idx[0] = 0; part_ids[0] = 0; ovis_getPartsize(&(part_sizes[0])); printf("Rank %d ps %f\n",zz->Proc, part_sizes[0]); /* clear out old part size info first */ Zoltan_LB_Set_Part_Sizes(zz, 0, -1, NULL, NULL, NULL); Zoltan_LB_Set_Part_Sizes(zz, 0, 1, part_ids, wgt_idx, part_sizes); } #endif wgt_dim = zz->Obj_Weight_Dim; part_dim = ((wgt_dim > 0) ? wgt_dim : 1); part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim * zz->LB.Num_Global_Parts); if (part_sizes == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); error = ZOLTAN_MEMERR; goto End; } /* Get part sizes. */ Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim, part_sizes); #ifdef ZOLTAN_OVIS /* if (ovisParameters.outputlevel > 3) */ { int myRank = zz->Proc; if (myRank == 0){ int i, j; for (i = 0; i < zz->LB.Num_Global_Parts; i++){ for (j = 0; j < part_dim; j++){ printf("Rank %d AG: part_sizes[%d] = %f (Num_Global_Parts = %d, part_dim = %d)\n",zz->Proc, (i*part_dim+j), part_sizes[i*part_dim+j],zz->LB.Num_Global_Parts, part_dim); } } } } #endif /* * Call the actual load-balancing function. */ error = zz->LB.LB_Fn(zz, part_sizes, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); ZOLTAN_FREE(&part_sizes); if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){ sprintf(msg, "Partitioning routine returned code %d.", error); #ifdef HOST_LINUX if ((error == ZOLTAN_MEMERR) && (Zoltan_Memory_Get_Debug() > 0)){ Zoltan_write_linux_meminfo(0, "State of /proc/meminfo after malloc failure\n", 0); } #endif ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } else if (error){ if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) { sprintf(msg, "Partitioning routine returned code %d.", error); ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning"); if (*num_import_objs >= 0) MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); else /* use export data */ MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); if (gmax == 0) { /* * Decomposition was not changed by the load balancing; no migration * is needed. */ if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s No changes to the decomposition due to partitioning; " "no migration is needed.\n", yo); /* * Reset num_import_objs and num_export_objs; don't want to return * -1 for the arrays that weren't returned by ZOLTAN_LB_FN. */ *num_import_objs = *num_export_objs = 0; if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){ /* * This parameter setting requires that all local objects * and their assignments appear in the export list. */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } goto End; } /* * Check whether we know the import data, export data, or both. * * If we were given the import data, * we know what the new decomposition should look like on the * processor, but we don't know which of our local objects we have * to export to other processors to establish the new decomposition. * Reverse the argument if we were given the export data. * * Unless we were given both maps, compute the inverse map. */ if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) { if (*num_import_objs >= 0) Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); if (*num_export_objs >= 0) Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); *num_import_objs = *num_export_objs = -1; } if (*num_import_objs >= 0){ if (*num_export_objs >= 0) { /* Both maps already available; nothing to do. */; } else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Export lists are requested; compute export map */ error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Method returned import lists, but only export lists were desired. */ /* Import lists not needed; free them. */ *num_import_objs = -1; Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); } } } else { /* (*num_import_objs < 0) */ if (*num_export_objs >= 0) { /* Only export lists have been returned. */ if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Compute import map */ error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Method returned export lists, but only import lists are desired. */ /* Export lists not needed; free them. */ *num_export_objs = -1; Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); } } } else { /* *num_export_objs < 0 && *num_import_objs < 0) */ if (zz->LB.Return_Lists) { /* No map at all available */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned " "neither import nor export data."); error = ZOLTAN_WARN; goto End; } } } if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* * Normally, Zoltan_LB returns in the export lists all local * objects that are moving off processor, or that are assigned * to a part on the local processor that is not the * default part. This setting of Return_Lists requests * that all local objects be included in the export list. */ if (*num_export_objs == 0){ /* all local objects are remaining on processor */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (*num_export_objs) { if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End; } else{ all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error); if (*num_export_objs < all_num_obj){ /* Create a lookup table for exported IDs */ ts = Zoltan_Recommended_Hash_Size(*num_export_objs); ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts); /* Create a list of all gids, lids and parts */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj, &all_global_ids, &all_local_ids, wgt_dim, &fdummy, &parts); if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){ ZOLTAN_FREE(&fdummy); if ((Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_procs, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) || (Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_to_part, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){ error = ZOLTAN_MEMERR; } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){ sprintf(msg, "Error building complete export list; " "%d returned by Zoltan_Get_Obj_List\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } gid = all_global_ids; for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){ idIdx = search_hash_table(zz, gid, ht, ts); if (idIdx >= 0){ export_all_procs[i] = (*export_procs)[idIdx]; export_all_to_part[i] = (*export_to_part)[idIdx]; } else{ export_all_procs[i] = zz->Proc; export_all_to_part[i] = parts[i]; } } free_hash_table(ht, ts); Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); Zoltan_Special_Free(zz, (void **)(void*)&parts, ZOLTAN_SPECIAL_MALLOC_INT); *export_global_ids = all_global_ids; *export_local_ids = all_local_ids; *export_procs = export_all_procs; *export_to_part = export_all_to_part; *num_export_objs = all_num_obj; } } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments"); end_time = Zoltan_Time(zz->Timer); lb_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc); for (i = 0; i < *num_import_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID])); printf(" To part: %4d", (*import_to_part != NULL ? (*import_to_part)[i] : zz->Proc)); printf(" From processor: %4d\n", (*import_procs)[i]); } printf("\n"); printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc); for (i = 0; i < *num_export_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID])); printf(" To part: %4d", (*export_to_part != NULL ? (*export_to_part)[i] : (*export_procs)[i])); printf(" To processor: %4d\n", (*export_procs)[i]); } Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* * If the Help_Migrate flag is set, perform migration for the application. */ if (zz->Migrate.Auto_Migrate) { ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration"); start_time = Zoltan_Time(zz->Timer); error = Zoltan_Migrate(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error in auto-migration; %d returned from " "Zoltan_Help_Migrate\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } end_time = Zoltan_Time(zz->Timer); lb_time[1] = end_time - start_time; ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration"); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0], "ZOLTAN Partition: "); if (zz->Migrate.Auto_Migrate) Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1], "ZOLTAN Migrate: "); } *changes = 1; End: ZOLTAN_TRACE_EXIT(zz, yo); return (error); }
int Zoltan_Migrate( ZZ *zz, /* Zoltan structure. */ int num_import, /* Number of non-local objects assigned to the processor in the new decomposition. */ ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects assigned to this processor in the new decomposition; this field can be NULL if the application doesn't provide import IDs.*/ ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects assigned to the processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_procs, /* Array of processor IDs of processors owning the non-local objects that are assigned to this processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_to_part, /* Array of partition numbers to which imported objects should be assigned. */ int num_export, /* Number of objs to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of objects to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of objects to be exported to other processors to establish the new decomposition. */ int *export_procs, /* Array of processor IDs to which objects will be exported to establish the new decomposition. */ int *export_to_part /* Array of partition numbers to which exported objects should be assigned. */ ) { /* * Routine to help perform migration. If migration pre-processing routine * (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn. * It then calls a function to obtain the size of the migrating objects * (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified * object packing routine (ZOLTAN_PACK_OBJ_FN) for each object * to be exported. It develops the needed communication map to move the * objects to other processors. It performs the communication according * to the map, and then calls an application-specified object unpacking * routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported. */ char *yo = "Zoltan_Migrate"; int num_gid_entries, num_lid_entries; /* lengths of global & local ids */ int *sizes = NULL; /* sizes (in bytes) of the object data for export. */ int id_size; /* size (in bytes) of ZOLTAN_GID + padding for alignment */ int tag_size; /* size (in bytes) of ZOLTAN_GID + one int (for message size) */ char *export_buf = NULL; /* buffer for packing export data. */ char *import_buf = NULL; /* buffer for receiving imported data. */ char *tmp; /* temporary pointer into buffers. */ int i; /* loop counter. */ int tmp_size; /* size of a single object's data. */ int *idx = NULL; /* index used for multi-fn packs and unpacks. */ int idx_cnt = 0; /* index counter for idx array. */ ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm buf */ ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass NULL to query functions when NUM_LID_ENTRIES=0. */ ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */ ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */ int msgtag, msgtag2; /* Tags for communication routines */ int total_send_size; /* Total size of outcoming message (in #items) */ int total_recv_size; /* Total size of incoming message (in #items) */ int aligned_int; /* size of an int padded for alignment */ int dest; /* temporary destination partition. */ int include_parts = 0; /* flag indicating whether partition info is provided */ int ierr = ZOLTAN_OK; int actual_num_exp = 0; int actual_exp_allocated = 0; ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */ int *actual_exp_procs = NULL; /* are changing partition but not */ int *actual_exp_to_part = NULL; /* processor may not be included. */ int actual_num_imp = 0; int actual_imp_allocated = 0; ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */ int *actual_imp_procs = NULL; /* are changing partition but not */ int *actual_imp_to_part = NULL; /* processor may not be included. */ ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { goto End; } /* * Check that all procs use the same id types. */ ierr = check_input(zz, ((num_export >= 0 && export_to_part) || (num_import >= 0 && import_to_part)), &include_parts); if (ierr != ZOLTAN_OK) goto End; num_gid_entries = zz->Num_GID; num_lid_entries = zz->Num_LID; /* * Check that all necessary query functions are available. */ if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (num_export >= 0) { /* Build the actual export arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &actual_num_exp, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part, &actual_exp_allocated); if (ierr < 0) goto End; /* Compute communication map based on actual exports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs, zz->Communicator, msgtag, &actual_num_imp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } } else if (num_import >= 0) { /* Build the actual import arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, &actual_num_imp, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part, &actual_imp_allocated); if (ierr < 0) goto End; /* Compute communication map based on imports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs, zz->Communicator, msgtag, &actual_num_exp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } /* Compute actual export lists for packing objects */ if (actual_num_exp > 0) { actual_exp_allocated = 1; actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp); actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp); actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp); if (include_parts) actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp); if (actual_exp_gids == NULL || (num_lid_entries && actual_exp_lids == NULL) || actual_exp_procs == NULL || (import_to_part != NULL && actual_exp_to_part == NULL)) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); ierr = ZOLTAN_MEMERR; goto End; } } msgtag2 = 32766; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) actual_exp_gids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) actual_exp_lids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part, (int) sizeof(int), (char *) actual_exp_to_part); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } /* Create inverse plan (i.e., plan based on exports) so can set * variable sizes. * (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable * but does not tell how large to allocate receive buffer. */ ierr = Zoltan_Comm_Invert_Plan(&imp_plan); exp_plan = imp_plan; imp_plan = NULL; } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Migrate.Pre_Migrate_PP != NULL) { zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Pre_Migrate != NULL) { zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_FN function."); goto End; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing"); id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE)); /* Note that alignment is not strictly necessary when ZOLTAN_ID_TYPE is int or unsigned int. */ aligned_int = Zoltan_Align(sizeof(int)); tag_size = id_size + aligned_int; /* * For each object, allow space for its global ID and its data plus * one int (for the object data size). * Zoltan will pack the global IDs; the application must pack the data * through the pack routine. Zoltan needs the global IDs for unpacking, * as the order of the data received during communication is not * necessarily the same order as import_global_ids[]. * Zoltan also needs to communicate the sizes of the objects because * only the sender knows the size of each object. */ if (actual_num_exp > 0) { sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!sizes) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Get_Obj_Size_Multi != NULL) { zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, sizes, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE_MULTI function."); goto End; } } else { for (i = 0; i < actual_num_exp; i++) { lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE function."); goto End; } } } total_send_size = 0; for (i = 0; i < actual_num_exp; i++) { sizes[i] = Zoltan_Align(sizes[i]); total_send_size += sizes[i] + tag_size; } export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char)); if (!export_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Pack_Obj_Multi != NULL) { /* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */ idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Pack the objects for export. */ idx_cnt = 0; tmp = export_buf; for (i = 0; i < actual_num_exp; i++) { /* Pack the object's global ID */ tmp_id = (ZOLTAN_ID_PTR) tmp; ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries])); tmp += id_size; /* Pack the object's size */ *((int *)tmp) = sizes[i]; tmp += aligned_int; /* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Pack_Obj_Multi != NULL) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n", zz->Proc, yo); } zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, (actual_exp_to_part!=NULL ? actual_exp_to_part : actual_exp_procs), sizes, idx, export_buf, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ_MULTI function."); goto End; } } else { tmp = export_buf + tag_size; for (i = 0; i < actual_num_exp; i++) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries])); printf("size = %d bytes\n", sizes[i]); } /* Pack the object's data */ lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i] : actual_exp_procs[i]); zz->Pack_Obj(zz->Pack_Obj_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, dest, sizes[i], tmp, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ function."); goto End; } tmp += sizes[i] + tag_size; } } ZOLTAN_FREE(&idx); tmp_id = NULL; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects"); /* Modify sizes[] to contain message sizes, not object sizes */ for (i=0; i<actual_num_exp; i++) { sizes[i] += tag_size; } msgtag--; ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize."); goto End; } if (actual_num_imp > 0) { import_buf = (char *) ZOLTAN_MALLOC(total_recv_size); if (!import_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Send the export data using the communication plan. */ msgtag2 = 32765; ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } /* * Free whatever memory we can. */ Zoltan_Comm_Destroy(&exp_plan); ZOLTAN_FREE(&export_buf); ZOLTAN_FREE(&sizes); ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication"); /* * Perform application-specified processing before unpacking the data. */ if (zz->Migrate.Mid_Migrate_PP != NULL) { zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Mid_Migrate != NULL) { zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_FN function."); goto End; } } /* * Unpack the object data. */ if (actual_num_imp > 0) { if (zz->Unpack_Obj_Multi != NULL) { /* Allocate and fill input arrays for Unpack_Obj_Multi. */ sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp); idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); if (!sizes || !tmp_id || !idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } tmp = import_buf; idx_cnt = 0; for (i = 0; i < actual_num_imp; i++) { /* Unpack the object's global ID */ ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp); tmp += id_size; /* Unpack the object's size */ sizes[i] = *((int *)tmp); tmp += aligned_int; /* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n", zz->Proc,yo); } zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries, actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr); ZOLTAN_FREE(&import_buf); ZOLTAN_FREE(&sizes); ZOLTAN_FREE(&tmp_id); ZOLTAN_FREE(&idx); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_MULTI_FN."); goto End; } } else { tmp = import_buf; for (i = 0; i < actual_num_imp; i++) { tmp_size = *((int *)(tmp + id_size)); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp); printf("size = %d bytes\n", tmp_size); } /* Unpack the object's data */ zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries, (ZOLTAN_ID_PTR) tmp, tmp_size, tmp + tag_size, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_FN."); goto End; } tmp += (tmp_size + tag_size); } ZOLTAN_FREE(&import_buf); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects"); if (zz->Migrate.Post_Migrate_PP != NULL) { zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Post_Migrate != NULL) { zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_FN function."); goto End; } } End: if (actual_exp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); } if (actual_imp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part); } if (ierr < 0) { if (exp_plan) Zoltan_Comm_Destroy(&exp_plan); Zoltan_Multifree(__FILE__, __LINE__, 5, &import_buf, &tmp_id, &sizes, &idx, &export_buf); } ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int *num_gid_entries, /* # of entries for a global id */ int *num_lid_entries, /* # of entries for a local id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm, /* inverse permutation of rank */ ZOS *order_info /* Method-specific ordering info. Currently not used. */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * iperm, inverse permutation of rank * order_info, a Zoltan Ordering Struct with additional info. * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; int *vtxdist; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strncpy(opt.order_type, "GLOBAL", MAX_PARAM_STRING_LEN); opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); if (opt.use_order_info == 0) order_info = NULL; /* * Check that the user has allocated space for the return args. */ if (!(gids && lids && rank && iperm)){ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL){ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. */ ierr = (*Order_fn)(zz, num_obj, gids, lids, rank, iperm, &opt, order_info); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN){ ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr){ /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK)){ /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, iperm, rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM)){ /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, rank, iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i, nobjs; nobjs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &i); Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < nobjs; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(*num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); printf("ZOLTAN: inverse permutation on Proc %d\n", zz->Proc); for (i = 0; i < nobjs; i++) { printf("iperm[%3d] = %3d\n", i, iperm[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
int Zoltan_Invert_Lists( ZZ *zz, /* Zoltan structure. */ int num_in, /* Number of objects in the input lists. */ ZOLTAN_ID_PTR in_global_ids, /* Array of input global IDs. */ ZOLTAN_ID_PTR in_local_ids, /* Array of input local IDs. */ int *in_procs, /* Array of processor IDs of processors owning the input objects. */ int *in_to_part, /* Optional: Array of partition numbers to which input objects should be assigned. */ int *num_out, /* Returned value: Number of output objs. */ ZOLTAN_ID_PTR *out_global_ids,/* Returned value: Array of global IDs of output objects. */ ZOLTAN_ID_PTR *out_local_ids, /* Returned value: Array of local IDs of output objects. */ int **out_procs, /* Returned value: Array of processor IDs to which output objects are assigned. */ int **out_to_part /* Optional: Returned value: Array of partition numbers to which output objects should be assigned. */ ) { /* * Routine to compute the inverse map. Can be used in two ways: * 1. Given, for each processor, a list of objects to be received by the * processor, compute the list of objects that the processor needs to send * to other processors to satisfy their needs. * 2. Given, for each processor, a list of objects to be sent to other * processors, compute the list of objects that the processor needs to receive * to satisfy its needs. */ char *yo = "Zoltan_Invert_Lists"; char msg[256]; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned communication routines */ int msgtag, msgtag2; /* Message tags for communication routines */ int num_gid_entries, num_lid_entries; /* Length of global and local ids */ int include_parts; /* Flag indicating whether to compute inverse list for partitions. */ int ierr, ret_ierr = ZOLTAN_OK; ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Check that all procs use the same id types. */ ierr = check_invert_input(zz, num_in, in_procs, in_to_part, &num_gid_entries, &num_lid_entries, &include_parts); if (ierr != ZOLTAN_OK) { ZOLTAN_TRACE_EXIT(zz, yo); return ierr; } /* Initialize returned arrays. */ *out_global_ids = NULL; *out_local_ids = NULL; *out_procs = NULL; if (include_parts) *out_to_part = NULL; /* * Compute communication map and num_out, the number of objs this * processor has to out to establish the new decomposition. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator, msgtag, num_out); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Create.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; goto End; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create"); /* * Allocate space for the object tags that need to be outed. Communicate * to get the list of objects to be outed. */ if (*num_out > 0) { if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out, ZOLTAN_SPECIAL_MALLOC_GID)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out, ZOLTAN_SPECIAL_MALLOC_LID)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out, ZOLTAN_SPECIAL_MALLOC_INT)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (include_parts) { if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out, ZOLTAN_SPECIAL_MALLOC_INT)) { ret_ierr = ZOLTAN_MEMERR; goto End; } } } /* * Use the communication plan to send global IDs, local IDs, and processor * numbers. Do in separate communications to avoid a memory copy and to * simplify implementation when a data type is added to the comm. package * (to support heterogeneous computing). */ msgtag2 = 32766; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) *out_global_ids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) *out_local_ids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } } Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, *out_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part, (int) sizeof(int), (char *) *out_to_part); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do"); End: Zoltan_Comm_Destroy(&comm_plan); if (ret_ierr == ZOLTAN_MEMERR) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID); Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID); Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT); if (include_parts) Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT); } ZOLTAN_TRACE_EXIT(zz, yo); return (ret_ierr); }
int Zoltan_Color_Test( ZZ *zz, /* Zoltan structure */ int *num_gid_entries, /* # of entries for a global id */ int *num_lid_entries, /* # of entries for a local id */ int num_obj, /* Input: number of objects */ ZOLTAN_ID_PTR global_ids, /* Input: global ids of the vertices */ /* The application must allocate enough space */ ZOLTAN_ID_PTR local_ids, /* Input: local ids of the vertices */ /* The application must allocate enough space */ int *color_exp /* Input: Colors assigned to local vertices */ ) { static char *yo = "color_test_fn"; int nvtx = num_obj; /* number of vertices */ int i, j; int ierr = ZOLTAN_OK; int ferr = ZOLTAN_OK; /* final error signal */ char coloring_problem; /* Input: which coloring to perform; currently only supports D1, D2 coloring and variants */ char coloring_problemStr[MAX_PARAM_STRING_LEN]; /* string version coloring problem name */ int ss=100; char comm_pattern='S', coloring_order='I', coloring_method='F'; int comm[2],gcomm[2]; int *color=NULL; int *adjproc=NULL, *xadj=NULL; ZOLTAN_GNO_TYPE gvtx; /* number of global vertices */ ZOLTAN_GNO_TYPE *vtxdist=NULL, *adjncy=NULL; ZG graph; ZOLTAN_GNO_TYPE *requested_GNOs = NULL; /* Return GNOs of the requested GIDs. */ int *loc_partialD2 = NULL; /* local binary array showing which vertices to be colored */ int *partialD2 = NULL; /* global binary array showing which vertices to be colored */ struct Zoltan_DD_Struct *dd_color; /* DDirectory for colors */ ZOLTAN_GNO_TYPE *local_GNOs = NULL; ZOLTAN_GNO_TYPE *global_GNOs = NULL; memset (&graph, 0, sizeof(ZG)); /* PARAMETER SETTINGS */ Zoltan_Bind_Param(Color_params, "COLORING_PROBLEM", (void *) &coloring_problemStr); Zoltan_Bind_Param(Color_params, "SUPERSTEP_SIZE", (void *) &ss); Zoltan_Bind_Param(Color_params, "COMM_PATTERN", (void *) &comm_pattern); Zoltan_Bind_Param(Color_params, "VERTEX_VISIT_ORDER", (void *) &coloring_order); Zoltan_Bind_Param(Color_params, "COLORING_METHOD", (void *) &coloring_method); strncpy(coloring_problemStr, "distance-1", MAX_PARAM_STRING_LEN); Zoltan_Assign_Param_Vals(zz->Params, Color_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* Check validity of parameters - they should be consistent with Zoltan_Color */ if (!strcasecmp(coloring_problemStr, "distance-1")) coloring_problem = '1'; else if (!strcasecmp(coloring_problemStr, "distance-2")) coloring_problem = '2'; else if (!strcasecmp(coloring_problemStr, "partial-distance-2") || !strcasecmp(coloring_problemStr, "bipartite")) coloring_problem = 'P'; else { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Unknown coloring requested. Using Distance-1 coloring."); coloring_problem = '1'; } if (ss == 0) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid superstep size. Using default value 100."); ss = 100; } if (comm_pattern != 'S' && comm_pattern != 'A') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid communication pattern. Using synchronous communication (S)."); comm_pattern = 'S'; } if (comm_pattern == 'A' && (coloring_problem == '2' || coloring_problem == 'P')) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Asynchronous communication pattern is not implemented for distance-2 coloring and its variants. Using synchronous communication (S)."); comm_pattern = 'S'; } if (coloring_order != 'I' && coloring_order != 'B' && coloring_order != 'U' && coloring_order != 'L' && coloring_order != 'N' && coloring_order != 'S') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring order. Using internal first coloring order (I)."); coloring_order = 'I'; } if (coloring_order == 'U' && (coloring_problem == '2' || coloring_problem == 'P')) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Interleaved coloring order is not implemented for distance-2 coloring and its variants. Using internal first coloring order (I)."); coloring_order = 'I'; } if (coloring_method !='F') { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring method. Using first fit method (F)."); coloring_method = 'F'; } /* Compute Max number of array entries per ID over all processors. This is a sanity-maintaining step; we don't want different processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; /* Return if this processor is not in the Zoltan structure's communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) return ZOLTAN_OK; /* BUILD THE GRAPH */ /* Check that the user has allocated space for the return args. */ if (!color_exp) ZOLTAN_COLOR_ERROR(ZOLTAN_FATAL, "Color argument is NULL. Please give colors of local vertices."); requested_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(num_obj * sizeof(ZOLTAN_GNO_TYPE)); Zoltan_ZG_Build (zz, &graph, 0, 1, num_obj, global_ids, requested_GNOs); Zoltan_ZG_Export (zz, &graph, &gvtx, &nvtx, NULL, NULL, &vtxdist, &xadj, &adjncy, &adjproc, NULL, NULL); if (gvtx > (ZOLTAN_GNO_TYPE)INT_MAX){ if (zz->Proc == 0){ fprintf(stderr, "Zoltan_Color_Test assumes number of vertices (%ld) is less than INT_MAX\n",gvtx); } ierr = ZOLTAN_FATAL; goto End; } KDDKDDKDD(zz->Proc, "COLORTEST DD"); /* Exchange global color information */ if (vtxdist[zz->Num_Proc] && !(color = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (nvtx && !(local_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(nvtx * sizeof(ZOLTAN_GNO_TYPE)))) MEMORY_ERROR; for (i=0; i<nvtx; ++i) local_GNOs[i] = i+vtxdist[zz->Proc]; if (vtxdist[zz->Num_Proc] && !(global_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(vtxdist[zz->Num_Proc] * sizeof(ZOLTAN_GNO_TYPE)))) MEMORY_ERROR; for (i=0; i<vtxdist[zz->Num_Proc]; ++i) global_GNOs[i] = i; ierr = Zoltan_DD_Create (&dd_color, zz->Communicator, sizeof(ZOLTAN_GNO_TYPE)/sizeof(ZOLTAN_ID_TYPE), 0, 0, num_obj, 0); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot construct DDirectory."); /* Put req obs with 1 but first inialize the rest with 0 */ ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)local_GNOs, NULL, NULL, color, nvtx); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory."); ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)requested_GNOs, NULL, NULL, color_exp, num_obj); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory."); /* Get requested colors from the DDirectory. */ ierr = Zoltan_DD_Find (dd_color, (ZOLTAN_ID_PTR)global_GNOs, NULL, NULL, color, vtxdist[zz->Num_Proc], NULL); if (ierr != ZOLTAN_OK) ZOLTAN_COLOR_ERROR(ierr, "Cannot find object in DDirectory."); /* Free DDirectory */ Zoltan_DD_Destroy(&dd_color); ZOLTAN_FREE(&local_GNOs); ZOLTAN_FREE(&global_GNOs); KDDKDDKDD(zz->Proc, "COLORTEST CHECK"); if (coloring_problem == 'P' || coloring_problem == '2') { if (vtxdist[zz->Num_Proc] && !(partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (vtxdist[zz->Num_Proc] && !(loc_partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int)))) MEMORY_ERROR; if (coloring_problem == 'P') { for (i=0; i<num_obj; ++i) { int gno=requested_GNOs[i]; loc_partialD2[gno] = 1; } MPI_Allreduce(loc_partialD2, partialD2, vtxdist[zz->Num_Proc], MPI_INT, MPI_LOR, zz->Communicator); } else { for (i=0; i<vtxdist[zz->Num_Proc]; ++i) partialD2[i] = 1; } } /* Check if there is an error in coloring */ if (coloring_problem == '1') { for (i=0; i<nvtx; i++) { int gno = i + (int)vtxdist[zz->Proc]; if (color[gno] <= 0) { /* object i is not colored properly */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]); break; } for (j = xadj[i]; j < xadj[i+1]; ++j) { int v = (int)adjncy[j]; if (color[gno] == color[v]) { /* neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, v:%d, cu:%d, cv:%d\n", gno, v, color[gno], color[v]); break; } } if (ierr == ZOLTAN_FATAL) break; } } else if (coloring_problem == '2' || coloring_problem == 'P') { for (i=0; i<nvtx; i++) { int gno = i + (int)vtxdist[zz->Proc]; if (partialD2[gno] && color[gno] <= 0) { /* object i is not colored properly */ ierr = ZOLTAN_FATAL; printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]); break; } for (j = xadj[i]; j < xadj[i+1]; ++j) { int v = (int)adjncy[j], k; if (partialD2[v] && color[v] <= 0) { ierr = ZOLTAN_FATAL; printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]); break; } if (partialD2[gno] && partialD2[v] && color[gno] == color[v]) { /* d-1 neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]); break; } for (k = j+1; k < xadj[i+1]; ++k) { int w = (int)adjncy[k]; if (partialD2[v] && partialD2[w] && color[v] == color[w]) { /* d-2 neighbors have the same color */ ierr = ZOLTAN_FATAL; printf("Error in coloring! d2-neigh: v:%d, w:%d, cv:%d, cw:%d pv:%d pw:%d\n", v, w, color[v], color[w], partialD2[v], partialD2[w]); break; } } } if (ierr == ZOLTAN_FATAL) break; } } else ZOLTAN_COLOR_ERROR(ZOLTAN_WARN, "Zoltan_Color_Test is only implemented for distance-1 and distance-2 coloring. Unknown coloring, skipping verification."); End: KDDKDDKDD(zz->Proc, "COLORTEST DONE"); if (ierr==ZOLTAN_FATAL) ierr = 2; MPI_Allreduce(&ierr, &ferr, 1, MPI_INT, MPI_MAX, zz->Communicator); if (ferr == 2) ierr = ZOLTAN_FATAL; else ierr = ZOLTAN_OK; Zoltan_ZG_Free (&graph); ZOLTAN_FREE(&adjproc); ZOLTAN_FREE(&color); ZOLTAN_FREE(&requested_GNOs); ZOLTAN_FREE(&partialD2); ZOLTAN_FREE(&loc_partialD2); return ierr; }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int num_gid_entries, /* # of entries for a global id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm /* iperm[rank[i]]=i, only for sequential ordering */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; int * vtxdist = NULL; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL, *local_iperm=NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value"); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.rank = rank; zz->Order.iperm = iperm; zz->Order.gids = gids; zz->Order.lids = lids; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); #ifdef HAVE_MPI strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN); #else strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); zz->Order.start_index = opt.start_index; /* * Check that the user has allocated space for the return args. */ if (!(gids && rank)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); /* strcpy(opt.order_type, "GLOBAL"); */ } #endif /* ZOLTAN_SCOTCH */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } if (!strcmp(opt.order_type, "GLOBAL")) strcpy (opt.order_type, "DIST"); if (!strcmp(opt.order_type, "LOCAL")) strcpy (opt.order_type, "SERIAL"); strcpy(zz->Order.order_type, opt.order_type); /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. * Compute gid according to the local graph. */ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 3, &local_gids, &local_rank, &local_iperm); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ if ((!(opt.return_args & RETURN_RANK) && (rank != NULL)) || (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) { ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr) { /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) { /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) { /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* I store : GNO, rank, iperm */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); ZOLTAN_FREE(&local_iperm); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
int Zoltan_Order ( struct Zoltan_Struct *zz, int num_gid_entries, int num_obj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR permuted_global_ids ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids. * num_gid_entries * Output: * permuted_global_ids * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { char msg[253]; sprintf(msg, "num_gid_entries=%d is not equal to parameter setting " "NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ #ifdef HAVE_MPI strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "GLOBAL"); #else strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN); strcpy(zz->Order.order_type, "LOCAL"); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* * Check that the user has allocated space for the return args. */ if (num_obj && !(gids && permuted_global_ids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } else if (!strcmp(opt.method, "LOCAL_HSFC")) { Order_fn = Zoltan_LocalHSFC_Order; strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/ } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PTSCOTCH")) { Order_fn = Zoltan_Scotch_Order; strcpy(zz->Order.order_type, "GLOBAL"); } #endif /* ZOLTAN_SCOTCH */ #ifdef ZOLTAN_HUND else if (!strcasecmp(opt.method, "HUND")) { ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL); goto End; } #endif /* ZOLTAN_HUND */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* TODO : Ask why useful ! */ /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /************************************ * Check for required query function ************************************/ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } /* TODO allocate all this stuff with the graph */ local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); /* * Call the actual ordering function. * Compute gid according to the local graph. */ ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 2, &local_gids, &local_rank); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* TODO: At this time, I consider rank == permuted_global_ids */ /* I store : GNO, rank, permuted GID */ /* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0); /* Hope a linear assignment will help a little */ if (local_num_obj) Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", permuted_global_ids[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } #ifdef ZOLTAN_HUND End: #endif /*ZOLTAN_HUND*/ ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }