int Zoltan_Reftree_Part(
ZZ *zz, /* The Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_import, /* Not computed, set to -1 */
ZOLTAN_ID_PTR *import_global_ids, /* Not computed */
ZOLTAN_ID_PTR *import_local_ids, /* Not computed */
int **import_procs, /* Not computed */
int **import_to_part, /* Not computed */
int *num_export, /* Number of objects to be exported */
ZOLTAN_ID_PTR *export_global_ids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *export_local_ids, /* local ids of objects to be exported */
int **export_procs, /* list of processors to export to */
int **export_to_partition /* list of partitions to export to */
)
{
char *yo = "Zoltan_Reftree_Part";
int ierr; /* error code returned by called routines */
int final_ierr; /* error code returned by this routine */
double time0 = 0, time1= 0, time2 = 0, time3 =0, time4 =0;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Initializations in case of early exit. */
*num_export = -1;
*num_import = -1;
final_ierr = ZOLTAN_OK;
/*
* initialize the tree (first call only)
*/
if (zz->LB.Data_Structure == NULL) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time0 = Zoltan_Time(zz->Timer);
ierr = Zoltan_Reftree_Init(zz);
if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned by Zoltan_Reftree_Init.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time1 = Zoltan_Time(zz->Timer);
} else {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) {
time1 = Zoltan_Time(zz->Timer);
time0 = time1 + 1.0;
}
}
/*
* build the refinement tree
*/
ierr = Zoltan_Reftree_Build(zz);
if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned by Zoltan_Reftree_Build.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time2 = Zoltan_Time(zz->Timer);
/*
* sum the weights in the tree
*/
ierr = Zoltan_Reftree_Sum_Weights(zz);
if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned by Zoltan_Reftree_Sum_Weights.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time3 = Zoltan_Time(zz->Timer);
/*
* determine the new partition
*/
ierr = Zoltan_Reftree_Partition(zz, part_sizes, num_export, export_global_ids,
export_local_ids, export_to_partition, export_procs);
if (ierr==ZOLTAN_FATAL || ierr==ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned by Zoltan_Reftree_Partition.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (ierr==ZOLTAN_WARN) final_ierr = ZOLTAN_WARN;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) time4 = Zoltan_Time(zz->Timer);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) {
if (time0 <= time1) {
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time1-time0,
"REFTREE Time to initialize :");
}
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time2-time1,
"REFTREE Time to build tree :");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time3-time2,
"REFTREE Time to sum weights:");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, time4-time3,
"REFTREE Time to partition :");
}
ZOLTAN_TRACE_EXIT(zz, yo);
return(final_ierr);
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int *num_gid_entries, /* # of entries for a global id */
int *num_lid_entries, /* # of entries for a local id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm, /* inverse permutation of rank */
ZOS *order_info /* Method-specific ordering info. Currently not used. */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* iperm, inverse permutation of rank
* order_info, a Zoltan Ordering Struct with additional info.
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
int *vtxdist;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strncpy(opt.order_type, "GLOBAL", MAX_PARAM_STRING_LEN);
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
if (opt.use_order_info == 0) order_info = NULL;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && lids && rank && iperm)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL){
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
*/
ierr = (*Order_fn)(zz, num_obj, gids, lids, rank, iperm, &opt, order_info);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN){
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr){
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK)){
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, iperm, rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM)){
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, rank, iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i, nobjs;
nobjs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &i);
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < nobjs; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(*num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
printf("ZOLTAN: inverse permutation on Proc %d\n", zz->Proc);
for (i = 0; i < nobjs; i++) {
printf("iperm[%3d] = %3d\n", i, iperm[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
static int Zoltan_LB(
ZZ *zz,
int include_parts, /* Flag indicating whether to generate
part informtion;
0 if called by Zoltan_LB_Balance,
1 if called by Zoltan_LB_Partition. */
int *changes, /* Set to zero or one depending on if
Zoltan determines a new
decomposition or not:
zero - No changes to the decomposition
were made by the load-balancing
algorithm; migration is not needed.
one - A new decomposition is suggested
by the load-balancer; migration is
needed to establish the new
decomposition. */
int *num_gid_entries, /* The number of array entries in a global ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Global_ID_Entries. */
int *num_lid_entries, /* The number of array entries in a local ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Local_ID_Entries. */
int *num_import_objs, /* The number of non-local objects in the
processor's new decomposition. */
ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
int **import_procs, /* Array of processor IDs for processors
currently owning non-local objects (i.e.,
objs to be imported) in this processor's
new decomposition. */
int **import_to_part, /* Partition to which the objects should be
imported. */
int *num_export_objs, /* The number of local objects that need to
be exported from the processor to establish
the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
int **export_procs, /* Array of destination processor IDs for
objects that need to be exported
to establish the new decomposition. */
int **export_to_part /* Partition to which objects should be
exported. */
)
{
/*
* Main load-balancing routine.
* Input: a Zoltan structure with appropriate function pointers set.
* Output:
* changes
* num_import_objs
* import_global_ids
* import_local_ids
* import_procs
* import_to_part
* num_export_objs
* export_global_ids
* export_local_ids
* export_procs
* export_to_part
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_LB";
int gmax; /* Maximum number of imported/exported objects
over all processors. */
int error = ZOLTAN_OK; /* Error code */
double start_time, end_time;
double lb_time[2] = {0.0,0.0};
char msg[256];
int comm[3],gcomm[3];
float *part_sizes = NULL, *fdummy = NULL;
int wgt_dim, part_dim;
int all_num_obj, i, ts, idIdx;
struct Hash_Node **ht;
int *export_all_procs, *export_all_to_part, *parts=NULL;
ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL;
ZOLTAN_ID_PTR gid;
#ifdef ZOLTAN_OVIS
struct OVIS_parameters ovisParameters;
#endif
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS){
printf("Build configuration:\n");
Zoltan_Print_Configuration(" ");
printf("\n");
Zoltan_Print_Key_Params(zz);
}
start_time = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_OVIS
Zoltan_OVIS_Setup(zz, &ovisParameters);
if (zz->Proc == 0)
printf("OVIS PARAMETERS %s %s %d %f\n",
ovisParameters.hello,
ovisParameters.dll,
ovisParameters.outputLevel,
ovisParameters.minVersion);
ovis_enabled(zz->Proc, ovisParameters.dll);
#endif
/*
* Compute Max number of array entries per ID over all processors.
* Compute Max number of return arguments for Zoltan_LB_Balance.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
comm[2] = zz->LB.Return_Lists;
MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
zz->LB.Return_Lists = gcomm[2];
/* assume no changes */
*changes = 0;
*num_import_objs = *num_export_objs = 0;
*import_global_ids = NULL;
*import_local_ids = NULL;
*import_procs = NULL;
*import_to_part = NULL;
*export_global_ids = NULL;
*export_local_ids = NULL;
*export_procs = NULL;
*export_to_part = NULL;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz))
goto End;
if (zz->LB.Method == NONE) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s Balancing method selected == NONE; no balancing performed\n",
yo);
error = ZOLTAN_WARN;
goto End;
}
/*
* Sync the random number generator across processors.
*/
Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator);
/* Since generating a new partition, need to free old mapping vector */
zz->LB.OldRemap = zz->LB.Remap;
zz->LB.Remap = NULL;
error = Zoltan_LB_Build_PartDist(zz);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN)
goto End;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
int i, np, fp;
for (i = 0; i < zz->Num_Proc; i++) {
Zoltan_LB_Proc_To_Part(zz, i, &np, &fp);
printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n",
zz->Proc, i, np, fp);
}
}
/*
* Generate parts sizes.
*/
#ifdef ZOLTAN_OVIS
/* set part sizes computed by OVIS, if requested. Processes set only their own value */
{
float part_sizes[1];
int part_ids[1], wgt_idx[1];
wgt_idx[0] = 0;
part_ids[0] = 0;
ovis_getPartsize(&(part_sizes[0]));
printf("Rank %d ps %f\n",zz->Proc, part_sizes[0]);
/* clear out old part size info first */
Zoltan_LB_Set_Part_Sizes(zz, 0, -1, NULL, NULL, NULL);
Zoltan_LB_Set_Part_Sizes(zz, 0, 1, part_ids, wgt_idx, part_sizes);
}
#endif
wgt_dim = zz->Obj_Weight_Dim;
part_dim = ((wgt_dim > 0) ? wgt_dim : 1);
part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim
* zz->LB.Num_Global_Parts);
if (part_sizes == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
/* Get part sizes. */
Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim,
part_sizes);
#ifdef ZOLTAN_OVIS
/* if (ovisParameters.outputlevel > 3) */
{
int myRank = zz->Proc;
if (myRank == 0){
int i, j;
for (i = 0; i < zz->LB.Num_Global_Parts; i++){
for (j = 0; j < part_dim; j++){
printf("Rank %d AG: part_sizes[%d] = %f (Num_Global_Parts = %d, part_dim = %d)\n",zz->Proc,
(i*part_dim+j), part_sizes[i*part_dim+j],zz->LB.Num_Global_Parts, part_dim);
}
}
}
}
#endif
/*
* Call the actual load-balancing function.
*/
error = zz->LB.LB_Fn(zz, part_sizes,
num_import_objs, import_global_ids, import_local_ids,
import_procs, import_to_part,
num_export_objs, export_global_ids, export_local_ids,
export_procs, export_to_part);
ZOLTAN_FREE(&part_sizes);
if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){
sprintf(msg, "Partitioning routine returned code %d.", error);
#ifdef HOST_LINUX
if ((error == ZOLTAN_MEMERR) && (Zoltan_Memory_Get_Debug() > 0)){
Zoltan_write_linux_meminfo(0, "State of /proc/meminfo after malloc failure\n", 0);
}
#endif
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
else if (error){
if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) {
sprintf(msg, "Partitioning routine returned code %d.", error);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning");
if (*num_import_objs >= 0)
MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
else /* use export data */
MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
if (gmax == 0) {
/*
* Decomposition was not changed by the load balancing; no migration
* is needed.
*/
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s No changes to the decomposition due to partitioning; "
"no migration is needed.\n", yo);
/*
* Reset num_import_objs and num_export_objs; don't want to return
* -1 for the arrays that weren't returned by ZOLTAN_LB_FN.
*/
*num_import_objs = *num_export_objs = 0;
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){
/*
* This parameter setting requires that all local objects
* and their assignments appear in the export list.
*/
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
goto End;
}
/*
* Check whether we know the import data, export data, or both.
*
* If we were given the import data,
* we know what the new decomposition should look like on the
* processor, but we don't know which of our local objects we have
* to export to other processors to establish the new decomposition.
* Reverse the argument if we were given the export data.
*
* Unless we were given both maps, compute the inverse map.
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) {
if (*num_import_objs >= 0)
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
if (*num_export_objs >= 0)
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
*num_import_objs = *num_export_objs = -1;
}
if (*num_import_objs >= 0){
if (*num_export_objs >= 0) {
/* Both maps already available; nothing to do. */;
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Export lists are requested; compute export map */
error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids,
*import_local_ids, *import_procs,
*import_to_part,
num_export_objs, export_global_ids,
export_local_ids, export_procs,
export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Method returned import lists, but only export lists were desired. */
/* Import lists not needed; free them. */
*num_import_objs = -1;
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
}
}
}
else { /* (*num_import_objs < 0) */
if (*num_export_objs >= 0) {
/* Only export lists have been returned. */
if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Compute import map */
error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids,
*export_local_ids, *export_procs,
*export_to_part,
num_import_objs, import_global_ids,
import_local_ids, import_procs,
import_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Method returned export lists, but only import lists are desired. */
/* Export lists not needed; free them. */
*num_export_objs = -1;
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
}
}
}
else { /* *num_export_objs < 0 && *num_import_objs < 0) */
if (zz->LB.Return_Lists) {
/* No map at all available */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned "
"neither import nor export data.");
error = ZOLTAN_WARN;
goto End;
}
}
}
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/*
* Normally, Zoltan_LB returns in the export lists all local
* objects that are moving off processor, or that are assigned
* to a part on the local processor that is not the
* default part. This setting of Return_Lists requests
* that all local objects be included in the export list.
*/
if (*num_export_objs == 0){
/* all local objects are remaining on processor */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (*num_export_objs) {
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End;
}
else{
all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error);
if (*num_export_objs < all_num_obj){
/* Create a lookup table for exported IDs */
ts = Zoltan_Recommended_Hash_Size(*num_export_objs);
ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts);
/* Create a list of all gids, lids and parts */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj,
&all_global_ids, &all_local_ids,
wgt_dim, &fdummy, &parts);
if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){
ZOLTAN_FREE(&fdummy);
if ((Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_procs,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) ||
(Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_to_part,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){
error = ZOLTAN_MEMERR;
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){
sprintf(msg, "Error building complete export list; "
"%d returned by Zoltan_Get_Obj_List\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
gid = all_global_ids;
for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){
idIdx = search_hash_table(zz, gid, ht, ts);
if (idIdx >= 0){
export_all_procs[i] = (*export_procs)[idIdx];
export_all_to_part[i] = (*export_to_part)[idIdx];
}
else{
export_all_procs[i] = zz->Proc;
export_all_to_part[i] = parts[i];
}
}
free_hash_table(ht, ts);
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
Zoltan_Special_Free(zz, (void **)(void*)&parts,
ZOLTAN_SPECIAL_MALLOC_INT);
*export_global_ids = all_global_ids;
*export_local_ids = all_local_ids;
*export_procs = export_all_procs;
*export_to_part = export_all_to_part;
*num_export_objs = all_num_obj;
}
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments");
end_time = Zoltan_Time(zz->Timer);
lb_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc);
for (i = 0; i < *num_import_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID]));
printf(" To part: %4d",
(*import_to_part != NULL ? (*import_to_part)[i]
: zz->Proc));
printf(" From processor: %4d\n", (*import_procs)[i]);
}
printf("\n");
printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc);
for (i = 0; i < *num_export_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID]));
printf(" To part: %4d",
(*export_to_part != NULL ? (*export_to_part)[i]
: (*export_procs)[i]));
printf(" To processor: %4d\n", (*export_procs)[i]);
}
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/*
* If the Help_Migrate flag is set, perform migration for the application.
*/
if (zz->Migrate.Auto_Migrate) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration");
start_time = Zoltan_Time(zz->Timer);
error = Zoltan_Migrate(zz,
*num_import_objs, *import_global_ids,
*import_local_ids, *import_procs, *import_to_part,
*num_export_objs, *export_global_ids,
*export_local_ids, *export_procs, *export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error in auto-migration; %d returned from "
"Zoltan_Help_Migrate\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
end_time = Zoltan_Time(zz->Timer);
lb_time[1] = end_time - start_time;
ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration");
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0],
"ZOLTAN Partition: ");
if (zz->Migrate.Auto_Migrate)
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1],
"ZOLTAN Migrate: ");
}
*changes = 1;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (error);
}
static int rib_fn(
ZZ *zz, /* The Zoltan structure with info for
the RIB balancer. */
int *num_import, /* Number of non-local objects assigned to
this processor in the new decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
num_import returns the number of input
objects as given by ZOLTAN_NUM_OBJ_FN. */
ZOLTAN_ID_PTR *import_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
this array contains GIDs for all input
objs as given by ZOLTAN_OBJ_LIST_FN.*/
ZOLTAN_ID_PTR *import_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
this array contains LIDs for all input
objs as given by ZOLTAN_OBJ_LIST_FN.*/
int **import_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
the returned array is NULL. */
int **import_to_part, /* Returned value: array of parts to
which objects are imported.
When LB.Return_Lists==CANDIDATE_LISTS,
the returned array is NULL. */
int *num_export, /* Returned value only when
LB.Return_Lists==CANDIDATE_LISTS; number of
input objs as given by ZOLTAN_NUM_OBJ_FN */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value only when
LB.Return_Lists==CANDIDATE_LISTS; for each
input obj (from ZOLTAN_OBJ_LIST_FN),
return a candidate obj from the part to which
the obj is assigned; used in PHG matching */
double overalloc, /* amount to overallocate by when realloc
of dot array must be done.
1.0 = no extra; 1.5 = 50% extra; etc. */
int wgtflag, /* No. of weights per dot supplied by user. */
int check_geom, /* Check input & output for consistency? */
int stats, /* Print timing & count summary? */
int gen_tree, /* (0) do not (1) do generate full treept */
int average_cuts, /* (0) don't (1) compute the cut to be the
average of the closest dots. */
float *part_sizes /* Input: Array of size
zz->Num_Global_Parts * max(zz->Obj_Weight_Dim, 1)
containing the percentage of work to be
assigned to each part. */
)
{
char yo[] = "rib_fn";
int proc,nprocs; /* my proc id, total # of procs */
struct Dot_Struct *dotpt; /* temporary pointer to local dot arrays */
int pdotnum; /* # of dots - decomposition changes it */
int *dotmark = NULL; /* which side of median for each dot */
int dotnum; /* number of dots */
int dotmax = 0; /* max # of dots arrays can hold */
int dottop; /* dots >= this index are new */
int proclower; /* 1st proc in lower set */
int procmid; /* 1st proc in upper set */
int partlower; /* 1st part in lower set */
int partmid; /* 1st part in upper set */
int set; /* which set processor is in = 0/1 */
int old_set; /* set processor was in last cut = 0/1 */
int root; /* part that stores last cut */
int num_procs; /* number of procs in current set */
int num_parts; /* number of parts in current set */
int ierr = ZOLTAN_OK; /* error flag. */
double *value = NULL; /* temp array for median_find */
double *wgts = NULL; /* temp array for serial_rib */
double valuehalf; /* median cut position */
double cm[3]; /* Center of mass of objects */
double evec[3]; /* Eigenvector defining direction */
int first_guess = 0; /* flag if first guess for median search */
int allocflag; /* have to re-allocate space */
double time1=0,time2=0; /* timers */
double time3=0,time4=0; /* timers */
double timestart=0,timestop=0; /* timers */
double timers[4]={0.,0.,0.,0.};
/* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
ZOLTAN_GNO_TYPE counters[7]; /* diagnostic counts
0 = unused
1 = # of dots sent
2 = # of dots received
3 = most dots this proc ever owns
4 = most dot memory this proc ever allocs
5 = # of times a previous cut is re-used
6 = # of reallocs of dot array */
int i, j; /* local variables */
int use_ids; /* When true, global and local IDs will be
stored along with dots in the RCB_STRUCT.
When false, storage, manipulation, and
communication of IDs is avoided.
Set by call to Zoltan_RB_Use_IDs(). */
RIB_STRUCT *rib = NULL; /* Pointer to data structures for RIB */
struct rib_tree *treept = NULL; /* tree of cuts - single cut on exit*/
double start_time, end_time;
double lb_time[2]={0,0};
int tfs[2], tmp_tfs[2]; /* added for Tflops_Special; max number
of procs and parts over all processors
in each iteration (while loop) of
parallel partitioning. */
int old_nprocs; /* added for Tflops_Special */
int old_nparts; /* added for Tflops_Special */
double valuelo; /* smallest value of value[i] */
double valuehi; /* largest value of value[i] */
double weight[RB_MAX_WGTS]; /* weight for current set */
double weightlo[RB_MAX_WGTS]; /* weight of lower side of cut */
double weighthi[RB_MAX_WGTS]; /* weight of upper side of cut */
double fractionlo[RB_MAX_WGTS]; /* desired wt in lower half */
int *dotlist = NULL; /* list of dots for find_median.
allocated above find_median for
better efficiency (don't necessarily
have to realloc for each find_median).*/
int rectilinear_blocks = 0; /* parameter for find_median (not used by rib) */
int fp=0; /* first part assigned to this proc. */
int np=0; /* number of parts assigned to this proc. */
int wgtdim; /* max(wgtflag,1) */
int *dindx = NULL, *tmpdindx = NULL;
/* MPI data types and user functions */
MPI_Comm local_comm, tmp_comm;
int free_comm = FALSE; /* Flag indicating whether MPI_Comm_free
should be called on local_comm at end. */
ZOLTAN_TRACE_ENTER(zz, yo);
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
MPI_Barrier(zz->Communicator);
timestart = time1 = Zoltan_Time(zz->Timer);
}
/* setup for parallel */
proc = zz->Proc;
nprocs = zz->Num_Proc;
num_parts = zz->LB.Num_Global_Parts;
/*
* Determine whether to store, manipulate, and communicate global and
* local IDs.
*/
use_ids = Zoltan_RB_Use_IDs(zz);
/*
* Build the RIB Data structure and
* set pointers to information in it.
*/
start_time = Zoltan_Time(zz->Timer);
ierr = Zoltan_RIB_Build_Structure(zz, &pdotnum, &dotmax, wgtflag, overalloc,
use_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RIB_Build_Structure.");
goto End;
}
rib = (RIB_STRUCT *) (zz->LB.Data_Structure);
treept = rib->Tree_Ptr;
end_time = Zoltan_Time(zz->Timer);
lb_time[0] = end_time - start_time;
start_time = end_time;
/* local copies of calling parameters */
dottop = dotnum = pdotnum;
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = dotnum;
counters[4] = dotmax;
counters[5] = 0;
counters[6] = 0;
/* Ensure there are dots */
MPI_Allreduce(&dotnum, &i, 1, MPI_INT, MPI_MAX, zz->Communicator);
if (i == 0){
if (proc == 0){
ZOLTAN_PRINT_WARN(proc, yo, "RIB partitioning called with no objects");
}
timestart = timestop = 0;
goto EndReporting;
}
/* If using RIB for matching, need to generate candidate lists.
* Candidate lists include input GIDs, LIDs as provided by the application.
* We need to capture that input here before we move any dots!
* We return it in the import lists.
* Candidates will be computed after partitioning and returned in the
* export lists.
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) {
ierr = Zoltan_RB_Candidates_Copy_Input(zz, dotnum,
rib->Global_IDs, rib->Local_IDs,
&rib->Dots,
num_import,
import_global_ids, import_local_ids,
import_procs, import_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc,yo,
"Error returned from Zoltan_RB_Return_Arguments.");
goto End;
}
}
/* create mark and list arrays for dots */
allocflag = 0;
if (dotmax > 0) {
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ierr = ZOLTAN_MEMERR;
goto End;
}
}
else {
dotmark = NULL;
value = NULL;
dotlist = NULL;
}
/* set dot weights = 1.0 if user didn't and determine total weight */
dotpt = &rib->Dots;
if (dotpt->nWeights == 0) {
weightlo[0] = (double) dotnum;
dotpt->uniformWeight = 1.0;
wgtdim = 1;
}
else {
double *wgt;
for (j=0; j<dotpt->nWeights; j++){
weightlo[j] = 0.0;
wgt = dotpt->Weight + j;
for (i=0; i < dotnum; i++){
weightlo[j] += *wgt;
wgt += dotpt->nWeights;
}
}
wgtdim = dotpt->nWeights;
}
MPI_Allreduce(weightlo, weight, wgtdim, MPI_DOUBLE, MPI_SUM, zz->Communicator);
if (check_geom) {
ierr = Zoltan_RB_check_geom_input(zz, dotpt, dotnum);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_check_geom_input");
goto End;
}
}
/* create local communicator for use in recursion */
if (zz->Tflops_Special)
local_comm = zz->Communicator;
else {
MPI_Comm_dup(zz->Communicator,&local_comm);
free_comm = TRUE;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
time2 = Zoltan_Time(zz->Timer);
timers[0] = time2 - time1;
}
/* recursively halve until just one part or proc in set */
old_nprocs = num_procs = nprocs;
old_nparts = num_parts;
partlower = 0;
root = 0;
old_set = 1;
ierr = Zoltan_LB_Proc_To_Part(zz, proc, &np, &fp);
for (i = fp; i < (fp + np); i++) {
treept[i].parent = 0;
treept[i].left_leaf = 0;
}
if (zz->Tflops_Special) {
proclower = 0;
tfs[0] = nprocs;
tfs[1] = num_parts;
}
while ((num_parts > 1 && num_procs > 1) ||
(zz->Tflops_Special && tfs[0] > 1 && tfs[1] > 1)) {
ierr = Zoltan_Divide_Machine(zz, zz->Obj_Weight_Dim, part_sizes,
proc, local_comm, &set,
&proclower, &procmid, &num_procs,
&partlower, &partmid, &num_parts,
fractionlo);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo, "Error in Zoltan_Divide_Machine.");
goto End;
}
/* tfs[0] is max number of processors in all sets over all processors -
* tfs[1] is max number of parts in all sets over all processors -
* force all processors to go through all levels of parallel rib */
if (zz->Tflops_Special) {
tmp_tfs[0] = num_procs;
tmp_tfs[1] = num_parts;
MPI_Allreduce(tmp_tfs, tfs, 2, MPI_INT, MPI_MAX, local_comm);
}
/* create mark array and active list for dots */
if (allocflag) {
allocflag = 0;
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ierr = ZOLTAN_MEMERR;
goto End;
}
}
dotpt = &rib->Dots;
if (old_nparts > 1 && old_nprocs > 1) { /* test added for Tflops_Special;
compute values only if looping
to decompose, not if looping to
keep Tflops_Special happy. */
ierr = compute_rib_direction(zz, zz->Tflops_Special, rib->Num_Geom,
&valuelo, &valuehi, dotpt, NULL, dotnum,
wgtflag, cm, evec, value,
local_comm, proc, old_nprocs, proclower);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from compute_rib_direction");
goto End;
}
}
else { /* For Tflops_Special: initialize value when looping only
for Tflops_Special */
for (i = 0; i < dotmax; i++)
value[i] = 0.0;
valuelo = valuehi = 0.0;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
time2 = Zoltan_Time(zz->Timer);
if (!Zoltan_RB_find_median(
zz->Tflops_Special, value, dotpt->Weight, dotpt->uniformWeight, dotmark, dotnum, proc,
fractionlo, local_comm, &valuehalf, first_guess,
nprocs, old_nprocs, proclower, old_nparts,
wgtflag, valuelo, valuehi, weight[0], weightlo,
weighthi, dotlist, rectilinear_blocks, average_cuts)) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_find_median.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (set) /* set weight for current part */
for (j=0; j<wgtdim; j++) weight[j] = weighthi[j];
else
for (j=0; j<wgtdim; j++) weight[j] = weightlo[j];
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
time3 = Zoltan_Time(zz->Timer);
/* store cut info in tree only if proc "owns" partmid */
/* test of partmid > 0 prevents treept[0] being set when this cut is
only removing low-numbered processors (proclower to procmid-1) that
have no parts in them from the processors remaining to
be partitioned. */
if (partmid > 0 && partmid == fp) {
treept[partmid].cm[0] = cm[0];
treept[partmid].cm[1] = cm[1];
treept[partmid].cm[2] = cm[2];
treept[partmid].ev[0] = evec[0];
treept[partmid].ev[1] = evec[1];
treept[partmid].ev[2] = evec[2];
treept[partmid].cut = valuehalf;
treept[partmid].parent = old_set ? -(root+1) : root+1;
/* The following two will get overwritten when the information
is assembled if this is not a terminal cut */
treept[partmid].left_leaf = -partlower;
treept[partmid].right_leaf = -partmid;
}
if (old_nprocs > 1 && partmid > 0 && partmid != partlower + old_nparts) {
/* old_nprocs > 1 test: Don't reset these values if proc is in loop only
* because of other procs for Tflops_Special.
* partmid > 0 test: Don't reset these values if low-numbered processors
* (proclower to procmid-1) have zero parts and this cut is removing
* them from the processors remaining to be partitioned.
* partmid != partlower + old_nparts test: Don't reset these values if
* cut is removing high-numbered processors with zero parts from
* the processors remaining to be partitioned.
*/
old_set = set;
root = partmid;
}
ierr = Zoltan_RB_Send_Outgoing(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotmark,
&dottop, &dotnum, &dotmax,
set, &allocflag, overalloc,
stats, counters, use_ids,
local_comm, proclower,
old_nprocs, partlower, partmid);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_Send_Outgoing.");
goto End;
}
/* create new communicators */
if (zz->Tflops_Special) {
if (set) {
proclower = procmid;
partlower = partmid;
}
old_nprocs = num_procs;
old_nparts = num_parts;
}
else {
if (set) partlower = partmid;
MPI_Comm_split(local_comm,set,proc,&tmp_comm);
MPI_Comm_free(&local_comm);
local_comm = tmp_comm;
old_nprocs = num_procs;
old_nparts = num_parts;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
time4 = Zoltan_Time(zz->Timer);
timers[1] += time2 - time1;
timers[2] += time3 - time2;
timers[3] += time4 - time3;
}
}
/* have recursed all the way to a single processor sub-domain */
/* Send dots to correct processors for their parts. This is needed
most notably when a processor has zero parts on it, but still has
some dots after the parallel partitioning. */
ierr = Zoltan_RB_Send_To_Part(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotmark, &dottop,
&dotnum, &dotmax, &allocflag, overalloc,
stats, counters, use_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_RB_Send_To_Part");
goto End;
}
/* All dots are now on the processors they will end up on; now generate
* more parts if needed. */
if (num_parts > 1) {
if (dotpt->nWeights)
wgts = (double *) ZOLTAN_MALLOC(dotpt->nWeights * dotnum * sizeof(double));
dindx = (int *) ZOLTAN_MALLOC(dotnum * 2 * sizeof(int));
tmpdindx = dindx + dotnum;
if (allocflag) {
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
for (i = 0; i < dotnum; i++)
dindx[i] = i;
ierr = serial_rib(zz, &rib->Dots, dotmark, dotlist, old_set, root,
rib->Num_Geom, weight[0], dotnum, num_parts,
&(dindx[0]), &(tmpdindx[0]), partlower,
proc, wgtflag, stats, gen_tree,
rectilinear_blocks, average_cuts,
treept, value, wgts, part_sizes);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from serial_rib");
goto End;
}
ZOLTAN_FREE(&wgts);
}
end_time = Zoltan_Time(zz->Timer);
lb_time[1] = end_time - start_time;
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
MPI_Barrier(zz->Communicator);
timestop = time1 = Zoltan_Time(zz->Timer);
}
/* error checking and statistics */
if (check_geom) {
ierr = Zoltan_RB_check_geom_output(zz, &rib->Dots, part_sizes, np, fp,
dotnum, pdotnum, NULL);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_check_geom_output");
goto End;
}
}
EndReporting:
/* update calling routine parameters */
start_time = Zoltan_Time(zz->Timer);
pdotnum = dotnum;
/* Perform remapping (if requested) */
if (zz->LB.Remap_Flag) {
ierr = Zoltan_RB_Remap(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotnum, &dotmax,
&allocflag, overalloc, stats, counters, use_ids);
/* Note: dottop is no longer valid after remapping. Remapping might
destroy the nice local-followed-by-non-local ordering of the
dots array. Do not use dottop after remapping. */
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_RB_Remap.");
goto End;
}
}
/* build return arguments */
if (zz->LB.Return_Lists != ZOLTAN_LB_NO_LISTS &&
zz->LB.Return_Lists != ZOLTAN_LB_CANDIDATE_LISTS) {
/* zz->LB.Return_Lists is true ==> use_ids is true */
ierr = Zoltan_RB_Return_Arguments(zz, rib->Global_IDs, rib->Local_IDs,
&rib->Dots, num_import,
import_global_ids, import_local_ids,
import_procs, import_to_part,
dotnum);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_Return_Arguments.");
goto End;
}
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) {
/* Select a candidate for each part and return it in the export_GIDs. */
ierr = Zoltan_RB_Candidates_Output(zz, dotnum, dindx,
rib->Global_IDs, rib->Local_IDs,
&rib->Dots,
*num_import, *import_global_ids,
num_export, export_global_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc,yo,
"Error returned from Zoltan_RB_Return_Candidates.");
goto End;
}
}
ZOLTAN_FREE(&dindx);
if (gen_tree) {
int *displ, *recvcount;
int sendcount;
struct rib_tree *treetmp = NULL; /* temporary tree of cuts; used to keep
valgrind from reporting overlapped
memory in MPI_Allgatherv */
treetmp = (struct rib_tree *)
ZOLTAN_MALLOC(zz->LB.Num_Global_Parts* sizeof(struct rib_tree));
displ = (int *) ZOLTAN_MALLOC(2 * zz->Num_Proc * sizeof(int));
if (!displ || !treetmp) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
recvcount = displ + zz->Num_Proc;
ierr = Zoltan_RB_Tree_Gatherv(zz, sizeof(struct rib_tree), &sendcount,
recvcount, displ);
/*
* Create copy of treept so that MPI_Allgatherv doesn't use same
* memory for sending and receiving; removes valgrind warning.
*/
for (i = 0; i < zz->LB.Num_Global_Parts; i++)
treetmp[i] = treept[i];
MPI_Allgatherv(&treetmp[fp], sendcount, MPI_BYTE, treept, recvcount, displ,
MPI_BYTE, zz->Communicator);
for (i = 1; i < zz->LB.Num_Global_Parts; i++){
if (treept[i].parent > 0)
treept[treept[i].parent - 1].left_leaf = i;
else if (treept[i].parent < 0)
treept[-treept[i].parent - 1].right_leaf = i;
}
ZOLTAN_FREE(&displ);
ZOLTAN_FREE(&treetmp);
}
else {
treept[0].right_leaf = -1;
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
print_rib_tree(zz, np, fp, &(treept[fp]));
end_time = Zoltan_Time(zz->Timer);
lb_time[0] += (end_time - start_time);
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
Zoltan_RB_stats(zz, timestop-timestart, &rib->Dots, dotnum,
part_sizes, timers, counters, stats, NULL, NULL, FALSE);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) {
if (zz->Proc == zz->Debug_Proc)
printf("ZOLTAN RIB Times: \n");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, lb_time[0],
"ZOLTAN Build: ");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, lb_time[1],
"ZOLTAN RIB: ");
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
/* zz->Debug_Level >= ZOLTAN_DEBUG_ALL ==> use_ids is true */
Zoltan_RB_Print_All(zz, rib->Global_IDs, &rib->Dots,
dotnum, *num_import,
*import_global_ids, *import_procs);
}
End:
/* Free memory allocated by the algorithm. */
if (free_comm) MPI_Comm_free(&local_comm);
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!gen_tree && /* don't need parts */
rib && (rib->Tran.Target_Dim < 0)) { /* don't need transformation */
/* Free all memory used. */
Zoltan_RIB_Free_Structure(zz);
}
else if (rib != NULL) {
/* Free only Dots and IDs; keep other structures. */
ZOLTAN_FREE(&(rib->Global_IDs));
ZOLTAN_FREE(&(rib->Local_IDs));
Zoltan_Free_And_Reset_Dot_Structure(&(rib->Dots));
}
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
ts = &(zt->Times[ts_idx]);
if (ts->Status > RUNNING) {
char msg[256];
sprintf(msg,
"Cannot start timer %d at %s:%d; timer already running from %s:%d.",
ts_idx, filename, lineno, ts->Start_File, ts->Start_Line);
FATALERROR(yo, msg)
}
ts->Status += RUNNING;
strncpy(ts->Start_File, filename, MAXNAMELEN);
ts->Start_Line = lineno;
if (ts->Use_Barrier)
MPI_Barrier(comm);
ts->Start_Time = Zoltan_Time(zt->Timer_Flag);
#ifdef VAMPIR
if (VT_begin(ts->vt_handle) != VT_OK)
FATALERROR(yo, "VT_begin failed.");
#endif
return ZOLTAN_OK;
}
/****************************************************************************/
int Zoltan_Timer_Stop(
ZTIMER *zt, /* Ptr to Timer object */
int ts_idx, /* Index of the timer to use */
MPI_Comm comm, /* Communicator to use for synchronization,
int Zoltan_ParMetis_Order(
ZZ *zz, /* Zoltan structure */
int num_obj, /* Number of (local) objects to order. */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */
int *iperm,
ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */
)
{
static char *yo = "Zoltan_ParMetis_Order";
int i, n, ierr;
ZOLTAN_Output_Order ord;
ZOLTAN_Third_Graph gr;
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't want to risk letting external
packages changing our communicator */
#endif
indextype numflag = 0;
int timer_p = 0;
int get_times = 0;
int use_timers = 0;
double times[5];
ZOLTAN_ID_PTR l_gids = NULL;
ZOLTAN_ID_PTR l_lids = NULL;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
ZOLTAN_TRACE_ENTER(zz, yo);
#ifdef ZOLTAN_PARMETIS
#if TPL_USE_DATATYPE != TPL_METIS_DATATYPES
#ifdef TPL_FLOAT_WEIGHT
i = 1;
#else
i = 0;
#endif
if ((sizeof(indextype) != sizeof(idxtype)) ||
(sizeof(weighttype) != sizeof(idxtype)) || i){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,
"Not supported: Multiple 3rd party libraries with incompatible "
"data types.");
return ZOLTAN_FATAL;
}
#endif
#endif
memset(&gr, 0, sizeof(ZOLTAN_Third_Graph));
memset(&ord, 0, sizeof(ZOLTAN_Output_Order));
memset(times, 0, sizeof(times));
ord.order_opt = order_opt;
if (!order_opt){
/* If for some reason order_opt is NULL, allocate a new ZOOS here. */
/* This should really never happen. */
order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
strcpy(order_opt->method,"PARMETIS");
}
ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord);
/* ParMetis only computes the rank vector */
order_opt->return_args = RETURN_RANK;
/* Check that num_obj equals the number of objects on this proc. */
/* This constraint may be removed in the future. */
n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){
/* Return error code */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
return(ZOLTAN_FATAL);
}
if (n != num_obj){
/* Currently this is a fatal error. */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the "
"number of objects.");
return(ZOLTAN_FATAL);
}
/* Do not use weights for ordering */
gr.obj_wgt_dim = -1;
gr.edge_wgt_dim = -1;
gr.num_obj = num_obj;
/* Check what ordering type is requested */
if (order_opt){
SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */
#ifdef ZOLTAN_PARMETIS
if ((strcmp(order_opt->method, "METIS") == 0))
#endif /* ZOLTAN_PARMETIS */
SET_LOCAL_GRAPH(&gr.graph_type);
}
gr.get_data = 1;
if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: "
"set ParMetis instead.");
return(ZOLTAN_FATAL);
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
return (ierr);
}
/* Allocate space for separator sizes */
if (IS_GLOBAL_GRAPH(gr.graph_type)) {
if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) {
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype));
if (ord.sep_sizes == NULL) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */
}
/* Allocate space for direct perm */
ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.rank){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
if (IS_LOCAL_GRAPH(gr.graph_type)){
/* Allocate space for inverse perm */
ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.iperm){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
}
else
ord.iperm = NULL;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_PARMETIS
if (IS_GLOBAL_GRAPH(gr.graph_type)){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library");
ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy,
&numflag, options, ord.rank, ord.sep_sizes, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else
#endif /* ZOLTAN_PARMETIS */
#if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS)
if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */
indextype numobj = gr.num_obj;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library");
order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0; /* Use default options for METIS. */
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options,
ord.iperm, ord.rank);
#else
METIS_SetDefaultOptions(options);
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, NULL, options,
ord.iperm, ord.rank); /* NULL is vwgt -- new interface in v4 */
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */
int numbloc;
int start;
int leaf;
int *converttab;
int levelmax;
levelmax = mylog2(zz->Num_Proc) + 1;
converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int));
memset(converttab, 0, zz->Num_Proc*2*sizeof(int));
/* Determine the first node in each separator, store it in zz->TPL_Order.start */
for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) {
int father;
father = zz->Num_Proc + numbloc;
converttab[start] = 2*numbloc;
zz->TPL_Order.leaves[leaf++]=start;
zz->TPL_Order.ancestor[start] = start + 2;
converttab[start+1] = 2*numbloc+1;
zz->TPL_Order.leaves[leaf++]=start+1;
zz->TPL_Order.ancestor[start+1] = start + 2;
start+=2;
do {
converttab[start] = father;
if (father %2 == 0) {
int nextoffset;
int level;
level = mylog2(2*zz->Num_Proc - 1 - father);
nextoffset = (1<<(levelmax-level));
zz->TPL_Order.ancestor[start] = start+nextoffset;
start++;
break;
}
else {
zz->TPL_Order.ancestor[start] = start+1;
start++;
father = zz->Num_Proc + father/2;
}
} while (father < 2*zz->Num_Proc - 1);
}
zz->TPL_Order.start[0] = 0;
zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1;
for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) {
int oldblock=converttab[numbloc-1];
zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock];
}
ZOLTAN_FREE(&converttab);
ZOLTAN_FREE(&ord.sep_sizes);
zz->TPL_Order.leaves[zz->Num_Proc] = -1;
zz->TPL_Order.nbr_leaves = zz->Num_Proc;
zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1;
}
else { /* No tree */
zz->TPL_Order.nbr_blocks = 0;
zz->TPL_Order.start = NULL;
zz->TPL_Order.ancestor = NULL;
zz->TPL_Order.leaves = NULL;
}
/* Correct because no redistribution */
memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE));
memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE));
ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL);
ZOLTAN_FREE(&l_gids);
ZOLTAN_FREE(&l_lids);
/* Get a time here */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){
memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i];
}
}
if ((ord.iperm != NULL) && (iperm != NULL)){
if (sizeof(indextype) == sizeof(int)){
memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
iperm[i] = (int)ord.iperm[i];
}
}
}
if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm);
ZOLTAN_FREE(&ord.rank);
/* Free all other "graph" stuff */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
realtype itr = 0.0;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
/* change our zz struct. */
#endif
indextype i;
realtype *imb_tols;
indextype ncon;
indextype edgecut;
indextype wgtflag;
indextype numflag = 0;
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
ZOLTAN_TRACE_ENTER(zz, yo);
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
if (sizeof(realtype) != sizeof(float)) {
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));
for (i = 0; i < tmp; i++)
prt.input_part_sizes[i] = (realtype) part_sizes[i];
/* KDD 2/2014: removed re-scaling part sizes so they sum to one.
* part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
* plus, the code here was wrong for multiple object weights.
* similar scaling code did not exist in the Scotch interface.
*/
prt.part_sizes = prt.input_part_sizes;
}
else
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
indextype j;
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
/* Now we can call ParMetis */
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
/* Use default options for METIS */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0;
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
#else
METIS_SetDefaultOptions(options);
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
prt.part_sizes, imb_tols, options,
&edgecut, prt.part);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) {
/* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int num_gid_entries, /* # of entries for a global id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm /* iperm[rank[i]]=i, only for sequential ordering */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
int * vtxdist = NULL;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL, *local_iperm=NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value");
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.rank = rank;
zz->Order.iperm = iperm;
zz->Order.gids = gids;
zz->Order.lids = lids;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
#ifdef HAVE_MPI
strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN);
#else
strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN);
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
zz->Order.start_index = opt.start_index;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && rank)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
/* strcpy(opt.order_type, "GLOBAL"); */
}
#endif /* ZOLTAN_SCOTCH */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
if (!strcmp(opt.order_type, "GLOBAL"))
strcpy (opt.order_type, "DIST");
if (!strcmp(opt.order_type, "LOCAL"))
strcpy (opt.order_type, "SERIAL");
strcpy(zz->Order.order_type, opt.order_type);
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 3,
&local_gids, &local_rank, &local_iperm);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
if ((!(opt.return_args & RETURN_RANK) && (rank != NULL))
|| (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) {
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr) {
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) {
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) {
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* I store : GNO, rank, iperm */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
ZOLTAN_FREE(&local_iperm);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
int Zoltan_Order (
struct Zoltan_Struct *zz,
int num_gid_entries,
int num_obj,
ZOLTAN_ID_PTR gids,
ZOLTAN_ID_PTR permuted_global_ids
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids.
* num_gid_entries
* Output:
* permuted_global_ids
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
char msg[253];
sprintf(msg, "num_gid_entries=%d is not equal to parameter setting "
"NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
#ifdef HAVE_MPI
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "GLOBAL");
#else
strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "LOCAL");
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
/*
* Check that the user has allocated space for the return args.
*/
if (num_obj && !(gids && permuted_global_ids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "LOCAL_HSFC"))
{
Order_fn = Zoltan_LocalHSFC_Order;
strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PTSCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_SCOTCH */
#ifdef ZOLTAN_HUND
else if (!strcasecmp(opt.method, "HUND")) {
ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL);
goto End;
}
#endif /* ZOLTAN_HUND */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/* TODO : Ask why useful ! */
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/************************************
* Check for required query function
************************************/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
/* TODO allocate all this stuff with the graph */
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 2,
&local_gids, &local_rank);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* TODO: At this time, I consider rank == permuted_global_ids */
/* I store : GNO, rank, permuted GID */
/* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0);
/* Hope a linear assignment will help a little */
if (local_num_obj)
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", permuted_global_ids[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
#ifdef ZOLTAN_HUND
End:
#endif /*ZOLTAN_HUND*/
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/* This function needs a distribution : rows then cols to work properly */
int
Zoltan_ZG_Build (ZZ* zz, ZG* graph, int local)
{
static char *yo = "Zoltan_ZG_Build";
int ierr = ZOLTAN_OK;
int diag;
int *diagarray=NULL;
Zoltan_matrix_options opt;
char symmetrization[MAX_PARAM_STRING_LEN+1];
char bipartite_type[MAX_PARAM_STRING_LEN+1];
char weigth_type[MAX_PARAM_STRING_LEN+1];
char matrix_build_type[MAX_PARAM_STRING_LEN+1];
int bipartite = 0;
#ifdef CC_TIMERS
double times[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
double gtimes[9]={0.,0.,0.,0.,0.,0.,0.,0.}; /* Used for timing measurements */
char *timenames[9]= {"", "setup", "matrix build", "diag", "symmetrize", "dist lin", "2D dist", "complete", "clean up"};
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
#endif /* CC_TIMERS */
ZOLTAN_TRACE_ENTER(zz, yo);
memset (graph, 0, sizeof(ZG));
/* Read graph build parameters */
Zoltan_Bind_Param(ZG_params, "GRAPH_SYMMETRIZE", (void *) &symmetrization);
Zoltan_Bind_Param(ZG_params, "GRAPH_SYM_WEIGHT", (void *) &weigth_type);
Zoltan_Bind_Param(ZG_params, "GRAPH_BIPARTITE_TYPE", (void *) &bipartite_type);
Zoltan_Bind_Param(ZG_params, "GRAPH_BUILD_TYPE", (void*) &matrix_build_type);
/* Set default values */
strncpy(symmetrization, "NONE", MAX_PARAM_STRING_LEN);
strncpy(bipartite_type, "OBJ", MAX_PARAM_STRING_LEN);
strncpy(weigth_type, "ADD", MAX_PARAM_STRING_LEN);
strncpy(matrix_build_type, "NORMAL", MAX_PARAM_STRING_LEN);
Zoltan_Assign_Param_Vals(zz->Params, ZG_params, zz->Debug_Level, zz->Proc,
zz->Debug_Proc);
Zoltan_Matrix2d_Init(&graph->mtx);
graph->mtx.comm = (PHGComm*)ZOLTAN_MALLOC (sizeof(PHGComm));
if (graph->mtx.comm == NULL) MEMORY_ERROR;
Zoltan_PHGComm_Init (graph->mtx.comm);
memset(&opt, 0, sizeof(Zoltan_matrix_options));
opt.enforceSquare = 1; /* We want a graph: square matrix */
if (!strcasecmp(weigth_type, "ADD"))
opt.pinwgtop = ADD_WEIGHT;
else if (!strcasecmp(weigth_type, "MAX"))
opt.pinwgtop = MAX_WEIGHT;
else if (!strcasecmp(weigth_type, "CMP"))
opt.pinwgtop = MAX_WEIGHT;
opt.pinwgt = 1;
opt.randomize = 0;
opt.local = local;
opt.keep_distribution = 1;
if (strcasecmp(symmetrization, "NONE")) {
opt.symmetrize = 1;
}
if (!strcasecmp(matrix_build_type, "FAST"))
opt.speed = MATRIX_FAST;
else if (!strcasecmp(matrix_build_type, "FAST_NO_DUP"))
opt.speed = MATRIX_NO_REDIST;
else
opt.speed = MATRIX_FULL_DD;
#ifdef CC_TIMERS
times[1] = Zoltan_Time(zz->Timer);
#endif
ierr = Zoltan_Matrix_Build(zz, &opt, &graph->mtx.mtx);
CHECK_IERR;
#ifdef CC_TIMERS
times[2] = Zoltan_Time(zz->Timer);
#endif
ierr = Zoltan_Matrix_Mark_Diag (zz, &graph->mtx.mtx, &diag, &diagarray);
CHECK_IERR;
if (diag) { /* Some Diagonal Terms have to be removed */
ierr = Zoltan_Matrix_Delete_nnz(zz, &graph->mtx.mtx, diag, diagarray);
ZOLTAN_FREE(&diagarray);
CHECK_IERR;
}
#ifdef CC_TIMERS
times[3] = Zoltan_Time(zz->Timer);
#endif
if (opt.symmetrize) {
if (!strcasecmp(symmetrization, "BIPARTITE"))
bipartite = 1;
ierr = Zoltan_Matrix_Sym(zz, &graph->mtx.mtx, bipartite);
CHECK_IERR;
}
#ifdef CC_TIMERS
times[4] = Zoltan_Time(zz->Timer);
#endif
ierr = Zoltan_Distribute_LinearY(zz, graph->mtx.comm);
CHECK_IERR;
#ifdef CC_TIMERS
times[5] = Zoltan_Time(zz->Timer);
MPI_Barrier(zz->Communicator);
#endif
ierr = Zoltan_Matrix2d_Distribute (zz, graph->mtx.mtx, &graph->mtx, 0);
CHECK_IERR;
#ifdef CC_TIMERS
times[6] = Zoltan_Time(zz->Timer);
#endif
ierr = Zoltan_Matrix_Complete(zz, &graph->mtx.mtx);
#ifdef CC_TIMERS
times[7] = Zoltan_Time(zz->Timer);
#endif
if (bipartite) {
int vertlno;
int limit;
int offset;
graph->bipartite = 1;
graph->fixed_vertices = graph->mtx.mtx.ybipart;
/* graph->fixed_vertices = (int*) ZOLTAN_MALLOC(graph->mtx.mtx.nY*sizeof(int)); */
/* if (graph->mtx.mtx.nY && graph->fixed_vertices == NULL) MEMORY_ERROR; */
/* limit = graph->mtx.mtx.offsetY; */
/* /\* What kind of vertices do we want to keep ? *\/ */
/* graph->fixObj = !strcasecmp(bipartite_type, "OBJ"); /\* Non-zero value means "objects" *\/ */
/* offset = graph->mtx.mtx.offsetY - graph->mtx.dist_y[graph->mtx.comm->myProc_y]; */
/* if (graph->fixObj) /\* What kind of vertices do we want to keep ? *\/ */
/* for (vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++ vertlno) */
/* graph->fixed_vertices[vertlno] = (vertlno < offset); */
/* else */
/* for (vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++ vertlno) */
/* graph->fixed_vertices[vertlno] = (vertlno >= offset); */
}
#ifdef CC_TIMERS
MPI_Barrier(zz->Communicator);
times[8] = Zoltan_Time(zz->Timer);
MPI_Reduce(times, gtimes, 9, MPI_DOUBLE, MPI_MAX, 0, zz->Communicator);
if (!zz->Proc) {
int i;
printf("Total Build Time in Proc-0: %.2lf Max: %.2lf\n", times[8]-times[0], gtimes[8]-times[0]);
for (i=1; i<9; ++i)
printf("%-13s in Proc-0: %8.2lf Max: %8.2lf\n", timenames[i], times[i]-times[i-1], gtimes[i]-gtimes[i-1]);
}
#endif
End:
ZOLTAN_FREE(&diagarray);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
