/*
* void Zoltan_Oct_insert_orphan(pRegion region)
*
* Insert orphan regions migrated from off processors, or to insert
* regions that lie on the boundary.
*/
static int Zoltan_Oct_insert_orphan(ZZ *zz, Region reg) {
pRList RootList; /* list of all local roots */
pOctant RootOct;
int rflag; /* flag to indicate region fits in octant */
int i, j; /* index counters */
double upper, /* upper bounds of the octant */
lower; /* lower bounds of the octant */
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
char *yo = "Zoltan_Oct_insert_orphan";
int ierr = ZOLTAN_OK;
if (OCT_info->OCT_dimension == 2)
i = 2; /* ignore z coordinates */
else
i = 3;
rflag = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info); /* get a list all local roots */
while((RootOct = RL_nextRootOctant(&RootList))) {
rflag = 1;
for (j=0; j<i; j++) {
lower = RootOct->min[j];
upper = RootOct->max[j];
if (reg.Coord[j]<lower || reg.Coord[j]>upper) {
/* if region coord lie outside bounds, then cannot fit */
rflag = 0;
break;
}
}
if(rflag == 1) {
/* region fits inside octant */
/* found a place to insert region */
Zoltan_Oct_subtree_insert(zz, RootOct, ®);
return ierr;
}
}
ierr = ZOLTAN_WARN;
ZOLTAN_TRACE_DETAIL(zz, yo, "could not insert region");
fprintf(stderr,"%s failed to insert %f %f %f on proc %d\n",yo, reg.Coord[0], reg.Coord[1], reg.Coord[2], zz->Proc);
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RL_printRootOctants(RootList);
return ierr;
}
int Zoltan_Migrate(
ZZ *zz, /* Zoltan structure. */
int num_import, /* Number of non-local objects assigned to the
processor in the new decomposition. */
ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects
assigned to this processor in the new
decomposition; this field can be NULL if
the application doesn't provide import IDs.*/
ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects
assigned to the processor in the new
decomposition; this field can be NULL if the
application does not provide import IDs. */
int *import_procs, /* Array of processor IDs of processors owning
the non-local objects that are assigned to
this processor in the new decomposition; this
field can be NULL if the application does
not provide import IDs. */
int *import_to_part, /* Array of partition numbers to which imported
objects should be assigned. */
int num_export, /* Number of objs to be exported
to other processors to establish the new
decomposition. */
ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of
objects to be exported to other processors
to establish the new decomposition. */
ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of
objects to be exported to other processors
to establish the new decomposition. */
int *export_procs, /* Array of processor IDs
to which objects will be exported
to establish the new decomposition. */
int *export_to_part /* Array of partition numbers to which exported
objects should be assigned. */
)
{
/*
* Routine to help perform migration. If migration pre-processing routine
* (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn.
* It then calls a function to obtain the size of the migrating objects
* (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified
* object packing routine (ZOLTAN_PACK_OBJ_FN) for each object
* to be exported. It develops the needed communication map to move the
* objects to other processors. It performs the communication according
* to the map, and then calls an application-specified object unpacking
* routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported.
*/
char *yo = "Zoltan_Migrate";
int num_gid_entries, num_lid_entries; /* lengths of global & local ids */
int *sizes = NULL; /* sizes (in bytes) of the object data for export. */
int id_size; /* size (in bytes) of ZOLTAN_GID + padding for
alignment */
int tag_size; /* size (in bytes) of ZOLTAN_GID + one int
(for message size) */
char *export_buf = NULL; /* buffer for packing export data. */
char *import_buf = NULL; /* buffer for receiving imported data. */
char *tmp; /* temporary pointer into buffers. */
int i; /* loop counter. */
int tmp_size; /* size of a single object's data. */
int *idx = NULL; /* index used for multi-fn packs and unpacks. */
int idx_cnt = 0; /* index counter for idx array. */
ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm
buf */
ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass
NULL to query functions when NUM_LID_ENTRIES=0. */
ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */
ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */
int msgtag, msgtag2; /* Tags for communication routines */
int total_send_size; /* Total size of outcoming message (in #items) */
int total_recv_size; /* Total size of incoming message (in #items) */
int aligned_int; /* size of an int padded for alignment */
int dest; /* temporary destination partition. */
int include_parts = 0; /* flag indicating whether partition info is
provided */
int ierr = ZOLTAN_OK;
int actual_num_exp = 0;
int actual_exp_allocated = 0;
ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */
int *actual_exp_procs = NULL; /* are changing partition but not */
int *actual_exp_to_part = NULL; /* processor may not be included. */
int actual_num_imp = 0;
int actual_imp_allocated = 0;
ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */
int *actual_imp_procs = NULL; /* are changing partition but not */
int *actual_imp_to_part = NULL; /* processor may not be included. */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
goto End;
}
/*
* Check that all procs use the same id types.
*/
ierr = check_input(zz,
((num_export >= 0 && export_to_part) ||
(num_import >= 0 && import_to_part)),
&include_parts);
if (ierr != ZOLTAN_OK)
goto End;
num_gid_entries = zz->Num_GID;
num_lid_entries = zz->Num_LID;
/*
* Check that all necessary query functions are available.
*/
if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (num_export >= 0) {
/* Build the actual export arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_export, export_global_ids, export_local_ids,
export_procs, export_to_part,
&actual_num_exp, &actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part,
&actual_exp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on actual exports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs,
zz->Communicator, msgtag, &actual_num_imp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
}
else if (num_import >= 0) {
/* Build the actual import arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_import, import_global_ids, import_local_ids,
import_procs, import_to_part,
&actual_num_imp, &actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part,
&actual_imp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on imports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs,
zz->Communicator, msgtag, &actual_num_exp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
/* Compute actual export lists for packing objects */
if (actual_num_exp > 0) {
actual_exp_allocated = 1;
actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp);
actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp);
actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp);
if (include_parts)
actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp);
if (actual_exp_gids == NULL ||
(num_lid_entries && actual_exp_lids == NULL) ||
actual_exp_procs == NULL ||
(import_to_part != NULL && actual_exp_to_part == NULL)) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
ierr = ZOLTAN_MEMERR;
goto End;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) actual_exp_gids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) actual_exp_lids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part,
(int) sizeof(int), (char *) actual_exp_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
/* Create inverse plan (i.e., plan based on exports) so can set
* variable sizes.
* (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable
* but does not tell how large to allocate receive buffer.
*/
ierr = Zoltan_Comm_Invert_Plan(&imp_plan);
exp_plan = imp_plan;
imp_plan = NULL;
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Migrate.Pre_Migrate_PP != NULL) {
zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Pre_Migrate != NULL) {
zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_FN function.");
goto End;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing");
id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE));
/* Note that alignment is not strictly necessary
when ZOLTAN_ID_TYPE is int or unsigned int. */
aligned_int = Zoltan_Align(sizeof(int));
tag_size = id_size + aligned_int;
/*
* For each object, allow space for its global ID and its data plus
* one int (for the object data size).
* Zoltan will pack the global IDs; the application must pack the data
* through the pack routine. Zoltan needs the global IDs for unpacking,
* as the order of the data received during communication is not
* necessarily the same order as import_global_ids[].
* Zoltan also needs to communicate the sizes of the objects because
* only the sender knows the size of each object.
*/
if (actual_num_exp > 0) {
sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!sizes) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_Size_Multi != NULL) {
zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids, sizes, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE_MULTI function.");
goto End;
}
}
else {
for (i = 0; i < actual_num_exp; i++) {
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]),
lid, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE function.");
goto End;
}
}
}
total_send_size = 0;
for (i = 0; i < actual_num_exp; i++) {
sizes[i] = Zoltan_Align(sizes[i]);
total_send_size += sizes[i] + tag_size;
}
export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char));
if (!export_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Pack_Obj_Multi != NULL) {
/* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */
idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Pack the objects for export.
*/
idx_cnt = 0;
tmp = export_buf;
for (i = 0; i < actual_num_exp; i++) {
/* Pack the object's global ID */
tmp_id = (ZOLTAN_ID_PTR) tmp;
ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries]));
tmp += id_size;
/* Pack the object's size */
*((int *)tmp) = sizes[i];
tmp += aligned_int;
/* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Pack_Obj_Multi != NULL) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n",
zz->Proc, yo);
}
zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids,
(actual_exp_to_part!=NULL ? actual_exp_to_part
: actual_exp_procs),
sizes, idx, export_buf, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ_MULTI function.");
goto End;
}
}
else {
tmp = export_buf + tag_size;
for (i = 0; i < actual_num_exp; i++) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries]));
printf("size = %d bytes\n", sizes[i]);
}
/* Pack the object's data */
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i]
: actual_exp_procs[i]);
zz->Pack_Obj(zz->Pack_Obj_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]), lid, dest,
sizes[i], tmp, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ function.");
goto End;
}
tmp += sizes[i] + tag_size;
}
}
ZOLTAN_FREE(&idx);
tmp_id = NULL;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects");
/* Modify sizes[] to contain message sizes, not object sizes */
for (i=0; i<actual_num_exp; i++) {
sizes[i] += tag_size;
}
msgtag--;
ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize.");
goto End;
}
if (actual_num_imp > 0) {
import_buf = (char *) ZOLTAN_MALLOC(total_recv_size);
if (!import_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Send the export data using the communication plan.
*/
msgtag2 = 32765;
ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
/*
* Free whatever memory we can.
*/
Zoltan_Comm_Destroy(&exp_plan);
ZOLTAN_FREE(&export_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication");
/*
* Perform application-specified processing before unpacking the data.
*/
if (zz->Migrate.Mid_Migrate_PP != NULL) {
zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Mid_Migrate != NULL) {
zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_FN function.");
goto End;
}
}
/*
* Unpack the object data.
*/
if (actual_num_imp > 0) {
if (zz->Unpack_Obj_Multi != NULL) {
/* Allocate and fill input arrays for Unpack_Obj_Multi. */
sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp);
idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
if (!sizes || !tmp_id || !idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
tmp = import_buf;
idx_cnt = 0;
for (i = 0; i < actual_num_imp; i++) {
/* Unpack the object's global ID */
ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp);
tmp += id_size;
/* Unpack the object's size */
sizes[i] = *((int *)tmp);
tmp += aligned_int;
/* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n",
zz->Proc,yo);
}
zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries,
actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr);
ZOLTAN_FREE(&import_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_FREE(&tmp_id);
ZOLTAN_FREE(&idx);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_MULTI_FN.");
goto End;
}
}
else {
tmp = import_buf;
for (i = 0; i < actual_num_imp; i++) {
tmp_size = *((int *)(tmp + id_size));
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp);
printf("size = %d bytes\n", tmp_size);
}
/* Unpack the object's data */
zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries,
(ZOLTAN_ID_PTR) tmp, tmp_size,
tmp + tag_size, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_FN.");
goto End;
}
tmp += (tmp_size + tag_size);
}
ZOLTAN_FREE(&import_buf);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects");
if (zz->Migrate.Post_Migrate_PP != NULL) {
zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Post_Migrate != NULL) {
zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_FN function.");
goto End;
}
}
End:
if (actual_exp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
}
if (actual_imp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part);
}
if (ierr < 0) {
if (exp_plan) Zoltan_Comm_Destroy(&exp_plan);
Zoltan_Multifree(__FILE__, __LINE__, 5,
&import_buf, &tmp_id, &sizes, &idx, &export_buf);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
static int Zoltan_LB(
ZZ *zz,
int include_parts, /* Flag indicating whether to generate
part informtion;
0 if called by Zoltan_LB_Balance,
1 if called by Zoltan_LB_Partition. */
int *changes, /* Set to zero or one depending on if
Zoltan determines a new
decomposition or not:
zero - No changes to the decomposition
were made by the load-balancing
algorithm; migration is not needed.
one - A new decomposition is suggested
by the load-balancer; migration is
needed to establish the new
decomposition. */
int *num_gid_entries, /* The number of array entries in a global ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Global_ID_Entries. */
int *num_lid_entries, /* The number of array entries in a local ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Local_ID_Entries. */
int *num_import_objs, /* The number of non-local objects in the
processor's new decomposition. */
ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
int **import_procs, /* Array of processor IDs for processors
currently owning non-local objects (i.e.,
objs to be imported) in this processor's
new decomposition. */
int **import_to_part, /* Partition to which the objects should be
imported. */
int *num_export_objs, /* The number of local objects that need to
be exported from the processor to establish
the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
int **export_procs, /* Array of destination processor IDs for
objects that need to be exported
to establish the new decomposition. */
int **export_to_part /* Partition to which objects should be
exported. */
)
{
/*
* Main load-balancing routine.
* Input: a Zoltan structure with appropriate function pointers set.
* Output:
* changes
* num_import_objs
* import_global_ids
* import_local_ids
* import_procs
* import_to_part
* num_export_objs
* export_global_ids
* export_local_ids
* export_procs
* export_to_part
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_LB";
int gmax; /* Maximum number of imported/exported objects
over all processors. */
int error = ZOLTAN_OK; /* Error code */
double start_time, end_time;
double lb_time[2] = {0.0,0.0};
char msg[256];
int comm[3],gcomm[3];
float *part_sizes = NULL, *fdummy = NULL;
int wgt_dim, part_dim;
int all_num_obj, i, ts, idIdx;
struct Hash_Node **ht;
int *export_all_procs, *export_all_to_part, *parts=NULL;
ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL;
ZOLTAN_ID_PTR gid;
#ifdef ZOLTAN_OVIS
struct OVIS_parameters ovisParameters;
#endif
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS){
printf("Build configuration:\n");
Zoltan_Print_Configuration(" ");
printf("\n");
Zoltan_Print_Key_Params(zz);
}
start_time = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_OVIS
Zoltan_OVIS_Setup(zz, &ovisParameters);
if (zz->Proc == 0)
printf("OVIS PARAMETERS %s %s %d %f\n",
ovisParameters.hello,
ovisParameters.dll,
ovisParameters.outputLevel,
ovisParameters.minVersion);
ovis_enabled(zz->Proc, ovisParameters.dll);
#endif
/*
* Compute Max number of array entries per ID over all processors.
* Compute Max number of return arguments for Zoltan_LB_Balance.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
comm[2] = zz->LB.Return_Lists;
MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
zz->LB.Return_Lists = gcomm[2];
/* assume no changes */
*changes = 0;
*num_import_objs = *num_export_objs = 0;
*import_global_ids = NULL;
*import_local_ids = NULL;
*import_procs = NULL;
*import_to_part = NULL;
*export_global_ids = NULL;
*export_local_ids = NULL;
*export_procs = NULL;
*export_to_part = NULL;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz))
goto End;
if (zz->LB.Method == NONE) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s Balancing method selected == NONE; no balancing performed\n",
yo);
error = ZOLTAN_WARN;
goto End;
}
/*
* Sync the random number generator across processors.
*/
Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator);
/* Since generating a new partition, need to free old mapping vector */
zz->LB.OldRemap = zz->LB.Remap;
zz->LB.Remap = NULL;
error = Zoltan_LB_Build_PartDist(zz);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN)
goto End;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
int i, np, fp;
for (i = 0; i < zz->Num_Proc; i++) {
Zoltan_LB_Proc_To_Part(zz, i, &np, &fp);
printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n",
zz->Proc, i, np, fp);
}
}
/*
* Generate parts sizes.
*/
#ifdef ZOLTAN_OVIS
/* set part sizes computed by OVIS, if requested. Processes set only their own value */
{
float part_sizes[1];
int part_ids[1], wgt_idx[1];
wgt_idx[0] = 0;
part_ids[0] = 0;
ovis_getPartsize(&(part_sizes[0]));
printf("Rank %d ps %f\n",zz->Proc, part_sizes[0]);
/* clear out old part size info first */
Zoltan_LB_Set_Part_Sizes(zz, 0, -1, NULL, NULL, NULL);
Zoltan_LB_Set_Part_Sizes(zz, 0, 1, part_ids, wgt_idx, part_sizes);
}
#endif
wgt_dim = zz->Obj_Weight_Dim;
part_dim = ((wgt_dim > 0) ? wgt_dim : 1);
part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim
* zz->LB.Num_Global_Parts);
if (part_sizes == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
/* Get part sizes. */
Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim,
part_sizes);
#ifdef ZOLTAN_OVIS
/* if (ovisParameters.outputlevel > 3) */
{
int myRank = zz->Proc;
if (myRank == 0){
int i, j;
for (i = 0; i < zz->LB.Num_Global_Parts; i++){
for (j = 0; j < part_dim; j++){
printf("Rank %d AG: part_sizes[%d] = %f (Num_Global_Parts = %d, part_dim = %d)\n",zz->Proc,
(i*part_dim+j), part_sizes[i*part_dim+j],zz->LB.Num_Global_Parts, part_dim);
}
}
}
}
#endif
/*
* Call the actual load-balancing function.
*/
error = zz->LB.LB_Fn(zz, part_sizes,
num_import_objs, import_global_ids, import_local_ids,
import_procs, import_to_part,
num_export_objs, export_global_ids, export_local_ids,
export_procs, export_to_part);
ZOLTAN_FREE(&part_sizes);
if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){
sprintf(msg, "Partitioning routine returned code %d.", error);
#ifdef HOST_LINUX
if ((error == ZOLTAN_MEMERR) && (Zoltan_Memory_Get_Debug() > 0)){
Zoltan_write_linux_meminfo(0, "State of /proc/meminfo after malloc failure\n", 0);
}
#endif
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
else if (error){
if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) {
sprintf(msg, "Partitioning routine returned code %d.", error);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning");
if (*num_import_objs >= 0)
MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
else /* use export data */
MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
if (gmax == 0) {
/*
* Decomposition was not changed by the load balancing; no migration
* is needed.
*/
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s No changes to the decomposition due to partitioning; "
"no migration is needed.\n", yo);
/*
* Reset num_import_objs and num_export_objs; don't want to return
* -1 for the arrays that weren't returned by ZOLTAN_LB_FN.
*/
*num_import_objs = *num_export_objs = 0;
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){
/*
* This parameter setting requires that all local objects
* and their assignments appear in the export list.
*/
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
goto End;
}
/*
* Check whether we know the import data, export data, or both.
*
* If we were given the import data,
* we know what the new decomposition should look like on the
* processor, but we don't know which of our local objects we have
* to export to other processors to establish the new decomposition.
* Reverse the argument if we were given the export data.
*
* Unless we were given both maps, compute the inverse map.
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) {
if (*num_import_objs >= 0)
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
if (*num_export_objs >= 0)
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
*num_import_objs = *num_export_objs = -1;
}
if (*num_import_objs >= 0){
if (*num_export_objs >= 0) {
/* Both maps already available; nothing to do. */;
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Export lists are requested; compute export map */
error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids,
*import_local_ids, *import_procs,
*import_to_part,
num_export_objs, export_global_ids,
export_local_ids, export_procs,
export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Method returned import lists, but only export lists were desired. */
/* Import lists not needed; free them. */
*num_import_objs = -1;
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
}
}
}
else { /* (*num_import_objs < 0) */
if (*num_export_objs >= 0) {
/* Only export lists have been returned. */
if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Compute import map */
error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids,
*export_local_ids, *export_procs,
*export_to_part,
num_import_objs, import_global_ids,
import_local_ids, import_procs,
import_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Method returned export lists, but only import lists are desired. */
/* Export lists not needed; free them. */
*num_export_objs = -1;
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
}
}
}
else { /* *num_export_objs < 0 && *num_import_objs < 0) */
if (zz->LB.Return_Lists) {
/* No map at all available */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned "
"neither import nor export data.");
error = ZOLTAN_WARN;
goto End;
}
}
}
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/*
* Normally, Zoltan_LB returns in the export lists all local
* objects that are moving off processor, or that are assigned
* to a part on the local processor that is not the
* default part. This setting of Return_Lists requests
* that all local objects be included in the export list.
*/
if (*num_export_objs == 0){
/* all local objects are remaining on processor */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (*num_export_objs) {
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End;
}
else{
all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error);
if (*num_export_objs < all_num_obj){
/* Create a lookup table for exported IDs */
ts = Zoltan_Recommended_Hash_Size(*num_export_objs);
ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts);
/* Create a list of all gids, lids and parts */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj,
&all_global_ids, &all_local_ids,
wgt_dim, &fdummy, &parts);
if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){
ZOLTAN_FREE(&fdummy);
if ((Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_procs,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) ||
(Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_to_part,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){
error = ZOLTAN_MEMERR;
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){
sprintf(msg, "Error building complete export list; "
"%d returned by Zoltan_Get_Obj_List\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
gid = all_global_ids;
for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){
idIdx = search_hash_table(zz, gid, ht, ts);
if (idIdx >= 0){
export_all_procs[i] = (*export_procs)[idIdx];
export_all_to_part[i] = (*export_to_part)[idIdx];
}
else{
export_all_procs[i] = zz->Proc;
export_all_to_part[i] = parts[i];
}
}
free_hash_table(ht, ts);
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
Zoltan_Special_Free(zz, (void **)(void*)&parts,
ZOLTAN_SPECIAL_MALLOC_INT);
*export_global_ids = all_global_ids;
*export_local_ids = all_local_ids;
*export_procs = export_all_procs;
*export_to_part = export_all_to_part;
*num_export_objs = all_num_obj;
}
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments");
end_time = Zoltan_Time(zz->Timer);
lb_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc);
for (i = 0; i < *num_import_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID]));
printf(" To part: %4d",
(*import_to_part != NULL ? (*import_to_part)[i]
: zz->Proc));
printf(" From processor: %4d\n", (*import_procs)[i]);
}
printf("\n");
printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc);
for (i = 0; i < *num_export_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID]));
printf(" To part: %4d",
(*export_to_part != NULL ? (*export_to_part)[i]
: (*export_procs)[i]));
printf(" To processor: %4d\n", (*export_procs)[i]);
}
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/*
* If the Help_Migrate flag is set, perform migration for the application.
*/
if (zz->Migrate.Auto_Migrate) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration");
start_time = Zoltan_Time(zz->Timer);
error = Zoltan_Migrate(zz,
*num_import_objs, *import_global_ids,
*import_local_ids, *import_procs, *import_to_part,
*num_export_objs, *export_global_ids,
*export_local_ids, *export_procs, *export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error in auto-migration; %d returned from "
"Zoltan_Help_Migrate\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
end_time = Zoltan_Time(zz->Timer);
lb_time[1] = end_time - start_time;
ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration");
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0],
"ZOLTAN Partition: ");
if (zz->Migrate.Auto_Migrate)
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1],
"ZOLTAN Migrate: ");
}
*changes = 1;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (error);
}
/*
* void malloc_new_objects();
*
* gets the tags being imported into this processor, and sets up the
* import_tags array, and the nrectags array.
*/
static int malloc_new_objects(ZZ *zz, int nsentags, pRegion exported_tags,
ZOLTAN_ID_PTR exported_gids,
ZOLTAN_ID_PTR exported_lids, int *tag_pids,
int *nstags, pRegion *ex_tags,
pRegion prev_tags, ZOLTAN_ID_PTR prev_gids,
ZOLTAN_ID_PTR prev_lids, int npimtags,
float *c3)
{
char *yo = "malloc_new_objects";
int i; /* index counter */
int nreceives; /* number of messages received */
pRegion t_b_exp; /* array of tags to be exported */
pRegion tmp = NULL;
ZOLTAN_ID_PTR tmp_gids = NULL;
ZOLTAN_ID_PTR tmp_lids = NULL;
int msgtag, msgtag2;
int j;
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
float im_load;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
im_load = 0;
msgtag = 32767;
ierr = Zoltan_Comm_Create(&comm_plan, nsentags, tag_pids, zz->Communicator,
msgtag, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_Comm_Create.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (nreceives > 0) {
tmp = (pRegion) ZOLTAN_MALLOC(nreceives * sizeof(Region));
tmp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, nreceives);
tmp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, nreceives);
if(tmp == NULL || !tmp_gids || (num_lid_entries && !tmp_lids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_tags,
sizeof(Region), (char *) tmp);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_gids,
sizeof(ZOLTAN_ID_TYPE)*num_gid_entries,
(char *) tmp_gids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
fprintf(stderr, "OCT %s Error %s returned from Zoltan_Comm_Do\n", yo,
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
if (num_lid_entries > 0) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_lids,
sizeof(ZOLTAN_ID_TYPE)*num_lid_entries,
(char *) tmp_lids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_Comm_Destroy.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
return(ierr);
}
/* get each message sent, and store region in import array */
j=0;
for (i=0; i<nreceives; i++) {
im_load += tmp[i].Weight;
if(tmp[i].newProc != zz->Proc) {
j++;
}
}
if((j + npimtags) != 0) { /* malloc import array */
if((t_b_exp = (pRegion)ZOLTAN_MALLOC((j+npimtags)*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
}
else
t_b_exp = NULL;
/* setup return pointer */
(*ex_tags) = t_b_exp;
j=0;
for (i=0; i<nreceives; i++) {
if(tmp[i].newProc != zz->Proc) {
t_b_exp[j] = tmp[i];
t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz);
if (!(t_b_exp[j].Global_ID) ||
(num_lid_entries && !(t_b_exp[j].Local_ID))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
&(tmp_gids[i*num_gid_entries]));
ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(tmp_lids[i*num_lid_entries]));
j++;
}
}
if(npimtags > 0) {
for(i=0; i<npimtags; i++) {
t_b_exp[j] = prev_tags[i];
t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz);
if (!(t_b_exp[j].Global_ID) ||
(num_lid_entries && !(t_b_exp[j].Local_ID))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
&(prev_gids[i*num_gid_entries]));
ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID,
&(prev_lids[i*num_lid_entries]));
j++;
}
}
*nstags = j;
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
if((*nstags == 0) && (*ex_tags != NULL)) {
ZOLTAN_TRACE_DETAIL(zz, yo,
"Fatal error, import tags not empty but no tags received\n");
return ZOLTAN_FATAL;
}
*c3 = im_load;
return ierr;
}
/*
* void tag_regions()
* Iterates through the list of octants on the processor and finds which
* are to be migrated. It then looks at the region list for those octants
* and stores the migrating regions into the export_tags array.
*/
static int tag_regions(ZZ *zz,
pOctant *octs,
int *newpids,
int nocts,
Region **exported_tags,
ZOLTAN_ID_PTR *exported_gids,
ZOLTAN_ID_PTR *exported_lids,
int *nsentags,
int **tag_pids,
Region **p_tags,
ZOLTAN_ID_PTR *p_gids,
ZOLTAN_ID_PTR *p_lids,
int *npimtags,
float *c2,
int *max_objs)
{
char *yo = "tag_regions";
int i; /* index counter */
pRegion regionlist; /* list of region on this processor */
int index; /* index counter */
int index2; /* yet another index counter */
int count; /* count of objects exported form this processor */
int count2; /* count of objects that are kept on processor */
int count3;
int *exported_pids; /* array of pids where regions are being exported to */
pRegion mtags; /* object tags of objects to be migrated */
pRegion ptags; /* tags of objects that were previously migrated */
float ex_load;
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
ex_load = 0;
(*max_objs) = 0;
if (!nsentags)
return ierr;
/* find how many objects have been exported */
count = 0;
/* find number of local objs to export */
count2 = 0;
count3 = 0;
for (i=0; i<nocts; i++) {
if(Zoltan_Oct_isTerminal(octs[i])) {
(*max_objs) += Zoltan_Oct_nRegions(octs[i]);
regionlist = Zoltan_Oct_regionlist(octs[i]);
while(regionlist != NULL) {
count3++;
if(regionlist->Proc != zz->Proc) {
count++;
if(newpids[i] != zz->Proc)
regionlist->newProc = newpids[i];
else
regionlist->newProc = zz->Proc;
}
else {
if(newpids[i] != zz->Proc) {
count2++;
regionlist->newProc = newpids[i];
}
else
regionlist->newProc = zz->Proc;
}
regionlist = regionlist->next; /* get next region */
}
}
}
#if 0
{
{
if (newpids[i]!=zz->Proc) {
count+=Zoltan_Oct_nRegions(octs[i]);
}
else {
pRegion regions;
regions = Zoltan_Oct_regionlist(octs[i]);
while(regions != NULL) {
if(regions->Proc != zz->Proc)
count2++;
regions = regions->next;
}
}
}
}
#endif
/* set up the return pointers */
*nsentags = count;
*npimtags = count2;
if (!exported_tags) {
return ierr;
}
if (count > 0) {
/* allocate some space */
if((mtags=(pRegion)ZOLTAN_MALLOC((unsigned)count*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((exported_pids = (int *)ZOLTAN_MALLOC((unsigned)count*sizeof(int))) ==
NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
return ZOLTAN_MEMERR;
}
*exported_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count);
*exported_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count);
if(!(*exported_gids) || (num_lid_entries && !(*exported_lids))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
return ZOLTAN_MEMERR;
}
}
else {
mtags = NULL;
exported_pids = NULL;
*exported_gids = NULL;
*exported_lids = NULL;
}
/* set up return pointers */
*exported_tags=mtags;
*tag_pids = exported_pids;
if (count2 > 0) {
/* allocate some space */
if((ptags=(pRegion)ZOLTAN_MALLOC((unsigned)count2*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
ZOLTAN_FREE(exported_gids);
ZOLTAN_FREE(exported_lids);
return ZOLTAN_MEMERR;
}
*p_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count2);
*p_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count2);
if(!(*p_gids) || (num_lid_entries && !(*p_lids))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
ZOLTAN_FREE(exported_gids);
ZOLTAN_FREE(exported_lids);
ZOLTAN_FREE(&ptags);
ZOLTAN_FREE(p_gids);
ZOLTAN_FREE(p_lids);
return ZOLTAN_MEMERR;
}
}
else {
ptags = NULL;
*p_gids = NULL;
*p_lids = NULL;
}
/* set up return pointers */
*p_tags=ptags;
index = index2 = 0;
for (i=0; i<nocts; i++) {
if(Zoltan_Oct_isTerminal(octs[i])) {
regionlist = Zoltan_Oct_regionlist(octs[i]);
while(regionlist != NULL) {
if(regionlist->Proc != zz->Proc) {
/* place information in the appropritate array */
mtags[index] = *regionlist;
ZOLTAN_SET_GID(zz, &((*exported_gids)[index*num_gid_entries]),
regionlist->Global_ID);
ZOLTAN_SET_LID(zz, &((*exported_lids)[index*num_lid_entries]),
regionlist->Local_ID);
/*ex_load += (float)(regionlist->Weight);*/
exported_pids[index] = regionlist->Proc;
index++; /* increment counter */
}
else if(newpids[i] != zz->Proc) {
ptags[index2] = *regionlist; /* get region information */
ZOLTAN_SET_GID(zz, &((*p_gids)[index2*num_gid_entries]),
regionlist->Global_ID);
ZOLTAN_SET_LID(zz, &((*p_lids)[index2*num_lid_entries]),
regionlist->Local_ID);
index2++; /* increment counter */
}
regionlist = regionlist->next; /* get next region */
}
}
}
if (index!=count) { /* error check */
ZOLTAN_TRACE_DETAIL(zz, yo,
"Fatal error, inconsistent number of regions.\n");
return ZOLTAN_FATAL;
}
*c2 = ex_load;
return ierr;
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int *num_gid_entries, /* # of entries for a global id */
int *num_lid_entries, /* # of entries for a local id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm, /* inverse permutation of rank */
ZOS *order_info /* Method-specific ordering info. Currently not used. */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* iperm, inverse permutation of rank
* order_info, a Zoltan Ordering Struct with additional info.
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
int *vtxdist;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strncpy(opt.order_type, "GLOBAL", MAX_PARAM_STRING_LEN);
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
if (opt.use_order_info == 0) order_info = NULL;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && lids && rank && iperm)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL){
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
*/
ierr = (*Order_fn)(zz, num_obj, gids, lids, rank, iperm, &opt, order_info);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN){
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr){
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK)){
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, iperm, rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM)){
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, rank, iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i, nobjs;
nobjs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &i);
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < nobjs; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(*num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
printf("ZOLTAN: inverse permutation on Proc %d\n", zz->Proc);
for (i = 0; i < nobjs; i++) {
printf("iperm[%3d] = %3d\n", i, iperm[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
int Zoltan_Invert_Lists(
ZZ *zz, /* Zoltan structure. */
int num_in, /* Number of objects in the input lists. */
ZOLTAN_ID_PTR in_global_ids, /* Array of input global IDs. */
ZOLTAN_ID_PTR in_local_ids, /* Array of input local IDs. */
int *in_procs, /* Array of processor IDs of processors owning
the input objects. */
int *in_to_part, /* Optional: Array of partition numbers to
which input objects should be assigned. */
int *num_out, /* Returned value: Number of output objs. */
ZOLTAN_ID_PTR *out_global_ids,/* Returned value: Array of global IDs of
output objects. */
ZOLTAN_ID_PTR *out_local_ids, /* Returned value: Array of local IDs of
output objects. */
int **out_procs, /* Returned value: Array of processor IDs
to which output objects are assigned. */
int **out_to_part /* Optional: Returned value: Array of
partition numbers to which output
objects should be assigned. */
)
{
/*
* Routine to compute the inverse map. Can be used in two ways:
* 1. Given, for each processor, a list of objects to be received by the
* processor, compute the list of objects that the processor needs to send
* to other processors to satisfy their needs.
* 2. Given, for each processor, a list of objects to be sent to other
* processors, compute the list of objects that the processor needs to receive
* to satisfy its needs.
*/
char *yo = "Zoltan_Invert_Lists";
char msg[256];
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned communication routines */
int msgtag, msgtag2; /* Message tags for communication routines */
int num_gid_entries, num_lid_entries; /* Length of global and local ids */
int include_parts; /* Flag indicating whether to compute
inverse list for partitions. */
int ierr, ret_ierr = ZOLTAN_OK;
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Check that all procs use the same id types.
*/
ierr = check_invert_input(zz, num_in, in_procs, in_to_part,
&num_gid_entries, &num_lid_entries, &include_parts);
if (ierr != ZOLTAN_OK) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
/* Initialize returned arrays. */
*out_global_ids = NULL;
*out_local_ids = NULL;
*out_procs = NULL;
if (include_parts) *out_to_part = NULL;
/*
* Compute communication map and num_out, the number of objs this
* processor has to out to establish the new decomposition.
*/
msgtag = 32767;
ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator,
msgtag, num_out);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Create.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
goto End;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create");
/*
* Allocate space for the object tags that need to be outed. Communicate
* to get the list of objects to be outed.
*/
if (*num_out > 0) {
if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out,
ZOLTAN_SPECIAL_MALLOC_GID)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out,
ZOLTAN_SPECIAL_MALLOC_LID)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (include_parts) {
if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
}
}
/*
* Use the communication plan to send global IDs, local IDs, and processor
* numbers. Do in separate communications to avoid a memory copy and to
* simplify implementation when a data type is added to the comm. package
* (to support heterogeneous computing).
*/
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) *out_global_ids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) *out_local_ids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
}
Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, *out_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part,
(int) sizeof(int), (char *) *out_to_part);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do");
End:
Zoltan_Comm_Destroy(&comm_plan);
if (ret_ierr == ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT);
if (include_parts)
Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ret_ierr);
}
int Zoltan_HG_Create_Mirror (
ZZ *zz,
HGraph *hg
)
{
int inlength, outlength; /* input/output array lengths */
int *index, *data; /* pointers to input information */
int *outindex, *outdata;
char *yo = "Zoltan_HG_Create_Mirror";
ZOLTAN_TRACE_ENTER(zz, yo);
/* determine which data to "mirror" and set corresponding data pointers. */
if (hg && (hg->nEdge == 0 || hg->hindex) && (hg->nPins == 0 || hg->hvertex)
&& !hg->vindex && !hg->vedge) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Have hindex; building vindex.");
inlength = hg->nEdge;
outlength = hg->nVtx;
index = hg->hindex;
data = hg->hvertex;
outindex = hg->vindex = (int*) ZOLTAN_MALLOC((hg->nVtx+1) * sizeof(int));
outdata = hg->vedge = (int*) ZOLTAN_MALLOC (hg->nPins * sizeof(int));
if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) {
Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->vindex, &hg->vedge);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
else if (hg && (hg->nVtx == 0 || hg->vindex) && (hg->nPins == 0 || hg->vedge)
&& !hg->hindex && !hg->hvertex) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Have vindex; building hindex.");
inlength = hg->nVtx;
outlength = hg->nEdge;
index = hg->vindex;
data = hg->vedge;
outindex = hg->hindex = (int*) ZOLTAN_MALLOC((hg->nEdge+1) * sizeof(int));
outdata = hg->hvertex = (int*) ZOLTAN_MALLOC(hg->nPins * sizeof(int));
if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) {
Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->hindex, &hg->hvertex);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
else {
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input error.");
return ZOLTAN_FATAL; /* unable to proceed */
}
Zoltan_HG_Mirror(inlength, index, data,
outlength, outindex, outdata);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_OK;
}
int Zoltan_ParMetis_Order(
ZZ *zz, /* Zoltan structure */
int num_obj, /* Number of (local) objects to order. */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */
int *iperm,
ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */
)
{
static char *yo = "Zoltan_ParMetis_Order";
int i, n, ierr;
ZOLTAN_Output_Order ord;
ZOLTAN_Third_Graph gr;
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't want to risk letting external
packages changing our communicator */
#endif
indextype numflag = 0;
int timer_p = 0;
int get_times = 0;
int use_timers = 0;
double times[5];
ZOLTAN_ID_PTR l_gids = NULL;
ZOLTAN_ID_PTR l_lids = NULL;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
ZOLTAN_TRACE_ENTER(zz, yo);
#ifdef ZOLTAN_PARMETIS
#if TPL_USE_DATATYPE != TPL_METIS_DATATYPES
#ifdef TPL_FLOAT_WEIGHT
i = 1;
#else
i = 0;
#endif
if ((sizeof(indextype) != sizeof(idxtype)) ||
(sizeof(weighttype) != sizeof(idxtype)) || i){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,
"Not supported: Multiple 3rd party libraries with incompatible "
"data types.");
return ZOLTAN_FATAL;
}
#endif
#endif
memset(&gr, 0, sizeof(ZOLTAN_Third_Graph));
memset(&ord, 0, sizeof(ZOLTAN_Output_Order));
memset(times, 0, sizeof(times));
ord.order_opt = order_opt;
if (!order_opt){
/* If for some reason order_opt is NULL, allocate a new ZOOS here. */
/* This should really never happen. */
order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
strcpy(order_opt->method,"PARMETIS");
}
ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord);
/* ParMetis only computes the rank vector */
order_opt->return_args = RETURN_RANK;
/* Check that num_obj equals the number of objects on this proc. */
/* This constraint may be removed in the future. */
n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){
/* Return error code */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
return(ZOLTAN_FATAL);
}
if (n != num_obj){
/* Currently this is a fatal error. */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the "
"number of objects.");
return(ZOLTAN_FATAL);
}
/* Do not use weights for ordering */
gr.obj_wgt_dim = -1;
gr.edge_wgt_dim = -1;
gr.num_obj = num_obj;
/* Check what ordering type is requested */
if (order_opt){
SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */
#ifdef ZOLTAN_PARMETIS
if ((strcmp(order_opt->method, "METIS") == 0))
#endif /* ZOLTAN_PARMETIS */
SET_LOCAL_GRAPH(&gr.graph_type);
}
gr.get_data = 1;
if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: "
"set ParMetis instead.");
return(ZOLTAN_FATAL);
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
return (ierr);
}
/* Allocate space for separator sizes */
if (IS_GLOBAL_GRAPH(gr.graph_type)) {
if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) {
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype));
if (ord.sep_sizes == NULL) {
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */
}
/* Allocate space for direct perm */
ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.rank){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
if (IS_LOCAL_GRAPH(gr.graph_type)){
/* Allocate space for inverse perm */
ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
if (!ord.iperm){
/* Not enough memory */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
}
else
ord.iperm = NULL;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_PARMETIS
if (IS_GLOBAL_GRAPH(gr.graph_type)){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library");
ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy,
&numflag, options, ord.rank, ord.sep_sizes, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else
#endif /* ZOLTAN_PARMETIS */
#if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS)
if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */
indextype numobj = gr.num_obj;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library");
order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0; /* Use default options for METIS. */
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options,
ord.iperm, ord.rank);
#else
METIS_SetDefaultOptions(options);
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, NULL, options,
ord.iperm, ord.rank); /* NULL is vwgt -- new interface in v4 */
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */
int numbloc;
int start;
int leaf;
int *converttab;
int levelmax;
levelmax = mylog2(zz->Num_Proc) + 1;
converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int));
memset(converttab, 0, zz->Num_Proc*2*sizeof(int));
/* Determine the first node in each separator, store it in zz->TPL_Order.start */
for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) {
int father;
father = zz->Num_Proc + numbloc;
converttab[start] = 2*numbloc;
zz->TPL_Order.leaves[leaf++]=start;
zz->TPL_Order.ancestor[start] = start + 2;
converttab[start+1] = 2*numbloc+1;
zz->TPL_Order.leaves[leaf++]=start+1;
zz->TPL_Order.ancestor[start+1] = start + 2;
start+=2;
do {
converttab[start] = father;
if (father %2 == 0) {
int nextoffset;
int level;
level = mylog2(2*zz->Num_Proc - 1 - father);
nextoffset = (1<<(levelmax-level));
zz->TPL_Order.ancestor[start] = start+nextoffset;
start++;
break;
}
else {
zz->TPL_Order.ancestor[start] = start+1;
start++;
father = zz->Num_Proc + father/2;
}
} while (father < 2*zz->Num_Proc - 1);
}
zz->TPL_Order.start[0] = 0;
zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1;
for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) {
int oldblock=converttab[numbloc-1];
zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock];
}
ZOLTAN_FREE(&converttab);
ZOLTAN_FREE(&ord.sep_sizes);
zz->TPL_Order.leaves[zz->Num_Proc] = -1;
zz->TPL_Order.nbr_leaves = zz->Num_Proc;
zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1;
}
else { /* No tree */
zz->TPL_Order.nbr_blocks = 0;
zz->TPL_Order.start = NULL;
zz->TPL_Order.ancestor = NULL;
zz->TPL_Order.leaves = NULL;
}
/* Correct because no redistribution */
memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE));
memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE));
ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL);
ZOLTAN_FREE(&l_gids);
ZOLTAN_FREE(&l_lids);
/* Get a time here */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){
memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i];
}
}
if ((ord.iperm != NULL) && (iperm != NULL)){
if (sizeof(indextype) == sizeof(int)){
memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype));
}
else{
for (i=0; i < gr.num_obj; i++){
iperm[i] = (int)ord.iperm[i];
}
}
}
if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm);
ZOLTAN_FREE(&ord.rank);
/* Free all other "graph" stuff */
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
realtype itr = 0.0;
indextype options[MAX_PARMETIS_OPTIONS];
char alg[MAX_PARAM_STRING_LEN];
#ifdef ZOLTAN_PARMETIS
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
/* change our zz struct. */
#endif
indextype i;
realtype *imb_tols;
indextype ncon;
indextype edgecut;
indextype wgtflag;
indextype numflag = 0;
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
ZOLTAN_TRACE_ENTER(zz, yo);
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
if (sizeof(realtype) != sizeof(float)) {
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));
for (i = 0; i < tmp; i++)
prt.input_part_sizes[i] = (realtype) part_sizes[i];
/* KDD 2/2014: removed re-scaling part sizes so they sum to one.
* part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
* plus, the code here was wrong for multiple object weights.
* similar scaling code did not exist in the Scotch interface.
*/
prt.part_sizes = prt.input_part_sizes;
}
else
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
indextype j;
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
/* Now we can call ParMetis */
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
indextype ndims = geo->ndims;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
"ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
/* Use default options for METIS */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
options[0] = 0;
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
#else
METIS_SetDefaultOptions(options);
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
prt.part_sizes, imb_tols, options,
&edgecut, prt.part);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) {
/* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
ZOLTAN_FREE(&global_ids);
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/*
* void Zoltan_Oct_gen_tree_from_input_data()
*
* This function will create a root node of on each processor which will
* then be used to create an octree with regions associated with it. The
* tree will then be balanced and the output used to balance "mesh regions"
* on several processors.
*/
static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag,
int *c1, int *c2, int *c3,
float *c0, int createpartree)
{
char *yo = "Zoltan_Oct_gen_tree_from_input_data";
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
COORD min, /* min coord bounds of objects */
max; /* max coord bounds of objects */
int num_extra; /* number of orphaned objects */
int num_objs; /* total number of local objects */
pRegion ptr, /* pointer to iterate trough region list */
ptr1; /* pointer to iterate trough region list */
pOctant root; /* root of the partition tree */
int i; /* index counter */
int count, /* count for leaf nodes in partition tree */
proc, /* proc leaf node of parition tree belongs to */
extra, /* extra leaf node flag, if not evenly divisible */
remainder; /* remainder of node, or processors to fill */
pOctant cursor, /* cursor to iterate through octant list */
cursor2, /* another cursor to iterate through octant list */
parent; /* parent of an octant */
int level, /* number of levels of refinement */
n, /* index counter */
part; /* partition counter */
Map *array; /* map of which processors own which octants */
int hold; /* used for calculating partition divisions */
int ierr = 0;
#ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX};
COORD global_min, global_max;
#endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */
int nroots = 0;
/*test*/
/* COORD gmin,gmax; */
OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* If there are no objects on this processor, do not create a root octant.
* The partitioner will probably assign objects to this processor
*/
if(zz->Get_Num_Obj == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register ZOLTAN_NUM_OBJ_FN function");
abort();
}
*c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr) {
fprintf(stderr, "OCT [%d] %s: Error returned from user defined "
"Get_Num_Obj function.\n", zz->Proc, yo);
exit (-1);
}
ptr1 = NULL;
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds");
/* Need A Function To Get The Bounds Of The Local Objects */
Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0);
#ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
/* For now, don't want to add the new query function to Zoltan. */
/* Zoltan_Oct_get_bounds appears to compute the global min and max from */
/* the object input. */
vector_set(OCT_info->OCT_gmin, min);
vector_set(OCT_info->OCT_gmax, max);
#else
/*test*/
/*getMaxBounds(&gmin, &gmax);*/
if(zz->Get_Bounds_Geom == NULL) {
fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
"Must register Get_Bounds_Geom function");
abort();
}
zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr);
MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3,
MPI_DOUBLE, MPI_MIN, zz->Communicator);
MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3,
MPI_DOUBLE, MPI_MAX, zz->Communicator);
vector_set(OCT_info->OCT_gmin, global_min);
vector_set(OCT_info->OCT_gmax, global_max);
#endif
/*
* the following code segment was added to create a pseudo global octree
* needed for the partitioner. The basic idea is to regroup all the
* regions into something close to an octree partitioning and build the
* tree from that.
* NOTE: This way of doing things is very costly, especially when calling
* this for the first time on a mesh not partitioned in an octree style
* partitioning.
*/
level = 0; /* initialize level count */
/*
* if more than 1 processor, need to find what level of refinement needed
* to initially partition bounding box among the processors
*/
if(zz->Num_Proc > 1) {
n = zz->Num_Proc;
if(OCT_info->OCT_dimension == 2)
hold = 4;
else
hold = 8;
remainder = hold;
for(; remainder > 0; level++) {
int pr = (int)POW(hold, level);
remainder = n - pr;
}
level--;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree");
if(createpartree) {
/* create the global root octant */
root = Zoltan_Oct_POct_new(OCT_info);
Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax);
/* Zoltan_Oct_setOrientation(root, 0); */
/* subdivide to as many levels as calculated */
for(i=0; i<level; i++) {
cursor = root;
while(cursor != NULL) {
if(Zoltan_Oct_isTerminal(cursor)) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
Zoltan_Oct_terminal_refine(zz, cursor, 0);
cursor = cursor2;
}
else
cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
}
}
#if 0
if(zz->Proc == 0)
for(i=0; i<8; i++)
if(Zoltan_Oct_child(root, i) == NULL)
fprintf(stderr,"NULL child pointer\n");
else
fprintf(stderr, "child %d exists\n", i);
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array");
/* this part creates the map array */
if(OCT_info->OCT_dimension == 2) {
hold = (int)POW(4, level); /* ignoring the z+ octants */
if(hold == 0)
hold = 1;
}
else
hold = (int)POW(8, level);
part = hold / zz->Num_Proc; /* how many octants per partition */
remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */
extra = zz->Num_Proc - remainder;/* where to start adding extra octants */
array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */
if(array == NULL) {
fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n",
zz->Proc);
abort();
}
/* initialize variables */
proc = 0;
count = 0;
i = 0;
cursor = root;
while(cursor != NULL) {
cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) {
if(proc == extra) {
part++;
extra = -1;
}
if(count != part) {
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
count++;
}
else {
count = 1;
proc++;
array[i].npid = proc;
array[i].list = RL_initRootList();
Zoltan_Oct_bounds(cursor, min, max);
vector_set(array[i].min, min);
vector_set(array[i].max, max);
}
if(proc == zz->Proc) {
array[i].npid = -1;
/* KDDKDD Added RL_freeList below. The
* KDDKDD implementation from RPI leaked memory because the
* KDDKDD test cases for setting array[i].list were not mutually
* KDDKDD exclusive. Freeing the list produces the result we got
* KDDKDD before, without the memory leak.
*/
/* LGG -- it seems to me that this array[i].list assignment is
* not really necessary. It looks as though it has already been
* assigned with the same information from the prev if-else
* commented out RL_freeList(), and RL_initRootList()
*/
/*RL_freeList(&(array[i].list));*/
/* KDDKDD End addition */
/*array[i].list = RL_initRootList();*/
parent = Zoltan_Oct_parent(cursor);
if(parent != NULL)
Zoltan_Oct_setchild(parent, cursor->which, NULL);
/* octant into local root list */
Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1);
Zoltan_Oct_setMapIdx(cursor, i);
nroots++;
/* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc);
octant into local root list */
}
i++;
}
cursor = cursor2;
}
RootList = Zoltan_Oct_POct_localroots(OCT_info);
RootOct = RL_nextRootOctant(&RootList);
if(RootOct != root) {
/* KDDKDDFREE changed root to &root to allow root to be reset to NULL */
Zoltan_Oct_POct_delTree(OCT_info,&root);
}
OCT_info->map = array;
OCT_info->mapsize = hold;
}
/*
* attach the regions to the root... Zoltan_Oct_fix will create the octree
* starting with the root and subdividing as needed
*/
num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans");
Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map,
c1, c2);
/* ZOLTAN_FREE(&array); */
while(ptr1 != NULL) {
ptr = ptr1->next;
ZOLTAN_FREE(&(ptr1->Global_ID));
ZOLTAN_FREE(&(ptr1->Local_ID));
ZOLTAN_FREE(&ptr1);
ptr1 = ptr;
}
ZOLTAN_TRACE_EXIT(zz, yo);
}
/*
* void lb_oct_init();
*
* initialize the calls needed to start the octree load balancing rounties
*/
static int lb_oct_init(
ZZ *zz, /* The Zoltan structure with info for
the OCTPART balancer. */
int *num_export, /* Number of non-local objs assigned to this
processor in the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition. */
int **export_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition. */
int **export_to_part, /* Returned value: array of partitions to
which objects are imported.
KDDKDD Assume #parts==#procs. */
int oct_dim, /* Dimension of method (2D or 3D) */
int oct_method, /* Flag specifying curve to be used. */
int oct_maxoctregions, /* max # of objects in leaves of octree. */
int oct_minoctregions, /* min # of objects in leaves of octree. */
int oct_output_level, /* Flag specifying amount of output. */
int oct_wgtflag, /* Flag specifying use of object weights. */
float *part_sizes /* Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
)
{
char *yo = "lb_oct_init";
OCT_Global_Info *OCT_info;
int nsentags; /* number of tags being sent */
pRegion export_regs; /* */
int nrectags; /* number of tags received */
int kk;
double time1,time2; /* timers */
double timestart,timestop; /* timers */
double timers[4]; /* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
int counters[6]; /* diagnostic counts
0 = # of median iterations
1 = # of objects sent
2 = # of objects received
3 = most objects this proc ever owns
*/
float c[4];
int createpartree = 0;
/*int num_gid_entries = zz->Num_GID;*/
/*int num_lid_entries = zz->Num_LID;*/
ZOLTAN_TRACE_ENTER(zz, yo);
MPI_Barrier(zz->Communicator);
timestart = MPI_Wtime();
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = 0;
counters[4] = 0;
counters[5] = 0;
c[0] = 0;
c[1] = 0;
c[2] = 0;
c[3] = 0;
timers[1] = 0.0;
timers[2] = 0.0;
timers[3] = 0.0;
nsentags = nrectags = 0;
if(zz->LB.Data_Structure == NULL) {
OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim);
Zoltan_Oct_set_method(OCT_info, oct_method);
Zoltan_Oct_set_maxregions(oct_maxoctregions);
Zoltan_Oct_set_minregions(oct_minoctregions);
createpartree = 1;
}
else {
OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
}
/* create the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data");
Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1],
&counters[2], &counters[3], &c[0],
createpartree);
time2 = MPI_Wtime();
timers[0] = time2 - time1; /* time took to create octree */
/* Zoltan_Oct_POct_printResults(OCT_info); */
/* partition the octree structure */
time1 = MPI_Wtime();
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition");
/* old call to dfs_paritition: */
#if 0
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]);
#else
/***************************
if(zz->Proc == 0) {
int debug_i;
for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) {
fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]);
}
}
****************************/
Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes);
#endif
time2 = MPI_Wtime();
timers[1] = time2 - time1; /* time took to partition octree */
if (oct_output_level > 2) {
Zoltan_Oct_Plots(zz);
}
/* set up tags for migrations */
time1 = MPI_Wtime();
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate");
Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags,
&c[2], &c[3], &counters[3], &counters[5]);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags");
if (zz->LB.Return_Lists) {
*num_export = nrectags;
if (nrectags > 0)
Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids,
export_procs, export_to_part, nrectags,
export_regs);
}
time2 = MPI_Wtime();
timers[2] = time2 - time1; /* time took to setup migration */
#if 0 /* KDDKDD -- Count is never used; why is it computed? */
{
/* count the number of objects on this processor */
pRList RootList; /* list of all local roots */
pOctant RootOct; /* root octree octant */
int count = 0;
RootList = Zoltan_Oct_POct_localroots(OCT_info);
while((RootOct = RL_nextRootOctant(&RootList))) {
while(RootOct) {
if(Zoltan_Oct_isTerminal(RootOct)) {
count += Zoltan_Oct_nRegions(RootOct);
}
RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
}
}
}
#endif
counters[4] = nsentags;
MPI_Barrier(zz->Communicator);
timestop = MPI_Wtime();
if (oct_output_level > 0) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats");
Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c,
oct_output_level);
}
for (kk = 0; kk < nrectags; kk++) {
ZOLTAN_FREE(&(export_regs[kk].Global_ID));
ZOLTAN_FREE(&(export_regs[kk].Local_ID));
}
ZOLTAN_FREE(&export_regs);
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear");
Zoltan_Oct_global_clear(OCT_info);
/* KDDKDD Don't understand how re-used octree will work, especially without
* KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything;
* KDDKDD we can move back to saving some of the tree later.
*/
Zoltan_Oct_Free_Structure(zz);
/* KDDKDD END */
/* Temporary return value until error codes are fully implemented. */
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int num_gid_entries, /* # of entries for a global id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm /* iperm[rank[i]]=i, only for sequential ordering */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
int * vtxdist = NULL;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL, *local_iperm=NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value");
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.rank = rank;
zz->Order.iperm = iperm;
zz->Order.gids = gids;
zz->Order.lids = lids;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
#ifdef HAVE_MPI
strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN);
#else
strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN);
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
zz->Order.start_index = opt.start_index;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && rank)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
/* strcpy(opt.order_type, "GLOBAL"); */
}
#endif /* ZOLTAN_SCOTCH */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
if (!strcmp(opt.order_type, "GLOBAL"))
strcpy (opt.order_type, "DIST");
if (!strcmp(opt.order_type, "LOCAL"))
strcpy (opt.order_type, "SERIAL");
strcpy(zz->Order.order_type, opt.order_type);
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 3,
&local_gids, &local_rank, &local_iperm);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
if ((!(opt.return_args & RETURN_RANK) && (rank != NULL))
|| (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) {
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr) {
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) {
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) {
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* I store : GNO, rank, iperm */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
ZOLTAN_FREE(&local_iperm);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
int Zoltan_Order (
struct Zoltan_Struct *zz,
int num_gid_entries,
int num_obj,
ZOLTAN_ID_PTR gids,
ZOLTAN_ID_PTR permuted_global_ids
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids.
* num_gid_entries
* Output:
* permuted_global_ids
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
char msg[253];
sprintf(msg, "num_gid_entries=%d is not equal to parameter setting "
"NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
#ifdef HAVE_MPI
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "GLOBAL");
#else
strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "LOCAL");
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
/*
* Check that the user has allocated space for the return args.
*/
if (num_obj && !(gids && permuted_global_ids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "LOCAL_HSFC"))
{
Order_fn = Zoltan_LocalHSFC_Order;
strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PTSCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_SCOTCH */
#ifdef ZOLTAN_HUND
else if (!strcasecmp(opt.method, "HUND")) {
ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL);
goto End;
}
#endif /* ZOLTAN_HUND */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/* TODO : Ask why useful ! */
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/************************************
* Check for required query function
************************************/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
/* TODO allocate all this stuff with the graph */
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 2,
&local_gids, &local_rank);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* TODO: At this time, I consider rank == permuted_global_ids */
/* I store : GNO, rank, permuted GID */
/* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0);
/* Hope a linear assignment will help a little */
if (local_num_obj)
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", permuted_global_ids[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
#ifdef ZOLTAN_HUND
End:
#endif /*ZOLTAN_HUND*/
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
