int Zoltan_Migrate(
ZZ *zz, /* Zoltan structure. */
int num_import, /* Number of non-local objects assigned to the
processor in the new decomposition. */
ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects
assigned to this processor in the new
decomposition; this field can be NULL if
the application doesn't provide import IDs.*/
ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects
assigned to the processor in the new
decomposition; this field can be NULL if the
application does not provide import IDs. */
int *import_procs, /* Array of processor IDs of processors owning
the non-local objects that are assigned to
this processor in the new decomposition; this
field can be NULL if the application does
not provide import IDs. */
int *import_to_part, /* Array of partition numbers to which imported
objects should be assigned. */
int num_export, /* Number of objs to be exported
to other processors to establish the new
decomposition. */
ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of
objects to be exported to other processors
to establish the new decomposition. */
ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of
objects to be exported to other processors
to establish the new decomposition. */
int *export_procs, /* Array of processor IDs
to which objects will be exported
to establish the new decomposition. */
int *export_to_part /* Array of partition numbers to which exported
objects should be assigned. */
)
{
/*
* Routine to help perform migration. If migration pre-processing routine
* (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn.
* It then calls a function to obtain the size of the migrating objects
* (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified
* object packing routine (ZOLTAN_PACK_OBJ_FN) for each object
* to be exported. It develops the needed communication map to move the
* objects to other processors. It performs the communication according
* to the map, and then calls an application-specified object unpacking
* routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported.
*/
char *yo = "Zoltan_Migrate";
int num_gid_entries, num_lid_entries; /* lengths of global & local ids */
int *sizes = NULL; /* sizes (in bytes) of the object data for export. */
int id_size; /* size (in bytes) of ZOLTAN_GID + padding for
alignment */
int tag_size; /* size (in bytes) of ZOLTAN_GID + one int
(for message size) */
char *export_buf = NULL; /* buffer for packing export data. */
char *import_buf = NULL; /* buffer for receiving imported data. */
char *tmp; /* temporary pointer into buffers. */
int i; /* loop counter. */
int tmp_size; /* size of a single object's data. */
int *idx = NULL; /* index used for multi-fn packs and unpacks. */
int idx_cnt = 0; /* index counter for idx array. */
ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm
buf */
ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass
NULL to query functions when NUM_LID_ENTRIES=0. */
ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */
ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */
int msgtag, msgtag2; /* Tags for communication routines */
int total_send_size; /* Total size of outcoming message (in #items) */
int total_recv_size; /* Total size of incoming message (in #items) */
int aligned_int; /* size of an int padded for alignment */
int dest; /* temporary destination partition. */
int include_parts = 0; /* flag indicating whether partition info is
provided */
int ierr = ZOLTAN_OK;
int actual_num_exp = 0;
int actual_exp_allocated = 0;
ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */
int *actual_exp_procs = NULL; /* are changing partition but not */
int *actual_exp_to_part = NULL; /* processor may not be included. */
int actual_num_imp = 0;
int actual_imp_allocated = 0;
ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */
int *actual_imp_procs = NULL; /* are changing partition but not */
int *actual_imp_to_part = NULL; /* processor may not be included. */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
goto End;
}
/*
* Check that all procs use the same id types.
*/
ierr = check_input(zz,
((num_export >= 0 && export_to_part) ||
(num_import >= 0 && import_to_part)),
&include_parts);
if (ierr != ZOLTAN_OK)
goto End;
num_gid_entries = zz->Num_GID;
num_lid_entries = zz->Num_LID;
/*
* Check that all necessary query functions are available.
*/
if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (num_export >= 0) {
/* Build the actual export arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_export, export_global_ids, export_local_ids,
export_procs, export_to_part,
&actual_num_exp, &actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part,
&actual_exp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on actual exports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs,
zz->Communicator, msgtag, &actual_num_imp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
}
else if (num_import >= 0) {
/* Build the actual import arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_import, import_global_ids, import_local_ids,
import_procs, import_to_part,
&actual_num_imp, &actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part,
&actual_imp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on imports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs,
zz->Communicator, msgtag, &actual_num_exp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
/* Compute actual export lists for packing objects */
if (actual_num_exp > 0) {
actual_exp_allocated = 1;
actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp);
actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp);
actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp);
if (include_parts)
actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp);
if (actual_exp_gids == NULL ||
(num_lid_entries && actual_exp_lids == NULL) ||
actual_exp_procs == NULL ||
(import_to_part != NULL && actual_exp_to_part == NULL)) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
ierr = ZOLTAN_MEMERR;
goto End;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) actual_exp_gids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) actual_exp_lids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part,
(int) sizeof(int), (char *) actual_exp_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
/* Create inverse plan (i.e., plan based on exports) so can set
* variable sizes.
* (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable
* but does not tell how large to allocate receive buffer.
*/
ierr = Zoltan_Comm_Invert_Plan(&imp_plan);
exp_plan = imp_plan;
imp_plan = NULL;
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Migrate.Pre_Migrate_PP != NULL) {
zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Pre_Migrate != NULL) {
zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_FN function.");
goto End;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing");
id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE));
/* Note that alignment is not strictly necessary
when ZOLTAN_ID_TYPE is int or unsigned int. */
aligned_int = Zoltan_Align(sizeof(int));
tag_size = id_size + aligned_int;
/*
* For each object, allow space for its global ID and its data plus
* one int (for the object data size).
* Zoltan will pack the global IDs; the application must pack the data
* through the pack routine. Zoltan needs the global IDs for unpacking,
* as the order of the data received during communication is not
* necessarily the same order as import_global_ids[].
* Zoltan also needs to communicate the sizes of the objects because
* only the sender knows the size of each object.
*/
if (actual_num_exp > 0) {
sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!sizes) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_Size_Multi != NULL) {
zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids, sizes, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE_MULTI function.");
goto End;
}
}
else {
for (i = 0; i < actual_num_exp; i++) {
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]),
lid, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE function.");
goto End;
}
}
}
total_send_size = 0;
for (i = 0; i < actual_num_exp; i++) {
sizes[i] = Zoltan_Align(sizes[i]);
total_send_size += sizes[i] + tag_size;
}
export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char));
if (!export_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Pack_Obj_Multi != NULL) {
/* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */
idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Pack the objects for export.
*/
idx_cnt = 0;
tmp = export_buf;
for (i = 0; i < actual_num_exp; i++) {
/* Pack the object's global ID */
tmp_id = (ZOLTAN_ID_PTR) tmp;
ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries]));
tmp += id_size;
/* Pack the object's size */
*((int *)tmp) = sizes[i];
tmp += aligned_int;
/* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Pack_Obj_Multi != NULL) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n",
zz->Proc, yo);
}
zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids,
(actual_exp_to_part!=NULL ? actual_exp_to_part
: actual_exp_procs),
sizes, idx, export_buf, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ_MULTI function.");
goto End;
}
}
else {
tmp = export_buf + tag_size;
for (i = 0; i < actual_num_exp; i++) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries]));
printf("size = %d bytes\n", sizes[i]);
}
/* Pack the object's data */
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i]
: actual_exp_procs[i]);
zz->Pack_Obj(zz->Pack_Obj_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]), lid, dest,
sizes[i], tmp, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ function.");
goto End;
}
tmp += sizes[i] + tag_size;
}
}
ZOLTAN_FREE(&idx);
tmp_id = NULL;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects");
/* Modify sizes[] to contain message sizes, not object sizes */
for (i=0; i<actual_num_exp; i++) {
sizes[i] += tag_size;
}
msgtag--;
ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize.");
goto End;
}
if (actual_num_imp > 0) {
import_buf = (char *) ZOLTAN_MALLOC(total_recv_size);
if (!import_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Send the export data using the communication plan.
*/
msgtag2 = 32765;
ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
/*
* Free whatever memory we can.
*/
Zoltan_Comm_Destroy(&exp_plan);
ZOLTAN_FREE(&export_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication");
/*
* Perform application-specified processing before unpacking the data.
*/
if (zz->Migrate.Mid_Migrate_PP != NULL) {
zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Mid_Migrate != NULL) {
zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_FN function.");
goto End;
}
}
/*
* Unpack the object data.
*/
if (actual_num_imp > 0) {
if (zz->Unpack_Obj_Multi != NULL) {
/* Allocate and fill input arrays for Unpack_Obj_Multi. */
sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp);
idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
if (!sizes || !tmp_id || !idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
tmp = import_buf;
idx_cnt = 0;
for (i = 0; i < actual_num_imp; i++) {
/* Unpack the object's global ID */
ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp);
tmp += id_size;
/* Unpack the object's size */
sizes[i] = *((int *)tmp);
tmp += aligned_int;
/* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n",
zz->Proc,yo);
}
zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries,
actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr);
ZOLTAN_FREE(&import_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_FREE(&tmp_id);
ZOLTAN_FREE(&idx);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_MULTI_FN.");
goto End;
}
}
else {
tmp = import_buf;
for (i = 0; i < actual_num_imp; i++) {
tmp_size = *((int *)(tmp + id_size));
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp);
printf("size = %d bytes\n", tmp_size);
}
/* Unpack the object's data */
zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries,
(ZOLTAN_ID_PTR) tmp, tmp_size,
tmp + tag_size, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_FN.");
goto End;
}
tmp += (tmp_size + tag_size);
}
ZOLTAN_FREE(&import_buf);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects");
if (zz->Migrate.Post_Migrate_PP != NULL) {
zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Post_Migrate != NULL) {
zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_FN function.");
goto End;
}
}
End:
if (actual_exp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
}
if (actual_imp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part);
}
if (ierr < 0) {
if (exp_plan) Zoltan_Comm_Destroy(&exp_plan);
Zoltan_Multifree(__FILE__, __LINE__, 5,
&import_buf, &tmp_id, &sizes, &idx, &export_buf);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Invert_Lists(
ZZ *zz, /* Zoltan structure. */
int num_in, /* Number of objects in the input lists. */
ZOLTAN_ID_PTR in_global_ids, /* Array of input global IDs. */
ZOLTAN_ID_PTR in_local_ids, /* Array of input local IDs. */
int *in_procs, /* Array of processor IDs of processors owning
the input objects. */
int *in_to_part, /* Optional: Array of partition numbers to
which input objects should be assigned. */
int *num_out, /* Returned value: Number of output objs. */
ZOLTAN_ID_PTR *out_global_ids,/* Returned value: Array of global IDs of
output objects. */
ZOLTAN_ID_PTR *out_local_ids, /* Returned value: Array of local IDs of
output objects. */
int **out_procs, /* Returned value: Array of processor IDs
to which output objects are assigned. */
int **out_to_part /* Optional: Returned value: Array of
partition numbers to which output
objects should be assigned. */
)
{
/*
* Routine to compute the inverse map. Can be used in two ways:
* 1. Given, for each processor, a list of objects to be received by the
* processor, compute the list of objects that the processor needs to send
* to other processors to satisfy their needs.
* 2. Given, for each processor, a list of objects to be sent to other
* processors, compute the list of objects that the processor needs to receive
* to satisfy its needs.
*/
char *yo = "Zoltan_Invert_Lists";
char msg[256];
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned communication routines */
int msgtag, msgtag2; /* Message tags for communication routines */
int num_gid_entries, num_lid_entries; /* Length of global and local ids */
int include_parts; /* Flag indicating whether to compute
inverse list for partitions. */
int ierr, ret_ierr = ZOLTAN_OK;
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Check that all procs use the same id types.
*/
ierr = check_invert_input(zz, num_in, in_procs, in_to_part,
&num_gid_entries, &num_lid_entries, &include_parts);
if (ierr != ZOLTAN_OK) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
/* Initialize returned arrays. */
*out_global_ids = NULL;
*out_local_ids = NULL;
*out_procs = NULL;
if (include_parts) *out_to_part = NULL;
/*
* Compute communication map and num_out, the number of objs this
* processor has to out to establish the new decomposition.
*/
msgtag = 32767;
ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator,
msgtag, num_out);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Create.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
goto End;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create");
/*
* Allocate space for the object tags that need to be outed. Communicate
* to get the list of objects to be outed.
*/
if (*num_out > 0) {
if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out,
ZOLTAN_SPECIAL_MALLOC_GID)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out,
ZOLTAN_SPECIAL_MALLOC_LID)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
if (include_parts) {
if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out,
ZOLTAN_SPECIAL_MALLOC_INT)) {
ret_ierr = ZOLTAN_MEMERR;
goto End;
}
}
}
/*
* Use the communication plan to send global IDs, local IDs, and processor
* numbers. Do in separate communications to avoid a memory copy and to
* simplify implementation when a data type is added to the comm. package
* (to support heterogeneous computing).
*/
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) *out_global_ids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) *out_local_ids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
}
Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, *out_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part,
(int) sizeof(int), (char *) *out_to_part);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ret_ierr = ierr;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do");
End:
Zoltan_Comm_Destroy(&comm_plan);
if (ret_ierr == ZOLTAN_MEMERR) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT);
if (include_parts)
Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ret_ierr);
}
static int local_HEs_from_export_lists(
ZZ *zz,
int remap_type, /* type of remapping to do: parts, procs, or none. */
int nobj, /* # objs the processor knows about (keep + exports) */
int *new_proc, /* On input, new processor assignment for each obj;
Upon return, remapped new proc assignment for
each obj. */
int *old_part, /* old partition assignments for each objs */
int *new_part, /* On input, new partition assignments for each objs.
Upon return, remapped new partition assignments */
int *HEcnt, /* # of HEs allocated. */
int **HEinfo /* Array of HE info; for each HE, two pins and
one edge weight. Stored as a single vector
to minimize communication calls. */
)
{
/* Routine to remap partitions (to new processors or new partition numbers)
* to reduce data movement.
* This routine assumes the load-balancing algorithm built export lists.
* Objects described are those that STARTED on zz->Proc due to load balancing.
* For all these objects, old_proc == zz->Proc.
*/
char *yo = "local_HEs_from_export_lists";
int ierr = ZOLTAN_OK;
int i, cnt, tmp;
int *tmp_HEinfo;
int my_proc = zz->Proc; /* This processor's rank. */
int nimp = 0;
int *imp_proc = NULL, /* Temporary arrays if inversion of export to */
*imp_old_part = NULL, /* import lists is needed. */
*imp_new_part = NULL;
int HEwgt_size; /* # of HE weights allocated. */
int *HEwgt = NULL; /* Array of HE weights. Initially includes
zero weights; later zero-weights are removed.*/
if (remap_type == ZOLTAN_LB_REMAP_PROCESSORS) {
/* Build HEs based on processor assignment.
* We know the old processor for all objects we are keeping and all
* export objects -- it is my_proc!
* We also know the new processor number for all objects initially on
* my_proc (since we built export lists.)
* This case is a special case of partition remapping; it is easy to
* build the hyperedges in this special case.
*/
HEwgt_size = zz->Num_Proc;
HEwgt = (int *) ZOLTAN_CALLOC(HEwgt_size, sizeof(int));
if (!HEwgt) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < nobj; i++)
HEwgt[new_proc[i]]++;
*HEcnt = 0;
for (i = 0; i < HEwgt_size; i++)
if (HEwgt[i] != 0) (*HEcnt)++;
ierr = malloc_HEinfo(zz, *HEcnt, HEinfo);
if (ierr < 0)
goto End;
tmp_HEinfo = *HEinfo;
cnt = 0;
for (i = 0; i < HEwgt_size; i++) {
if (HEwgt[i] != 0) {
tmp = cnt * HEINFO_ENTRIES;
tmp_HEinfo[tmp] = my_proc; /* Old processor number */
tmp_HEinfo[tmp+1] = i; /* New processor number */
tmp_HEinfo[tmp+2] = HEwgt[i]; /* shift non-zero weights down. */
cnt++;
}
}
}
else { /* ZOLTAN_LB_REMAP_PARTS */
/* Cannot renumber partitions given export lists without summing HE weights
* across processors. This summation is not straightforward. Also, a
* potentially large number of HEs may exist
* (max_old_partition_number * zz->Num_Global_Parts). Rather than build
* this large matrix, just compute import lists from the export lists
* and run the import-list algorithm.
*/
ZOLTAN_COMM_OBJ *plan;
int msg_tag = 22345;
ierr = Zoltan_Comm_Create(&plan, nobj, new_proc, zz->Communicator,
msg_tag, &nimp);
if (nimp > 0) {
imp_proc = (int *) ZOLTAN_MALLOC(3 * nimp * sizeof(int));
imp_old_part = imp_proc + nimp;
imp_new_part = imp_old_part + nimp;
if (!imp_proc) {
ierr = ZOLTAN_MEMERR;
ZOLTAN_PRINT_ERROR(my_proc, yo, "Memory error.");
goto End;
}
}
ierr = Zoltan_Comm_Info(plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, imp_proc, NULL);
msg_tag++;
ierr = Zoltan_Comm_Do(plan, msg_tag, (char *) old_part, sizeof(int),
(char *) imp_old_part);
msg_tag++;
ierr = Zoltan_Comm_Do(plan, msg_tag, (char *) new_part, sizeof(int),
(char *) imp_new_part);
Zoltan_Comm_Destroy(&plan);
ierr = local_HEs_from_import_lists(zz, remap_type, nimp, imp_proc,
imp_old_part, imp_new_part,
HEcnt, HEinfo);
}
End:
if (HEwgt) ZOLTAN_FREE(&HEwgt);
if (imp_proc) ZOLTAN_FREE(&imp_proc);
return ierr;
}
void check_comm_info(
ZOLTAN_COMM_OBJ *plan,
struct Data *my_send_data,
int nvals_recv,
int my_proc
)
{
int *info_tmp = NULL, /* Temp bufs for verifying Zoltan_Comm_Info */
*info_tmp_send = NULL,
*info_tmp_recv = NULL;
int *info_send_list = NULL,
*info_send_procs = NULL,
*info_send_lengths = NULL;
int info_nsend = 0,
info_send_nvals = 0,
info_send_size = 0,
info_max_send_size = 0;
int *info_recv_list = NULL,
*info_recv_procs = NULL,
*info_recv_lengths = NULL;
int info_nrecv = 0,
info_recv_nvals = 0,
info_recv_size = 0,
info_total_recv_size = 0;
int info_self_msg = 0;
int i;
Zoltan_Comm_Info(plan, &info_nsend, NULL, NULL,
&info_send_nvals, NULL, NULL, &info_nrecv, NULL,
NULL, &info_recv_nvals, NULL, NULL, &info_self_msg);
if (info_send_nvals != my_send_data->nvals)
printf("%d Error in Zoltan_Comm_Info info_send_nvals %d != %d\n",
my_proc, info_send_nvals, my_send_data->nvals);
if (info_recv_nvals != nvals_recv)
printf("%d Error in Zoltan_Comm_Info info_recv_nvals %d != %d\n",
my_proc, info_recv_nvals, nvals_recv);
info_send_size = 2 * (info_nsend + info_self_msg) + info_send_nvals;
info_send_procs = (int *) ZOLTAN_MALLOC(info_send_size * sizeof(int));
info_send_lengths = info_send_procs + (info_nsend + info_self_msg);
info_send_list = info_send_lengths + (info_nsend + info_self_msg);
info_recv_size = 2 * (info_nrecv + info_self_msg) + info_recv_nvals;
info_recv_procs = (int *) ZOLTAN_MALLOC(info_recv_size * sizeof(int));
info_recv_lengths = info_recv_procs + (info_nrecv + info_self_msg);
info_recv_list = info_recv_lengths + (info_nrecv + info_self_msg);
Zoltan_Comm_Info(plan, &info_nsend, info_send_procs, info_send_lengths,
&info_send_nvals, &info_max_send_size, info_send_list,
&info_nrecv, info_recv_procs, info_recv_lengths,
&info_recv_nvals, &info_total_recv_size,
info_recv_list, &info_self_msg);
for (i = 0; i < info_send_nvals; i++)
if (info_send_list[i] != my_send_data->proc_dest[i])
printf("%d Error in Zoltan_Comm_Info send_list[%d]: %d != %d\n",
my_proc, i, info_send_list[i], my_send_data->proc_dest[i]);
info_tmp = (int *) ZOLTAN_MALLOC((info_send_nvals + info_recv_nvals)
* sizeof(int));
info_tmp_send = info_tmp;
info_tmp_recv = info_tmp_send + info_send_nvals;
for (i = 0; i < info_send_nvals; i++)
info_tmp_send[i] = my_proc;
Zoltan_Comm_Do(plan, 12, (char *) info_tmp_send, sizeof(int),
(char *) info_tmp_recv);
for (i = 0; i < info_recv_nvals; i++)
if (info_recv_list[i] != info_tmp_recv[i])
printf("%d Error in Zoltan_Comm_Info recv_list[%d]: %d != %d\n",
my_proc, i, info_recv_list[i], info_tmp_recv[i]);
ZOLTAN_FREE(&info_tmp);
ZOLTAN_FREE(&info_recv_procs);
ZOLTAN_FREE(&info_send_procs);
}
