/* This function may work on any distribution of the bipartite graph */
int
Zoltan_ZG_Register(ZZ* zz, ZG* graph, int* properties)
{
static char *yo = "Zoltan_ZG_Register";
int ierr = ZOLTAN_OK;
int *props;
struct Zoltan_DD_Struct *dd;
int size;
ZOLTAN_ID_PTR GID;
ZOLTAN_TRACE_ENTER(zz, yo);
size = graph->mtx.mtx.nY;
dd = graph->mtx.mtx.ddY;
if (graph->bipartite) { /* Need to construct another properties array with only the fixed elements ! */
int vertlno;
if (graph->fixObj) {
dd = graph->mtx.mtx.ddX;
}
props = (int*)ZOLTAN_MALLOC(sizeof(int)*size);
if (graph->mtx.mtx.nY && props == NULL) MEMORY_ERROR;
GID = ZOLTAN_MALLOC_GID_ARRAY(zz, size);
if (size && GID == NULL) MEMORY_ERROR;
for (size = 0, vertlno = 0 ; vertlno < graph->mtx.mtx.nY ; ++vertlno) {
if (graph->fixed_vertices[vertlno]) {
props[size] = properties[vertlno];
ZOLTAN_SET_GID(zz, GID+ size*zz->Num_GID,
graph->mtx.mtx.yGID+vertlno*zz->Num_GID);
size ++;
}
}
}
else {
props = properties;
GID = graph->mtx.mtx.yGID;
if (graph->mtx.mtx.ddY == NULL) {
ierr = Zoltan_DD_Create (&graph->mtx.mtx.ddY, zz->Communicator, 1, zz->Num_GID,
1, graph->mtx.mtx.globalY/zz->Num_Proc, 0);
CHECK_IERR;
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(graph->mtx.mtx.ddY, graph->mtx.mtx.globalX/zz->Num_Proc);
}
dd = graph->mtx.mtx.ddY;
}
/* Make our new numbering public */
ierr = Zoltan_DD_Update (dd, GID, NULL, NULL, props, size);
CHECK_IERR;
End:
if (graph->bipartite) {
ZOLTAN_FREE(&props);
ZOLTAN_FREE(&GID);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int
Zoltan_Matrix_Complete(ZZ* zz,Zoltan_matrix* m)
{
static char *yo = "Zoltan_Matrix_Complete";
int ierr = ZOLTAN_OK;
if(m->completed)
return (ZOLTAN_OK);
ZOLTAN_TRACE_ENTER(zz, yo);
if (m->yend != m->ystart +1 ) {/* Not in compact mode yet */
int y;
/* I have to also rewrites all the pinarrays */
for (y = 1 ; y <= m->nY ; ++y) {
int length;
if (m->ystart[y] == m->yend[y-1]) /* No hole */
continue;
length = m->yend[y]-m->ystart[y];
memcpy(m->pinGNO+m->yend[y-1], m->pinGNO+m->ystart[y], length*sizeof(ZOLTAN_GNO_TYPE));
memcpy(m->pinwgt+m->yend[y-1]*m->pinwgtdim,
m->pinGNO+m->ystart[y]*m->pinwgtdim, length*sizeof(float)*m->pinwgtdim);
m->ystart[y] = m->yend[y-1];
m->yend[y] = m->ystart[y] + length;
}
ZOLTAN_FREE(&m->yend);
m->yend = m->ystart + 1;
}
/* Update data directories */
m->yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY);
m->ypid = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int));
if (m->bipartite)
m->ybipart = (int*) ZOLTAN_MALLOC(m->nY * sizeof(int));
if (m->nY && ((m->yGID == NULL) || (m->ypid == NULL) || (m->bipartite && m->ybipart == NULL)))
MEMORY_ERROR;
/* Get Informations about Y */
Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, m->yGID, (char *)m->ypid, m->ybipart,
m->nY, NULL);
if (m->ddY != m->ddX) {
Zoltan_DD_Destroy(&m->ddY);
m->ddY = NULL;
}
m->completed = 1;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/* Return an array of locally owned GID */
ZOLTAN_ID_PTR Zoltan_Matrix_Get_GID(ZZ* zz, Zoltan_matrix* m)
{
ZOLTAN_ID_PTR yGID;
yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY);
if (m->nY && yGID == NULL)
return (NULL);
/* Get Informations about Y */
Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, NULL, NULL,
m->nY, NULL);
return (yGID);
}
static int actual_arrays(
ZZ *zz,
int num_gid_entries,
int num_lid_entries,
int num,
ZOLTAN_ID_PTR gids,
ZOLTAN_ID_PTR lids,
int *procs,
int *to_part,
int *actual_num,
ZOLTAN_ID_PTR *actual_gids,
ZOLTAN_ID_PTR *actual_lids,
int **actual_procs,
int **actual_to_part,
int *actual_allocated
)
{
char *yo = "actual_arrays";
int i, j;
/*
* Test whether to pack objects that have changed partition
* but not changed processor.
* If packing them, the actual objects == objects passed to this function.
* If not packing them, build arrays with them stripped out.
*/
*actual_allocated = 0;
if (!(zz->Migrate.Only_Proc_Changes)) {
/* Pack all objects, even if they are not changing processor. */
*actual_num = num;
*actual_gids = gids;
*actual_lids = lids;
*actual_procs = procs;
*actual_to_part = to_part;
}
else { /* zz->Migrate.Only_Proc_Changes */
/* Pack only objects that are actually changing processor. */
*actual_num = 0;
for (i = 0; i < num; i++)
if (procs[i] != zz->Proc)
(*actual_num)++;
if (*actual_num == num) {
/* Number of actual objects == number of objects in input arrays. */
/* No stripping needed. */
*actual_gids = gids;
*actual_lids = lids;
*actual_procs = procs;
*actual_to_part = to_part;
}
else if (*actual_num != num && *actual_num > 0) {
/* Number of actual_num < num. Build arrays */
/* containing only actual objects. */
*actual_allocated = 1;
*actual_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, *actual_num);
*actual_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, *actual_num);
*actual_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * (*actual_num));
if (to_part != NULL)
*actual_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*(*actual_num));
if (*actual_gids == NULL || (num_lid_entries && *actual_lids == NULL) ||
*actual_procs == NULL ||
(to_part != NULL && *actual_to_part == NULL)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory Error.");
Zoltan_Multifree(__FILE__, __LINE__, 4,
actual_gids, actual_lids,
actual_procs, actual_to_part);
return (ZOLTAN_MEMERR);
}
for (j = 0, i = 0; i < num; i++) {
if (procs[i] != zz->Proc) {
ZOLTAN_SET_GID(zz,
*actual_gids + j*num_gid_entries,
gids + i*num_gid_entries);
if (num_lid_entries)
ZOLTAN_SET_LID(zz,
*actual_lids + j*num_lid_entries,
lids + i*num_lid_entries);
(*actual_procs)[j] = procs[i];
if (to_part) (*actual_to_part)[j] = to_part[i];
j++;
}
}
}
}
return ZOLTAN_OK;
}
int Zoltan_Migrate(
ZZ *zz, /* Zoltan structure. */
int num_import, /* Number of non-local objects assigned to the
processor in the new decomposition. */
ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects
assigned to this processor in the new
decomposition; this field can be NULL if
the application doesn't provide import IDs.*/
ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects
assigned to the processor in the new
decomposition; this field can be NULL if the
application does not provide import IDs. */
int *import_procs, /* Array of processor IDs of processors owning
the non-local objects that are assigned to
this processor in the new decomposition; this
field can be NULL if the application does
not provide import IDs. */
int *import_to_part, /* Array of partition numbers to which imported
objects should be assigned. */
int num_export, /* Number of objs to be exported
to other processors to establish the new
decomposition. */
ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of
objects to be exported to other processors
to establish the new decomposition. */
ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of
objects to be exported to other processors
to establish the new decomposition. */
int *export_procs, /* Array of processor IDs
to which objects will be exported
to establish the new decomposition. */
int *export_to_part /* Array of partition numbers to which exported
objects should be assigned. */
)
{
/*
* Routine to help perform migration. If migration pre-processing routine
* (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn.
* It then calls a function to obtain the size of the migrating objects
* (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified
* object packing routine (ZOLTAN_PACK_OBJ_FN) for each object
* to be exported. It develops the needed communication map to move the
* objects to other processors. It performs the communication according
* to the map, and then calls an application-specified object unpacking
* routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported.
*/
char *yo = "Zoltan_Migrate";
int num_gid_entries, num_lid_entries; /* lengths of global & local ids */
int *sizes = NULL; /* sizes (in bytes) of the object data for export. */
int id_size; /* size (in bytes) of ZOLTAN_GID + padding for
alignment */
int tag_size; /* size (in bytes) of ZOLTAN_GID + one int
(for message size) */
char *export_buf = NULL; /* buffer for packing export data. */
char *import_buf = NULL; /* buffer for receiving imported data. */
char *tmp; /* temporary pointer into buffers. */
int i; /* loop counter. */
int tmp_size; /* size of a single object's data. */
int *idx = NULL; /* index used for multi-fn packs and unpacks. */
int idx_cnt = 0; /* index counter for idx array. */
ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm
buf */
ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass
NULL to query functions when NUM_LID_ENTRIES=0. */
ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */
ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */
int msgtag, msgtag2; /* Tags for communication routines */
int total_send_size; /* Total size of outcoming message (in #items) */
int total_recv_size; /* Total size of incoming message (in #items) */
int aligned_int; /* size of an int padded for alignment */
int dest; /* temporary destination partition. */
int include_parts = 0; /* flag indicating whether partition info is
provided */
int ierr = ZOLTAN_OK;
int actual_num_exp = 0;
int actual_exp_allocated = 0;
ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */
int *actual_exp_procs = NULL; /* are changing partition but not */
int *actual_exp_to_part = NULL; /* processor may not be included. */
int actual_num_imp = 0;
int actual_imp_allocated = 0;
ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */
ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */
int *actual_imp_procs = NULL; /* are changing partition but not */
int *actual_imp_to_part = NULL; /* processor may not be included. */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
goto End;
}
/*
* Check that all procs use the same id types.
*/
ierr = check_input(zz,
((num_export >= 0 && export_to_part) ||
(num_import >= 0 && import_to_part)),
&include_parts);
if (ierr != ZOLTAN_OK)
goto End;
num_gid_entries = zz->Num_GID;
num_lid_entries = zz->Num_LID;
/*
* Check that all necessary query functions are available.
*/
if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
"ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function "
"to use the migration-help tools.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (num_export >= 0) {
/* Build the actual export arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_export, export_global_ids, export_local_ids,
export_procs, export_to_part,
&actual_num_exp, &actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part,
&actual_exp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on actual exports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs,
zz->Communicator, msgtag, &actual_num_imp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
}
else if (num_import >= 0) {
/* Build the actual import arrays */
ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
num_import, import_global_ids, import_local_ids,
import_procs, import_to_part,
&actual_num_imp, &actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part,
&actual_imp_allocated);
if (ierr < 0)
goto End;
/* Compute communication map based on imports. */
msgtag = 32767;
ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs,
zz->Communicator, msgtag, &actual_num_exp);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
goto End;
}
/* Compute actual export lists for packing objects */
if (actual_num_exp > 0) {
actual_exp_allocated = 1;
actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp);
actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp);
actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp);
if (include_parts)
actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp);
if (actual_exp_gids == NULL ||
(num_lid_entries && actual_exp_lids == NULL) ||
actual_exp_procs == NULL ||
(import_to_part != NULL && actual_exp_to_part == NULL)) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
ierr = ZOLTAN_MEMERR;
goto End;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids,
(int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
(char *) actual_exp_gids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
if (num_lid_entries) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids,
(int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
(char *) actual_exp_lids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL);
if (include_parts) {
msgtag2--;
ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part,
(int) sizeof(int), (char *) actual_exp_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
}
/* Create inverse plan (i.e., plan based on exports) so can set
* variable sizes.
* (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable
* but does not tell how large to allocate receive buffer.
*/
ierr = Zoltan_Comm_Invert_Plan(&imp_plan);
exp_plan = imp_plan;
imp_plan = NULL;
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (zz->Migrate.Pre_Migrate_PP != NULL) {
zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Pre_Migrate != NULL) {
zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PRE_MIGRATE_FN function.");
goto End;
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing");
id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE));
/* Note that alignment is not strictly necessary
when ZOLTAN_ID_TYPE is int or unsigned int. */
aligned_int = Zoltan_Align(sizeof(int));
tag_size = id_size + aligned_int;
/*
* For each object, allow space for its global ID and its data plus
* one int (for the object data size).
* Zoltan will pack the global IDs; the application must pack the data
* through the pack routine. Zoltan needs the global IDs for unpacking,
* as the order of the data received during communication is not
* necessarily the same order as import_global_ids[].
* Zoltan also needs to communicate the sizes of the objects because
* only the sender knows the size of each object.
*/
if (actual_num_exp > 0) {
sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!sizes) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_Size_Multi != NULL) {
zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids, sizes, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE_MULTI function.");
goto End;
}
}
else {
for (i = 0; i < actual_num_exp; i++) {
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]),
lid, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_OBJ_SIZE function.");
goto End;
}
}
}
total_send_size = 0;
for (i = 0; i < actual_num_exp; i++) {
sizes[i] = Zoltan_Align(sizes[i]);
total_send_size += sizes[i] + tag_size;
}
export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char));
if (!export_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Pack_Obj_Multi != NULL) {
/* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */
idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
if (!idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Pack the objects for export.
*/
idx_cnt = 0;
tmp = export_buf;
for (i = 0; i < actual_num_exp; i++) {
/* Pack the object's global ID */
tmp_id = (ZOLTAN_ID_PTR) tmp;
ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries]));
tmp += id_size;
/* Pack the object's size */
*((int *)tmp) = sizes[i];
tmp += aligned_int;
/* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Pack_Obj_Multi != NULL) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n",
zz->Proc, yo);
}
zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data,
num_gid_entries, num_lid_entries, actual_num_exp,
actual_exp_gids, actual_exp_lids,
(actual_exp_to_part!=NULL ? actual_exp_to_part
: actual_exp_procs),
sizes, idx, export_buf, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ_MULTI function.");
goto End;
}
}
else {
tmp = export_buf + tag_size;
for (i = 0; i < actual_num_exp; i++) {
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries]));
printf("size = %d bytes\n", sizes[i]);
}
/* Pack the object's data */
lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i]
: actual_exp_procs[i]);
zz->Pack_Obj(zz->Pack_Obj_Data,
num_gid_entries, num_lid_entries,
&(actual_exp_gids[i*num_gid_entries]), lid, dest,
sizes[i], tmp, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_PACK_OBJ function.");
goto End;
}
tmp += sizes[i] + tag_size;
}
}
ZOLTAN_FREE(&idx);
tmp_id = NULL;
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects");
/* Modify sizes[] to contain message sizes, not object sizes */
for (i=0; i<actual_num_exp; i++) {
sizes[i] += tag_size;
}
msgtag--;
ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize.");
goto End;
}
if (actual_num_imp > 0) {
import_buf = (char *) ZOLTAN_MALLOC(total_recv_size);
if (!import_buf) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
/*
* Send the export data using the communication plan.
*/
msgtag2 = 32765;
ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
goto End;
}
/*
* Free whatever memory we can.
*/
Zoltan_Comm_Destroy(&exp_plan);
ZOLTAN_FREE(&export_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication");
/*
* Perform application-specified processing before unpacking the data.
*/
if (zz->Migrate.Mid_Migrate_PP != NULL) {
zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Mid_Migrate != NULL) {
zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_MID_MIGRATE_FN function.");
goto End;
}
}
/*
* Unpack the object data.
*/
if (actual_num_imp > 0) {
if (zz->Unpack_Obj_Multi != NULL) {
/* Allocate and fill input arrays for Unpack_Obj_Multi. */
sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp);
idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
if (!sizes || !tmp_id || !idx) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
tmp = import_buf;
idx_cnt = 0;
for (i = 0; i < actual_num_imp; i++) {
/* Unpack the object's global ID */
ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp);
tmp += id_size;
/* Unpack the object's size */
sizes[i] = *((int *)tmp);
tmp += aligned_int;
/* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */
idx_cnt += tag_size;
if (idx != NULL) {
idx[i] = idx_cnt;
}
tmp += sizes[i];
idx_cnt += sizes[i];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n",
zz->Proc,yo);
}
zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries,
actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr);
ZOLTAN_FREE(&import_buf);
ZOLTAN_FREE(&sizes);
ZOLTAN_FREE(&tmp_id);
ZOLTAN_FREE(&idx);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_MULTI_FN.");
goto End;
}
}
else {
tmp = import_buf;
for (i = 0; i < actual_num_imp; i++) {
tmp_size = *((int *)(tmp + id_size));
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo);
ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp);
printf("size = %d bytes\n", tmp_size);
}
/* Unpack the object's data */
zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries,
(ZOLTAN_ID_PTR) tmp, tmp_size,
tmp + tag_size, &ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_UNPACK_OBJ_FN.");
goto End;
}
tmp += (tmp_size + tag_size);
}
ZOLTAN_FREE(&import_buf);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects");
if (zz->Migrate.Post_Migrate_PP != NULL) {
zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs, import_to_part,
num_export, export_global_ids,
export_local_ids, export_procs, export_to_part,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_PP_FN function.");
goto End;
}
}
if (zz->Migrate.Post_Migrate != NULL) {
zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data,
num_gid_entries, num_lid_entries,
num_import, import_global_ids,
import_local_ids, import_procs,
num_export, export_global_ids,
export_local_ids, export_procs,
&ierr);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
"ZOLTAN_POST_MIGRATE_FN function.");
goto End;
}
}
End:
if (actual_exp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_exp_gids, &actual_exp_lids,
&actual_exp_procs, &actual_exp_to_part);
}
if (actual_imp_allocated) {
Zoltan_Multifree(__FILE__, __LINE__, 4,
&actual_imp_gids, &actual_imp_lids,
&actual_imp_procs, &actual_imp_to_part);
}
if (ierr < 0) {
if (exp_plan) Zoltan_Comm_Destroy(&exp_plan);
Zoltan_Multifree(__FILE__, __LINE__, 5,
&import_buf, &tmp_id, &sizes, &idx, &export_buf);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/*
* void malloc_new_objects();
*
* gets the tags being imported into this processor, and sets up the
* import_tags array, and the nrectags array.
*/
static int malloc_new_objects(ZZ *zz, int nsentags, pRegion exported_tags,
ZOLTAN_ID_PTR exported_gids,
ZOLTAN_ID_PTR exported_lids, int *tag_pids,
int *nstags, pRegion *ex_tags,
pRegion prev_tags, ZOLTAN_ID_PTR prev_gids,
ZOLTAN_ID_PTR prev_lids, int npimtags,
float *c3)
{
char *yo = "malloc_new_objects";
int i; /* index counter */
int nreceives; /* number of messages received */
pRegion t_b_exp; /* array of tags to be exported */
pRegion tmp = NULL;
ZOLTAN_ID_PTR tmp_gids = NULL;
ZOLTAN_ID_PTR tmp_lids = NULL;
int msgtag, msgtag2;
int j;
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
float im_load;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
im_load = 0;
msgtag = 32767;
ierr = Zoltan_Comm_Create(&comm_plan, nsentags, tag_pids, zz->Communicator,
msgtag, &nreceives);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_Comm_Create.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (nreceives > 0) {
tmp = (pRegion) ZOLTAN_MALLOC(nreceives * sizeof(Region));
tmp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, nreceives);
tmp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, nreceives);
if(tmp == NULL || !tmp_gids || (num_lid_entries && !tmp_lids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
msgtag2 = 32766;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_tags,
sizeof(Region), (char *) tmp);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_gids,
sizeof(ZOLTAN_ID_TYPE)*num_gid_entries,
(char *) tmp_gids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
fprintf(stderr, "OCT %s Error %s returned from Zoltan_Comm_Do\n", yo,
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
if (num_lid_entries > 0) {
msgtag2--;
ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) exported_lids,
sizeof(ZOLTAN_ID_TYPE)*num_lid_entries,
(char *) tmp_lids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return(ierr);
}
}
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_Comm_Destroy.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
return(ierr);
}
/* get each message sent, and store region in import array */
j=0;
for (i=0; i<nreceives; i++) {
im_load += tmp[i].Weight;
if(tmp[i].newProc != zz->Proc) {
j++;
}
}
if((j + npimtags) != 0) { /* malloc import array */
if((t_b_exp = (pRegion)ZOLTAN_MALLOC((j+npimtags)*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
}
else
t_b_exp = NULL;
/* setup return pointer */
(*ex_tags) = t_b_exp;
j=0;
for (i=0; i<nreceives; i++) {
if(tmp[i].newProc != zz->Proc) {
t_b_exp[j] = tmp[i];
t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz);
if (!(t_b_exp[j].Global_ID) ||
(num_lid_entries && !(t_b_exp[j].Local_ID))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
&(tmp_gids[i*num_gid_entries]));
ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID, &(tmp_lids[i*num_lid_entries]));
j++;
}
}
if(npimtags > 0) {
for(i=0; i<npimtags; i++) {
t_b_exp[j] = prev_tags[i];
t_b_exp[j].Global_ID = ZOLTAN_MALLOC_GID(zz);
t_b_exp[j].Local_ID = ZOLTAN_MALLOC_LID(zz);
if (!(t_b_exp[j].Global_ID) ||
(num_lid_entries && !(t_b_exp[j].Local_ID))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
return ZOLTAN_MEMERR;
}
ZOLTAN_SET_GID(zz, t_b_exp[j].Global_ID,
&(prev_gids[i*num_gid_entries]));
ZOLTAN_SET_LID(zz, t_b_exp[j].Local_ID,
&(prev_lids[i*num_lid_entries]));
j++;
}
}
*nstags = j;
ZOLTAN_FREE(&tmp);
ZOLTAN_FREE(&tmp_gids);
ZOLTAN_FREE(&tmp_lids);
if((*nstags == 0) && (*ex_tags != NULL)) {
ZOLTAN_TRACE_DETAIL(zz, yo,
"Fatal error, import tags not empty but no tags received\n");
return ZOLTAN_FATAL;
}
*c3 = im_load;
return ierr;
}
/*
* void tag_regions()
* Iterates through the list of octants on the processor and finds which
* are to be migrated. It then looks at the region list for those octants
* and stores the migrating regions into the export_tags array.
*/
static int tag_regions(ZZ *zz,
pOctant *octs,
int *newpids,
int nocts,
Region **exported_tags,
ZOLTAN_ID_PTR *exported_gids,
ZOLTAN_ID_PTR *exported_lids,
int *nsentags,
int **tag_pids,
Region **p_tags,
ZOLTAN_ID_PTR *p_gids,
ZOLTAN_ID_PTR *p_lids,
int *npimtags,
float *c2,
int *max_objs)
{
char *yo = "tag_regions";
int i; /* index counter */
pRegion regionlist; /* list of region on this processor */
int index; /* index counter */
int index2; /* yet another index counter */
int count; /* count of objects exported form this processor */
int count2; /* count of objects that are kept on processor */
int count3;
int *exported_pids; /* array of pids where regions are being exported to */
pRegion mtags; /* object tags of objects to be migrated */
pRegion ptags; /* tags of objects that were previously migrated */
float ex_load;
int ierr = ZOLTAN_OK;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
ex_load = 0;
(*max_objs) = 0;
if (!nsentags)
return ierr;
/* find how many objects have been exported */
count = 0;
/* find number of local objs to export */
count2 = 0;
count3 = 0;
for (i=0; i<nocts; i++) {
if(Zoltan_Oct_isTerminal(octs[i])) {
(*max_objs) += Zoltan_Oct_nRegions(octs[i]);
regionlist = Zoltan_Oct_regionlist(octs[i]);
while(regionlist != NULL) {
count3++;
if(regionlist->Proc != zz->Proc) {
count++;
if(newpids[i] != zz->Proc)
regionlist->newProc = newpids[i];
else
regionlist->newProc = zz->Proc;
}
else {
if(newpids[i] != zz->Proc) {
count2++;
regionlist->newProc = newpids[i];
}
else
regionlist->newProc = zz->Proc;
}
regionlist = regionlist->next; /* get next region */
}
}
}
#if 0
{
{
if (newpids[i]!=zz->Proc) {
count+=Zoltan_Oct_nRegions(octs[i]);
}
else {
pRegion regions;
regions = Zoltan_Oct_regionlist(octs[i]);
while(regions != NULL) {
if(regions->Proc != zz->Proc)
count2++;
regions = regions->next;
}
}
}
}
#endif
/* set up the return pointers */
*nsentags = count;
*npimtags = count2;
if (!exported_tags) {
return ierr;
}
if (count > 0) {
/* allocate some space */
if((mtags=(pRegion)ZOLTAN_MALLOC((unsigned)count*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((exported_pids = (int *)ZOLTAN_MALLOC((unsigned)count*sizeof(int))) ==
NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
return ZOLTAN_MEMERR;
}
*exported_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count);
*exported_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count);
if(!(*exported_gids) || (num_lid_entries && !(*exported_lids))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
return ZOLTAN_MEMERR;
}
}
else {
mtags = NULL;
exported_pids = NULL;
*exported_gids = NULL;
*exported_lids = NULL;
}
/* set up return pointers */
*exported_tags=mtags;
*tag_pids = exported_pids;
if (count2 > 0) {
/* allocate some space */
if((ptags=(pRegion)ZOLTAN_MALLOC((unsigned)count2*sizeof(Region)))==NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
ZOLTAN_FREE(exported_gids);
ZOLTAN_FREE(exported_lids);
return ZOLTAN_MEMERR;
}
*p_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, count2);
*p_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, count2);
if(!(*p_gids) || (num_lid_entries && !(*p_lids))) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient Memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&mtags);
ZOLTAN_FREE(&exported_pids);
ZOLTAN_FREE(exported_gids);
ZOLTAN_FREE(exported_lids);
ZOLTAN_FREE(&ptags);
ZOLTAN_FREE(p_gids);
ZOLTAN_FREE(p_lids);
return ZOLTAN_MEMERR;
}
}
else {
ptags = NULL;
*p_gids = NULL;
*p_lids = NULL;
}
/* set up return pointers */
*p_tags=ptags;
index = index2 = 0;
for (i=0; i<nocts; i++) {
if(Zoltan_Oct_isTerminal(octs[i])) {
regionlist = Zoltan_Oct_regionlist(octs[i]);
while(regionlist != NULL) {
if(regionlist->Proc != zz->Proc) {
/* place information in the appropritate array */
mtags[index] = *regionlist;
ZOLTAN_SET_GID(zz, &((*exported_gids)[index*num_gid_entries]),
regionlist->Global_ID);
ZOLTAN_SET_LID(zz, &((*exported_lids)[index*num_lid_entries]),
regionlist->Local_ID);
/*ex_load += (float)(regionlist->Weight);*/
exported_pids[index] = regionlist->Proc;
index++; /* increment counter */
}
else if(newpids[i] != zz->Proc) {
ptags[index2] = *regionlist; /* get region information */
ZOLTAN_SET_GID(zz, &((*p_gids)[index2*num_gid_entries]),
regionlist->Global_ID);
ZOLTAN_SET_LID(zz, &((*p_lids)[index2*num_lid_entries]),
regionlist->Local_ID);
index2++; /* increment counter */
}
regionlist = regionlist->next; /* get next region */
}
}
}
if (index!=count) { /* error check */
ZOLTAN_TRACE_DETAIL(zz, yo,
"Fatal error, inconsistent number of regions.\n");
return ZOLTAN_FATAL;
}
*c2 = ex_load;
return ierr;
}
static int _Zoltan_Get_Obj_List(
ZZ *zz,
int *num_obj,
ZOLTAN_ID_PTR *global_ids,
ZOLTAN_ID_PTR *local_ids,
int wdim,
float **objwgts,
int **parts,
int special_malloc
)
{
char *yo = "_Zoltan_Get_Obj_List";
int i, n;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
int alloced_gids = 0, alloced_lids = 0;
int gid_off, lid_off;
ZOLTAN_ID_PTR lid, next_lid; /* Temporary pointers to local IDs; used to pass
NULL to query functions when
NUM_LID_ENTRIES == 0. */
float *next_objwgt; /* Temporarry pointer to an object weight; used
to pass NULL to query functions when wdim=0 */
int ierr = ZOLTAN_OK;
ZOLTAN_TRACE_ENTER(zz, yo);
*num_obj = 0;
*objwgts = NULL;
if (zz->Get_Num_Obj != NULL) {
*num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
goto End;
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (*num_obj > 0) {
/*
* Test global_ids and local_ids for NULL.
* Should be NULL for doing partitioning.
* Should not be NULL if doing ordering.
*/
if (special_malloc){
if (*global_ids == NULL) {
Zoltan_Special_Malloc(zz, (void **)global_ids,
*num_obj, ZOLTAN_SPECIAL_MALLOC_GID);
alloced_gids = 1;
}
if (*local_ids == NULL) {
Zoltan_Special_Malloc(zz, (void **)local_ids,
*num_obj, ZOLTAN_SPECIAL_MALLOC_LID);
alloced_lids = 1;
}
Zoltan_Special_Malloc(zz, (void **)parts,
*num_obj, ZOLTAN_SPECIAL_MALLOC_INT);
}
else{
if (*global_ids == NULL) {
*global_ids = ZOLTAN_MALLOC_GID_ARRAY(zz, *num_obj);
alloced_gids = 1;
}
if (*local_ids == NULL) {
*local_ids = ZOLTAN_MALLOC_LID_ARRAY(zz, *num_obj);
alloced_lids = 1;
}
*parts = (int *) ZOLTAN_MALLOC(*num_obj * sizeof(int));
}
if (wdim > 0)
*objwgts = (float*) ZOLTAN_MALLOC (sizeof(float) * *num_obj * wdim);
if ((*global_ids == NULL) || (num_lid_entries > 0 && *local_ids == NULL) ||
(*parts == NULL) ||
(wdim > 0 && *objwgts == NULL)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory Error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
if (zz->Get_Obj_List != NULL){
/* Get object list directly */
zz->Get_Obj_List(zz->Get_Obj_List_Data,
num_gid_entries, num_lid_entries,
*global_ids, *local_ids,
wdim, *objwgts, &ierr);
}
else if ((zz->Get_First_Obj != NULL) && (zz->Get_Next_Obj != NULL)){
/* Use iterator functions to loop through object list */
if (zz->Get_First_Obj(zz->Get_First_Obj_Data,
num_gid_entries, num_lid_entries,
*global_ids, *local_ids,
wdim, *objwgts, &ierr)){
n = *num_obj;
i = 0;
while (!ierr && (i<n-1)){
gid_off = i * num_gid_entries;
lid_off = i * num_lid_entries;
lid = (num_lid_entries ? &((*local_ids)[lid_off]) : NULL);
next_lid = (num_lid_entries ? &((*local_ids)[lid_off+num_lid_entries])
: NULL);
next_objwgt = (wdim ? (*objwgts) + (i+1)*wdim : NULL);
zz->Get_Next_Obj(zz->Get_Next_Obj_Data,
num_gid_entries, num_lid_entries,
&((*global_ids)[gid_off]), lid,
&((*global_ids)[gid_off+num_gid_entries]),
next_lid,
wdim, next_objwgt, &ierr);
i++;
}
}
}
else { /* No way to get objects */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_OBJ_LIST_FN or "
"ZOLTAN_FIRST_OBJ_FN/ZOLTAN_NEXT_OBJ_FN.");
ierr = ZOLTAN_FATAL;
goto End;
}
/* Get partition information for objects. */
/* Call user-callback if provided; otherwise, all parts == zz->Proc */
if (zz->Get_Part == NULL && zz->Get_Part_Multi == NULL) {
for (i = 0; i < *num_obj; i++)
(*parts)[i] = zz->Proc;
}
else if (zz->Get_Part_Multi) {
zz->Get_Part_Multi(zz->Get_Part_Multi_Data,
num_gid_entries, num_lid_entries, *num_obj,
*global_ids, *local_ids, *parts, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_PART_MULTI_FN");
goto End;
}
}
else {
for (i = 0; i < *num_obj; i++) {
lid = (num_lid_entries ? &((*local_ids)[i*num_lid_entries]) : NULL);
(*parts)[i] = zz->Get_Part(zz->Get_Part_Data,
num_gid_entries, num_lid_entries,
&((*global_ids)[i*num_gid_entries]), lid, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_PART_FN");
goto End;
}
}
}
}
End:
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no lists returned.");
if (special_malloc){
if (alloced_gids)
Zoltan_Special_Free(zz, (void **)global_ids, ZOLTAN_SPECIAL_MALLOC_GID);
if (alloced_lids)
Zoltan_Special_Free(zz, (void **)local_ids, ZOLTAN_SPECIAL_MALLOC_LID);
Zoltan_Special_Free(zz, (void **)parts, ZOLTAN_SPECIAL_MALLOC_INT);
}
else{
if (alloced_gids) ZOLTAN_FREE(global_ids);
if (alloced_lids) ZOLTAN_FREE(local_ids);
ZOLTAN_FREE(parts);
}
ZOLTAN_FREE(objwgts);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
/* Performs a permutation of the matrix, perm_y A perm_y^t.
* TODO: at this time we only do symmetric permutations (don't know xGNO !).
*/
int
Zoltan_Matrix_Permute(ZZ* zz, Zoltan_matrix *m, int* perm_y)
{
static char *yo = "Zoltan_Matrix_Permute";
int ierr = ZOLTAN_OK;
int *pinGNO = NULL;
ZOLTAN_ID_PTR yGID=NULL;
float *ywgt=NULL;
struct Zoltan_DD_Struct *dd;
ZOLTAN_TRACE_ENTER(zz, yo);
/* First apply y permutation */
if (m->completed) { /* We directly know the good arrays */
yGID = m->yGID;
ywgt = m->ywgt;
if (m->ddY == NULL || m->ddY != m->ddX) { /* We have to create again the DD */
/* We have to define ddY : yGNO, yGID, ywgt */
ierr = Zoltan_DD_Create (&m->ddY, zz->Communicator, 1, zz->Num_GID,
m->ywgtdim*sizeof(float)/sizeof(int), m->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(m->ddY, m->globalY/zz->Num_Proc);
}
}
else { /* We have to get these fields */
/* Update data directories */
yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, m->nY);
ywgt = (float*) ZOLTAN_MALLOC(m->nY * sizeof(float) * m->ywgtdim);
if (m->nY && (yGID == NULL || (m->ywgtdim && ywgt == NULL)))
MEMORY_ERROR;
/* Get Informations about Y */
Zoltan_DD_Find (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL,
m->nY, NULL);
}
memcpy (m->yGNO, perm_y, m->nY*sizeof(int));
/* Get Informations about Y */
Zoltan_DD_Update (m->ddY, (ZOLTAN_ID_PTR)m->yGNO, yGID, (ZOLTAN_ID_PTR)ywgt, NULL,
m->nY);
ZOLTAN_FREE (&yGID);
ZOLTAN_FREE (&ywgt);
/* We have to define dd : old_yGNO, new_yGNO */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, 1, 1, 0, m->globalY/zz->Num_Proc, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, m->globalY/zz->Num_Proc);
Zoltan_DD_Update (dd, (ZOLTAN_ID_PTR)m->yGNO, (ZOLTAN_ID_PTR)perm_y, NULL, NULL, m->nY);
pinGNO = (int*)ZOLTAN_MALLOC(m->nPins*sizeof(int));
if (m->nPins && pinGNO == NULL)
MEMORY_ERROR;
Zoltan_DD_Find (dd, (ZOLTAN_ID_PTR)m->pinGNO, (ZOLTAN_ID_PTR)pinGNO, NULL, NULL,
m->nY, NULL);
ZOLTAN_FREE(&m->pinGNO);
m->pinGNO = pinGNO;
pinGNO = NULL;
End:
ZOLTAN_FREE (&pinGNO);
ZOLTAN_FREE (&yGID);
ZOLTAN_FREE (&ywgt);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Order_Get_GID_Order(
struct Zoltan_Struct *zz,
ZOLTAN_ID_PTR global_ids,
ZOLTAN_ID_PTR order_ids
)
{
int * proctab;
ZOLTAN_ID_PTR sendgidtab;
int * sendtab, *recvtab;
int i;
int nrecv;
struct Zoltan_Comm_Obj *plan;
int ierr = ZOLTAN_OK;
int *vtxdist;
int tag = 24061986;
int offset;
zz->Order.gidrank = order_ids;
if (!strcmp(zz->Order.order_type, "LOCAL")) {
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
order_ids[i] = global_ids[zz->Order.rank[i] - zz->Order.start_index];
}
return (ZOLTAN_OK);
}
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr != ZOLTAN_OK) return (ierr);
proctab = (int*) ZOLTAN_MALLOC(3*zz->Order.nbr_objects*sizeof(int));
sendgidtab = ZOLTAN_MALLOC_GID_ARRAY(zz, 2*zz->Order.nbr_objects);
if (proctab == NULL) {
ZOLTAN_FREE(&vtxdist);
return (ZOLTAN_MEMERR);
}
if (sendgidtab == NULL) {
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ZOLTAN_MEMERR);
}
sendtab = proctab + zz->Order.nbr_objects;
recvtab = sendtab + zz->Order.nbr_objects;
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
proctab[i] = Zoltan_Get_Processor_Graph(vtxdist, zz->Num_Proc, zz->Order.rank[i] - zz->Order.start_index);
sendtab[i] = zz->Order.rank[i] - zz->Order.start_index;
}
ierr = Zoltan_Comm_Create(&plan, zz->Order.nbr_objects, proctab, zz->Communicator, tag++,
&nrecv);
if (ierr != ZOLTAN_OK) {
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
ierr = Zoltan_Comm_Do(plan, tag++, (char *) sendtab, sizeof(int), (char *) recvtab);
if (ierr != ZOLTAN_OK) {
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
offset = vtxdist[zz->Proc];
for (i = 0 ; i < zz->Order.nbr_objects ; ++i) {
int position;
position = recvtab[i]-offset;
ZOLTAN_SET_GID(zz, &sendgidtab[i], &global_ids[position]);
}
ierr = Zoltan_Comm_Do_Reverse(plan, tag++, (char *) sendgidtab, zz->Num_GID*sizeof(int), NULL,
(char *) order_ids);
Zoltan_Comm_Destroy(&plan);
ZOLTAN_FREE(&sendgidtab);
ZOLTAN_FREE(&proctab);
ZOLTAN_FREE(&vtxdist);
return (ierr);
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int num_gid_entries, /* # of entries for a global id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm /* iperm[rank[i]]=i, only for sequential ordering */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
int * vtxdist = NULL;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL, *local_iperm=NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value");
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.rank = rank;
zz->Order.iperm = iperm;
zz->Order.gids = gids;
zz->Order.lids = lids;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
#ifdef HAVE_MPI
strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN);
#else
strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN);
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
zz->Order.start_index = opt.start_index;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && rank)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
/* strcpy(opt.order_type, "GLOBAL"); */
}
#endif /* ZOLTAN_SCOTCH */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
if (!strcmp(opt.order_type, "GLOBAL"))
strcpy (opt.order_type, "DIST");
if (!strcmp(opt.order_type, "LOCAL"))
strcpy (opt.order_type, "SERIAL");
strcpy(zz->Order.order_type, opt.order_type);
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 3,
&local_gids, &local_rank, &local_iperm);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
if ((!(opt.return_args & RETURN_RANK) && (rank != NULL))
|| (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) {
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr) {
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) {
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) {
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* I store : GNO, rank, iperm */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
ZOLTAN_FREE(&local_iperm);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions,
ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
int *npids, int nregions, int *c2)
{
char *yo = "Zoltan_Oct_migreg_migrate_regions";
int i; /* index counter */
int ierr = ZOLTAN_OK;
int n_import;
ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */
Region *import_objs = NULL; /* Array of import objects used to request
the objs from other processors. */
ZOLTAN_ID_PTR import_gids = NULL; /* Array of global IDs of import_objs. */
ZOLTAN_ID_PTR import_lids = NULL; /* Array of local IDs of import_objs. */
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator,
MIGMIGREGCommCreate, &n_import);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
*c2 = n_import;
if (n_import > 0) {
import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region));
import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import);
import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import);
if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
}
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region),
(char *) import_objs);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids,
sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
if (num_lid_entries > 0) {
ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids,
sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
return (ierr);
}
}
for (i=0; i<n_import; i++) {
import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]);
import_objs[i].Local_ID = (num_lid_entries
? &(import_lids[i*num_lid_entries])
: NULL);
Zoltan_Oct_insert_orphan(zz, import_objs[i]);
}
ZOLTAN_FREE(&import_objs);
ZOLTAN_FREE(&import_gids);
ZOLTAN_FREE(&import_lids);
ierr = Zoltan_Comm_Destroy(&comm_plan);
if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
return ierr;
}
int Zoltan_Oct_migreg_migrate_orphans(ZZ *zz, pRegion RegionList, int nregions,
int level, Map *array, int *c1, int *c2) {
int i, j, k; /* index counters */
pRegion ptr; /* region in the mesh */
COORD origin; /* centroid coordinate information */
pRegion *regions = NULL; /* an array of regions */
int *npids = NULL;
Region *regions2 = NULL; /* an array of regions */
int *npids2 = NULL;
int nreg; /* number of regions */
COORD min, /* minimum bounds of an octant */
max; /* maximum bounds of an octant */
COORD cmin, /* minimum bounds of a child octant */
cmax; /* maximum bounds of a child octant */
COORD rmin, /* minimum bounds of a remote octant */
rmax; /* maximum bounds of a remote octant */
int new_num;
int n;
int dir = 0;
pRList RootList;
pOctant RootOct;
OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
char *yo = "Zoltan_Oct_migreg_migrate_orphans_static";
int ierr = ZOLTAN_OK;
ZOLTAN_ID_PTR gids2, lids2;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
if(nregions > 0) {
/* create the array of messages to be sent to other processors */
/* Array = (Message *) ZOLTAN_MALLOC(nregions * sizeof(Message)); */
if((regions = (pRegion *) ZOLTAN_MALLOC(nregions * sizeof(pRegion))) == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if((npids = (int *) ZOLTAN_MALLOC(nregions * sizeof(int))) == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
ZOLTAN_FREE(®ions);
return ZOLTAN_MEMERR;
}
}
ptr = RegionList;
n = nreg = 0;
while((ptr != NULL) && (nregions > 0)) {
if(ptr->attached == 1) {
/* if region already attached to an octant, then skip to next region */
ptr = ptr->next;
continue;
}
/* region not attached, have to find which processor to send to */
j=0;
dir = 0;
vector_set(min, OCT_info->OCT_gmin);
vector_set(max, OCT_info->OCT_gmax);
/*
* for each level of refinement, find which child region belongs to.
* translate which child to which entry in map array.
*/
for(i=0; i<level; i++) {
Zoltan_Oct_bounds_to_origin(min, max, origin);
if(OCT_info->OCT_dimension == 2)
j = j * 4;
else
j = j * 8;
k = Zoltan_Oct_child_which(OCT_info,origin, ptr->Coord);
new_num = Zoltan_Oct_convert_idx_from_map(OCT_info, dir, k);
dir = Zoltan_Oct_get_child_dir(OCT_info, dir, new_num);
j += new_num;
Zoltan_Oct_child_bounds(min, max, origin, k, cmin, cmax);
vector_set(min, cmin);
vector_set(max, cmax);
}
/* inform message which processor to send to */
npids[n] = array[j].npid;
RootList = array[j].list;
while((RootOct = RL_nextRootOctant(&RootList))) {
Zoltan_Oct_bounds(RootOct,rmin,rmax);
if (Zoltan_Oct_in_box_closure(OCT_info, ptr->Coord ,rmin, rmax)) {
npids[n] = RootOct->npid;
break;
}
}
if((npids[n] != -1) && (npids[n] != zz->Proc)) {
Zoltan_Oct_copy_info(zz, ptr, &(regions[n++]));
}
else {
Zoltan_Oct_insert_orphan(zz, *ptr);
}
nreg++; /* increment region counter */
ptr = ptr->next; /* look at next region */
}
/*
* if regions looked at != number of regions in region list,
* then there is an error
*/
if (nreg!=nregions) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "regions found != to expected number of regions");
return ZOLTAN_FATAL;
}
regions2 = (Region *) ZOLTAN_MALLOC(n * sizeof(Region));
gids2 = ZOLTAN_MALLOC_GID_ARRAY(zz, n);
lids2 = ZOLTAN_MALLOC_LID_ARRAY(zz, n);
npids2 = (int *) ZOLTAN_MALLOC(n * sizeof(int));
for(i=0; i<n; i++) {
npids2[i] = npids[i];
vector_set(regions2[i].Coord, regions[i]->Coord);
regions2[i].Weight = regions[i]->Weight;
regions2[i].Global_ID = &(gids2[i*num_gid_entries]);
regions2[i].Local_ID = (num_lid_entries
? &(lids2[i*num_lid_entries])
: NULL);
ZOLTAN_SET_GID(zz, &(gids2[i*num_gid_entries]), regions[i]->Global_ID);
ZOLTAN_SET_LID(zz, &(lids2[i*num_lid_entries]), regions[i]->Local_ID);
regions2[i].Proc = regions[i]->Proc;
regions2[i].attached = 0;
}
*c1 = n;
/* migrate the orphan regions according to the message array */
Zoltan_Oct_migreg_migrate_regions(zz, regions2, gids2, lids2, npids2, n, c2);
for (i=0; i < n; i++) {
ZOLTAN_FREE(&(regions[i]->Global_ID));
ZOLTAN_FREE(&(regions[i]->Local_ID));
ZOLTAN_FREE(&(regions[i]));
}
ZOLTAN_FREE(®ions);
ZOLTAN_FREE(&npids);
ZOLTAN_FREE(®ions2);
ZOLTAN_FREE(&gids2);
ZOLTAN_FREE(&lids2);
ZOLTAN_FREE(&npids2);
return ierr;
}
int Zoltan_Order (
struct Zoltan_Struct *zz,
int num_gid_entries,
int num_obj,
ZOLTAN_ID_PTR gids,
ZOLTAN_ID_PTR permuted_global_ids
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids.
* num_gid_entries
* Output:
* permuted_global_ids
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
char msg[253];
sprintf(msg, "num_gid_entries=%d is not equal to parameter setting "
"NUM_GID_ENTRIES=%d\n", num_gid_entries, zz->Num_GID);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
#ifdef HAVE_MPI
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "GLOBAL");
#else
strncpy(opt.method, "METIS", MAX_PARAM_STRING_LEN);
strcpy(zz->Order.order_type, "LOCAL");
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
/*
* Check that the user has allocated space for the return args.
*/
if (num_obj && !(gids && permuted_global_ids)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "LOCAL_HSFC"))
{
Order_fn = Zoltan_LocalHSFC_Order;
strcpy(zz->Order.order_type, "LOCAL"); /*MMW, not sure about this*/
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PTSCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
strcpy(zz->Order.order_type, "GLOBAL");
}
#endif /* ZOLTAN_SCOTCH */
#ifdef ZOLTAN_HUND
else if (!strcasecmp(opt.method, "HUND")) {
ierr = Zoltan_HUND(zz, num_gid_entries, num_obj, gids, permuted_global_ids, NULL);
goto End;
}
#endif /* ZOLTAN_HUND */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/* TODO : Ask why useful ! */
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/************************************
* Check for required query function
************************************/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
/* TODO allocate all this stuff with the graph */
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, NULL, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 2,
&local_gids, &local_rank);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* TODO: At this time, I consider rank == permuted_global_ids */
/* I store : GNO, rank, permuted GID */
/* MMW: perhaps don't ever use graph here since we need to support geometric orderings, otherwise need if/else */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, 0, local_num_obj, 0);
/* Hope a linear assignment will help a little */
if (local_num_obj)
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, NULL, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)permuted_global_ids, NULL, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", permuted_global_ids[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
#ifdef ZOLTAN_HUND
End:
#endif /*ZOLTAN_HUND*/
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int
Zoltan_PHG_2ways_hyperedge_partition (
ZZ *zz, /* Input : Zoltan data structure */
HGraph *hg,
Partition parts,
Zoltan_PHG_Tree *tree,
struct Zoltan_DD_Struct * gnoToGID,
struct Zoltan_DD_Struct **dd,
int *numParts,
int **sizeParts
)
{
int ierr = ZOLTAN_OK;
char *yo = "Zoltan_PHG_2ways_hyperedge_partition";
int nEdge, hEdge;
int *interval;
int *partnumber = NULL;
int tree_size;
int *rowpart =NULL;
int *rowGNO = NULL;
ZOLTAN_ID_PTR rowGID=NULL;
int index;
int offset;
ZOLTAN_TRACE_ENTER(zz, yo);
nEdge = hg->nEdge;
fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge);
interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int));
if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR;
tree_size = get_tree_size(tree) + 1;
for (index = 0 ; index < nEdge ; ++index){
SET_MIN_NODE(interval, index, tree_size);
SET_MAX_NODE(interval, index, -1);
}
/* Update interval with the local knowledge */
/* XXX: I loop on the hyperedges, as I think it's more cache friendly
* and it allows me to discoupled the computation if a non blocking MPI_Reduce is
* available
*/
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){
int part;
int max = -1; /* Trick : we use the initialized values */
int min = tree_size + 1;
for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) {
part = parts[hg->hvertex[index]];
max = MAX(max, part);
min = MIN(min, part);
}
SET_MIN_NODE(interval, hEdge, min);
SET_MAX_NODE(interval, hEdge, max);
}
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm);
/* Now I have to compute the partition of hyperedges according to the "interval"
* and the tree */
/* First, compute the partition number corresponding to the nodes in the tree */
partnumber = compute_part_number(tree);
if (partnumber == NULL) {
ierr = ZOLTAN_FATAL;
goto End;
}
(*numParts) = get_tree_size(tree);
rowpart = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int));
if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR;
rowGNO = (int*) ZOLTAN_MALLOC(nEdge*sizeof(int));
if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR;
(*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int));
if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR;
offset = hg->dist_y[hg->comm->myProc_y];
/* Then we search we is the hyperedge in the tree */
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
int node;
node = find_interval_in_tree(tree, interval+2*hEdge);
rowpart[hEdge] = partnumber[node];
(*sizeParts)[rowpart[hEdge]] ++;
rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge);
#if 0
fprintf (stderr, "%d : %d (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]);
#endif
}
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
/* Compute number of elements per parts */
/* TODO: support processor which are not part of the distribution */
/* Update results to view the complete hyperedges */
Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm);
/* Export results to data directory */
/* First, get the GIDs of our edges */
rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge);
if (nEdge && rowGID == NULL) MEMORY_ERROR;
ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL,
nEdge, NULL);
ZOLTAN_FREE(&rowGNO);
ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0);
CHECK_IERR;
/* Make our new numbering public */
Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, (ZOLTAN_ID_PTR) rowpart, NULL, NULL, nEdge);
#ifdef CEDRIC_PRINT
for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]);
}
#endif
End:
ZOLTAN_FREE(&rowGID);
ZOLTAN_FREE(&rowGNO);
ZOLTAN_FREE(&rowpart);
if (partnumber != NULL)
partnumber += 1;
ZOLTAN_FREE(&partnumber);
ZOLTAN_FREE(&interval);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
/*
* Each processor:
* owns a set of pins (nonzeros)
* may provide some edge weights
*
* We assume that no two processes will supply the same pin.
* But more than one process may supply pins for the same edge.
*/
static int
matrix_get_edges(ZZ *zz, Zoltan_matrix *matrix, ZOLTAN_ID_PTR *yGID, ZOLTAN_ID_PTR *pinID, int nX,
ZOLTAN_ID_PTR *xGID, ZOLTAN_ID_PTR *xLID, int **xGNO, float **xwgt)
{
static char *yo = "Zoltan_Matrix_Build";
int ierr = ZOLTAN_OK;
int hypergraph_callbacks = 0, graph_callbacks = 0;
int *nbors_proc = NULL; /* Pointers are global for the function to ensure proper free */
int *edgeSize = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Get_HG_Size_CS && zz->Get_HG_CS) {
hypergraph_callbacks = 1;
}
if ((zz->Get_Num_Edges != NULL || zz->Get_Num_Edges_Multi != NULL) &&
(zz->Get_Edge_List != NULL || zz->Get_Edge_List_Multi != NULL)) {
graph_callbacks = 1;
}
if (graph_callbacks && hypergraph_callbacks){
/* if (hgraph_model == GRAPH) */
/* hypergraph_callbacks = 0; */
graph_callbacks = 1; /* I prefer graph (allow to do "inplace") ! */
}
if (hypergraph_callbacks) {
matrix->redist = 1;
if (matrix->opts.speed == MATRIX_FULL_DD)
ZOLTAN_FREE(xGID);
else
*xGID = NULL;
ZOLTAN_FREE(xLID);
ZOLTAN_FREE(xGNO);
ierr = Zoltan_Hypergraph_Queries(zz, &matrix->nY,
&matrix->nPins, yGID, &matrix->ystart,
pinID);
CHECK_IERR;
matrix->yend = matrix->ystart + 1;
}
else if (graph_callbacks) {
int max_edges = 0;
int vertex;
int numGID, numLID;
matrix->opts.enforceSquare = 1;
matrix->nY = nX; /* It is square ! */
matrix->yGNO = *xGNO;
*xGNO = NULL;
*yGID = NULL;
matrix->ywgtdim = zz->Obj_Weight_Dim;
*xwgt = NULL;
numGID = zz->Num_GID;
numLID = zz->Num_LID;
/* TODO : support local graphs */
/* TODO : support weights ! */
/* Get edge data */
Zoltan_Get_Num_Edges_Per_Obj(zz, matrix->nY, *xGID, *xLID,
&edgeSize, &max_edges, &matrix->nPins);
(*pinID) = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nPins);
nbors_proc = (int *)ZOLTAN_MALLOC(matrix->nPins * sizeof(int));
if (matrix->nPins && ((*pinID) == NULL || nbors_proc == NULL))
MEMORY_ERROR;
matrix->pinwgt = (float*)ZOLTAN_MALLOC(matrix->nPins*matrix->pinwgtdim*sizeof(float));
if (matrix->nPins && matrix->pinwgtdim && matrix->pinwgt == NULL)
MEMORY_ERROR;
if (zz->Get_Edge_List_Multi) {
zz->Get_Edge_List_Multi(zz->Get_Edge_List_Multi_Data,
numGID, numLID,
matrix->nY, *xGID, *xLID,
edgeSize,
(*pinID), nbors_proc, matrix->pinwgtdim,
matrix->pinwgt, &ierr);
}
else {
int edge;
for (vertex = 0, edge = 0 ; vertex < matrix->nY ; ++vertex) {
zz->Get_Edge_List(zz->Get_Edge_List_Data, numGID, numLID,
(*xGID)+vertex*numGID, (*xLID)+vertex*numLID,
(*pinID)+edge*numGID, nbors_proc+edge, matrix->pinwgtdim,
matrix->pinwgt+edge*matrix->pinwgtdim, &ierr);
edge += edgeSize[vertex];
}
}
CHECK_IERR;
/* Not Useful anymore */
ZOLTAN_FREE(xLID);
if (matrix->opts.speed == MATRIX_FULL_DD)
ZOLTAN_FREE(xGID);
else
*xGID = NULL;
ZOLTAN_FREE(&nbors_proc);
/* Now construct CSR indexing */
matrix->ystart = (int*) ZOLTAN_MALLOC((matrix->nY+1)*sizeof(int));
if (matrix->ystart == NULL)
MEMORY_ERROR;
matrix->ystart[0] = 0;
matrix->yend = matrix->ystart + 1;
for (vertex = 0 ; vertex < matrix->nY ; ++vertex)
matrix->ystart[vertex+1] = matrix->ystart[vertex] + edgeSize[vertex];
}
else {
FATAL_ERROR ("You have to define Hypergraph or Graph queries");
}
if (matrix->opts.enforceSquare) {
matrix->globalY = matrix->globalX;
matrix->ddY = matrix->ddX;
matrix->ywgtdim = zz->Obj_Weight_Dim;
}
End:
ZOLTAN_FREE(&edgeSize);
ZOLTAN_FREE(&nbors_proc);
ZOLTAN_FREE(xLID);
ZOLTAN_FREE(xGID);
ZOLTAN_FREE(xGNO);
ZOLTAN_FREE(xwgt);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
static int Zoltan_Reftree_Sum_Weights(ZZ *zz)
{
/*
* Function to sum the weights in the refinement tree. On input the
* refinement tree should be valid and have weight set. On output the
* values in summed_weight at each node is the sum of the weights in the
* subtree with that node as root.
* This function also sets assigned_to_me for interior nodes to be
* 1 if the entire subtree is assigned to this processor
* 0 if none of the subtree is assigned to this processor
* -1 if some of the subtree is assigned to this processor
*/
char *yo = "Zoltan_Reftree_Sum_Weights";
ZOLTAN_REFTREE *root; /* Root of the refinement tree */
int wdim; /* Dimension of the weight array */
int i,j; /* loop counters */
int count; /* counter */
ZOLTAN_ID_PTR leaf_list = NULL;
/* leaves for which some proc requests weight */
ZOLTAN_ID_PTR all_leaflist = NULL;
/* leaf_list from all processors */
int reqsize; /* length of leaf_list */
int *reqsize_all; /* reqsize from all processors */
int sum_reqsize; /* sum of all reqsize */
int *displs; /* running sum of all reqsize */
int my_start; /* position in leaf_list of this proc's list */
int nproc; /* number of processors */
ZOLTAN_REFTREE *node; /* a node in the refinement tree */
struct Zoltan_Reftree_hash_node **hashtab; /* hash table */
int hashsize; /* dimension of hash table */
float *send_float; /* sending message of floats */
float *req_weights; /* the requested weights */
int num_gid_entries = zz->Num_GID; /* Number of array entries in a global ID */
ZOLTAN_TRACE_ENTER(zz, yo);
/*
* set the root and hash table
*/
root = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->reftree_root;
if (root == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Refinement tree not defined.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_FATAL);
}
hashtab = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table;
hashsize = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table_size;
/*
* Determine the dimension of the weight array
*/
if (zz->Obj_Weight_Dim == 0) {
wdim = 1;
} else {
wdim = zz->Obj_Weight_Dim;
}
/*
* In the first pass, sum the weights of the nodes that are assigned to
* this processor, and count the leaves that are not.
*/
count = 0;
for (i=0; i<root->num_child; i++) {
Zoltan_Reftree_Sum_My_Weights(zz,&(root->children[i]),&count,wdim);
}
root->assigned_to_me = -1;
/*
* Make a list of the leaves that are not assigned to this processor
*/
if (count == 0)
leaf_list = ZOLTAN_MALLOC_GID(zz);
else
leaf_list = ZOLTAN_MALLOC_GID_ARRAY(zz, count);
if (leaf_list == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
count = 0;
Zoltan_Reftree_List_Other_Leaves(zz, root,leaf_list,&count);
/*
* Get the unknown leaf weights from other processors.
*/
nproc = zz->Num_Proc;
reqsize = count;
/*
* Build a list of all processor's request list by concatinating them in
* the order of the processor ranks
*/
/*
* Determine the request size of all processors
*/
reqsize_all = (int *)ZOLTAN_MALLOC(nproc*sizeof(int));
displs = (int *)ZOLTAN_MALLOC(nproc*sizeof(int));
if (reqsize_all == NULL || displs == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 3, &displs,
&reqsize_all,
&leaf_list);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
MPI_Allgather((void *)&reqsize,1,MPI_INT,(void *)reqsize_all,1,MPI_INT,
zz->Communicator);
displs[0] = 0;
for (i=1; i<nproc; i++) displs[i] = displs[i-1]+reqsize_all[i-1];
sum_reqsize = displs[nproc-1] + reqsize_all[nproc-1];
my_start = displs[zz->Proc];
/*
* If sum_reqsize is 0, nothing needs to be communciated
*/
if (sum_reqsize == 0) {
Zoltan_Multifree(__FILE__, __LINE__, 3, &displs,
&reqsize_all,
&leaf_list);
}
else {
/*
* Gather the request list from all processors
*/
all_leaflist = ZOLTAN_MALLOC_GID_ARRAY(zz, sum_reqsize);
if (all_leaflist == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 4, &all_leaflist,
&displs,
&reqsize_all,
&leaf_list);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
/* KDDKDD Changed MPI_BYTE to ZOLTAN_ID_MPI_TYPE */
/* Account for number of array entries in an ID. */
for (i=0; i<nproc; i++) {
reqsize_all[i] = reqsize_all[i]*num_gid_entries;
displs[i] = displs[i]*num_gid_entries;
}
MPI_Allgatherv((void *)leaf_list,reqsize*num_gid_entries,ZOLTAN_ID_MPI_TYPE,
(void *)all_leaflist,reqsize_all,displs,ZOLTAN_ID_MPI_TYPE,
zz->Communicator);
ZOLTAN_FREE(&displs);
ZOLTAN_FREE(&leaf_list);
for (i=0; i<nproc; i++) reqsize_all[i] = reqsize_all[i]/num_gid_entries;
/*
* Create a list with the partial sums this processor has
*/
send_float = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*sum_reqsize);
if (send_float == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 3, &send_float,
&all_leaflist,
&reqsize_all);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
for (i=0; i<sum_reqsize; i++) {
node = Zoltan_Reftree_hash_lookup(zz, hashtab,
&(all_leaflist[i*num_gid_entries]),
hashsize);
if (node == NULL)
for (j=0; j<wdim; j++) send_float[i*wdim+j] = 0.0;
else
for (j=0; j<wdim; j++) send_float[i*wdim+j] = node->my_sum_weight[j];
}
/*
* Sum the weights over all the processors
*/
if (reqsize == 0)
req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim);
else
req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*reqsize);
if (req_weights == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
Zoltan_Multifree(__FILE__, __LINE__, 4, &req_weights,
&send_float,
&all_leaflist,
&reqsize_all);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
MPI_Reduce_scatter((void *)send_float, (void *)req_weights, reqsize_all,
MPI_FLOAT, MPI_SUM, zz->Communicator);
ZOLTAN_FREE(&send_float);
ZOLTAN_FREE(&reqsize_all);
/*
* Set the weights this processor requested
*/
for (i=0; i<count; i++) {
node = Zoltan_Reftree_hash_lookup(zz, hashtab,
&(all_leaflist[(i+my_start)*num_gid_entries]),
hashsize);
for (j=0; j<wdim; j++) node->summed_weight[j] = req_weights[i*wdim+j];
}
ZOLTAN_FREE(&req_weights);
ZOLTAN_FREE(&all_leaflist);
}
/*
* All the leaves now have summed_weight set.
* Sum the weights throughout the tree.
*/
Zoltan_Reftree_Sum_All_Weights(zz,root,wdim);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
int
Zoltan_Matrix_Sym(ZZ* zz, Zoltan_matrix *matrix, int bipartite)
{
static char *yo = "Zoltan_Matrix_Sym";
int ierr = ZOLTAN_OK;
Zoltan_Arc *tr_tab = NULL;
int i, j, cnt;
ZOLTAN_ID_PTR yGID = NULL;
int *ypid=NULL;
float *pinwgt=NULL;
int * ybipart = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (bipartite || !matrix->opts.enforceSquare) {
matrix->bipartite = 1;
matrix->redist = 1;
}
if (matrix->ywgtdim != zz->Obj_Weight_Dim)
FATAL_ERROR("Cannot form bipartite graph: vertex and edge weights are not consistant");
matrix->opts.symmetrize = 1;
/* Update the data directories */
tr_tab = (Zoltan_Arc*) ZOLTAN_MALLOC(sizeof(Zoltan_Arc)*(matrix->nPins*2+matrix->nY));
if (matrix->nPins && tr_tab == NULL) MEMORY_ERROR;
pinwgt = (float*)ZOLTAN_MALLOC((matrix->nPins*2+matrix->nY)*matrix->pinwgtdim*sizeof(float));
for (i = 0 ; i < 2 ; ++i) /* Copy pin weights */
memcpy(pinwgt + i*matrix->nPins*matrix->pinwgtdim*sizeof(float),
matrix->pinwgt, matrix->nPins*matrix->pinwgtdim*sizeof(float));
for (i=0, cnt = 0 ; i < matrix->nY ; ++i) {
for (j = matrix->ystart[i] ; j < matrix->yend[i] ; ++j) {
tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */
tr_tab[cnt].GNO[1] = matrix->pinGNO[j];
memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim,
matrix->pinwgtdim*sizeof(float));
cnt ++;
tr_tab[cnt].GNO[0] = matrix->pinGNO[j]; /* Symmetric arc */
tr_tab[cnt].GNO[1] = matrix->yGNO[i] + bipartite*matrix->globalX; /* new ordering */
memcpy(pinwgt + cnt*matrix->pinwgtdim, matrix->pinwgt+j*matrix->pinwgtdim,
matrix->pinwgtdim*sizeof(float));
cnt ++;
}
if (matrix->ystart[i] == matrix->yend[i]) { /* Singleton */
tr_tab[cnt].GNO[0] = matrix->yGNO[i] + bipartite*matrix->globalX; /* Normal arc */
tr_tab[cnt].GNO[1] = -1;
cnt ++;
}
}
ZOLTAN_FREE(&matrix->pinwgt);
Zoltan_Matrix_Remove_DupArcs(zz, cnt, tr_tab, pinwgt, matrix);
ZOLTAN_FREE(&tr_tab);
ZOLTAN_FREE(&pinwgt);
if (bipartite) {
int endX;
int * yGNO = NULL;
/* Update data directories */
yGID = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nY);
ypid = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int));
ybipart = (int*) ZOLTAN_MALLOC(matrix->nY*sizeof(int));
for (endX = 0 ; endX < matrix->nY ; ++endX) {
if (matrix->yGNO[endX] >= matrix->globalX)
break;
ybipart[endX] = 0;
}
/* Get Informations about X */
Zoltan_DD_Find (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, NULL,
endX, NULL);
yGNO = (int*)ZOLTAN_MALLOC(endX*sizeof(int));
for (i = endX ; i < matrix->nY ; ++i) {
yGNO[i-endX] = matrix->yGNO[i] - matrix->globalX;
/* TODO: add a something to have the correct ypid */
ybipart[endX] = 1;
}
/* Get Informations about Y */
Zoltan_DD_Find (matrix->ddY, (ZOLTAN_ID_PTR)yGNO,
yGID + endX*zz->Num_GID,
NULL, NULL,
matrix->nY-endX, NULL);
if (matrix->ddY != matrix->ddX)
Zoltan_DD_Destroy (&matrix->ddY);
Zoltan_DD_Destroy (&matrix->ddX);
matrix->globalX += matrix->globalY;
matrix->globalY = matrix->globalX;
/* I store : xGNO, xGID, xpid, bipart */
ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, 1, zz->Num_GID,
1, matrix->globalX/zz->Num_Proc, 0);
matrix->ddY = matrix->ddX;
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
/* Associate all the data with our xyGNO */
Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, (ZOLTAN_ID_PTR)ypid, ybipart,
matrix->nY);
}
End:
ZOLTAN_FREE(&ybipart);
ZOLTAN_FREE(&ypid);
ZOLTAN_FREE(&pinwgt);
ZOLTAN_FREE(&yGID);
ZOLTAN_FREE(&tr_tab);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
