void migrate_pre_process(void *data, int num_gid_entries, int num_lid_entries,
int num_import,
ZOLTAN_ID_PTR import_global_ids,
ZOLTAN_ID_PTR import_local_ids, int *import_procs,
int *import_to_part,
int num_export, ZOLTAN_ID_PTR export_global_ids,
ZOLTAN_ID_PTR export_local_ids, int *export_procs,
int *export_to_part,
int *ierr)
{
int lid = num_lid_entries-1;
int gid = num_gid_entries-1;
char msg[256];
*ierr = ZOLTAN_OK;
if (data == NULL) {
*ierr = ZOLTAN_FATAL;
return;
}
MESH_INFO_PTR mesh = (MESH_INFO_PTR) data;
ELEM_INFO_PTR elements = mesh->elements;
/*
* Set some flags. Assume if true for one element, true for all elements.
* Note that some procs may have no elements.
*/
int k = 0;
if (elements[0].edge_wgt != NULL)
k = 1;
/* Make sure all procs have the same value */
MPI_Allreduce(&k, &Use_Edge_Wgts, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
/*
* For all elements, update adjacent elements' processor information.
* That way, when perform migration, will be migrating updated adjacency
* information.
*/
int proc = 0;
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
/*
* Build New_Elem_Index array and list of processor assignments.
*/
New_Elem_Index_Size = mesh->num_elems + num_import - num_export;
if (mesh->elem_array_len > New_Elem_Index_Size)
New_Elem_Index_Size = mesh->elem_array_len;
New_Elem_Index = new ZOLTAN_ID_TYPE [New_Elem_Index_Size];
int *proc_ids = NULL;
char *change = NULL;
if (mesh->num_elems > 0) {
proc_ids = new int [mesh->num_elems];
change = new char [mesh->num_elems];
if (New_Elem_Index == NULL || proc_ids == NULL || change == NULL) {
Gen_Error(0, "fatal: insufficient memory");
*ierr = ZOLTAN_MEMERR;
if (proc_ids) delete [] proc_ids;
if (change) delete [] change;
if (New_Elem_Index)
{
delete [] New_Elem_Index;
New_Elem_Index = NULL;
}
return;
}
for (int i = 0; i < mesh->num_elems; i++) {
New_Elem_Index[i] = elements[i].globalID;
proc_ids[i] = proc;
change[i] = 0;
}
}
for (int i = mesh->num_elems; i < New_Elem_Index_Size; i++) {
New_Elem_Index[i] = ZOLTAN_ID_INVALID;
}
for (int i = 0; i < num_export; i++) {
int exp_elem = 0;
if (num_lid_entries)
exp_elem = export_local_ids[lid+i*num_lid_entries];
else /* testing num_lid_entries == 0 */
search_by_global_id(mesh, export_global_ids[gid+i*num_gid_entries],
&exp_elem);
if (export_procs[i] != proc) {
/* Export is moving to a new processor */
New_Elem_Index[exp_elem] = ZOLTAN_ID_INVALID;
proc_ids[exp_elem] = export_procs[i];
}
}
for (int i = 0; i < num_import; i++) {
if (import_procs[i] != proc) {
/* Import is moving from a new processor, not just from a new partition */
/* search for first free location */
int j=0;
for (j = 0; j < New_Elem_Index_Size; j++)
if (New_Elem_Index[j] == ZOLTAN_ID_INVALID) break;
New_Elem_Index[j] = import_global_ids[gid+i*num_gid_entries];
}
}
/*
* Update local information
*/
/* Set change flag for elements whose adjacent elements are being exported */
for (int i = 0; i < num_export; i++) {
int exp_elem = 0;
if (num_lid_entries)
exp_elem = export_local_ids[lid+i*num_lid_entries];
else /* testing num_lid_entries == 0 */
search_by_global_id(mesh, export_global_ids[gid+i*num_gid_entries],
&exp_elem);
elements[exp_elem].my_part = export_to_part[i];
if (export_procs[i] == proc)
continue; /* No adjacency changes needed if export is changing
only partition, not processor. */
for (int j = 0; j < elements[exp_elem].adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (elements[exp_elem].adj[j] == ZOLTAN_ID_INVALID) continue;
/* Set change flag for adjacent local elements. */
if (elements[exp_elem].adj_proc[j] == proc) {
change[elements[exp_elem].adj[j]] = 1;
}
}
}
/* Change adjacency information in marked elements */
for (int i = 0; i < mesh->num_elems; i++) {
if (change[i] == 0) continue;
/* loop over marked element's adjacencies; look for ones that are moving */
for (int j = 0; j < elements[i].adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (elements[i].adj[j] == ZOLTAN_ID_INVALID) continue;
if (elements[i].adj_proc[j] == proc) {
/* adjacent element is local; check whether it is moving. */
int new_proc = proc_ids[elements[i].adj[j]];
if (new_proc != proc) {
/* Adjacent element is being exported; update this adjacency entry */
elements[i].adj[j] = elements[elements[i].adj[j]].globalID;
elements[i].adj_proc[j] = new_proc;
}
}
}
}
delete [] change;
/*
* Update off-processor information
*/
int maxlen = 0;
int *send_vec = NULL;
for (int i = 0; i < mesh->necmap; i++)
maxlen += mesh->ecmap_cnt[i];
if (maxlen > 0) {
send_vec = new int [maxlen];
if (send_vec == NULL) {
Gen_Error(0, "fatal: insufficient memory");
*ierr = ZOLTAN_MEMERR;
delete [] proc_ids;
delete [] change;
return;
}
/* Load send vector */
for (int i = 0; i < maxlen; i++)
send_vec[i] = proc_ids[mesh->ecmap_elemids[i]];
}
delete [] proc_ids;
int *recv_vec = NULL;
if (maxlen > 0)
recv_vec = new int [maxlen];
/* Perform boundary exchange */
boundary_exchange(mesh, 1, send_vec, recv_vec);
/* Unload receive vector */
int offset = 0;
for (int i = 0; i < mesh->necmap; i++) {
for (int j = 0; j < mesh->ecmap_cnt[i]; j++, offset++) {
if (recv_vec[offset] == mesh->ecmap_id[i]) {
/* off-processor element is not changing processors. */
/* no changes are needed in the local data structure. */
continue;
}
/* Change processor assignment in local element's adjacency list */
int bor_elem = mesh->ecmap_elemids[offset];
for (k = 0; k < elements[bor_elem].adj_len; k++) {
/* Skip NULL adjacencies (sides that are not adj to another elem). */
if (elements[bor_elem].adj[k] == ZOLTAN_ID_INVALID) continue;
if (elements[bor_elem].adj[k] == mesh->ecmap_neighids[offset] &&
elements[bor_elem].adj_proc[k] == mesh->ecmap_id[i]) {
elements[bor_elem].adj_proc[k] = recv_vec[offset];
if (recv_vec[offset] == proc) {
/* element is moving to this processor; */
/* convert adj from global to local ID. */
int idx = in_list(mesh->ecmap_neighids[offset],New_Elem_Index_Size,
New_Elem_Index);
if (idx == -1) {
sprintf(msg, "fatal: unable to locate element " ZOLTAN_ID_SPEC " in "
"New_Elem_Index", mesh->ecmap_neighids[offset]);
Gen_Error(0, msg);
*ierr = ZOLTAN_FATAL;
if (send_vec) delete [] send_vec;
if (recv_vec) delete [] recv_vec;
return;
}
elements[bor_elem].adj[k] = idx;
}
break; /* from k loop */
}
}
}
}
if (recv_vec) delete [] recv_vec;
if (send_vec) delete [] send_vec;
/*
* Allocate space (if needed) for the new element data.
*/
if (mesh->elem_array_len < New_Elem_Index_Size) {
mesh->elem_array_len = New_Elem_Index_Size;
// We don't use C++ new/delete here, because this was malloc'd
// in some C code.
mesh->elements = (ELEM_INFO_PTR) realloc (mesh->elements,
mesh->elem_array_len * sizeof(ELEM_INFO));
if (mesh->elements == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return;
}
/* initialize the new spots */
for (int i = mesh->num_elems; i < mesh->elem_array_len; i++)
initialize_element(&(mesh->elements[i]));
}
}
void migrate_pre_process(void *data, int num_gid_entries, int num_lid_entries,
int num_import,
ZOLTAN_ID_PTR import_global_ids,
ZOLTAN_ID_PTR import_local_ids, int *import_procs,
int *import_to_part,
int num_export, ZOLTAN_ID_PTR export_global_ids,
ZOLTAN_ID_PTR export_local_ids, int *export_procs,
int *export_to_part,
int *ierr)
{
int i, j, k, idx, maxlen, proc, offset;
int *proc_ids = NULL; /* Temp array of processor assignments for elements.*/
char *change = NULL; /* Temp array indicating whether local element's adj
list must be updated due to a nbor's migration. */
int new_proc; /* New processor assignment for nbor element. */
int exp_elem; /* index of an element being exported */
int bor_elem; /* index of an element along the processor border */
int *send_vec = NULL, *recv_vec = NULL; /* Communication vecs. */
MESH_INFO_PTR mesh;
ELEM_INFO_PTR elements;
int lid = num_lid_entries-1;
int gid = num_gid_entries-1;
char msg[256];
*ierr = ZOLTAN_OK;
if (data == NULL) {
*ierr = ZOLTAN_FATAL;
return;
}
mesh = (MESH_INFO_PTR) data;
elements = mesh->elements;
for (i=0; i < mesh->num_elems; i++) {
/* don't migrate a pointer created on this process */
safe_free((void **)(void *)&(elements[i].adj_blank));
}
/*
* Set some flags. Assume if true for one element, true for all elements.
* Note that some procs may have no elements.
*/
if (elements[0].edge_wgt != NULL)
k = 1;
else
k = 0;
/* Make sure all procs have the same value */
MPI_Allreduce(&k, &Use_Edge_Wgts, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
/* NOT IMPLEMENTED: blanking information is not sent along. Subsequent
lb_eval may be incorrect, since imported elements may have blanked
adjacencies.
if (mesh->blank_count > 0)
k = 1;
else
k = 0;
MPI_Allreduce(&k, &Vertex_Blanking, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
*/
/*
* For all elements, update adjacent elements' processor information.
* That way, when perform migration, will be migrating updated adjacency
* information.
*/
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
/*
* Build New_Elem_Index array and list of processor assignments.
*/
New_Elem_Index_Size = mesh->num_elems + num_import - num_export;
if (mesh->elem_array_len > New_Elem_Index_Size)
New_Elem_Index_Size = mesh->elem_array_len;
New_Elem_Index = (int *) malloc(New_Elem_Index_Size * sizeof(int));
New_Elem_Hash_Table = (int *) malloc(New_Elem_Index_Size * sizeof(int));
New_Elem_Hash_Nodes = (struct New_Elem_Hash_Node *)
malloc(New_Elem_Index_Size * sizeof(struct New_Elem_Hash_Node));
if (New_Elem_Index == NULL ||
New_Elem_Hash_Table == NULL || New_Elem_Hash_Nodes == NULL) {
Gen_Error(0, "fatal: insufficient memory");
*ierr = ZOLTAN_MEMERR;
return;
}
for (i = 0; i < New_Elem_Index_Size; i++)
New_Elem_Hash_Table[i] = -1;
for (i = 0; i < New_Elem_Index_Size; i++) {
New_Elem_Hash_Nodes[i].globalID = -1;
New_Elem_Hash_Nodes[i].localID = -1;
New_Elem_Hash_Nodes[i].next = -1;
}
if (mesh->num_elems > 0) {
proc_ids = (int *) malloc(mesh->num_elems * sizeof(int));
change = (char *) malloc(mesh->num_elems * sizeof(char));
if (New_Elem_Index == NULL || proc_ids == NULL || change == NULL ||
New_Elem_Hash_Table == NULL || New_Elem_Hash_Nodes == NULL) {
Gen_Error(0, "fatal: insufficient memory");
*ierr = ZOLTAN_MEMERR;
return;
}
for (i = 0; i < mesh->num_elems; i++) {
New_Elem_Index[i] = elements[i].globalID;
insert_in_hash(elements[i].globalID, i);
proc_ids[i] = proc;
change[i] = 0;
}
}
for (i = mesh->num_elems; i < New_Elem_Index_Size; i++) {
New_Elem_Index[i] = -1;
}
for (i = 0; i < num_export; i++) {
if (num_lid_entries)
exp_elem = export_local_ids[lid+i*num_lid_entries];
else /* testing num_lid_entries == 0 */
search_by_global_id(mesh, export_global_ids[gid+i*num_gid_entries],
&exp_elem);
if (export_procs[i] != proc) {
/* Export is moving to a new processor */
New_Elem_Index[exp_elem] = -1;
remove_from_hash(export_global_ids[gid+i*num_gid_entries]);
proc_ids[exp_elem] = export_procs[i];
}
}
j = 0;
for (i = 0; i < num_import; i++) {
if (import_procs[i] != proc) {
/* Import is moving from a new processor, not just from a new partition */
/* search for first free location */
for ( ; j < New_Elem_Index_Size; j++)
if (New_Elem_Index[j] == -1) break;
New_Elem_Index[j] = import_global_ids[gid+i*num_gid_entries];
insert_in_hash((int) import_global_ids[gid+i*num_gid_entries], j);
}
}
/*
* Update local information
*/
/* Set change flag for elements whose adjacent elements are being exported */
for (i = 0; i < num_export; i++) {
if (num_lid_entries)
exp_elem = export_local_ids[lid+i*num_lid_entries];
else /* testing num_lid_entries == 0 */
search_by_global_id(mesh, export_global_ids[gid+i*num_gid_entries],
&exp_elem);
elements[exp_elem].my_part = export_to_part[i];
if (export_procs[i] == proc)
continue; /* No adjacency changes needed if export is changing
only partition, not processor. */
for (j = 0; j < elements[exp_elem].adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (elements[exp_elem].adj[j] == -1) continue;
/* Set change flag for adjacent local elements. */
if (elements[exp_elem].adj_proc[j] == proc) {
change[elements[exp_elem].adj[j]] = 1;
}
}
}
/* Change adjacency information in marked elements */
for (i = 0; i < mesh->num_elems; i++) {
if (change[i] == 0) continue;
/* loop over marked element's adjacencies; look for ones that are moving */
for (j = 0; j < elements[i].adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (elements[i].adj[j] == -1) continue;
if (elements[i].adj_proc[j] == proc) {
/* adjacent element is local; check whether it is moving. */
if ((new_proc = proc_ids[elements[i].adj[j]]) != proc) {
/* Adjacent element is being exported; update this adjacency entry */
elements[i].adj[j] = elements[elements[i].adj[j]].globalID;
elements[i].adj_proc[j] = new_proc;
}
}
}
}
safe_free((void **)(void *) &change);
/*
* Update off-processor information
*/
maxlen = 0;
for (i = 0; i < mesh->necmap; i++)
maxlen += mesh->ecmap_cnt[i];
if (maxlen > 0) {
send_vec = (int *) malloc(maxlen * sizeof(int));
if (send_vec == NULL) {
Gen_Error(0, "fatal: insufficient memory");
*ierr = ZOLTAN_MEMERR;
return;
}
/* Load send vector */
for (i = 0; i < maxlen; i++)
send_vec[i] = proc_ids[mesh->ecmap_elemids[i]];
}
safe_free((void **)(void *) &proc_ids);
if (maxlen > 0)
recv_vec = (int *) malloc(maxlen * sizeof(int));
/* Perform boundary exchange */
boundary_exchange(mesh, 1, send_vec, recv_vec);
/* Unload receive vector */
offset = 0;
for (i = 0; i < mesh->necmap; i++) {
for (j = 0; j < mesh->ecmap_cnt[i]; j++, offset++) {
if (recv_vec[offset] == mesh->ecmap_id[i]) {
/* off-processor element is not changing processors. */
/* no changes are needed in the local data structure. */
continue;
}
/* Change processor assignment in local element's adjacency list */
bor_elem = mesh->ecmap_elemids[offset];
for (k = 0; k < elements[bor_elem].adj_len; k++) {
/* Skip NULL adjacencies (sides that are not adj to another elem). */
if (elements[bor_elem].adj[k] == -1) continue;
if (elements[bor_elem].adj[k] == mesh->ecmap_neighids[offset] &&
elements[bor_elem].adj_proc[k] == mesh->ecmap_id[i]) {
elements[bor_elem].adj_proc[k] = recv_vec[offset];
if (recv_vec[offset] == proc) {
/* element is moving to this processor; */
/* convert adj from global to local ID. */
idx = find_in_hash(mesh->ecmap_neighids[offset]);
if (idx >= 0)
idx = New_Elem_Hash_Nodes[idx].localID;
else {
sprintf(msg, "fatal: unable to locate element %d in "
"New_Elem_Index", mesh->ecmap_neighids[offset]);
Gen_Error(0, msg);
*ierr = ZOLTAN_FATAL;
return;
}
elements[bor_elem].adj[k] = idx;
}
break; /* from k loop */
}
}
}
}
safe_free((void **)(void *) &recv_vec);
safe_free((void **)(void *) &send_vec);
/*
* Allocate space (if needed) for the new element data.
*/
if (mesh->elem_array_len < New_Elem_Index_Size) {
mesh->elem_array_len = New_Elem_Index_Size;
mesh->elements = (ELEM_INFO_PTR) realloc (mesh->elements,
mesh->elem_array_len * sizeof(ELEM_INFO));
if (mesh->elements == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return;
}
/* initialize the new spots */
for (i = mesh->num_elems; i < mesh->elem_array_len; i++)
initialize_element(&(mesh->elements[i]));
}
}
