int build_elem_comm_maps(int proc, MESH_INFO_PTR mesh)
{
/*
* Build element communication maps, given a distributed mesh.
* This routine builds initial communication maps for Chaco input
* (for Nemesis, initial communication maps are read from the Nemesis file)
* and rebuilds communication maps after data migration.
*
* One communication map per neighboring processor is built.
* The corresponding maps on neighboring processors
* must be sorted in the same order, so that neighboring processors do not
* have to use ghost elements. For each communication map's pair of
* processors, the lower-numbered processor determines the order of the
* elements in the communication map. The sort key is the elements' global
* IDs on the lower-number processor; the secondary key is the neighboring
* elements global IDs. The secondary key is used when a single element
* must communicate with more than one neighbor.
*/
const char *yo = "build_elem_comm_maps";
int i, j;
ELEM_INFO *elem;
ZOLTAN_ID_TYPE iadj_elem;
int iadj_proc;
int indx;
int num_alloc_maps;
int max_adj = 0;
int max_adj_per_map;
int cnt, offset;
int *sindex = NULL;
int tmp;
struct map_list_head *tmp_maps = NULL, *map = NULL;
DEBUG_TRACE_START(proc, yo);
/*
* Free the old maps, if they exist.
*/
if (mesh->ecmap_id != NULL) {
safe_free((void **) &(mesh->ecmap_id));
safe_free((void **) &(mesh->ecmap_cnt));
safe_free((void **) &(mesh->ecmap_elemids));
safe_free((void **) &(mesh->ecmap_sideids));
safe_free((void **) &(mesh->ecmap_neighids));
mesh->necmap = 0;
}
/*
* Look for off-processor adjacencies.
* Loop over all elements
*/
num_alloc_maps = MAP_ALLOC;
mesh->ecmap_id = (int *) malloc(num_alloc_maps * sizeof(int));
mesh->ecmap_cnt = (int *) malloc(num_alloc_maps * sizeof(int));
tmp_maps = (struct map_list_head*) malloc(num_alloc_maps
* sizeof(struct map_list_head));
if (mesh->ecmap_id == NULL || mesh->ecmap_cnt == NULL || tmp_maps == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
DEBUG_TRACE_END(proc, yo);
return 0;
}
for (i = 0; i < mesh->num_elems; i++) {
elem = &(mesh->elements[i]);
for (j = 0; j < elem->adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (elem->adj[j] == ZOLTAN_ID_INVALID) continue;
iadj_elem = elem->adj[j];
iadj_proc = elem->adj_proc[j];
if (iadj_proc != proc) {
/*
* Adjacent element is off-processor.
* Add this element to the temporary data structure for
* the appropriate neighboring processor.
*/
if ((indx = in_list2(iadj_proc, mesh->necmap, mesh->ecmap_id)) == -1) {
/*
* Start a new communication map.
*/
if (mesh->necmap >= num_alloc_maps) {
num_alloc_maps += MAP_ALLOC;
mesh->ecmap_id = (int *) realloc(mesh->ecmap_id,
num_alloc_maps * sizeof(int));
mesh->ecmap_cnt = (int *) realloc(mesh->ecmap_cnt,
num_alloc_maps * sizeof(int));
tmp_maps = (struct map_list_head *) realloc(tmp_maps,
num_alloc_maps * sizeof(struct map_list_head));
if (mesh->ecmap_id == NULL || mesh->ecmap_cnt == NULL ||
tmp_maps == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
DEBUG_TRACE_END(proc, yo);
return 0;
}
}
mesh->ecmap_id[mesh->necmap] = iadj_proc;
mesh->ecmap_cnt[mesh->necmap] = 0;
map = &(tmp_maps[mesh->necmap]);
map->glob_id = (ZOLTAN_ID_TYPE *) malloc(MAP_ALLOC * sizeof(ZOLTAN_ID_TYPE));
map->elem_id = (int *) malloc(MAP_ALLOC * sizeof(int));
map->side_id = (int *) malloc(MAP_ALLOC * sizeof(int));
map->neigh_id = (ZOLTAN_ID_TYPE *) malloc(MAP_ALLOC * sizeof(ZOLTAN_ID_TYPE));
if (map->glob_id == NULL || map->elem_id == NULL ||
map->side_id == NULL || map->neigh_id == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
DEBUG_TRACE_END(proc, yo);
return 0;
}
map->map_alloc_size = MAP_ALLOC;
indx = mesh->necmap;
mesh->necmap++;
}
/* Add to map for indx. */
map = &(tmp_maps[indx]);
if (mesh->ecmap_cnt[indx] >= map->map_alloc_size) {
map->map_alloc_size += MAP_ALLOC;
map->glob_id = (ZOLTAN_ID_TYPE *) realloc(map->glob_id, map->map_alloc_size * sizeof(ZOLTAN_ID_TYPE));
map->elem_id = (int *) realloc(map->elem_id,
map->map_alloc_size * sizeof(int));
map->side_id = (int *) realloc(map->side_id,
map->map_alloc_size * sizeof(int));
map->neigh_id = (ZOLTAN_ID_TYPE *) realloc(map->neigh_id, map->map_alloc_size * sizeof(ZOLTAN_ID_TYPE));
if (map->glob_id == NULL || map->elem_id == NULL ||
map->side_id == NULL || map->neigh_id == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
DEBUG_TRACE_END(proc, yo);
return 0;
}
}
tmp = mesh->ecmap_cnt[indx];
map->glob_id[tmp] = elem->globalID;
map->elem_id[tmp] = i;
map->side_id[tmp] = j+1; /* side is determined by position in
adj array (+1 since not 0-based). */
map->neigh_id[tmp] = iadj_elem;
mesh->ecmap_cnt[indx]++;
max_adj++;
}
}
}
/*
* If no communication maps, don't need to do anything else.
*/
if (mesh->necmap > 0) {
/*
* Allocate data structure for element communication map arrays.
*/
mesh->ecmap_elemids = (int *) malloc(max_adj * sizeof(int));
mesh->ecmap_sideids = (int *) malloc(max_adj * sizeof(int));
mesh->ecmap_neighids = (ZOLTAN_ID_TYPE *) malloc(max_adj * sizeof(ZOLTAN_ID_TYPE));
/*
* Allocate temporary memory for sort index.
*/
max_adj_per_map = 0;
for (i = 0; i < mesh->necmap; i++)
if (mesh->ecmap_cnt[i] > max_adj_per_map)
max_adj_per_map = mesh->ecmap_cnt[i];
sindex = (int *) malloc(max_adj_per_map * sizeof(int));
cnt = 0;
for (i = 0; i < mesh->necmap; i++) {
map = &(tmp_maps[i]);
for (j = 0; j < mesh->ecmap_cnt[i]; j++)
sindex[j] = j;
/*
* Sort the map so that adjacent processors have the same ordering
* for the communication.
* Assume the ordering of the lower-numbered processor in the pair
* of communicating processors.
*/
if (proc < mesh->ecmap_id[i])
quicksort_pointer_inc_id_id(sindex, map->glob_id, map->neigh_id,
0, mesh->ecmap_cnt[i]-1);
else
quicksort_pointer_inc_id_id(sindex, map->neigh_id, map->glob_id,
0, mesh->ecmap_cnt[i]-1);
/*
* Copy sorted data into elem map arrays.
*/
offset = cnt;
for (j = 0; j < mesh->ecmap_cnt[i]; j++) {
mesh->ecmap_elemids[offset] = map->elem_id[sindex[j]];
mesh->ecmap_sideids[offset] = map->side_id[sindex[j]];
mesh->ecmap_neighids[offset] = map->neigh_id[sindex[j]];
offset++;
}
cnt += mesh->ecmap_cnt[i];
}
}
/* Free temporary data structure. */
for (i = 0; i < mesh->necmap; i++) {
safe_free((void **) &(tmp_maps[i].glob_id));
safe_free((void **) &(tmp_maps[i].elem_id));
safe_free((void **) &(tmp_maps[i].side_id));
safe_free((void **) &(tmp_maps[i].neigh_id));
}
safe_free((void **) &tmp_maps);
safe_free((void **) &sindex);
if (Test.DDirectory)
compare_maps_with_ddirectory_results(proc, mesh);
DEBUG_TRACE_END(proc, yo);
return 1;
}
static void sort_and_compare_maps(
int proc,
int nbor_proc,
MESH_INFO_PTR mesh,
struct map_list_head *map,
int map_size,
int *sindex
)
{
/*
* Routine to sort a given communication map for a single neighbor processor
* and compare it to the actual communication map for that processor
* (generated by build_elem_comm_maps).
* If the DDirectory were used to build comm maps, this routine could be
* modified to do the assignments to mesh->ecmap_*, rather than do comparisons
* with it.
*/
int i, j;
int cnt = 0;
int indx;
/*
* Sort the given map according to element ids.
* Primary key is determined as in build_elem_comm_maps.
*/
for (i = 0; i < map_size; i++)
sindex[i] = i;
if (proc < nbor_proc)
quicksort_pointer_inc_id_id(sindex, map->glob_id, map->neigh_id,
0, map_size-1);
else
quicksort_pointer_inc_id_id(sindex, map->neigh_id, map->glob_id,
0, map_size-1);
/*
* Compute offset into mesh communication maps for the given nbor proc.
*/
if ((indx = in_list2(nbor_proc, mesh->necmap, mesh->ecmap_id)) == -1) {
printf("%d DDirectory Test: Comm map for nbor proc %d does not exist\n",
proc, nbor_proc);
return;
}
cnt = 0;
for (i = 0; i < indx; i++)
cnt += mesh->ecmap_cnt[i];
/*
* Compare given map to mesh communication map for this nbor proc.
* If the DDirectory code were used to construct maps, assignments
* would be done here (rather than comparisons).
*/
if (map_size != mesh->ecmap_cnt[indx]) {
printf("%d DDirectory Test: Different map size for nbor_proc %d: "
"%d != %d\n", proc, nbor_proc, map_size, mesh->ecmap_cnt[indx]);
return;
}
for (i = 0; i < map_size; i++) {
j = sindex[i];
if (map->elem_id[j] != mesh->ecmap_elemids[i+cnt]) {
printf("%d DDirectory Test: Different element IDs for nbor_proc %d: "
"%d != %d\n", proc, nbor_proc, map->elem_id[j],
mesh->ecmap_elemids[i+cnt]);
}
}
for (i = 0; i < map_size; i++) {
j = sindex[i];
if (map->side_id[j] != mesh->ecmap_sideids[i+cnt]) {
printf("%d DDirectory Test: Different side IDs for nbor_proc %d: "
"%d != %d\n", proc, nbor_proc, map->side_id[j],
mesh->ecmap_sideids[i+cnt]);
}
}
for (i = 0; i < map_size; i++) {
j = sindex[i];
if (map->neigh_id[j] != mesh->ecmap_neighids[i+cnt]) {
printf("%d DDirectory Test: Different neigh IDs for nbor_proc %d: "
ZOLTAN_ID_SPEC " != " ZOLTAN_ID_SPEC "\n", proc, nbor_proc, map->neigh_id[j],
mesh->ecmap_neighids[i+cnt]);
}
}
if (Debug_Driver > 3) {
printf("%d ************* DDirectory Map for %d ***************\n",
proc, nbor_proc);
printf("Local ID\tSide ID\tGlobal ID\tNeigh ID\n");
for (i = 0; i < map_size; i++) {
j = sindex[i];
printf("\t%d\t%d\t" ZOLTAN_ID_SPEC "\t" ZOLTAN_ID_SPEC "\n",
map->elem_id[j], map->side_id[j],
map->glob_id[j], map->neigh_id[j]);
}
}
}
/*--------------------------------------------------------------------------*/
int output_results(const char *cmd_file,
const char *tag,
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
/*
* For the first swipe at this, don't try to create a new
* exodus/nemesis file or anything. Just get the global ids,
* sort them, and print them to a new ascii file.
*/
{
/* Local declarations. */
const char *yo = "output_results";
char par_out_fname[FILENAME_MAX+1], ctemp[FILENAME_MAX+1];
char cmsg[256];
ZOLTAN_ID_TYPE *global_ids = NULL;
ZOLTAN_ID_TYPE *parts = NULL;
ZOLTAN_ID_TYPE *perm = NULL;
ZOLTAN_ID_TYPE *invperm = NULL;
int *index = NULL;
int i, j;
FILE *fp;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
if (mesh->num_elems) {
global_ids = (ZOLTAN_ID_TYPE*) malloc(4 * mesh->num_elems * sizeof(ZOLTAN_ID_TYPE) + mesh->num_elems * sizeof(int));
if (!global_ids) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
parts = global_ids + mesh->num_elems;
perm = parts + mesh->num_elems;
invperm = perm + mesh->num_elems;
index = (int*)(invperm + mesh->num_elems);
}
for (i = j = 0; i < mesh->elem_array_len; i++) {
if (mesh->elements[i].globalID != ZOLTAN_ID_INVALID) {
global_ids[j] = (ZOLTAN_ID_TYPE) mesh->elements[i].globalID;
parts[j] =(ZOLTAN_ID_TYPE) mesh->elements[i].my_part;
perm[j] = (ZOLTAN_ID_TYPE)mesh->elements[i].perm_value;
invperm[j] = (ZOLTAN_ID_TYPE)mesh->elements[i].invperm_value;
index[j] = j;
j++;
}
}
quicksort_pointer_inc_id_id(index, global_ids, NULL,
0, mesh->num_elems-1);
/* generate the parallel filename for this processor */
strcpy(ctemp, pio_info->pexo_fname);
strcat(ctemp, ".");
strcat(ctemp, tag);
gen_par_filename(ctemp, par_out_fname, pio_info, Proc, Num_Proc);
fp = fopen(par_out_fname, "w");
if (fp == NULL){
sprintf(cmsg, "Error in %s; %s can not be opened for writing.", yo, par_out_fname);
Gen_Error(0, cmsg);
return 0;
}
if (Proc == 0)
echo_cmd_file(fp, cmd_file);
fprintf(fp, "Global element ids assigned to processor %d\n", Proc);
fprintf(fp, "GID\tPart\tPerm\tIPerm\n");
for (i = 0; i < mesh->num_elems; i++) {
j = index[i];
fprintf(fp, ZOLTAN_ID_SPEC "\t%d\t%d\t%d\n",
global_ids[j], (int)parts[j], (int)perm[j], (int)invperm[j]);
}
fclose(fp);
free(global_ids);
if (Output.Mesh_Info_File) {
ELEM_INFO_PTR current_element;
int total_nodes = 0;
float *x, *y, *z;
int k;
ZOLTAN_ID_TYPE prev_id;
for (i = 0; i < mesh->num_elems; i++) {
total_nodes += mesh->eb_nnodes[mesh->elements[i].elem_blk];
}
global_ids = (ZOLTAN_ID_TYPE *) malloc(total_nodes * (sizeof(ZOLTAN_ID_TYPE)+sizeof(int)));
index = (int*)(global_ids + total_nodes);
x = (float *) calloc(3 * total_nodes, sizeof(float));
y = x + total_nodes;
z = y + total_nodes;
for (k = 0, i = 0; i < mesh->num_elems; i++) {
current_element = &(mesh->elements[i]);
for (j = 0; j < mesh->eb_nnodes[current_element->elem_blk]; j++) {
global_ids[k] = (ZOLTAN_ID_TYPE)(current_element->connect[j]);
x[k] = current_element->coord[j][0];
if (mesh->num_dims > 1)
y[k] = current_element->coord[j][1];
if (mesh->num_dims > 2)
z[k] = current_element->coord[j][2];
index[k] = k;
k++;
}
}
quicksort_pointer_inc_id_id(index, global_ids, NULL,
0, total_nodes-1);
strcat(par_out_fname, ".mesh");
fp = fopen(par_out_fname, "w");
fprintf(fp, "Vertex IDs and coordinates\n");
prev_id = ZOLTAN_ID_INVALID;
for (k = 0; k < total_nodes; k++) {
j = (int)index[k];
if (global_ids[j] == prev_id)
continue;
prev_id = global_ids[j];
fprintf(fp, " " ZOLTAN_ID_SPEC " (%e, %e, %e)\n", global_ids[j], x[j], y[j], z[j]);
}
fprintf(fp, "\n");
fprintf(fp, "Element connectivity:\n");
for (i = 0; i < mesh->num_elems; i++) {
current_element = &(mesh->elements[i]);
fprintf(fp, " " ZOLTAN_ID_SPEC " (", current_element->globalID);
for (j = 0; j < mesh->eb_nnodes[current_element->elem_blk]; j++) {
fprintf(fp, ZOLTAN_ID_SPEC " ", current_element->connect[j]);
}
fprintf(fp, ")\n");
}
fclose(fp);
free(global_ids);
free(x);
}
DEBUG_TRACE_END(Proc, yo);
return 1;
}
