void migrate_post_process(void *data, int num_gid_entries, int num_lid_entries,
int num_import,
ZOLTAN_ID_PTR import_global_ids,
ZOLTAN_ID_PTR import_local_ids, int *import_procs,
int *import_to_part,
int num_export, ZOLTAN_ID_PTR export_global_ids,
ZOLTAN_ID_PTR export_local_ids, int *export_procs,
int *export_to_part,
int *ierr)
{
if (data == NULL) {
*ierr = ZOLTAN_FATAL;
return;
}
MESH_INFO_PTR mesh = (MESH_INFO_PTR) data;
ELEM_INFO *elements = mesh->elements;
int proc = 0, num_proc = 0;
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
MPI_Comm_size(MPI_COMM_WORLD, &num_proc);
/* compact elements array, as the application expects the array to be dense */
for (int i = 0; i < New_Elem_Index_Size; i++) {
if (New_Elem_Index[i] != ZOLTAN_ID_INVALID) continue;
/* Don't want to shift all elements down one position to fill the */
/* blank spot -- too much work to adjust adjacencies! So find the */
/* last element in the array and move it to the blank spot. */
int last = 0;
for (last = New_Elem_Index_Size-1; last >= 0; last--)
if (New_Elem_Index[last] != ZOLTAN_ID_INVALID) break;
/* If (last < i), array is already dense; i is just in some blank spots */
/* at the end of the array. Quit the compacting. */
if (last < i) break;
/* Copy elements[last] to elements[i]. */
elements[i] = elements[last];
/* Adjust adjacencies for local elements. Off-processor adjacencies */
/* don't matter here. */
for (int j = 0; j < elements[i].adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (elements[i].adj[j] == ZOLTAN_ID_INVALID) continue;
ZOLTAN_ID_TYPE adj_elem = elements[i].adj[j];
/* See whether adjacent element is local; if so, adjust its entry */
/* for local element i. */
if (elements[i].adj_proc[j] == proc) {
for (int k = 0; k < elements[adj_elem].adj_len; k++) {
if (elements[adj_elem].adj[k] == (ZOLTAN_ID_TYPE)last &&
elements[adj_elem].adj_proc[k] == proc) {
/* found adjacency entry for element last; change it to i */
elements[adj_elem].adj[k] = (ZOLTAN_ID_TYPE)i;
break;
}
}
}
}
/* Update New_Elem_Index */
New_Elem_Index[i] = New_Elem_Index[last];
New_Elem_Index[last] = ZOLTAN_ID_INVALID;
/* clear elements[last] */
elements[last].globalID = ZOLTAN_ID_INVALID;
elements[last].border = 0;
elements[last].my_part = -1;
elements[last].perm_value = -1;
elements[last].invperm_value = -1;
elements[last].nadj = 0;
elements[last].adj_len = 0;
elements[last].elem_blk = -1;
for (int k=0; k<MAX_CPU_WGTS; k++)
elements[last].cpu_wgt[k] = 0;
elements[last].mem_wgt = 0;
elements[last].avg_coord[0] = elements[last].avg_coord[1]
= elements[last].avg_coord[2] = 0.;
elements[last].coord = NULL;
elements[last].connect = NULL;
elements[last].adj = NULL;
elements[last].adj_proc = NULL;
elements[last].edge_wgt = NULL;
}
if (New_Elem_Index != NULL) {
delete [] New_Elem_Index;
New_Elem_Index = NULL;
}
New_Elem_Index_Size = 0;
if (!build_elem_comm_maps(proc, mesh)) {
Gen_Error(0, "Fatal: error rebuilding elem comm maps");
}
if (mesh->data_type == HYPERGRAPH && !update_elem_dd(mesh)) {
Gen_Error(0, "Fatal: error updating element dd");
}
if (mesh->data_type == HYPERGRAPH && mesh->hvertex_proc &&
!update_hvertex_proc(mesh)) {
Gen_Error(0, "Fatal: error updating hyperedges");
}
if (Debug_Driver > 3)
print_distributed_mesh(proc, num_proc, mesh);
}
static int setup_mesh_struct(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob, /* problem description */
MESH_INFO_PTR mesh, /* mesh information for the problem */
PARIO_INFO_PTR pio_info, /* element distribution info*/
ZOLTAN_ID_TYPE gnvtxs, /* global number of vertices across all procs*/
int nvtxs, /* number of vertices in local graph */
int *start, /* start of edge list for each vertex */
ZOLTAN_ID_TYPE *adj, /* edge list data */
int vwgt_dim, /* # of weights per vertex */
float *vwgts, /* vertex weight list data */
int ewgt_dim, /* # of weights per edge */
float *ewgts, /* edge weight list data */
int ndim, /* dimension of the geometry */
float *x, /* x-coordinates of the vertices */
float *y, /* y-coordinates of the vertices */
float *z /* z-coordinates of the vertices */
)
{
const char *yo = "setup_mesh_struct";
int i, j, k;
ZOLTAN_ID_TYPE elem_id;
ZOLTAN_ID_TYPE min_vtx;
DEBUG_TRACE_START(Proc, yo);
/* Initialize mesh structure for Chaco mesh. */
mesh->data_type = ZOLTAN_GRAPH;
mesh->vwgt_dim = vwgt_dim;
mesh->ewgt_dim = ewgt_dim;
mesh->num_elems = nvtxs;
mesh->elem_array_len = mesh->num_elems + 5;
mesh->num_dims = ndim;
mesh->num_el_blks = 1;
mesh->eb_etypes = (int *) malloc (4 * mesh->num_el_blks * sizeof(int));
if (!mesh->eb_etypes) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_ids = mesh->eb_etypes + mesh->num_el_blks;
mesh->eb_nnodes = mesh->eb_ids + mesh->num_el_blks;
mesh->eb_nattrs = mesh->eb_nnodes + mesh->num_el_blks;
mesh->eb_cnts = (ZOLTAN_ID_TYPE *) malloc (mesh->num_el_blks * sizeof(ZOLTAN_ID_TYPE));
if (!mesh->eb_cnts) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_names = (char **) malloc (mesh->num_el_blks * sizeof(char *));
if (!mesh->eb_names) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_etypes[0] = -1;
mesh->eb_ids[0] = 1;
mesh->eb_cnts[0] = (ZOLTAN_ID_TYPE)nvtxs;
mesh->eb_nattrs[0] = 0;
mesh->hindex = (int *) malloc(sizeof(int));
mesh->hindex[0] = 0;
mesh->eb_nnodes[0] = 1;
/* allocate space for name */
mesh->eb_names[0] = (char *) malloc(16* sizeof(char));
if (!mesh->eb_names[0]) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
strcpy(mesh->eb_names[0], "random-graph");
/* allocate the element structure array */
mesh->elements = (ELEM_INFO_PTR) malloc (mesh->elem_array_len * sizeof(ELEM_INFO));
if (!(mesh->elements)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/* write element data */
for (i = 0; i < mesh->elem_array_len; i++)
initialize_element(&(mesh->elements[i]));
min_vtx = local_to_global_id_map(0, Proc);
for (i = 0; i < nvtxs; i++) {
mesh->elements[i].globalID = local_to_global_id_map(i, Proc);
if (vwgts != NULL){
for (j=0; j<vwgt_dim; j++) {
mesh->elements[i].cpu_wgt[j] = vwgts[i*vwgt_dim+j];
}
}
else
mesh->elements[i].cpu_wgt[0] = 1.0;
mesh->elements[i].elem_blk = 0;
mesh->elements[i].my_part = Proc;
if (mesh->num_dims > 0) {
/* One set of coords per element. */
mesh->elements[i].connect = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE));
mesh->elements[i].connect[0] = mesh->elements[i].globalID;
mesh->elements[i].coord = (float **) malloc(sizeof(float *));
mesh->elements[i].coord[0] = (float *) calloc(mesh->num_dims, sizeof(float));
mesh->elements[i].coord[0][0] = x[i];
mesh->elements[i].avg_coord[0] = x[i];
if (mesh->num_dims > 1) {
mesh->elements[i].coord[0][1] = y[i];
mesh->elements[i].avg_coord[1] = y[i];
if (mesh->num_dims > 2) {
mesh->elements[i].coord[0][2] = z[i];
mesh->elements[i].avg_coord[2] = z[i];
}
}
}
}
for (i = 0; i < nvtxs; i++) {
/* now start with the adjacencies */
if (start != NULL)
mesh->elements[i].nadj = start[i+1] - start[i];
else
mesh->elements[i].nadj = 0;
if (mesh->elements[i].nadj > 0) {
mesh->elements[i].adj_len = mesh->elements[i].nadj;
mesh->elements[i].adj = (ZOLTAN_ID_TYPE *) malloc (mesh->elements[i].nadj * sizeof(ZOLTAN_ID_TYPE));
mesh->elements[i].adj_proc = (int *) malloc (mesh->elements[i].nadj * sizeof(int));
if (!(mesh->elements[i].adj) || !(mesh->elements[i].adj_proc)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (ewgts != NULL) {
mesh->elements[i].edge_wgt = (float *) malloc (mesh->elements[i].nadj * sizeof(float));
if (!(mesh->elements[i].edge_wgt)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
else
mesh->elements[i].edge_wgt = NULL;
for (j = 0; j < mesh->elements[i].nadj; j++) {
elem_id = adj[start[i] + j];
k = global_to_proc_owner_map(elem_id, Num_Proc, Proc);
/*
* if the adjacent element is on this processor
* then find the local id for that element
*/
if (k == Proc)
mesh->elements[i].adj[j] = elem_id-min_vtx;
else /* use the global id */
mesh->elements[i].adj[j] = elem_id;
mesh->elements[i].adj_proc[j] = k;
if (ewgts != NULL)
mesh->elements[i].edge_wgt[j] = ewgts[start[i] + j];
}
} /* End: "if (mesh->elements[i].nadj > 0)" */
} /* End: "for (i = 0; i < mesh->num_elems; i++)" */
if (!build_elem_comm_maps(Proc, mesh)) {
Gen_Error(0, "Fatal: error building initial elem comm maps");
return 0;
}
if (Debug_Driver > 3)
print_distributed_mesh(Proc, Num_Proc, mesh);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
int chaco_fill_elements(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob, /* problem description */
MESH_INFO_PTR mesh, /* mesh information for the problem */
int gnvtxs, /* global number of vertices across all procs*/
int nvtxs, /* number of vertices in local graph */
int *start, /* start of edge list for each vertex */
int *adj, /* edge list data */
int vwgt_dim, /* # of weights per vertex */
float *vwgts, /* vertex weight list data */
int ewgt_dim, /* # of weights per edge */
float *ewgts, /* edge weight list data */
int ndim, /* dimension of the geometry */
float *x, /* x-coordinates of the vertices */
float *y, /* y-coordinates of the vertices */
float *z, /* z-coordinates of the vertices */
short *assignments, /* assignments from Chaco file; may be NULL */
int base /* smallest vertex number to use;
base == 1 for Chaco;
may be 0 or 1 for HG files. */
)
{
/* Local declarations. */
int i, j, k, elem_id, *local_ids = NULL;
int num_vtx, min_vtx, max_vtx;
int *vtx_list = NULL;
const char *yo = "chaco_fill_elements";
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
chaco_init_local_ids(&local_ids, &vtx_list, &min_vtx, &max_vtx, &num_vtx,
gnvtxs, assignments, base);
for (i = 0; i < num_vtx; i++) {
mesh->elements[i].globalID = vtx_list[i]+base; /* GlobalIDs are 1-based
in Chaco; may be 0-based
or 1-based in HG files */
if (vwgts != NULL){
for (j=0; j<vwgt_dim; j++) {
mesh->elements[i].cpu_wgt[j] = vwgts[i*vwgt_dim+j];
}
}
else
mesh->elements[i].cpu_wgt[0] = 1.0;
mesh->elements[i].elem_blk = 0; /* only one elem block for all vertices */
if (assignments)
mesh->elements[i].my_part = assignments[vtx_list[i]];
else
mesh->elements[i].my_part = Proc; /* Init partition is starting proc.*/
if (mesh->num_dims > 0) {
/* One set of coords per element. */
mesh->elements[i].connect = (int *) malloc(sizeof(int));
mesh->elements[i].connect[0] = mesh->elements[i].globalID;
mesh->elements[i].coord = (float **) malloc(sizeof(float *));
mesh->elements[i].coord[0] = (float *) calloc(mesh->num_dims,
sizeof(float));
mesh->elements[i].coord[0][0] = x[i];
mesh->elements[i].avg_coord[0] = x[i];
if (mesh->num_dims > 1) {
mesh->elements[i].coord[0][1] = y[i];
mesh->elements[i].avg_coord[1] = y[i];
if (mesh->num_dims > 2) {
mesh->elements[i].coord[0][2] = z[i];
mesh->elements[i].avg_coord[2] = z[i];
}
}
}
}
for (i = 0; i < num_vtx; i++) {
/* now start with the adjacencies */
if (start != NULL)
mesh->elements[i].nadj = start[i+1] - start[i];
else
mesh->elements[i].nadj = 0;
if (mesh->elements[i].nadj > 0) {
mesh->elements[i].adj_len = mesh->elements[i].nadj;
mesh->elements[i].adj = (int *) malloc (mesh->elements[i].nadj * sizeof(int));
mesh->elements[i].adj_proc = (int *) malloc (mesh->elements[i].nadj * sizeof(int));
if (!(mesh->elements[i].adj) || !(mesh->elements[i].adj_proc)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (ewgts != NULL) {
mesh->elements[i].edge_wgt = (float *) malloc (mesh->elements[i].nadj * sizeof(float));
if (!(mesh->elements[i].edge_wgt)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
else
mesh->elements[i].edge_wgt = NULL;
for (j = 0; j < mesh->elements[i].nadj; j++) {
elem_id = adj[start[i] + j] - (1-base); /* Chaco is 1-based;
HG may be 0 or 1 based. */
/* determine which processor the adjacent vertex is on */
k = ch_dist_proc(elem_id, assignments, base);
/*
* if the adjacent element is on this processor
* then find the local id for that element
*/
if (k == Proc)
mesh->elements[i].adj[j] = local_ids[elem_id-base-min_vtx];
else /* use the global id */
mesh->elements[i].adj[j] = elem_id;
mesh->elements[i].adj_proc[j] = k;
if (ewgts != NULL)
mesh->elements[i].edge_wgt[j] = ewgts[start[i] + j];
}
} /* End: "if (mesh->elements[i].nadj > 0)" */
} /* End: "for (i = 0; i < mesh->num_elems; i++)" */
safe_free((void **)(void *) &vtx_list);
safe_free((void **)(void *) &local_ids);
if (!build_elem_comm_maps(Proc, mesh)) {
Gen_Error(0, "Fatal: error building initial elem comm maps");
return 0;
}
if (Debug_Driver > 3)
print_distributed_mesh(Proc, Num_Proc, mesh);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
void migrate_post_process(void *data, int num_gid_entries, int num_lid_entries,
int num_import,
LB_ID_PTR import_global_ids,
LB_ID_PTR import_local_ids, int *import_procs,
int num_export, LB_ID_PTR export_global_ids,
LB_ID_PTR export_local_ids, int *export_procs,
int *ierr)
{
MESH_INFO_PTR mesh;
ELEM_INFO *elements;
int proc, num_proc;
int i, j, k, last;
int adj_elem;
if (data == NULL) {
*ierr = LB_FATAL;
return;
}
mesh = (MESH_INFO_PTR) data;
elements = mesh->elements;
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
MPI_Comm_size(MPI_COMM_WORLD, &num_proc);
/* compact elements array, as the application expects the array to be dense */
for (i = 0; i < New_Elem_Index_Size; i++) {
if (New_Elem_Index[i] != -1) continue;
/* Don't want to shift all elements down one position to fill the */
/* blank spot -- too much work to adjust adjacencies! So find the */
/* last element in the array and move it to the blank spot. */
for (last = New_Elem_Index_Size-1; last >= 0; last--)
if (New_Elem_Index[last] != -1) break;
/* If (last < i), array is already dense; i is just in some blank spots */
/* at the end of the array. Quit the compacting. */
if (last < i) break;
/* Copy elements[last] to elements[i]. */
elements[i] = elements[last];
/* Adjust adjacencies for local elements. Off-processor adjacencies */
/* don't matter here. */
for (j = 0; j < elements[i].adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (elements[i].adj[j] == -1) continue;
adj_elem = elements[i].adj[j];
/* See whether adjacent element is local; if so, adjust its entry */
/* for local element i. */
if (elements[i].adj_proc[j] == proc) {
for (k = 0; k < elements[adj_elem].adj_len; k++) {
if (elements[adj_elem].adj[k] == last &&
elements[adj_elem].adj_proc[k] == proc) {
/* found adjacency entry for element last; change it to i */
elements[adj_elem].adj[k] = i;
break;
}
}
}
}
/* Update New_Elem_Index */
New_Elem_Index[i] = New_Elem_Index[last];
New_Elem_Index[last] = -1;
/* clear elements[last] */
elements[last].globalID = -1;
elements[last].border = 0;
elements[last].nadj = 0;
elements[last].adj_len = 0;
elements[last].elem_blk = -1;
for (k=0; k<MAX_CPU_WGTS; k++)
elements[last].cpu_wgt[k] = 0;
elements[last].mem_wgt = 0;
elements[last].coord = NULL;
elements[last].connect = NULL;
elements[last].adj = NULL;
elements[last].adj_proc = NULL;
elements[last].edge_wgt = NULL;
}
if (New_Elem_Index != NULL) safe_free((void **) &New_Elem_Index);
New_Elem_Index_Size = 0;
if (!build_elem_comm_maps(proc, mesh)) {
Gen_Error(0, "Fatal: error rebuilding elem comm maps");
}
if (Debug_Driver > 3)
print_distributed_mesh(proc, num_proc, mesh);
}