static int read_comm_map_info(int pexoid, int Proc, PROB_INFO_PTR prob,
MESH_INFO_PTR mesh)
{
/* Local declarations. */
char *yo = "read_comm_map_info";
int ielem, imap, loc_elem, iblk, max_len, offset, index;
int nnodei, nnodeb, nnodee, nelemi, nelemb, nncmap;
int *int_elem, *bor_elem;
int *proc_ids;
int *gids, *my_procs, *recv_procs;
int ierr, nrecv;
int msg = 200;
int sid;
ELEM_INFO_PTR elements = mesh->elements;
ZOLTAN_COMM_OBJ *comm_obj;
E_Type etype;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
if (ne_get_loadbal_param(pexoid, &nnodei, &nnodeb, &nnodee,
&nelemi, &nelemb, &nncmap, &(mesh->necmap), Proc) < 0) {
Gen_Error(0, "fatal: Error returned from ne_get_loadbal_param");
return 0;
}
/*
* get the list of the border elements in order to set
* the border flag in the element structures
*/
int_elem = (int *) malloc ((nelemi + nelemb) * sizeof(int));
if (!int_elem) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
bor_elem = int_elem + nelemi;
if (ne_get_elem_map(pexoid, int_elem, bor_elem, Proc) < 0) {
Gen_Error(0, "fatal: Error returned from ne_get_elem_map");
return 0;
}
for (ielem = 0; ielem < nelemb; ielem++) {
elements[bor_elem[ielem]-1].border = 1;
}
free(int_elem);
/*
* For now, only get the elemental communication maps,
* since, in the driver, elements are only considered
* adjacent if they share a face (same definition used
* in element communication maps). Eventually, the ability
* to consider elements that are connected by any nodes
* adjacent will have to be added. When that happens,
* the nodal communication maps will be needed.
*/
mesh->ecmap_cnt = (int *) malloc (mesh->necmap * sizeof(int));
mesh->ecmap_id = (int *) malloc(mesh->necmap * sizeof(int));
if (!mesh->ecmap_cnt || !mesh->ecmap_id) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (ne_get_cmap_params(pexoid, NULL, NULL, mesh->ecmap_id,
mesh->ecmap_cnt, Proc) < 0) {
Gen_Error(0, "fatal: Error returned from ne_get_cmap_params");
return 0;
}
max_len = 0;
for (imap = 0; imap < mesh->necmap; imap++)
max_len += mesh->ecmap_cnt[imap];
proc_ids = (int *) malloc(4 * max_len * sizeof(int));
gids = proc_ids + max_len;
my_procs = gids + max_len;
recv_procs = my_procs + max_len;
mesh->ecmap_elemids = (int *) malloc(max_len * sizeof(int));
mesh->ecmap_sideids = (int *) malloc(max_len * sizeof(int));
mesh->ecmap_neighids = (int *) malloc(max_len * sizeof(int));
if (!mesh->ecmap_elemids || !mesh->ecmap_sideids || !mesh->ecmap_neighids) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
offset = 0;
for (imap = 0; imap < mesh->necmap; imap++) {
if(ne_get_elem_cmap(pexoid, mesh->ecmap_id[imap],
&(mesh->ecmap_elemids[offset]),
&(mesh->ecmap_sideids[offset]),
&(proc_ids[offset]), Proc) < 0) {
Gen_Error(0, "fatal: Error returned from ne_get_elem_cmap");
return 0;
}
offset += mesh->ecmap_cnt[imap];
} /* End: "for (imap = 0; imap < mesh->necmap; imap++)" */
/*
* Decrement the ecmap_elemids by one for zero-based local numbering.
* Convert the element ids to global ids to send to
* the neighboring processor.
*/
for (ielem = 0; ielem < max_len; ielem++) {
mesh->ecmap_elemids[ielem]--;
gids[ielem] = elements[mesh->ecmap_elemids[ielem]].globalID;
my_procs[ielem] = Proc;
}
/*
* Now communicate with other processor to get global IDs
* for the adjacent elements in this communication map.
*/
ierr = Zoltan_Comm_Create(&comm_obj, max_len, proc_ids, MPI_COMM_WORLD,
msg, &nrecv);
if (ierr != ZOLTAN_OK) {
Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Create");
return 0;
}
if (nrecv != max_len) {
/* Sanity check; this should never happen. */
Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Create");
return 0;
}
/* Exchange ids to neighbors.
* Assuming messages will be stored in order of processor number in
* ecmap_neighids.
*/
ierr = Zoltan_Comm_Do(comm_obj, msg+1, (char *) gids, sizeof(int),
(char *) (mesh->ecmap_neighids));
/* Exchange sanity check information.
* Allows to check assumption that messages are stored in order of
* processor number.
*/
if (ierr == ZOLTAN_OK)
ierr = Zoltan_Comm_Do(comm_obj, msg+2, (char *) my_procs, sizeof(int),
(char *) recv_procs);
if (ierr != ZOLTAN_OK) {
Gen_Error(0, "fatal: Error returned from Zoltan_Comm_Do");
return 0;
}
ierr = Zoltan_Comm_Destroy(&comm_obj);
/* Sanity check: messages stored in order of processor number. */
for (ielem = 0; ielem < max_len; ielem++) {
if (proc_ids[ielem] != recv_procs[ielem]) {
Gen_Error(0, "fatal: Sanity check failed; assumption wrong");
return 0;
}
}
/* now process all of the element ids that have been received */
offset = 0;
for (imap = 0; imap < mesh->necmap; imap++) {
for (ielem = 0; ielem < mesh->ecmap_cnt[imap]; ielem++) {
index = ielem + offset;
/* translate from element id in the communication map to local elem id */
loc_elem = mesh->ecmap_elemids[index];
iblk = elements[loc_elem].elem_blk;
etype = (E_Type) (mesh->eb_etypes[iblk]);
(elements[loc_elem].nadj)++;
if(elements[loc_elem].nadj > elements[loc_elem].adj_len) {
/* Shouldn't happen as long as only side adjacencies are used. */
/* adj_len == number of sides. */
/* Space should already be allocated for the adjacencies read */
/* from the communication maps. */
Gen_Error(0, "fatal: Number of adj greater than adj_len");
return 0;
}
/* Store adjacency info in the adj entry corresponding to this side. */
sid = mesh->ecmap_sideids[index] - 1;
elements[loc_elem].adj[sid] = mesh->ecmap_neighids[index];
elements[loc_elem].adj_proc[sid] = mesh->ecmap_id[imap];
elements[loc_elem].edge_wgt[sid] =
(float) get_elem_info(NSNODES, etype, mesh->ecmap_sideids[index]);
} /* End: "for (ielem = 0; ielem < mesh->ecmap_cnt[imap]; ielem++)" */
offset += mesh->ecmap_cnt[imap];
} /* End: "for for (imap = 0; imap < mesh->necmap; imap++)" */
free (proc_ids);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
int chaco_input_geom(
ZOLTAN_FILE *fingeom, /* geometry input file */
char *geomname, /* name of geometry file */
int nvtxs, /* number of coordinates to read */
int *igeom, /* dimensionality of geometry */
float **x, /* coordinates of vertices */
float **y,
float **z
)
{
const char *yo = "chaco_input_geom";
float xc, yc, zc =0; /* first x, y, z coordinate */
int nread; /* number of lines of coordinates read */
int flag; /* any bad data at end of file? */
int line_num; /* counts input lines in file */
int end_flag; /* return conditional */
int ndims; /* number of values in an input line */
int i=0; /* loop counter */
DEBUG_TRACE_START(0, yo);
*x = *y = *z = NULL;
line_num = 0;
end_flag = 1;
while (end_flag == 1) {
xc = read_val(fingeom, &end_flag);
++line_num;
}
if (end_flag == -1) {
printf("No values found in geometry file `%s'\n", geomname);
ZOLTAN_FILE_close(fingeom);
return (1);
}
ndims = 1;
yc = read_val(fingeom, &end_flag);
if (end_flag == 0) {
ndims = 2;
zc = read_val(fingeom, &end_flag);
if (end_flag == 0) {
ndims = 3;
read_val(fingeom, &end_flag);
if (!end_flag) {
printf("Too many values on input line of geometry file `%s'\n",
geomname);
printf(" Maximum dimensionality is 3\n");
ZOLTAN_FILE_close(fingeom);
return (1);
}
}
}
*igeom = ndims;
*x = (float *) malloc((unsigned) nvtxs * sizeof(float));
(*x)[0] = xc;
if (ndims > 1) {
*y = (float *) malloc((unsigned) nvtxs * sizeof(float));
(*y)[0] = yc;
}
if (ndims > 2) {
*z = (float *) malloc((unsigned) nvtxs * sizeof(float));
(*z)[0] = zc;
}
for (nread = 1; nread < nvtxs; nread++) {
++line_num;
if (ndims == 1) {
i = ZOLTAN_FILE_scanf(fingeom, "%f", &((*x)[nread]));
}
else if (ndims == 2) {
i = ZOLTAN_FILE_scanf(fingeom, "%f%f", &((*x)[nread]), &((*y)[nread]));
}
else if (ndims == 3) {
i = ZOLTAN_FILE_scanf(fingeom, "%f%f%f", &((*x)[nread]), &((*y)[nread]),
&((*z)[nread]));
}
if (i == EOF) {
printf("Too few lines of values in geometry file; nvtxs=%d, but only %d read\n",
nvtxs, nread);
ZOLTAN_FILE_close(fingeom);
return (1);
}
else if (i != ndims) {
printf("Wrong number of values in line %d of geometry file `%s'\n",
line_num, geomname);
ZOLTAN_FILE_close(fingeom);
return (1);
}
}
/* Check for spurious extra stuff in file. */
flag = 0;
end_flag = 0;
while (!flag && end_flag != -1) {
read_val(fingeom, &end_flag);
if (!end_flag)
flag = 1;
}
if (flag && Debug_Chaco_Input) {
printf("Warning: possible error in geometry file `%s'\n", geomname);
printf(" Numerical data found after expected end of file\n");
}
ZOLTAN_FILE_close(fingeom);
DEBUG_TRACE_END(0, yo);
return (0);
}
int chaco_input_graph(
ZOLTAN_FILE *fin, /* input file */
char *inname, /* name of input file */
int **start, /* start of edge list for each vertex */
int **adjacency, /* edge list data */
int *nvtxs, /* number of vertices in graph */
int *vwgt_dim, /* # of vertex weights per node */
float **vweights, /* vertex weight list data */
int *ewgt_dim, /* # of edge weights per edge */
float **eweights /* edge weight list data */
)
{
const char *yo = "chaco_input_graph";
int *adjptr; /* loops through adjacency data */
float *ewptr; /* loops through edge weight data */
int narcs; /* number of edges expected in graph */
int nedges; /* twice number of edges really in graph */
int nedge; /* loops through edges for each vertex */
int flag; /* condition indicator */
int found_flag; /* is vertex found in adjacency list? */
int skip_flag; /* should this edge be ignored? */
int end_flag; /* indicates end of line or file */
int vtx; /* vertex in graph */
int line_num; /* line number in input file */
int sum_edges; /* total number of edges read so far */
int option = 0; /* input option */
int using_ewgts; /* are edge weights in input file? */
int using_vwgts; /* are vertex weights in input file? */
int vtxnums; /* are vertex numbers in input file? */
int vertex; /* current vertex being read */
int new_vertex; /* new vertex being read */
float weight; /* weight being read */
float eweight; /* edge weight being read */
int neighbor; /* neighbor of current vertex */
int self_edge; /* is a self edge encountered? */
int ignore_me; /* is this edge being ignored? */
int ignored; /* how many edges are ignored? */
int error_flag; /* error reading input? */
int j; /* loop counters */
DEBUG_TRACE_START(0, yo);
/* Read first line of input (= nvtxs, narcs, option). */
/* The (decimal) digits of the option variable mean: 1's digit not zero => input
edge weights 10's digit not zero => input vertex weights 100's digit not zero
=> include vertex numbers */
*start = NULL;
*adjacency = NULL;
*vweights = NULL;
*eweights = NULL;
error_flag = 0;
line_num = 0;
/* Read any leading comment lines */
end_flag = 1;
while (end_flag == 1) {
*nvtxs = read_int(fin, &end_flag);
++line_num;
}
if (*nvtxs <= 0) {
printf("ERROR in graph file `%s':", inname);
printf(" Invalid number of vertices (%d).\n", *nvtxs);
ZOLTAN_FILE_close(fin);
return(1);
}
narcs = read_int(fin, &end_flag);
if (narcs < 0) {
printf("ERROR in graph file `%s':", inname);
printf(" Invalid number of expected edges (%d).\n", narcs);
ZOLTAN_FILE_close(fin);
return(1);
}
/* Check if vertex or edge weights are used */
if (!end_flag) {
option = read_int(fin, &end_flag);
}
using_ewgts = option - 10 * (option / 10);
option /= 10;
using_vwgts = option - 10 * (option / 10);
option /= 10;
vtxnums = option - 10 * (option / 10);
/* Get weight dimensions from Chaco option */
(*vwgt_dim) = using_vwgts;
(*ewgt_dim) = using_ewgts;
/* Read weight dimensions if they are specified separately */
if (!end_flag && using_vwgts==1){
j = read_int(fin, &end_flag);
if (!end_flag) (*vwgt_dim) = j;
}
if (!end_flag && using_ewgts==1){
j = read_int(fin, &end_flag);
if (!end_flag) (*ewgt_dim) = j;
}
/* Discard rest of line */
while (!end_flag)
j = read_int(fin, &end_flag);
/* Allocate space for rows and columns. */
*start = (int *) malloc((unsigned) (*nvtxs + 1) * sizeof(int));
if (narcs != 0)
*adjacency = (int *) malloc((unsigned) (2 * narcs + 1) * sizeof(int));
else
*adjacency = NULL;
if (using_vwgts)
*vweights = (float *) malloc((unsigned) (*nvtxs) * (*vwgt_dim) * sizeof(float));
else
*vweights = NULL;
if (using_ewgts)
*eweights = (float *)
malloc((unsigned) (2 * narcs + 1) * (*ewgt_dim) * sizeof(float));
else
*eweights = NULL;
adjptr = *adjacency;
ewptr = *eweights;
self_edge = 0;
ignored = 0;
sum_edges = 0;
nedges = 0;
(*start)[0] = 0;
vertex = 0;
vtx = 0;
new_vertex = TRUE;
while ((using_vwgts || vtxnums || narcs) && end_flag != -1) {
++line_num;
/* If multiple input lines per vertex, read vertex number. */
if (vtxnums) {
j = read_int(fin, &end_flag);
if (end_flag) {
if (vertex == *nvtxs)
break;
printf("ERROR in graph file `%s':", inname);
printf(" no vertex number in line %d.\n", line_num);
ZOLTAN_FILE_close(fin);
return (1);
}
if (j != vertex && j != vertex + 1) {
printf("ERROR in graph file `%s':", inname);
printf(" out-of-order vertex number in line %d.\n", line_num);
ZOLTAN_FILE_close(fin);
return (1);
}
if (j != vertex) {
new_vertex = TRUE;
vertex = j;
}
else
new_vertex = FALSE;
}
else
vertex = ++vtx;
if (vertex > *nvtxs)
break;
/* If vertices are weighted, read vertex weight. */
if (using_vwgts && new_vertex) {
for (j=0; j<(*vwgt_dim); j++){
weight = read_val(fin, &end_flag);
if (end_flag) {
printf("ERROR in graph file `%s':", inname);
printf(" not enough weights for vertex %d.\n", vertex);
ZOLTAN_FILE_close(fin);
return (1);
}
if ((weight < 0) && Debug_Chaco_Input) {
printf("ERROR in graph file `%s':", inname);
printf(" negative weight entered for vertex %d.\n", vertex);
ZOLTAN_FILE_close(fin);
return (1);
}
(*vweights)[(vertex-1)*(*vwgt_dim)+j] = weight;
}
}
nedge = 0;
/* Read number of adjacent vertex. */
neighbor = read_int(fin, &end_flag);
while (!end_flag) {
skip_flag = FALSE;
ignore_me = FALSE;
if (Debug_Chaco_Input){
if (neighbor > *nvtxs) {
printf("ERROR in graph file `%s':", inname);
printf(" nvtxs=%d, but edge (%d,%d) was input.\n",
*nvtxs, vertex, neighbor);
ZOLTAN_FILE_close(fin);
return (1);
}
if (neighbor < 0) {
printf("ERROR in graph file `%s':", inname);
printf(" negative vertex in edge (%d,%d).\n",
vertex, neighbor);
ZOLTAN_FILE_close(fin);
return (1);
}
if (neighbor == vertex) {
if (!self_edge && Debug_Chaco_Input) {
printf("WARNING: Self edge (%d, %d) being ignored.\n",
vertex, vertex);
}
skip_flag = TRUE;
++self_edge;
}
/* Check if adjacency is repeated. */
if (!skip_flag) {
found_flag = FALSE;
for (j = (*start)[vertex - 1]; !found_flag && j < sum_edges + nedge; j++) {
if ((*adjacency)[j] == neighbor)
found_flag = TRUE;
}
if (found_flag) {
printf("WARNING: Multiple occurences of edge (%d,%d) ignored\n",
vertex, neighbor);
skip_flag = TRUE;
if (!ignore_me) {
ignore_me = TRUE;
++ignored;
}
}
}
}
if (using_ewgts) { /* Read edge weight if it's being input. */
for (j=0; j<(*ewgt_dim); j++){
eweight = read_val(fin, &end_flag);
if (end_flag) {
printf("ERROR in graph file `%s':", inname);
printf(" not enough weights for edge (%d,%d).\n", vertex, neighbor);
ZOLTAN_FILE_close(fin);
return (1);
}
if (eweight <= 0 && Debug_Chaco_Input) {
printf("WARNING: Bad weight entered for edge (%d,%d). Edge ignored.\n",
vertex, neighbor);
skip_flag = TRUE;
if (!ignore_me) {
ignore_me = TRUE;
++ignored;
}
}
else {
*ewptr++ = eweight;
}
}
}
if (Debug_Chaco_Input){
/* Check for edge only entered once. */
if (neighbor < vertex && !skip_flag) {
found_flag = FALSE;
for (j = (*start)[neighbor - 1]; !found_flag && j < (*start)[neighbor]; j++) {
if ((*adjacency)[j] == vertex)
found_flag = TRUE;
}
if (!found_flag) {
printf("ERROR in graph file `%s':", inname);
printf(" Edge (%d, %d) entered but not (%d, %d)\n",
vertex, neighbor, neighbor, vertex);
error_flag = 1;
}
}
}
/* Add edge to data structure. */
if (!skip_flag) {
if (++nedges > 2*narcs) {
printf("ERROR in graph file `%s':", inname);
printf(" at least %d adjacencies entered, but nedges = %d\n",
nedges, narcs);
ZOLTAN_FILE_close(fin);
return (1);
}
*adjptr++ = neighbor;
nedge++;
}
/* Read number of next adjacent vertex. */
neighbor = read_int(fin, &end_flag);
}
sum_edges += nedge;
(*start)[vertex] = sum_edges;
}
/* Make sure there's nothing else in file. */
flag = FALSE;
while (!flag && end_flag != -1) {
read_int(fin, &end_flag);
if (!end_flag)
flag = TRUE;
}
if (flag && Debug_Chaco_Input) {
printf("WARNING: Possible error in graph file `%s'\n", inname);
printf(" Data found after expected end of file\n");
}
(*start)[*nvtxs] = sum_edges;
if (self_edge > 1 && Debug_Chaco_Input) {
printf("WARNING: %d self edges were read and ignored.\n", self_edge);
}
if (vertex != 0) { /* Normal file was read. */
/* Make sure narcs was reasonable. */
if (nedges + 2 * self_edge != 2 * narcs &&
nedges + 2 * self_edge + ignored != 2 * narcs &&
nedges + self_edge != 2 * narcs &&
nedges + self_edge + ignored != 2 * narcs &&
nedges != 2 * narcs &&
nedges + ignored != 2 * narcs &&
Debug_Chaco_Input) {
printf("WARNING: I expected %d edges entered twice, but I only count %d.\n",
narcs, nedges);
}
}
else {
/* Graph was empty */
free(*start);
if (*adjacency != NULL)
free(*adjacency);
if (*vweights != NULL)
free(*vweights);
if (*eweights != NULL)
free(*eweights);
*start = NULL;
*adjacency = NULL;
}
ZOLTAN_FILE_close(fin);
DEBUG_TRACE_END(0, yo);
return (error_flag);
}
/* Read from file and set up hypergraph. */
int read_hypergraph_file(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh
)
{
/* Local declarations. */
const char *yo = "read_hypergraph_file";
char cmesg[256];
int i, gnvtxs, distributed_pins = 0, edge, vertex, nextEdge;
int nvtxs = 0, gnhedges = 0, nhedges = 0, npins = 0;
int vwgt_dim=0, hewgt_dim=0, vtx, edgeSize, global_npins;
int *hindex = NULL, *hvertex = NULL, *hvertex_proc = NULL;
int *hgid = NULL;
float *hewgts = NULL, *vwgts = NULL;
ZOLTAN_FILE* fp = NULL;
int base = 0; /* Smallest vertex number; usually zero or one. */
char filename[256];
/* Variables that allow graph-based functions to be reused. */
/* If no chaco.graph or chaco.coords files exist, values are NULL or 0,
* since graph is not being built. If chaco.graph and/or chaco.coords
* exist, these arrays are filled and values stored in mesh.
* Including these files allows for comparison of HG methods with other
* methods, along with visualization of results and comparison of
* LB_Eval results.
*/
int ch_nvtxs = 0; /* Temporary values for chaco_read_graph. */
#ifdef KDDKDD
int ch_vwgt_dim = 0; /* Their values are ignored, as vertex */
#endif
float *ch_vwgts = NULL; /* info is provided by hypergraph file. */
int *ch_start = NULL, *ch_adj = NULL, ch_ewgt_dim = 0;
short *ch_assignments = NULL;
float *ch_ewgts = NULL;
int ch_ndim = 0;
float *ch_x = NULL, *ch_y = NULL, *ch_z = NULL;
int ch_no_geom = TRUE; /* Assume no geometry info is given; reset if
it is provided. */
int file_error = 0;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
if (Proc == 0) {
/* Open and read the hypergraph file. */
if (pio_info->file_type == HYPERGRAPH_FILE)
sprintf(filename, "%s.hg", pio_info->pexo_fname);
else if (pio_info->file_type == MATRIXMARKET_FILE)
sprintf(filename, "%s.mtx", pio_info->pexo_fname);
else {
sprintf(cmesg, "fatal: invalid file type %d", pio_info->file_type);
Gen_Error(0, cmesg);
return 0;
}
fp = ZOLTAN_FILE_open(filename, "r", pio_info->file_comp);
file_error = (fp == NULL);
}
MPI_Bcast(&file_error, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (file_error) {
sprintf(cmesg,
"fatal: Could not open hypergraph file %s",pio_info->pexo_fname);
Gen_Error(0, cmesg);
return 0;
}
if (pio_info->file_type == HYPERGRAPH_FILE) {
/* read the array in on processor 0 */
if (Proc == 0) {
if (HG_readfile(Proc, fp, &nvtxs, &nhedges, &npins,
&hindex, &hvertex, &vwgt_dim, &vwgts,
&hewgt_dim, &hewgts, &base) != 0){
Gen_Error(0, "fatal: Error returned from HG_readfile");
return 0;
}
}
}
else if (pio_info->file_type == MATRIXMARKET_FILE) {
/*
* pio_info->chunk_reader == 0 (the usual case)
* process 0 will read entire file in MM_readfile,
* and will distribute vertices in chaco_dist_graph and pins in
* dist_hyperedges later. (distributed_pins==0)
*
* pio_info->chunk_reader == 1 ("initial read = chunks" in zdrive.inp)
* process 0 will read the file in chunks, and will send vertices
* and pins to other processes before reading the next chunk, all
* in MM_readfile. (distributed_pins==1)
*/
if (MM_readfile(Proc, Num_Proc, fp, pio_info,
&nvtxs, /* global number of vertices */
&nhedges, /* global number of hyperedges */
&npins, /* local number of pins */
&hindex, &hvertex, &vwgt_dim, &vwgts,
&hewgt_dim, &hewgts, &ch_start, &ch_adj,
&ch_ewgt_dim, &ch_ewgts, &base, &global_npins)) {
Gen_Error(0, "fatal: Error returned from MM_readfile");
return 0;
}
if (Proc == 0) ZOLTAN_FILE_close(fp);
if ((Num_Proc > 1) && pio_info->chunk_reader && (global_npins > Num_Proc)){
distributed_pins = 1;
}
else{
distributed_pins = 0;
}
}
#ifdef KDDKDD
{
/* If CHACO graph file is available, read it. */
sprintf(filename, "%s.graph", pio_info->pexo_fname);
fp = ZOLTAN_FILE_open(filename, "r", pio_info->file_comp);
file_error =
#ifndef ZOLTAN_COMPRESS
(fp == NULL);
#else
fp.error;
#endif
if (!file_error) {
/* CHACO graph file is available. */
/* Assuming hypergraph vertices are same as chaco vertices. */
/* Chaco vertices and their weights are ignored in rest of function. */
if (chaco_input_graph(fp, filename, &ch_start, &ch_adj, &ch_nvtxs,
&ch_vwgt_dim, &ch_vwgts, &ch_ewgt_dim, &ch_ewgts) != 0) {
Gen_Error(0, "fatal: Error returned from chaco_input_graph");
return 0;
}
}
else
ch_nvtxs = nvtxs;
/* If coordinate file is available, read it. */
sprintf(filename, "%s.coords", pio_info->pexo_fname);
fp = ZOLTAN_FILE_open(filename, "r", pio_info->file_comp);
file_error =
#ifndef ZOLTAN_COMPRESS
(fp == NULL);
#else
fp.error;
#endif
if (!file_error) {
/* CHACO coordinates file is available. */
ch_no_geom = FALSE;
if (chaco_input_geom(fpkdd, filename, ch_nvtxs, &ch_ndim,
&ch_x, &ch_y, &ch_z) != 0) {
Gen_Error(0, "fatal: Error returned from chaco_input_geom");
return 0;
}
}
}
#else /* KDDKDD */
ch_nvtxs = nvtxs;
#endif /* KDDKDD */
{
/* Read Chaco assignment file, if requested */
if (pio_info->init_dist_type == INITIAL_FILE) {
sprintf(filename, "%s.assign", pio_info->pexo_fname);
fp = ZOLTAN_FILE_open(filename, "r", pio_info->file_comp);
if (fp == NULL) {
sprintf(cmesg, "Error: Could not open Chaco assignment file %s; "
"initial distribution cannot be read",
filename);
Gen_Error(0, cmesg);
return 0;
}
else {
/* read the coordinates in on processor 0 */
ch_assignments = (short *) malloc(nvtxs * sizeof(short));
if (nvtxs && !ch_assignments) {
Gen_Error(0, "fatal: memory error in read_hypergraph_file");
return 0;
}
/* closes fpassign when done */
if (chaco_input_assign(fp, filename, ch_nvtxs, ch_assignments) != 0){
Gen_Error(0, "fatal: Error returned from chaco_input_assign");
return 0;
}
}
}
}
MPI_Bcast(&base, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (distributed_pins){
gnhedges = nhedges;
nhedges = 0;
hewgt_dim = 0;
hewgts = NULL;
for (edge=0; edge<gnhedges; edge++){
edgeSize = hindex[edge+1] - hindex[edge];
if (edgeSize > 0) nhedges++;
}
hgid = (int *)malloc(nhedges * sizeof(int));
hvertex_proc = (int *)malloc(npins * sizeof(int));
nextEdge=0;
vtx=0;
for (edge=0; edge<gnhedges; edge++){
edgeSize = hindex[edge+1] - hindex[edge];
if (edgeSize > 0){
hgid[nextEdge] = edge+1;
if (nextEdge < edge){
hindex[nextEdge+1] = hindex[nextEdge] + edgeSize;
}
for (vertex=0; vertex<edgeSize; vertex++,vtx++){
hvertex_proc[vtx] = ch_dist_proc(hvertex[vtx], NULL, 1);
}
nextEdge++;
}
}
gnvtxs = nvtxs;
nvtxs = ch_dist_num_vtx(Proc, NULL);
if (ch_start){ /* need to include only vertices this process owns */
for (i=0,vertex=0; i<gnvtxs; i++){
if ((ch_start[i+1] > ch_start[vertex]) || /* vtx has adjacencies so it's mine */
(ch_dist_proc(i, NULL, 0) == Proc)) /* my vtx with no adjacencies */
{
if (i > vertex){
ch_start[vertex+1] = ch_start[i+1];
}
vertex++;
}
}
}
#if 0
debug_lists(Proc, Num_Proc, nhedges, hindex, hvertex, hvertex_proc, hgid);
#endif
} else{
/* Distribute hypergraph graph */
/* Use hypergraph vertex information and chaco edge information. */
if (!chaco_dist_graph(MPI_COMM_WORLD, pio_info, 0, &gnvtxs, &nvtxs,
&ch_start, &ch_adj, &vwgt_dim, &vwgts, &ch_ewgt_dim, &ch_ewgts,
&ch_ndim, &ch_x, &ch_y, &ch_z, &ch_assignments) != 0) {
Gen_Error(0, "fatal: Error returned from chaco_dist_graph");
return 0;
}
if (!dist_hyperedges(MPI_COMM_WORLD, pio_info, 0, base, gnvtxs, &gnhedges,
&nhedges, &hgid, &hindex, &hvertex, &hvertex_proc,
&hewgt_dim, &hewgts, ch_assignments)) {
Gen_Error(0, "fatal: Error returned from dist_hyperedges");
return 0;
}
}
/* Initialize mesh structure for Hypergraph. */
mesh->data_type = HYPERGRAPH;
mesh->num_elems = nvtxs;
mesh->vwgt_dim = vwgt_dim;
mesh->ewgt_dim = ch_ewgt_dim;
mesh->elem_array_len = mesh->num_elems + 5;
mesh->num_dims = ch_ndim;
mesh->num_el_blks = 1;
mesh->gnhedges = gnhedges;
mesh->nhedges = nhedges;
mesh->hewgt_dim = hewgt_dim;
mesh->hgid = hgid;
mesh->hindex = hindex;
mesh->hvertex = hvertex;
mesh->hvertex_proc = hvertex_proc;
mesh->heNumWgts = nhedges;
mesh->heWgtId = NULL;
mesh->hewgts = hewgts;
mesh->eb_etypes = (int *) malloc (5 * mesh->num_el_blks * sizeof(int));
if (!mesh->eb_etypes) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_ids = mesh->eb_etypes + mesh->num_el_blks;
mesh->eb_cnts = mesh->eb_ids + mesh->num_el_blks;
mesh->eb_nnodes = mesh->eb_cnts + mesh->num_el_blks;
mesh->eb_nattrs = mesh->eb_nnodes + mesh->num_el_blks;
mesh->eb_names = (char **) malloc (mesh->num_el_blks * sizeof(char *));
if (!mesh->eb_names) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_etypes[0] = -1;
mesh->eb_ids[0] = 1;
mesh->eb_cnts[0] = nvtxs;
mesh->eb_nattrs[0] = 0;
/*
* Each element has one set of coordinates (i.e., node) if a coords file
* was provided; zero otherwise.
*/
MPI_Bcast( &ch_no_geom, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (ch_no_geom)
mesh->eb_nnodes[0] = 0;
else
mesh->eb_nnodes[0] = 1;
/* allocate space for name */
mesh->eb_names[0] = (char *) malloc((MAX_STR_LENGTH+1) * sizeof(char));
if (!mesh->eb_names[0]) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
strcpy(mesh->eb_names[0], "hypergraph");
/* allocate the element structure array */
mesh->elements = (ELEM_INFO_PTR) malloc (mesh->elem_array_len
* sizeof(ELEM_INFO));
if (!(mesh->elements)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/*
* initialize all of the element structs as unused by
* setting the globalID to -1
*/
for (i = 0; i < mesh->elem_array_len; i++)
initialize_element(&(mesh->elements[i]));
/*
* now fill the element structure array with the
* information from the Chaco file
* Use hypergraph vertex information and chaco edge information.
*/
if (!chaco_fill_elements(Proc, Num_Proc, prob, mesh, gnvtxs, nvtxs,
ch_start, ch_adj, vwgt_dim, vwgts, ch_ewgt_dim, ch_ewgts,
ch_ndim, ch_x, ch_y, ch_z, ch_assignments, base)) {
Gen_Error(0, "fatal: Error returned from chaco_fill_elements");
return 0;
}
#if 0
debug_elements(Proc, Num_Proc, mesh->num_elems,mesh->elements);
#endif
safe_free((void **)(void *) &vwgts);
safe_free((void **)(void *) &ch_ewgts);
safe_free((void **)(void *) &ch_vwgts);
safe_free((void **)(void *) &ch_x);
safe_free((void **)(void *) &ch_y);
safe_free((void **)(void *) &ch_z);
safe_free((void **)(void *) &ch_start);
safe_free((void **)(void *) &ch_adj);
safe_free((void **)(void *) &ch_assignments);
if (Debug_Driver > 3)
print_distributed_mesh(Proc, Num_Proc, mesh);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
/*--------------------------------------------------------------------------*/
int output_results(const char *cmd_file,
const char *tag,
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
/*
* For the first swipe at this, don't try to create a new
* exodus/nemesis file or anything. Just get the global ids,
* sort them, and print them to a new ascii file.
*/
{
/* Local declarations. */
const char *yo = "output_results";
char par_out_fname[FILENAME_MAX+1], ctemp[FILENAME_MAX+1];
char cmsg[256];
int *global_ids = NULL;
int *parts = NULL;
int *perm = NULL;
int *invperm = NULL;
int *index = NULL;
int i, j;
FILE *fp;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
if (mesh->num_elems) {
global_ids = (int *) malloc(5 * mesh->num_elems * sizeof(int));
if (!global_ids) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
parts = global_ids + mesh->num_elems;
perm = parts + mesh->num_elems;
invperm = perm + mesh->num_elems;
index = invperm + mesh->num_elems;
}
for (i = j = 0; i < mesh->elem_array_len; i++) {
if (mesh->elements[i].globalID >= 0) {
global_ids[j] = mesh->elements[i].globalID;
parts[j] = mesh->elements[i].my_part;
perm[j] = mesh->elements[i].perm_value;
invperm[j] = mesh->elements[i].invperm_value;
index[j] = j;
j++;
}
}
sort_index(mesh->num_elems, global_ids, index);
/* generate the parallel filename for this processor */
strcpy(ctemp, pio_info->pexo_fname);
strcat(ctemp, ".");
strcat(ctemp, tag);
gen_par_filename(ctemp, par_out_fname, pio_info, Proc, Num_Proc);
fp = fopen(par_out_fname, "w");
if (fp == NULL){
sprintf(cmsg, "Error in %s; %s can not be opened for writing.", yo, par_out_fname);
Gen_Error(0, cmsg);
return 0;
}
if (Proc == 0)
echo_cmd_file(fp, cmd_file);
fprintf(fp, "Global element ids assigned to processor %d\n", Proc);
fprintf(fp, "GID\tPart\tPerm\tIPerm\n");
for (i = 0; i < mesh->num_elems; i++) {
j = index[i];
fprintf(fp, "%d\t%d\t%d\t%d\n", global_ids[j], parts[j], perm[j], invperm[j]);
}
fclose(fp);
free(global_ids);
if (Output.Mesh_Info_File) {
ELEM_INFO_PTR current_element;
int total_nodes = 0;
float *x, *y, *z;
int k;
int prev_id;
for (i = 0; i < mesh->num_elems; i++) {
total_nodes += mesh->eb_nnodes[mesh->elements[i].elem_blk];
}
global_ids = (int *) malloc(2 * total_nodes * sizeof(int));
index = global_ids + total_nodes;
x = (float *) calloc(3 * total_nodes, sizeof(float));
y = x + total_nodes;
z = y + total_nodes;
for (k = 0, i = 0; i < mesh->num_elems; i++) {
current_element = &(mesh->elements[i]);
for (j = 0; j < mesh->eb_nnodes[current_element->elem_blk]; j++) {
global_ids[k] = current_element->connect[j];
x[k] = current_element->coord[j][0];
if (mesh->num_dims > 1)
y[k] = current_element->coord[j][1];
if (mesh->num_dims > 2)
z[k] = current_element->coord[j][2];
index[k] = k;
k++;
}
}
sort_index(total_nodes, global_ids, index);
strcat(par_out_fname, ".mesh");
fp = fopen(par_out_fname, "w");
fprintf(fp, "Vertex IDs and coordinates\n");
prev_id = -1;
for (k = 0; k < total_nodes; k++) {
j = index[k];
if (global_ids[j] == prev_id)
continue;
prev_id = global_ids[j];
fprintf(fp, " %d (%e, %e, %e)\n", global_ids[j], x[j], y[j], z[j]);
}
fprintf(fp, "\n");
fprintf(fp, "Element connectivity:\n");
for (i = 0; i < mesh->num_elems; i++) {
current_element = &(mesh->elements[i]);
fprintf(fp, " %d (", current_element->globalID);
for (j = 0; j < mesh->eb_nnodes[current_element->elem_blk]; j++) {
fprintf(fp, "%d ", current_element->connect[j]);
}
fprintf(fp, ")\n");
}
fclose(fp);
free(global_ids);
free(x);
}
DEBUG_TRACE_END(Proc, yo);
return 1;
}
int chaco_fill_elements(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob, /* problem description */
MESH_INFO_PTR mesh, /* mesh information for the problem */
int gnvtxs, /* global number of vertices across all procs*/
int nvtxs, /* number of vertices in local graph */
int *start, /* start of edge list for each vertex */
int *adj, /* edge list data */
int vwgt_dim, /* # of weights per vertex */
float *vwgts, /* vertex weight list data */
int ewgt_dim, /* # of weights per edge */
float *ewgts, /* edge weight list data */
int ndim, /* dimension of the geometry */
float *x, /* x-coordinates of the vertices */
float *y, /* y-coordinates of the vertices */
float *z, /* z-coordinates of the vertices */
short *assignments, /* assignments from Chaco file; may be NULL */
int base /* smallest vertex number to use;
base == 1 for Chaco;
may be 0 or 1 for HG files. */
)
{
/* Local declarations. */
int i, j, k, elem_id, *local_ids = NULL;
int num_vtx, min_vtx, max_vtx;
int *vtx_list = NULL;
const char *yo = "chaco_fill_elements";
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
chaco_init_local_ids(&local_ids, &vtx_list, &min_vtx, &max_vtx, &num_vtx,
gnvtxs, assignments, base);
for (i = 0; i < num_vtx; i++) {
mesh->elements[i].globalID = vtx_list[i]+base; /* GlobalIDs are 1-based
in Chaco; may be 0-based
or 1-based in HG files */
if (vwgts != NULL){
for (j=0; j<vwgt_dim; j++) {
mesh->elements[i].cpu_wgt[j] = vwgts[i*vwgt_dim+j];
}
}
else
mesh->elements[i].cpu_wgt[0] = 1.0;
mesh->elements[i].elem_blk = 0; /* only one elem block for all vertices */
if (assignments)
mesh->elements[i].my_part = assignments[vtx_list[i]];
else
mesh->elements[i].my_part = Proc; /* Init partition is starting proc.*/
if (mesh->num_dims > 0) {
/* One set of coords per element. */
mesh->elements[i].connect = (int *) malloc(sizeof(int));
mesh->elements[i].connect[0] = mesh->elements[i].globalID;
mesh->elements[i].coord = (float **) malloc(sizeof(float *));
mesh->elements[i].coord[0] = (float *) calloc(mesh->num_dims,
sizeof(float));
mesh->elements[i].coord[0][0] = x[i];
mesh->elements[i].avg_coord[0] = x[i];
if (mesh->num_dims > 1) {
mesh->elements[i].coord[0][1] = y[i];
mesh->elements[i].avg_coord[1] = y[i];
if (mesh->num_dims > 2) {
mesh->elements[i].coord[0][2] = z[i];
mesh->elements[i].avg_coord[2] = z[i];
}
}
}
}
for (i = 0; i < num_vtx; i++) {
/* now start with the adjacencies */
if (start != NULL)
mesh->elements[i].nadj = start[i+1] - start[i];
else
mesh->elements[i].nadj = 0;
if (mesh->elements[i].nadj > 0) {
mesh->elements[i].adj_len = mesh->elements[i].nadj;
mesh->elements[i].adj = (int *) malloc (mesh->elements[i].nadj * sizeof(int));
mesh->elements[i].adj_proc = (int *) malloc (mesh->elements[i].nadj * sizeof(int));
if (!(mesh->elements[i].adj) || !(mesh->elements[i].adj_proc)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (ewgts != NULL) {
mesh->elements[i].edge_wgt = (float *) malloc (mesh->elements[i].nadj * sizeof(float));
if (!(mesh->elements[i].edge_wgt)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
else
mesh->elements[i].edge_wgt = NULL;
for (j = 0; j < mesh->elements[i].nadj; j++) {
elem_id = adj[start[i] + j] - (1-base); /* Chaco is 1-based;
HG may be 0 or 1 based. */
/* determine which processor the adjacent vertex is on */
k = ch_dist_proc(elem_id, assignments, base);
/*
* if the adjacent element is on this processor
* then find the local id for that element
*/
if (k == Proc)
mesh->elements[i].adj[j] = local_ids[elem_id-base-min_vtx];
else /* use the global id */
mesh->elements[i].adj[j] = elem_id;
mesh->elements[i].adj_proc[j] = k;
if (ewgts != NULL)
mesh->elements[i].edge_wgt[j] = ewgts[start[i] + j];
}
} /* End: "if (mesh->elements[i].nadj > 0)" */
} /* End: "for (i = 0; i < mesh->num_elems; i++)" */
safe_free((void **)(void *) &vtx_list);
safe_free((void **)(void *) &local_ids);
if (!build_elem_comm_maps(Proc, mesh)) {
Gen_Error(0, "Fatal: error building initial elem comm maps");
return 0;
}
if (Debug_Driver > 3)
print_distributed_mesh(Proc, Num_Proc, mesh);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
static int setup_mesh_struct(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob, /* problem description */
MESH_INFO_PTR mesh, /* mesh information for the problem */
PARIO_INFO_PTR pio_info, /* element distribution info*/
ZOLTAN_ID_TYPE gnvtxs, /* global number of vertices across all procs*/
int nvtxs, /* number of vertices in local graph */
int *start, /* start of edge list for each vertex */
ZOLTAN_ID_TYPE *adj, /* edge list data */
int vwgt_dim, /* # of weights per vertex */
float *vwgts, /* vertex weight list data */
int ewgt_dim, /* # of weights per edge */
float *ewgts, /* edge weight list data */
int ndim, /* dimension of the geometry */
float *x, /* x-coordinates of the vertices */
float *y, /* y-coordinates of the vertices */
float *z /* z-coordinates of the vertices */
)
{
const char *yo = "setup_mesh_struct";
int i, j, k;
ZOLTAN_ID_TYPE elem_id;
ZOLTAN_ID_TYPE min_vtx;
DEBUG_TRACE_START(Proc, yo);
/* Initialize mesh structure for Chaco mesh. */
mesh->data_type = ZOLTAN_GRAPH;
mesh->vwgt_dim = vwgt_dim;
mesh->ewgt_dim = ewgt_dim;
mesh->num_elems = nvtxs;
mesh->elem_array_len = mesh->num_elems + 5;
mesh->num_dims = ndim;
mesh->num_el_blks = 1;
mesh->eb_etypes = (int *) malloc (4 * mesh->num_el_blks * sizeof(int));
if (!mesh->eb_etypes) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_ids = mesh->eb_etypes + mesh->num_el_blks;
mesh->eb_nnodes = mesh->eb_ids + mesh->num_el_blks;
mesh->eb_nattrs = mesh->eb_nnodes + mesh->num_el_blks;
mesh->eb_cnts = (ZOLTAN_ID_TYPE *) malloc (mesh->num_el_blks * sizeof(ZOLTAN_ID_TYPE));
if (!mesh->eb_cnts) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_names = (char **) malloc (mesh->num_el_blks * sizeof(char *));
if (!mesh->eb_names) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_etypes[0] = -1;
mesh->eb_ids[0] = 1;
mesh->eb_cnts[0] = (ZOLTAN_ID_TYPE)nvtxs;
mesh->eb_nattrs[0] = 0;
mesh->hindex = (int *) malloc(sizeof(int));
mesh->hindex[0] = 0;
mesh->eb_nnodes[0] = 1;
/* allocate space for name */
mesh->eb_names[0] = (char *) malloc(16* sizeof(char));
if (!mesh->eb_names[0]) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
strcpy(mesh->eb_names[0], "random-graph");
/* allocate the element structure array */
mesh->elements = (ELEM_INFO_PTR) malloc (mesh->elem_array_len * sizeof(ELEM_INFO));
if (!(mesh->elements)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/* write element data */
for (i = 0; i < mesh->elem_array_len; i++)
initialize_element(&(mesh->elements[i]));
min_vtx = local_to_global_id_map(0, Proc);
for (i = 0; i < nvtxs; i++) {
mesh->elements[i].globalID = local_to_global_id_map(i, Proc);
if (vwgts != NULL){
for (j=0; j<vwgt_dim; j++) {
mesh->elements[i].cpu_wgt[j] = vwgts[i*vwgt_dim+j];
}
}
else
mesh->elements[i].cpu_wgt[0] = 1.0;
mesh->elements[i].elem_blk = 0;
mesh->elements[i].my_part = Proc;
if (mesh->num_dims > 0) {
/* One set of coords per element. */
mesh->elements[i].connect = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE));
mesh->elements[i].connect[0] = mesh->elements[i].globalID;
mesh->elements[i].coord = (float **) malloc(sizeof(float *));
mesh->elements[i].coord[0] = (float *) calloc(mesh->num_dims, sizeof(float));
mesh->elements[i].coord[0][0] = x[i];
mesh->elements[i].avg_coord[0] = x[i];
if (mesh->num_dims > 1) {
mesh->elements[i].coord[0][1] = y[i];
mesh->elements[i].avg_coord[1] = y[i];
if (mesh->num_dims > 2) {
mesh->elements[i].coord[0][2] = z[i];
mesh->elements[i].avg_coord[2] = z[i];
}
}
}
}
for (i = 0; i < nvtxs; i++) {
/* now start with the adjacencies */
if (start != NULL)
mesh->elements[i].nadj = start[i+1] - start[i];
else
mesh->elements[i].nadj = 0;
if (mesh->elements[i].nadj > 0) {
mesh->elements[i].adj_len = mesh->elements[i].nadj;
mesh->elements[i].adj = (ZOLTAN_ID_TYPE *) malloc (mesh->elements[i].nadj * sizeof(ZOLTAN_ID_TYPE));
mesh->elements[i].adj_proc = (int *) malloc (mesh->elements[i].nadj * sizeof(int));
if (!(mesh->elements[i].adj) || !(mesh->elements[i].adj_proc)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (ewgts != NULL) {
mesh->elements[i].edge_wgt = (float *) malloc (mesh->elements[i].nadj * sizeof(float));
if (!(mesh->elements[i].edge_wgt)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
else
mesh->elements[i].edge_wgt = NULL;
for (j = 0; j < mesh->elements[i].nadj; j++) {
elem_id = adj[start[i] + j];
k = global_to_proc_owner_map(elem_id, Num_Proc, Proc);
/*
* if the adjacent element is on this processor
* then find the local id for that element
*/
if (k == Proc)
mesh->elements[i].adj[j] = elem_id-min_vtx;
else /* use the global id */
mesh->elements[i].adj[j] = elem_id;
mesh->elements[i].adj_proc[j] = k;
if (ewgts != NULL)
mesh->elements[i].edge_wgt[j] = ewgts[start[i] + j];
}
} /* End: "if (mesh->elements[i].nadj > 0)" */
} /* End: "for (i = 0; i < mesh->num_elems; i++)" */
if (!build_elem_comm_maps(Proc, mesh)) {
Gen_Error(0, "Fatal: error building initial elem comm maps");
return 0;
}
if (Debug_Driver > 3)
print_distributed_mesh(Proc, Num_Proc, mesh);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
static int input_assign_inv(
ZOLTAN_FILE* finassign, /* input assignment file */
char *inassignname, /* name of input assignment file */
int nvtxs, /* number of vertices to output */
short *assignment) /* values to be printed */
{
const char *yo = "input_assign_inv";
int set; /* set number being read */
int size; /* number of vertices in set */
int total; /* total number of vertices read */
int done; /* have I hit end of file yet? */
int end_flag; /* return flag from read_int() */
int i, j, k; /* loop counter */
DEBUG_TRACE_START(0, yo);
/* Get the assignment vector one line at a time, checking as you go. */
/* Initialize assignment to help error detection. */
for (i = 0; i < nvtxs; i++) {
assignment[i] = -1;
}
/* First read past any comments at top. */
total = 0;
set = 0;
end_flag = 1;
while (end_flag == 1) {
size = read_int(finassign, &end_flag);
}
if (end_flag == -1) {
printf("ERROR: In assignment file `%s'\n", inassignname);
printf(" No values found\n");
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
if (size < 0) {
printf("ERROR: In assignment file `%s'\n", inassignname);
printf(" Size of set %d less than zero (%d)\n", set, size);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
if (total + size > nvtxs) {
printf("ERROR: In assignment file `%s'\n", inassignname);
printf(" Total set sizes greater than nvtxs (%d)\n", nvtxs);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
done = FALSE;
while (!done && total < nvtxs) {
for (i = 1; i <= size; i++) {
j = ZOLTAN_FILE_scanf(finassign, "%d", &k);
if (j != 1) {
printf("ERROR: Too few values in assignment file `%s'.\n",
inassignname);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
if (k <= 0 || k > nvtxs) {
printf("ERROR: In assignment file `%s'\n", inassignname);
printf(" Entry %d of set %d invalid (%d)\n",
total + i, set, k);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
if ((int) assignment[k - 1] != -1) {
printf("ERROR: In assignment file `%s'\n", inassignname);
printf(" Vertex %d assigned to multiple sets\n", k);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
assignment[k - 1] = (short) set;
}
total += size;
j = ZOLTAN_FILE_scanf(finassign, "%d", &size);
++set;
if (j != 1) {
if (total != nvtxs) {
printf("ERROR: Too few values in assignment file `%s'.\n",
inassignname);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
else {
done = TRUE;
size = 0;
}
}
if (size < 0) {
printf("ERROR: In assignment file `%s'\n", inassignname);
printf(" Size of set %d less than zero (%d)\n", set, size);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
if (total + size > nvtxs) {
printf("ERROR: In assignment file `%s'\n", inassignname);
printf(" Total set sizes greater than nvtxs (%d)\n", nvtxs);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
}
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (0);
}
int build_elem_comm_maps(int proc, MESH_INFO_PTR mesh)
{
/*
* Build element communication maps, given a distributed mesh.
* This routine builds initial communication maps for Chaco input
* (for Nemesis, initial communication maps are read from the Nemesis file)
* and rebuilds communication maps after data migration.
*
* One communication map per neighboring processor is built.
* The corresponding maps on neighboring processors
* must be sorted in the same order, so that neighboring processors do not
* have to use ghost elements. For each communication map's pair of
* processors, the lower-numbered processor determines the order of the
* elements in the communication map. The sort key is the elements' global
* IDs on the lower-number processor; the secondary key is the neighboring
* elements global IDs. The secondary key is used when a single element
* must communicate with more than one neighbor.
*/
const char *yo = "build_elem_comm_maps";
int i, j;
ELEM_INFO *elem;
ZOLTAN_ID_TYPE iadj_elem;
int iadj_proc;
int indx;
int num_alloc_maps;
int max_adj = 0;
int max_adj_per_map;
int cnt, offset;
int *sindex = NULL;
int tmp;
struct map_list_head *tmp_maps = NULL, *map = NULL;
DEBUG_TRACE_START(proc, yo);
/*
* Free the old maps, if they exist.
*/
if (mesh->ecmap_id != NULL) {
safe_free((void **) &(mesh->ecmap_id));
safe_free((void **) &(mesh->ecmap_cnt));
safe_free((void **) &(mesh->ecmap_elemids));
safe_free((void **) &(mesh->ecmap_sideids));
safe_free((void **) &(mesh->ecmap_neighids));
mesh->necmap = 0;
}
/*
* Look for off-processor adjacencies.
* Loop over all elements
*/
num_alloc_maps = MAP_ALLOC;
mesh->ecmap_id = (int *) malloc(num_alloc_maps * sizeof(int));
mesh->ecmap_cnt = (int *) malloc(num_alloc_maps * sizeof(int));
tmp_maps = (struct map_list_head*) malloc(num_alloc_maps
* sizeof(struct map_list_head));
if (mesh->ecmap_id == NULL || mesh->ecmap_cnt == NULL || tmp_maps == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
DEBUG_TRACE_END(proc, yo);
return 0;
}
for (i = 0; i < mesh->num_elems; i++) {
elem = &(mesh->elements[i]);
for (j = 0; j < elem->adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (elem->adj[j] == ZOLTAN_ID_INVALID) continue;
iadj_elem = elem->adj[j];
iadj_proc = elem->adj_proc[j];
if (iadj_proc != proc) {
/*
* Adjacent element is off-processor.
* Add this element to the temporary data structure for
* the appropriate neighboring processor.
*/
if ((indx = in_list2(iadj_proc, mesh->necmap, mesh->ecmap_id)) == -1) {
/*
* Start a new communication map.
*/
if (mesh->necmap >= num_alloc_maps) {
num_alloc_maps += MAP_ALLOC;
mesh->ecmap_id = (int *) realloc(mesh->ecmap_id,
num_alloc_maps * sizeof(int));
mesh->ecmap_cnt = (int *) realloc(mesh->ecmap_cnt,
num_alloc_maps * sizeof(int));
tmp_maps = (struct map_list_head *) realloc(tmp_maps,
num_alloc_maps * sizeof(struct map_list_head));
if (mesh->ecmap_id == NULL || mesh->ecmap_cnt == NULL ||
tmp_maps == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
DEBUG_TRACE_END(proc, yo);
return 0;
}
}
mesh->ecmap_id[mesh->necmap] = iadj_proc;
mesh->ecmap_cnt[mesh->necmap] = 0;
map = &(tmp_maps[mesh->necmap]);
map->glob_id = (ZOLTAN_ID_TYPE *) malloc(MAP_ALLOC * sizeof(ZOLTAN_ID_TYPE));
map->elem_id = (int *) malloc(MAP_ALLOC * sizeof(int));
map->side_id = (int *) malloc(MAP_ALLOC * sizeof(int));
map->neigh_id = (ZOLTAN_ID_TYPE *) malloc(MAP_ALLOC * sizeof(ZOLTAN_ID_TYPE));
if (map->glob_id == NULL || map->elem_id == NULL ||
map->side_id == NULL || map->neigh_id == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
DEBUG_TRACE_END(proc, yo);
return 0;
}
map->map_alloc_size = MAP_ALLOC;
indx = mesh->necmap;
mesh->necmap++;
}
/* Add to map for indx. */
map = &(tmp_maps[indx]);
if (mesh->ecmap_cnt[indx] >= map->map_alloc_size) {
map->map_alloc_size += MAP_ALLOC;
map->glob_id = (ZOLTAN_ID_TYPE *) realloc(map->glob_id, map->map_alloc_size * sizeof(ZOLTAN_ID_TYPE));
map->elem_id = (int *) realloc(map->elem_id,
map->map_alloc_size * sizeof(int));
map->side_id = (int *) realloc(map->side_id,
map->map_alloc_size * sizeof(int));
map->neigh_id = (ZOLTAN_ID_TYPE *) realloc(map->neigh_id, map->map_alloc_size * sizeof(ZOLTAN_ID_TYPE));
if (map->glob_id == NULL || map->elem_id == NULL ||
map->side_id == NULL || map->neigh_id == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
DEBUG_TRACE_END(proc, yo);
return 0;
}
}
tmp = mesh->ecmap_cnt[indx];
map->glob_id[tmp] = elem->globalID;
map->elem_id[tmp] = i;
map->side_id[tmp] = j+1; /* side is determined by position in
adj array (+1 since not 0-based). */
map->neigh_id[tmp] = iadj_elem;
mesh->ecmap_cnt[indx]++;
max_adj++;
}
}
}
/*
* If no communication maps, don't need to do anything else.
*/
if (mesh->necmap > 0) {
/*
* Allocate data structure for element communication map arrays.
*/
mesh->ecmap_elemids = (int *) malloc(max_adj * sizeof(int));
mesh->ecmap_sideids = (int *) malloc(max_adj * sizeof(int));
mesh->ecmap_neighids = (ZOLTAN_ID_TYPE *) malloc(max_adj * sizeof(ZOLTAN_ID_TYPE));
/*
* Allocate temporary memory for sort index.
*/
max_adj_per_map = 0;
for (i = 0; i < mesh->necmap; i++)
if (mesh->ecmap_cnt[i] > max_adj_per_map)
max_adj_per_map = mesh->ecmap_cnt[i];
sindex = (int *) malloc(max_adj_per_map * sizeof(int));
cnt = 0;
for (i = 0; i < mesh->necmap; i++) {
map = &(tmp_maps[i]);
for (j = 0; j < mesh->ecmap_cnt[i]; j++)
sindex[j] = j;
/*
* Sort the map so that adjacent processors have the same ordering
* for the communication.
* Assume the ordering of the lower-numbered processor in the pair
* of communicating processors.
*/
if (proc < mesh->ecmap_id[i])
quicksort_pointer_inc_id_id(sindex, map->glob_id, map->neigh_id,
0, mesh->ecmap_cnt[i]-1);
else
quicksort_pointer_inc_id_id(sindex, map->neigh_id, map->glob_id,
0, mesh->ecmap_cnt[i]-1);
/*
* Copy sorted data into elem map arrays.
*/
offset = cnt;
for (j = 0; j < mesh->ecmap_cnt[i]; j++) {
mesh->ecmap_elemids[offset] = map->elem_id[sindex[j]];
mesh->ecmap_sideids[offset] = map->side_id[sindex[j]];
mesh->ecmap_neighids[offset] = map->neigh_id[sindex[j]];
offset++;
}
cnt += mesh->ecmap_cnt[i];
}
}
/* Free temporary data structure. */
for (i = 0; i < mesh->necmap; i++) {
safe_free((void **) &(tmp_maps[i].glob_id));
safe_free((void **) &(tmp_maps[i].elem_id));
safe_free((void **) &(tmp_maps[i].side_id));
safe_free((void **) &(tmp_maps[i].neigh_id));
}
safe_free((void **) &tmp_maps);
safe_free((void **) &sindex);
if (Test.DDirectory)
compare_maps_with_ddirectory_results(proc, mesh);
DEBUG_TRACE_END(proc, yo);
return 1;
}
int create_a_graph(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
{
const char *yo = "create_a_graph";
/* The graph (and geometry) is created in parallel by each process, as opposed to being
* created by process 0 and then dealt out to the other processes. This allows us to
* create graphs where the number of vertices is larger than a number which would
* fit in the memory of one process.
*
* Geometrically the graph is a cylinder extending in the z-direction.
*
* Each process creates points along a circle in an x-y plane, and knows which process has the
* plane adjacent to it and what global ID has been assigned to the neighbors with the same
* x and y value as its points. So adjacency information is easily created.
*/
ZOLTAN_ID_TYPE i, nvtxs, gnvtxs, num4;
ZOLTAN_ID_TYPE gid;
long left=0, right=0;
int vwgt_dim=0, ewgt_dim=0;
int ndim = 0, next;
int *start;
ZOLTAN_ID_TYPE *adj = NULL;
float *ewgts = NULL, *vwgts = NULL;
float *x = NULL, *y = NULL, *z = NULL;
double theta, delta, radius, m, length, step;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
gnvtxs = pio_info->init_size;
ndim = pio_info->init_dim;
vwgt_dim = pio_info->init_vwgt_dim;
if (vwgt_dim<1) vwgt_dim=1; /* For now, insist on 1 or more weights. */
/* for simplicity coerce number of vertices on a process to a multiple of 4 */
nvtxs = gnvtxs / Num_Proc;
if (nvtxs > 4){
num4 = nvtxs / 4;
nvtxs = num4 * 4;
}
else{
num4 = 1;
nvtxs = 4;
}
gnvtxs = (ZOLTAN_ID_TYPE)nvtxs * Num_Proc;
if (Proc == 0){
printf("create_a_graph: Graph will have " ZOLTAN_ID_SPEC " vertices, " ZOLTAN_ID_SPEC " on each process\n",
gnvtxs, nvtxs);
}
/* Each process has the same number of vertices. Let's determine their global IDs */
initialize_vertex_global_id_info(nvtxs, Num_Proc);
/* Calculate vertex coordinates and adjacencies */
x = (float *) malloc(nvtxs * sizeof(float));
if (ndim > 1) y = (float *) malloc(nvtxs * sizeof(float));
if (ndim > 2) z = (float *) malloc(nvtxs * sizeof(float));
vwgts = (float *) malloc(vwgt_dim*nvtxs * sizeof(float));
if (!x || (ndim > 1 && !y) || (ndim > 2 && !z) || !vwgts) {
Gen_Error(0, "fatal: Error allocating memory.");
return 0;
}
if (ndim == 1){
/* a line */
step = 1.0 / 500.0;
length = (double)nvtxs * step;
x[0] = length * (float)Proc;
for (i=1; i < nvtxs; i++){
x[i] = x[i+1] + step;
}
}
else if (ndim == 2){
/* a circle */
radius = (double)nvtxs/500.0;
theta = (2 * M_PI ) / (double)Num_Proc;
delta = theta / (double)nvtxs;
m = (theta * Proc);
for (i=0; i < nvtxs; i++, m += delta){
x[i] = radius * cos(m);
y[i] = radius * sin(m);
}
}
else if (ndim == 3){
/* a cylinder */
radius = (double)nvtxs/500.0;
delta = M_PI_2 / (double)(num4 + 1);
theta = delta;
i = 0;
while (theta < M_PI_2){
/* points along first quadrant of a circle in the plane z=Proc */
x[i] = radius * cos(theta);
y[i] = radius * sin(theta);
z[i] = (float)Proc;
theta += delta;
i++;
}
for (i=0; i < num4; i++){
/* second quadrant */
x[num4+i] = -x[num4 - i - 1];
y[num4+i] = y[num4 - i - 1];
z[num4+i] = (float)Proc;
/* third quadrant */
x[2*num4+i] = -x[i];
y[2*num4+i] = -y[i];
z[2*num4+i] = (float)Proc;
/* third quadrant */
x[3*num4+i] = x[num4 - i - 1];
y[3*num4+i] = -y[num4 - i - 1];
z[3*num4+i] = (float)Proc;
}
}
for (i = 0; i < nvtxs; i++) {
if (pio_info->init_vwgt_dim == 0)
/* Unit weights if no weights were requested. */
vwgts[i] = 1.0;
else {
int jj;
srand(0);
for (jj = 0; jj < vwgt_dim; jj++) {
/* Only assign one of the weight dimensions a weight>0. */
/* Modify to get more complicated test cases. */
if (jj == (int)(i%vwgt_dim))
vwgts[i*vwgt_dim+jj] = ((float) rand())/RAND_MAX;
else
vwgts[i*vwgt_dim+jj] = 0.0;
}
}
}
start = (int *)malloc(sizeof(int) * (nvtxs + 1));
if (ndim < 3){
/* each vertex has one or two neighbors */
adj = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2 * nvtxs);
start[0] = 0;
next = 0;
for (i=0; i < nvtxs; i++){
gid = local_to_global_id_map(i, Proc);
if (ndim == 1){
left = gid - 1;
right = gid + 1;
}
else if (ndim == 2){
left = (gid == 0) ? (gnvtxs - 1) : (gid - 1);
right = (gid == gnvtxs - 1) ? 0 : (gid + 1);
}
start[i+1] = start[i];
if (left >= 0){
adj[next++] = left;
start[i+1]++;
}
if (right < gnvtxs){
adj[next++] = left;
start[i+1]++;
}
}
}
else{
/* each vertex has 2 neighbors on this process, and one or two off process */
adj = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 4 * nvtxs);
start[0] = 0;
next = 0;
for (i=0; i < nvtxs; i++){
gid = local_to_global_id_map(i, Proc);
left = (i == 0) ? local_to_global_id_map(nvtxs-1, Proc) : local_to_global_id_map(i-1, Proc);
right = (i == nvtxs-1) ? local_to_global_id_map(0, Proc) : local_to_global_id_map(i+1, Proc);
start[i+1] = start[i];
adj[next++] = left;
start[i+1]++;
adj[next++] = right;
start[i+1]++;
left = gid - nvtxs;
right = gid + nvtxs;
if (left >= 0){
adj[next++] = left;
start[i+1]++;
}
if (right < gnvtxs){
adj[next++] = right;
start[i+1]++;
}
}
}
/* TODO - edge weights? */
if (!setup_mesh_struct(Proc, Num_Proc, prob, mesh, pio_info, gnvtxs, nvtxs,
start, adj, vwgt_dim, vwgts, ewgt_dim, ewgts,
ndim, x, y, z)){
Gen_Error(0, "fatal: Error returned from chaco_setup_mesh_struct");
return 0;
}
safe_free((void **) &adj);
safe_free((void **) &vwgts);
safe_free((void **) &ewgts);
safe_free((void **) &start);
safe_free((void **) &x);
safe_free((void **) &y);
safe_free((void **) &z);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
int migrate_elements(
int Proc,
MESH_INFO_PTR mesh,
struct LB_Struct *lb,
int num_gid_entries,
int num_lid_entries,
int num_imp,
LB_ID_PTR imp_gids,
LB_ID_PTR imp_lids,
int *imp_procs,
int num_exp,
LB_ID_PTR exp_gids,
LB_ID_PTR exp_lids,
int *exp_procs)
{
/* Local declarations. */
char *yo = "migrate_elements";
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
/*
* register migration functions
*/
if (LB_Set_Fn(lb, LB_PRE_MIGRATE_FN_TYPE, (void (*)()) migrate_pre_process,
(void *) mesh) == LB_FATAL) {
Gen_Error(0, "fatal: error returned from LB_Set_Fn()\n");
return 0;
}
if (LB_Set_Fn(lb, LB_POST_MIGRATE_FN_TYPE, (void (*)()) migrate_post_process,
(void *) mesh) == LB_FATAL) {
Gen_Error(0, "fatal: error returned from LB_Set_Fn()\n");
return 0;
}
if (LB_Set_Fn(lb, LB_OBJ_SIZE_FN_TYPE, (void (*)()) migrate_elem_size,
(void *) mesh) == LB_FATAL) {
Gen_Error(0, "fatal: error returned from LB_Set_Fn()\n");
return 0;
}
if (LB_Set_Fn(lb, LB_PACK_OBJ_FN_TYPE, (void (*)()) migrate_pack_elem,
(void *) mesh) == LB_FATAL) {
Gen_Error(0, "fatal: error returned from LB_Set_Fn()\n");
return 0;
}
if (LB_Set_Fn(lb, LB_UNPACK_OBJ_FN_TYPE, (void (*)()) migrate_unpack_elem,
(void *) mesh) == LB_FATAL) {
Gen_Error(0, "fatal: error returned from LB_Set_Fn()\n");
return 0;
}
if (LB_Help_Migrate(lb,
num_imp, imp_gids, imp_lids, imp_procs,
num_exp, exp_gids, exp_lids, exp_procs) == LB_FATAL) {
Gen_Error(0, "fatal: error returned from LB_Help_Migrate()\n");
return 0;
}
DEBUG_TRACE_END(Proc, yo);
return 1;
}
/* Read "matrixmarket plus", the format written by Zoltan_Generate_Files.
*
* This format is our own extension of the NIST Matrix Market file
* format. We wished to store vertex and edge weights, and also
* pin, vertex weight and edge weight ownership data in the file.
* Here are some rules from the NIST design document:
* 1. lines are limited to 1024 characters
* 2. blank lines may appear anywhere after the first line
* 3. numeric data on a line is separated by one or more blanks
* 4. real data is in floating-point decimal format, can use "e" notation
* 5. all indices are 1-based
* 6. character data may be upper or lower case.
*
* The contents of the file reflects the data returned by the
* application in the hypergraph query functions. In particular:
*
* Each process supplied some subset of pins to Zoltan. Each owned
* some of the vertices and supplied weights for those. Each may have
* supplied weights for edges. The edges need not be the edges of
* their pins. More than one process may have supplied a weight for
* the same edge.
*/
int read_mtxplus_file(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh
)
{
/* Local declarations. */
const char *yo = "read_mtxplus_file";
char filename[256], cmesg[256];
struct stat statbuf;
int rc, fsize, i, j;
char *filebuf=NULL;
FILE* fp;
int nGlobalEdges, nGlobalVtxs, vtxWDim, edgeWDim;
int nMyPins, nMyVtx, nMyEdgeWgts;
int *myPinI, *myPinJ, *myVtxNum, *myEWGno;
float *myVtxWgts, *myEdgeWgts;
int status;
int numHEdges;
int *edgeGno, *edgeIdx, *pinGno;
DEBUG_TRACE_START(Proc, yo);
/* Process 0 reads the file and broadcasts it */
if (Proc == 0) {
fsize = 0;
sprintf(filename, "%s.mtxp", pio_info->pexo_fname);
if (pio_info->file_comp == GZIP)
sprintf(filename, "%s.gz", filename);
rc = stat(filename, &statbuf);
if (rc == 0){
fsize = statbuf.st_size;
fp = fopen(filename, "r");
if (!fp){
fsize = 0;
}
else{
filebuf = (char *)malloc(fsize+1);
rc = fread(filebuf, 1, fsize, fp);
if (rc != fsize){
free(filebuf);
fsize = 0;
fp = NULL;
}
else{
filebuf[fsize] = 0;
fsize++;
}
fclose(fp);
}
}
}
MPI_Bcast(&fsize, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (fsize == 0) {
sprintf(cmesg, "fatal: Could not open/read hypergraph file %s", filename);
Gen_Error(0, cmesg);
return 0;
}
if (Proc > 0){
filebuf = (char *)malloc(fsize);
}
MPI_Bcast(filebuf, fsize, MPI_BYTE, 0, MPI_COMM_WORLD);
/* Each process reads through the file, obtaining it's
* pins, vertex weights and edge weights. The file lists
* global IDs for the vertices and edges. These will be
* assigned global numbers based on the order they appear
* in the file. The global numbers begin with zero.
* Returns 1 on success, 0 on failure.
*/
rc = process_mtxp_file(pio_info, filebuf, fsize, Num_Proc, Proc,
&nGlobalEdges, &nGlobalVtxs, &vtxWDim, &edgeWDim,
&nMyPins, &myPinI, &myPinJ,
&nMyVtx, &myVtxNum, &myVtxWgts,
&nMyEdgeWgts, &myEWGno, &myEdgeWgts);
free(filebuf);
MPI_Allreduce(&rc, &status, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (status != Num_Proc){
return 0;
}
/*
* From the lists of pins, create edge lists. (Unless
* the initial pin distribution is by column, in which
* case we will test the hypergraph query interface's
* ability to accept pins by column rather than row.)
*/
if (pio_info->init_dist_pins != INITIAL_COL){ /* CRS */
rc = create_edge_lists(nMyPins, myPinI, myPinJ,
&numHEdges, &edgeGno, &edgeIdx, &pinGno);
mesh->format = ZOLTAN_COMPRESSED_EDGE;
}
else{ /* CCS */
/* actually creating vertex lists, since we switched
* the role of I and J in the argument list.
*/
rc = create_edge_lists(nMyPins, myPinJ, myPinI,
&numHEdges, &edgeGno, &edgeIdx, &pinGno);
mesh->format = ZOLTAN_COMPRESSED_VERTEX;
}
MPI_Allreduce(&rc, &status, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (status != Num_Proc){
return 0;
}
safe_free((void **)(void *)&myPinI);
safe_free((void **)(void *)&myPinJ);
/* Initialize mesh structure for Hypergraph. */
mesh->data_type = HYPERGRAPH;
mesh->num_elems = nMyVtx;
mesh->vwgt_dim = vtxWDim;
mesh->ewgt_dim = 0;
mesh->elem_array_len = mesh->num_elems + 5;
mesh->num_dims = 0;
mesh->num_el_blks = 1;
mesh->gnhedges = nGlobalEdges;
mesh->nhedges = numHEdges; /* (or num vertices if CCS) */
mesh->hewgt_dim = edgeWDim;
mesh->hgid = edgeGno; /* (or vertex gno if CCS) */
mesh->hindex = edgeIdx; /* (or vertex index if CCS) */
mesh->hvertex = pinGno; /* (or gno of pin edge if CCS) */
mesh->hvertex_proc = NULL; /* don't know don't care */
mesh->heNumWgts = nMyEdgeWgts;
mesh->heWgtId = myEWGno;
mesh->hewgts = myEdgeWgts;
mesh->eb_etypes = (int *) malloc (5 * mesh->num_el_blks * sizeof(int));
if (!mesh->eb_etypes) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_ids = mesh->eb_etypes + mesh->num_el_blks;
mesh->eb_cnts = mesh->eb_ids + mesh->num_el_blks;
mesh->eb_nnodes = mesh->eb_cnts + mesh->num_el_blks;
mesh->eb_nattrs = mesh->eb_nnodes + mesh->num_el_blks;
mesh->eb_names = (char **) malloc (mesh->num_el_blks * sizeof(char *));
if (!mesh->eb_names) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_etypes[0] = -1;
mesh->eb_ids[0] = 1;
mesh->eb_cnts[0] = nGlobalVtxs;
mesh->eb_nattrs[0] = 0;
mesh->eb_nnodes[0] = 0;
/* allocate space for name */
mesh->eb_names[0] = (char *) malloc((MAX_STR_LENGTH+1) * sizeof(char));
if (!mesh->eb_names[0]) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
strcpy(mesh->eb_names[0], "hypergraph");
/* allocate the element structure array */
mesh->elements = (ELEM_INFO_PTR) malloc (mesh->elem_array_len
* sizeof(ELEM_INFO));
if (!(mesh->elements)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/*
* Write the element structure with the vertices and weights
*/
for (i = 0; i < mesh->elem_array_len; i++) {
initialize_element(&(mesh->elements[i]));
if (i < mesh->num_elems){
mesh->elements[i].globalID = myVtxNum[i];
mesh->elements[i].my_part = Proc;
for (j=0; j<vtxWDim; j++){
mesh->elements[i].cpu_wgt[j] = myVtxWgts[i*vtxWDim + j];
}
}
}
safe_free((void **)(void *) &myVtxWgts);
safe_free((void **)(void *) &myVtxNum);
if (Debug_Driver > 3)
print_distributed_mesh(Proc, Num_Proc, mesh);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
static int dist_hyperedges(
MPI_Comm comm, /* MPI Communicator */
PARIO_INFO_PTR pio_info, /* Parallel IO info */
int host_proc, /* processor where all the data is initially */
int base, /* indicates whether input is 0-based or
1-based (i.e., is lowest vertex number
0 or 1?). */
int gnvtxs, /* global number of vertices */
int *gnhedges, /* global number of hyperedges */
int *nhedges, /* local number of hyperedges */
int **hgid, /* global hyperedge numbers */
int **hindex, /* Starting hvertex entry for hyperedges */
int **hvertex, /* Array of vertices in hyperedges; returned
values are global IDs for vertices */
int **hvertex_proc, /* Array of processor assignments for
vertices in hvertex. */
int *hewgt_dim, /* number of weights per hyperedge */
float **hewgts, /* hyperedge weight list data */
short *assignments /* assignments from Chaco file; may be NULL */
)
{
/*
* Distribute hyperedges from one processor to all processors.
* Vertex distribution is assumed already done through chaco_dist_graph.
* The memory for the hyperedges on the host node is freed
* and fresh memory is allocated for the distributed hyperedges.
*/
const char *yo = "dist_hyperedges";
int nprocs, myproc, i, h, p = 0;
int *old_hindex = NULL, *old_hvertex = NULL, *old_hvertex_proc = NULL;
int *size = NULL, *num_send = NULL;
int *send = NULL;
int *send_hgid = NULL, *send_hindex = NULL;
int *send_hvertex = NULL, *send_hvertex_proc = NULL;
int max_size, max_num_send;
int hcnt[2];
int hecnt, hvcnt;
float *old_hewgts = NULL;
float *send_hewgts = NULL;
MPI_Status status;
int num_dist_procs;
int hedge_init_dist_type;
hedge_init_dist_type = (pio_info->init_dist_type != INITIAL_FILE
? pio_info->init_dist_type
: INITIAL_LINEAR);
/* Determine number of processors and my rank. */
MPI_Comm_size (comm, &nprocs );
MPI_Comm_rank (comm, &myproc );
DEBUG_TRACE_START(myproc, yo);
if (pio_info->init_dist_procs > 0 && pio_info->init_dist_procs <= nprocs)
num_dist_procs = pio_info->init_dist_procs;
else
/* Reset num_dist_procs if not set validly by input */
num_dist_procs = nprocs;
/* Broadcast to all procs */
MPI_Bcast( hewgt_dim, 1, MPI_INT, host_proc, comm);
MPI_Bcast( nhedges, 1, MPI_INT, host_proc, comm);
*gnhedges = *nhedges;
/* Initialize */
if (*gnhedges == 0) {
*hindex = (int *) malloc(sizeof(int));
if (!(*hindex)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
(*hindex)[0] = 0;
*hvertex = NULL;
*hvertex_proc = NULL;
return 1;
}
if (nprocs == 1) {
*nhedges = *gnhedges;
*hgid = (int *) malloc(*gnhedges * sizeof(int));
*hvertex_proc = (int *) malloc((*hindex)[*gnhedges] * sizeof(int));
if ((*gnhedges && !(*hgid)) || ((*hindex)[*gnhedges] && !(*hvertex_proc))) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
for (h = 0; h < *gnhedges; h++)
(*hgid)[h] = h + base; /* Want the same numbering than for vertices */
for (h = 0; h < (*hindex)[*gnhedges]; h++)
(*hvertex_proc)[h] = 0;
return 1;
}
if (myproc == host_proc) {
/* Store pointers to original data */
old_hindex = *hindex;
old_hvertex = *hvertex;
old_hewgts = *hewgts;
old_hvertex_proc = (int *) malloc(old_hindex[*gnhedges] * sizeof(int));
/* Allocate space for size and send flags */
size = (int *) calloc(2 * nprocs, sizeof(int));
num_send = size + nprocs;
send = (int *) malloc(*gnhedges * sizeof(int *));
if ((old_hindex[*gnhedges] && !old_hvertex_proc) || !size ||
(*gnhedges && !send)) {
Gen_Error(0, "fatal: memory error");
return 0;
}
/* Determine to which processors hyperedges should be sent */
for (h = 0; h < *gnhedges; h++) {
if (hedge_init_dist_type == INITIAL_CYCLIC)
p = h % num_dist_procs;
else if (hedge_init_dist_type == INITIAL_LINEAR)
p = (int) ((float) h * (float) num_dist_procs / (float)(*gnhedges));
else if (hedge_init_dist_type == INITIAL_OWNER)
p = ch_dist_proc(old_hvertex[old_hindex[h]], assignments, base);
size[p] += (old_hindex[h+1] - old_hindex[h]);
num_send[p]++;
send[h] = p;
for (i = old_hindex[h]; i < old_hindex[h+1]; i++)
old_hvertex_proc[i] = ch_dist_proc(old_hvertex[i], assignments, base);
}
/* Determine size of send buffers and allocate them. */
max_size = 0;
max_num_send = 0;
for (p = 0; p < nprocs; p++) {
if (size[p] > max_size) max_size = size[p];
if (num_send[p] > max_num_send) max_num_send = num_send[p];
}
send_hgid = (int *) malloc((max_num_send) * sizeof(int));
send_hindex = (int *) malloc((max_num_send+1) * sizeof(int));
send_hvertex = (int *) malloc(max_size * sizeof(int));
send_hvertex_proc = (int *) malloc(max_size * sizeof(int));
if (*hewgt_dim)
send_hewgts = (float *) malloc(max_num_send*(*hewgt_dim)*sizeof(float));
if ((max_num_send && !send_hgid) || !send_hindex ||
(max_size && (!send_hvertex || !send_hvertex_proc)) ||
(max_num_send && *hewgt_dim && !send_hewgts)) {
Gen_Error(0, "fatal: memory error in dist_hyperedges");
return 0;
}
/* Load and send data */
for (p = 0; p < nprocs; p++) {
if (p == myproc) continue;
hecnt = 0;
hvcnt = 0;
send_hindex[0] = 0;
for (h = 0; h < *gnhedges; h++) {
if (send[h]==p) {
send_hgid[hecnt] = h + base; /* Want the same numbering than for vertices */
send_hindex[hecnt+1] = send_hindex[hecnt]
+ (old_hindex[h+1] - old_hindex[h]);
for (i = 0; i < *hewgt_dim; i++)
send_hewgts[hecnt*(*hewgt_dim)+i] = old_hewgts[h*(*hewgt_dim)+i];
hecnt++;
for (i = old_hindex[h]; i < old_hindex[h+1]; i++) {
send_hvertex[hvcnt] = old_hvertex[i];
send_hvertex_proc[hvcnt] = old_hvertex_proc[i];
hvcnt++;
}
}
}
hcnt[0] = hecnt;
hcnt[1] = hvcnt;
MPI_Send(hcnt, 2, MPI_INT, p, 1, comm);
MPI_Send(send_hgid, hecnt, MPI_INT, p, 2, comm);
MPI_Send(send_hindex, hecnt+1, MPI_INT, p, 3, comm);
MPI_Send(send_hvertex, hvcnt, MPI_INT, p, 4, comm);
MPI_Send(send_hvertex_proc, hvcnt, MPI_INT, p, 5, comm);
if (*hewgt_dim)
MPI_Send(send_hewgts, hecnt*(*hewgt_dim), MPI_FLOAT, p, 6, comm);
}
safe_free((void **)(void *) &send_hgid);
safe_free((void **)(void *) &send_hindex);
safe_free((void **)(void *) &send_hvertex);
safe_free((void **)(void *) &send_hvertex_proc);
safe_free((void **)(void *) &send_hewgts);
/* Copy data owned by myproc into new local storage */
*nhedges = num_send[myproc];
*hgid = (int *) malloc(*nhedges * sizeof(int));
*hindex = (int *) malloc((*nhedges+1) * sizeof(int));
*hvertex = (int *) malloc(size[myproc] * sizeof(int));
*hvertex_proc = (int *) malloc(size[myproc] * sizeof(int));
if (*hewgt_dim)
*hewgts = (float *) malloc(*nhedges * *hewgt_dim * sizeof(float));
if ((*nhedges && !(*hgid)) || !(*hindex) ||
(size[myproc] && (!(*hvertex) || !(*hvertex_proc))) ||
(*nhedges && *hewgt_dim && !(*hewgts))) {
Gen_Error(0, "fatal: memory error in dist_hyperedges");
return 0;
}
hecnt = 0;
hvcnt = 0;
(*hindex)[0] = 0;
for (h = 0; h < *gnhedges; h++) {
if (send[h]==myproc) {
(*hgid)[hecnt] = h + base; /* Want the same numbering than for vertices */
(*hindex)[hecnt+1] = (*hindex)[hecnt]
+ (old_hindex[h+1] - old_hindex[h]);
for (i = 0; i < *hewgt_dim; i++)
(*hewgts)[hecnt*(*hewgt_dim)+i] = old_hewgts[h*(*hewgt_dim)+i];
hecnt++;
for (i = old_hindex[h]; i < old_hindex[h+1]; i++) {
(*hvertex_proc)[hvcnt] = old_hvertex_proc[i];
(*hvertex)[hvcnt] = old_hvertex[i]; /* Global index */
hvcnt++;
}
}
}
safe_free((void **)(void *) &send);
safe_free((void **)(void *) &size);
safe_free((void **)(void *) &old_hindex);
safe_free((void **)(void *) &old_hvertex);
safe_free((void **)(void *) &old_hvertex_proc);
safe_free((void **)(void *) &old_hewgts);
}
else {
/* host_proc != myproc; receive hedge data from host_proc */
MPI_Recv(hcnt, 2, MPI_INT, host_proc, 1, comm, &status);
*nhedges = hcnt[0];
*hgid = (int *) malloc(hcnt[0] * sizeof(int));
*hindex = (int *) malloc((hcnt[0]+1) * sizeof(int));
*hvertex = (int *) malloc(hcnt[1] * sizeof(int));
*hvertex_proc = (int *) malloc(hcnt[1] * sizeof(int));
if (*hewgt_dim)
*hewgts = (float *) malloc(hcnt[0] * *hewgt_dim * sizeof(float));
if ((hcnt[0] && !(*hgid)) || !(*hindex) ||
(hcnt[1] && (!(*hvertex) || !(*hvertex_proc))) ||
(hcnt[0] && *hewgt_dim && !(*hewgts))) {
Gen_Error(0, "fatal: memory error in dist_hyperedges");
return 0;
}
MPI_Recv(*hgid, hcnt[0], MPI_INT, host_proc, 2, comm, &status);
MPI_Recv(*hindex, hcnt[0]+1, MPI_INT, host_proc, 3, comm, &status);
MPI_Recv(*hvertex, hcnt[1], MPI_INT, host_proc, 4, comm, &status);
MPI_Recv(*hvertex_proc, hcnt[1], MPI_INT, host_proc, 5, comm, &status);
if (*hewgt_dim)
MPI_Recv(*hewgts, hcnt[0]* *hewgt_dim, MPI_FLOAT, host_proc, 6, comm,
&status);
}
DEBUG_TRACE_END(myproc, yo);
return 1;
}
int chaco_dist_graph(
MPI_Comm comm, /* MPI Communicator */
PARIO_INFO_PTR pio_info, /* Parallel IO info */
int host_proc, /* processor where all the data is initially */
int *gnvtxs, /* number of vertices in global graph */
int *nvtxs, /* number of vertices in local graph */
int **xadj, /* start of edge list for each vertex */
int **adjncy, /* edge list data */
int *vwgt_dim, /* number of weights per vertex */
float **vwgts, /* vertex weight list data */
int *ewgt_dim, /* number of weights per edge */
float **ewgts, /* edge weight list data */
int *ndim, /* dimension of the geometry */
float **x, /* x-coordinates of the vertices */
float **y, /* y-coordinates of the vertices */
float **z, /* z-coordinates of the vertices */
short **assignments /* assignments from Chaco file; may be NULL */
)
{
const char *yo = "chaco_dist_graph";
int nprocs, myproc, i, j, k, n, p, nedges, nsend, max_nvtxs, v, adj_cnt;
int offset, use_graph, nvtx_edges;
int *old_xadj = NULL, *old_adjncy = NULL, *size = NULL;
int *send_xadj = NULL, *send_adjncy = NULL;
int *vtx_list = NULL;
float *old_x = NULL, *old_y = NULL, *old_z = NULL;
float *send_x = NULL, *send_y = NULL, *send_z = NULL;
float *old_vwgts = NULL, *old_ewgts = NULL;
float *send_vwgts = NULL, *send_ewgts = NULL;
MPI_Status status;
/* Determine number of processors and my rank. */
MPI_Comm_size (comm, &nprocs );
MPI_Comm_rank (comm, &myproc );
DEBUG_TRACE_START(myproc, yo);
/* Initialize */
use_graph = (*xadj != NULL);
/* Handle serial case and return. */
if (nprocs == 1) {
/* Set values expected to be returned by this function. */
/* All array pointers are unchanged. */
*gnvtxs = *nvtxs;
/* Initialize distribution, so other routines using it work. */
ch_dist_init(nprocs, *gnvtxs, pio_info, assignments, host_proc, comm);
return 1;
}
/* Broadcast to all procs */
MPI_Bcast( vwgt_dim, 1, MPI_INT, host_proc, comm);
MPI_Bcast( ewgt_dim, 1, MPI_INT, host_proc, comm);
MPI_Bcast( &use_graph, 1, MPI_INT, host_proc, comm);
MPI_Bcast( ndim, 1, MPI_INT, host_proc, comm);
MPI_Bcast( nvtxs, 1, MPI_INT, host_proc, comm);
*gnvtxs = *nvtxs;
/* Initialize the chaco distribution on all processors */
ch_dist_init(nprocs, *gnvtxs, pio_info, assignments, host_proc, comm);
/* Store pointers to original data */
if (myproc == host_proc) {
old_xadj = *xadj;
*xadj = NULL;
old_adjncy = *adjncy;
*adjncy = NULL;
old_x = *x;
old_y = *y;
old_z = *z;
}
/* Allocate space for new distributed graph data */
n = *nvtxs = ch_dist_num_vtx(myproc, *assignments);
if (use_graph) {
*xadj = (int *) malloc((n+1)*sizeof(int));
if (*xadj == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
if (*vwgt_dim){
old_vwgts = *vwgts;
*vwgts = NULL;
if (n > 0) {
*vwgts = (float *) malloc(n*(*vwgt_dim)*sizeof(float));
if (*vwgts == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
}
if (*ndim > 0) {
*x = (float *) malloc(n*sizeof(float));
if (*ndim > 1) {
*y = (float *) malloc(n*sizeof(float));
if (*ndim > 2) {
*z = (float *) malloc(n*sizeof(float));
}
}
}
/*
* Distribute vertex data (xadj, coordinates, etc ) to all procs
*/
if (myproc == host_proc){
/* Allocate space for send buffers (size = max num vtx per proc ) */
max_nvtxs = ch_dist_max_num_vtx(*assignments);
if (use_graph) {
send_xadj = (int *) malloc((max_nvtxs+1)*sizeof(int));
if (send_xadj == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
if (*vwgt_dim) {
send_vwgts = (float *) malloc(max_nvtxs*(*vwgt_dim)*sizeof(float));
if (send_vwgts == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
if (*ndim > 0) {
send_x = (float *) malloc(max_nvtxs*sizeof(float));
if (send_x == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (*ndim > 1) {
send_y = (float *) malloc(max_nvtxs*sizeof(float));
if (send_y == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (*ndim > 2) {
send_z = (float *) malloc(max_nvtxs*sizeof(float));
if (send_z == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
}
}
/* Allocate space for list of vertices on a given processor */
vtx_list = (int *) malloc(max_nvtxs*sizeof(int));
if (vtx_list == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/* Allocate array to accumulate number of edges to be sent to each proc. */
if (use_graph) {
size = (int *) malloc(nprocs*sizeof(int));
if (size == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
/* For each processor, gather its vertex information and send it. */
for (p = 0; p < nprocs; p++){
if (use_graph) size[p] = 0;
/* Get list of vertices to be assigned to processor p */
ch_dist_vtx_list(vtx_list, &nsend, p, *assignments);
if (p == myproc){
/* Loop over vertices assigned to myproc; copy the vertex */
/* data into local arrays. */
if (use_graph) (*xadj)[0] = 0;
for (i = 0; i < nsend; i++) {
v = vtx_list[i];
if (use_graph) {
size[p] += old_xadj[v+1]-old_xadj[v];
(*xadj)[i+1] = (*xadj)[i] + old_xadj[v+1] - old_xadj[v];
}
if (*vwgt_dim){
for (j=0; j<*vwgt_dim; j++)
(*vwgts)[i*(*vwgt_dim)+j] = old_vwgts[v*(*vwgt_dim)+j];
}
if (*ndim > 0) {
(*x)[i] = old_x[v];
if (*ndim > 1) {
(*y)[i] = old_y[v];
if (*ndim > 2) {
(*z)[i] = old_z[v];
}
}
}
}
}
else {
/* Loop over vertices assigned to proc p to gather */
/* vertex data into send buffers */
if (use_graph) send_xadj[0] = 0;
for (i = 0; i < nsend; i++) {
v = vtx_list[i];
if (use_graph) {
size[p] += old_xadj[v+1]-old_xadj[v];
send_xadj[i+1] = send_xadj[i] + old_xadj[v+1] - old_xadj[v];
}
if (*vwgt_dim){
for (j=0; j<*vwgt_dim; j++)
send_vwgts[i*(*vwgt_dim)+j] = old_vwgts[v*(*vwgt_dim)+j];
}
if (*ndim > 0) {
send_x[i] = old_x[v];
if (*ndim > 1) {
send_y[i] = old_y[v];
if (*ndim > 2) {
send_z[i] = old_z[v];
}
}
}
}
/* Send vertex data to proc p. */
if (use_graph)
MPI_Send(send_xadj, nsend+1, MPI_INT, p, 1, comm);
if (*vwgt_dim)
MPI_Send(send_vwgts, nsend*(*vwgt_dim), MPI_FLOAT, p, 2, comm);
if (*ndim > 0) {
MPI_Send(send_x, nsend, MPI_FLOAT, p, 3, comm);
if (*ndim > 1) {
MPI_Send(send_y, nsend, MPI_FLOAT, p, 4, comm);
if (*ndim > 2) {
MPI_Send(send_z, nsend, MPI_FLOAT, p, 5, comm);
}
}
}
}
}
safe_free((void **)(void *) &send_xadj);
safe_free((void **)(void *) &send_vwgts);
safe_free((void **)(void *) &send_x);
safe_free((void **)(void *) &send_y);
safe_free((void **)(void *) &send_z);
}
else {
/* host_proc != myproc; receive vertex data from host_proc */
if (use_graph)
MPI_Recv (*xadj, (*nvtxs)+1, MPI_INT, host_proc, 1, comm, &status);
if (*vwgt_dim)
MPI_Recv (*vwgts, (*nvtxs)*(*vwgt_dim), MPI_FLOAT, host_proc, 2, comm, &status);
if (*ndim > 0) {
MPI_Recv(*x, *nvtxs, MPI_FLOAT, host_proc, 3, comm, &status);
if (*ndim > 1) {
MPI_Recv(*y, *nvtxs, MPI_FLOAT, host_proc, 4, comm, &status);
if (*ndim > 2) {
MPI_Recv(*z, *nvtxs, MPI_FLOAT, host_proc, 5, comm, &status);
}
}
}
}
/*
* Distribute edge data to all procs
*/
if (use_graph) {
if (*ewgt_dim) {
old_ewgts = *ewgts;
*ewgts = NULL;
}
/* Allocate space for edge data */
nedges = (*xadj)[ *nvtxs];
if (nedges > 0) {
*adjncy = (int *) malloc(nedges * sizeof (int));
if (*adjncy == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (*ewgt_dim){
*ewgts = (float *) malloc(nedges*(*ewgt_dim) * sizeof (float));
if (*ewgts == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
}
/* Gather and send/receive edge data */
if (myproc == host_proc){
/* For each processor, gather its edge data and send it. */
for (p = 0; p < nprocs; p++){
if (size[p] == 0) continue;
/* Get list of vertices to be assigned to processor p */
ch_dist_vtx_list(vtx_list, &nsend, p, *assignments);
adj_cnt = 0;
if (p == myproc) {
/* Loop over vertices assigned to myproc copy the edge */
/* data into local arrays. */
for (i = 0; i < nsend; i++) {
v = vtx_list[i];
offset = old_xadj[v];
nvtx_edges = old_xadj[v+1] - old_xadj[v];
for (j = 0; j < nvtx_edges; j++) {
(*adjncy)[adj_cnt] = old_adjncy[offset+j];
if (*ewgt_dim){
for (k=0; k<*ewgt_dim; k++)
(*ewgts)[adj_cnt*(*ewgt_dim)+k] = old_ewgts[(offset+j)*(*ewgt_dim)+k];
}
adj_cnt++;
}
}
}
else { /* p != myproc */
/* allocate send buffers; size = num edges to send to proc p */
nvtx_edges = 0;
for (i = 0; i < nsend; i++) {
v = vtx_list[i];
nvtx_edges += old_xadj[v+1] - old_xadj[v];
}
send_adjncy = (int *) malloc(nvtx_edges * sizeof(int));
if (send_adjncy == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (*ewgt_dim) {
send_ewgts = (float *) malloc(nvtx_edges*(*ewgt_dim) * sizeof(float));
if (send_ewgts == NULL) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
}
/* Loop over vertices assigned to proc p to gather */
/* edge data into send buffers */
for (i = 0; i < nsend; i++) {
v = vtx_list[i];
offset = old_xadj[v];
nvtx_edges = old_xadj[v+1] - old_xadj[v];
for (j = 0; j < nvtx_edges; j++) {
send_adjncy[adj_cnt] = old_adjncy[offset+j];
if (*ewgt_dim){
for (k=0; k<*ewgt_dim; k++)
send_ewgts[adj_cnt*(*ewgt_dim)+k] = old_ewgts[(offset+j)*(*ewgt_dim)+k];
}
adj_cnt++;
}
}
/* Send edge data to proc p. */
MPI_Send(send_adjncy, size[p], MPI_INT, p, 6, comm);
if (*ewgt_dim)
MPI_Send(send_ewgts, size[p]*(*ewgt_dim), MPI_FLOAT, p, 7, comm);
safe_free((void **)(void *) &send_adjncy);
safe_free((void **)(void *) &send_ewgts);
}
}
}
else {
/* host_proc != myproc; receive edge data from host_proc */
if (nedges > 0) {
MPI_Recv (*adjncy, nedges, MPI_INT, host_proc, 6, comm, &status);
if (*ewgt_dim)
MPI_Recv (*ewgts, nedges*(*ewgt_dim), MPI_FLOAT, host_proc, 7, comm, &status);
}
}
}
/* Free space on host proc */
if (myproc == host_proc){
safe_free((void **)(void *) &old_xadj);
safe_free((void **)(void *) &old_adjncy);
safe_free((void **)(void *) &old_vwgts);
safe_free((void **)(void *) &old_ewgts);
safe_free((void **)(void *) &old_x);
safe_free((void **)(void *) &old_y);
safe_free((void **)(void *) &old_z);
safe_free((void **)(void *) &vtx_list);
}
if (size != NULL) safe_free((void **)(void *) &size);
DEBUG_TRACE_END(myproc, yo);
return 1;
}
int create_random_input(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
{
/* Local declarations. */
const char *yo = "create_random_input";
int i, j, nvtxs, gnvtxs;
int vwgt_dim=0, ewgt_dim=0;
int ndim = 0;
int *start = NULL, *adj = NULL;
int no_geom = FALSE;
float *ewgts = NULL, *vwgts = NULL;
float *x = NULL, *y = NULL, *z = NULL;
short *assignments = NULL;
char filename[256];
FILE *fpg = NULL, *fpc = NULL; /* Files to echo random input */
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
if (Proc == 0) {
/* Adapted from read_chaco_graph; build same values as read_chaco_graph
* and then let random input be distributed as a Chaco graph would be. */
/* read the array in on processor 0 */
nvtxs = pio_info->init_size;
ndim = pio_info->init_dim;
vwgt_dim = pio_info->init_vwgt_dim;
if (vwgt_dim<1) vwgt_dim=1; /* For now, insist on 1 or more weights. */
if (nvtxs <= OUTPUT_FILES_MAX_NVTXS) {
sprintf(filename, "%s.graph", pio_info->pexo_fname);
fpg = fopen(filename, "w");
sprintf(filename, "%s.coords", pio_info->pexo_fname);
fpc = fopen(filename, "w");
}
if (nvtxs > 0) {
/* Allocate space for vertex weights and coordinates. */
x = (float *) malloc(nvtxs * sizeof(double));
if (ndim > 1) y = (float *) malloc(nvtxs * sizeof(double));
if (ndim > 2) z = (float *) malloc(nvtxs * sizeof(double));
vwgts = (float *) malloc(vwgt_dim*nvtxs * sizeof(float));
if (!x || (ndim > 1 && !y) || (ndim > 2 && !z) || !vwgts) {
Gen_Error(0, "fatal: Error allocating memory.");
return 0;
}
/* Generate random coordinates and weights. */
srand(0);
switch (ndim) {
case 1:
for (i = 0; i < nvtxs; i++) {
x[i] = ((float) rand())/RAND_MAX;
if (fpc != NULL) fprintf(fpc, "%e\n", x[i]);
}
break;
case 2:
for (i = 0; i < nvtxs; i++) {
x[i] = ((float) rand())/RAND_MAX;
y[i] = ((float) rand())/RAND_MAX;
if (fpc != NULL) fprintf(fpc, "%e %e\n", x[i], y[i]);
}
break;
case 3:
for (i = 0; i < nvtxs; i++) {
x[i] = ((float) rand())/RAND_MAX;
y[i] = ((float) rand())/RAND_MAX;
z[i] = ((float) rand())/RAND_MAX;
if (fpc != NULL) fprintf(fpc, "%e %e %e\n", x[i], y[i], z[i]);
}
break;
}
for (i = 0; i < nvtxs; i++) {
if (pio_info->init_vwgt_dim == 0)
/* Unit weights if no weights were requested. */
vwgts[i] = 1.0;
else
for (j = 0; j < vwgt_dim; j++) {
/* Only assign one of the weight dimensions a weight>0. */
/* Modify to get more complicated test cases. */
if (j == i%vwgt_dim)
vwgts[i*vwgt_dim+j] = ((float) rand())/RAND_MAX;
else
vwgts[i*vwgt_dim+j] = 0.0;
}
}
/* KDDSJP: Need to add edge info here. Later. For now, set no edges */
ewgt_dim = 0;
start = (int *) malloc((nvtxs+1) * sizeof(int));
adj = NULL;
ewgts = NULL;
for (i = 0; i < nvtxs; i++) {
start[i] = 0;
}
start[nvtxs] = 0;
if (fpg != NULL) {
if (vwgt_dim==1)
fprintf(fpg, "%d %d 010\n", nvtxs, start[nvtxs]);
else
fprintf(fpg, "%d %d 010 %d\n", nvtxs, start[nvtxs], vwgt_dim);
}
for (i = 0; i < nvtxs; i++) {
if (fpg != NULL)
for (j = 0; j < vwgt_dim; j++)
fprintf(fpg, "%e ", vwgts[i*vwgt_dim+j]);
/* KDDSJP Print edges here */
if (fpg != NULL) fprintf(fpg, "\n");
}
}
if (fpg != NULL) fclose(fpg);
if (fpc != NULL) fclose(fpc);
}
/* Distribute graph */
if (!chaco_dist_graph(MPI_COMM_WORLD, pio_info, 0, &gnvtxs, &nvtxs,
&start, &adj, &vwgt_dim, &vwgts, &ewgt_dim, &ewgts,
&ndim, &x, &y, &z, &assignments)) {
Gen_Error(0, "fatal: Error returned from chaco_dist_graph");
return 0;
}
if (!chaco_setup_mesh_struct(Proc, Num_Proc, prob, mesh, gnvtxs, nvtxs,
start, adj, vwgt_dim, vwgts, ewgt_dim, ewgts,
ndim, x, y, z, assignments, 1, no_geom)) {
Gen_Error(0, "fatal: Error returned from chaco_setup_mesh_struct");
return 0;
}
safe_free((void **)(void *) &adj);
safe_free((void **)(void *) &vwgts);
safe_free((void **)(void *) &ewgts);
safe_free((void **)(void *) &start);
safe_free((void **)(void *) &x);
safe_free((void **)(void *) &y);
safe_free((void **)(void *) &z);
safe_free((void **)(void *) &assignments);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
/*--------------------------------------------------------------------------*/
int output_gnu(const char *cmd_file,
const char *tag,
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
/*
* For 2D problems, output files that can be read by gnuplot for looking at
* results.
* We'll do 3D problems later.
*
* One gnuplot file is written for each partition.
* When number of partitions == number of processors, there is one file per
* processor.
*
* For Chaco input files, the file written contains coordinates of owned
* nodes and all nodes in that partition connected to the owned nodes. When
* drawn "with linespoints", the subdomains are drawn, but lines connecting the
* subdomains are not drawn.
*
* For Nemesis input files, the file written contains the coordinates of
* each node of owned elements. When drawn "with lines", the element outlines
* for each owned element are drawn.
*
* In addition, processor 0 writes a gnuplot command file telling gnuplot how
* to process the individual coordinate files written. This file can be used
* with the gnuplot "load" command to simplify generation of the gnuplot.
*/
{
/* Local declarations. */
const char *yo = "output_gnu";
char par_out_fname[FILENAME_MAX+1], ctemp[FILENAME_MAX+1];
ELEM_INFO *current_elem, *nbor_elem;
int nbor, num_nodes;
const char *datastyle = NULL;
int i, j, nelems;
int prev_part = -1;
int max_part = -1;
float locMaxX = INT_MIN;
float locMinX = INT_MAX;
float locMaxY = INT_MIN;
float locMinY = INT_MAX;
float globMaxX = INT_MIN;
float globMinX = INT_MAX;
float globMaxY = INT_MIN;
float globMinY = INT_MAX;
int gmax_part = Num_Proc-1;
int gnum_part = Num_Proc;
int *parts = NULL;
int *index = NULL;
int *elem_index = NULL;
FILE *fp = NULL;
/***************************** BEGIN EXECUTION ******************************/
if(Output.Gnuplot < 0)
{
Gen_Error(0,"warning: 'gnuplot output' parameter set to invalid negative value.");
return 0;
}
DEBUG_TRACE_START(Proc, yo);
if (mesh->num_dims > 2) {
Gen_Error(0, "warning: cannot generate gnuplot data for 3D problems.");
DEBUG_TRACE_END(Proc, yo);
return 0;
}
if (mesh->eb_nnodes[0] == 0) {
/* No coordinate information is available. */
Gen_Error(0, "warning: cannot generate gnuplot data when no coordinate"
" input is given.");
DEBUG_TRACE_END(Proc, yo);
return 0;
}
/*
* Build arrays of partition number to sort by. Index and elem_index arrays
* will be used even when plotting by processor numbers (for generality),
* so build it regardless.
*/
nelems = mesh->num_elems - mesh->blank_count;
if (nelems > 0) {
parts = (int *) malloc(3 * nelems * sizeof(int));
index = parts + nelems;
elem_index = index + nelems;
for (j = 0, i = 0; i < mesh->elem_array_len; i++) {
current_elem = &(mesh->elements[i]);
if (current_elem->globalID >= 0) {
if (mesh->blank_count && (mesh->blank[i] == 1)) continue;
if (current_elem->my_part > max_part) max_part = current_elem->my_part;
parts[j] = (Output.Plot_Partition ? current_elem->my_part : Proc);
index[j] = j;
elem_index[j] = i;
j++;
}
}
}
if (Output.Plot_Partition) {
/* Sort by partition numbers. Assumes # parts >= # proc. */
if (nelems > 0)
sort_index(nelems, parts, index);
MPI_Allreduce(&max_part, &gmax_part, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
gnum_part = gmax_part + 1;
}
/* generate the parallel filename for this processor */
strcpy(ctemp, pio_info->pexo_fname);
strcat(ctemp, ".");
strcat(ctemp, tag);
strcat(ctemp, ".gnu");
if (pio_info->file_type == CHACO_FILE ||
pio_info->file_type == NO_FILE_POINTS ||
pio_info->file_type == NO_FILE_TRIANGLES ||
pio_info->file_type == HYPERGRAPH_FILE) {
/*
* For each node of Chaco graph, print the coordinates of the node.
* Then, for each neighboring node on the processor, print the neighbor's
* coordinates.
*/
datastyle = "linespoints";
for (i = 0; i < nelems; i++) {
current_elem = &(mesh->elements[elem_index[index[i]]]);
if (parts[index[i]] != prev_part) {
if (fp != NULL) fclose(fp);
gen_par_filename(ctemp, par_out_fname, pio_info,
parts[index[i]], Num_Proc);
fp = fopen(par_out_fname, "w");
prev_part = parts[index[i]];
}
/* Include the point itself, so that even if there are no edges,
* the point will appear. */
fprintf(fp, "\n%e %e\n",
current_elem->coord[0][0], current_elem->coord[0][1]);
/* save max and min x/y coords */
if(current_elem->coord[0][0] < locMinX)
{
locMinX = current_elem->coord[0][0];
}
if(current_elem->coord[0][0] > locMaxX)
{
locMaxX = current_elem->coord[0][0];
}
if(current_elem->coord[0][1] < locMinY)
{
locMinY = current_elem->coord[0][1];
}
if(current_elem->coord[0][1] > locMaxY)
{
locMaxY = current_elem->coord[0][1];
}
if (Output.Gnuplot>1)
{
for (j = 0; j < current_elem->nadj; j++) {
if (current_elem->adj_proc[j] == Proc) { /* Nbor is on same proc */
if (mesh->blank_count && (mesh->blank[current_elem->adj[j]] == 1))
continue;
if (!Output.Plot_Partition ||
mesh->elements[current_elem->adj[j]].my_part ==
current_elem->my_part) {
/* Not plotting partitions, or nbor is in same partition */
/* Plot the edge. Need to include current point and nbor point
* for each edge. */
fprintf(fp, "\n%e %e\n",
current_elem->coord[0][0], current_elem->coord[0][1]);
nbor = current_elem->adj[j];
nbor_elem = &(mesh->elements[nbor]);
fprintf(fp, "%e %e\n",
nbor_elem->coord[0][0], nbor_elem->coord[0][1]);
}
}
}
}
}
MPI_Reduce(&locMinX,&globMinX,1,MPI_FLOAT,MPI_MIN,0,MPI_COMM_WORLD);
MPI_Reduce(&locMinY,&globMinY,1,MPI_FLOAT,MPI_MIN,0,MPI_COMM_WORLD);
MPI_Reduce(&locMaxX,&globMaxX,1,MPI_FLOAT,MPI_MAX,0,MPI_COMM_WORLD);
MPI_Reduce(&locMaxY,&globMaxY,1,MPI_FLOAT,MPI_MAX,0,MPI_COMM_WORLD);
}
else if (pio_info->file_type == NEMESIS_FILE) { /* Nemesis input file */
/*
* For each element of Nemesis input file, print the coordinates of its
* nodes. No need to follow neighbors, as decomposition is by elements.
*/
double sum[2];
datastyle = "lines";
for (i = 0; i < nelems; i++) {
current_elem = &(mesh->elements[elem_index[index[i]]]);
if (parts[index[i]] != prev_part) {
if (fp != NULL) fclose(fp);
gen_par_filename(ctemp, par_out_fname, pio_info,
parts[index[i]], Num_Proc);
fp = fopen(par_out_fname, "w");
prev_part = parts[index[i]];
}
num_nodes = mesh->eb_nnodes[current_elem->elem_blk];
sum[0] = sum[1] = 0.0;
for (j = 0; j < num_nodes; j++) {
fprintf(fp, "%e %e\n",
current_elem->coord[j][0], current_elem->coord[j][1]);
sum[0] += current_elem->coord[j][0];
sum[1] += current_elem->coord[j][1];
}
fprintf(fp, "%e %e\n", current_elem->coord[0][0],
current_elem->coord[0][1]);
fprintf(fp, "\n");
/* Print small + in center of element */
sum[0] /= num_nodes;
sum[1] /= num_nodes;
fprintf(fp, "%e %e\n", sum[0] - 0.001, sum[1]);
fprintf(fp, "%e %e\n\n", sum[0] + 0.001, sum[1]);
fprintf(fp, "%e %e\n", sum[0], sum[1] - 0.001);
fprintf(fp, "%e %e\n\n", sum[0], sum[1] + 0.001);
}
}
if (nelems == 0 && !Output.Plot_Partition) {
/* Open a file just so one exists; satisfies the gnuload file. */
gen_par_filename(ctemp, par_out_fname, pio_info, Proc, Num_Proc);
fp = fopen(par_out_fname, "w");
}
if (fp != NULL) fclose(fp);
safe_free((void **)(void *) &parts);
if (Proc == 0) {
/* Write gnu master file with gnu commands for plotting */
strcpy(ctemp, pio_info->pexo_fname);
strcat(ctemp, ".");
strcat(ctemp, tag);
strcat(ctemp, ".gnuload");
fp = fopen(ctemp, "w");
fprintf(fp, "set nokey\n");
fprintf(fp, "set nolabel\n");
fprintf(fp, "set noxzeroaxis\n");
fprintf(fp, "set noyzeroaxis\n");
fprintf(fp, "set noxtics\n");
fprintf(fp, "set noytics\n");
fprintf(fp, "set data style %s\n", datastyle);
/* resize range so that there is a 5% border around data */
fprintf(fp, "set xrange [%f:%f] \n ",globMinX-(globMaxX-globMinX)/20
,globMaxX+(globMaxX-globMinX)/20);
fprintf(fp, "set yrange [%f:%f] \n ",globMinY-(globMaxY-globMinY)/20
,globMaxY+(globMaxY-globMinY)/20);
fprintf(fp, "plot ");
strcpy(ctemp, pio_info->pexo_fname);
strcat(ctemp, ".");
strcat(ctemp, tag);
strcat(ctemp, ".gnu");
for (i = 0; i < gnum_part; i++) {
gen_par_filename(ctemp, par_out_fname, pio_info, i, Num_Proc);
fprintf(fp, "\"%s\"", par_out_fname);
if (i != gnum_part-1) {
fprintf(fp, ",\\\n");
}
}
fprintf(fp, "\n");
fclose(fp);
}
DEBUG_TRACE_END(Proc, yo);
return 1;
}
int create_random_triangles(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
{
/* Local declarations. */
const char *yo = "create_random_input";
int i, j, w, nvtxs, gnvtxs, ntri;
int vwgt_dim=0, ewgt_dim=0;
int ndim = 0;
int *start = NULL, *adj = NULL;
int no_geom = FALSE;
float *ewgts = NULL, *vwgts = NULL;
float *x = NULL, *y = NULL, *z = NULL;
float wgt, diff;
short *assignments = NULL;
char filename[256];
FILE *fpg = NULL, *fpc = NULL; /* Files to echo random input */
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
if (Proc == 0) {
/* Adapted from read_chaco_graph; build same values as read_chaco_graph
* and then let random input be distributed as a Chaco graph would be. */
/* read the array in on processor 0 */
ntri = pio_info->init_size;
nvtxs = ntri * 3;
ndim = pio_info->init_dim;
vwgt_dim = pio_info->init_vwgt_dim;
if (vwgt_dim<1) vwgt_dim=1; /* For now, insist on 1 or more weights. */
if (ntri <= OUTPUT_FILES_MAX_NVTXS) {
sprintf(filename, "%s.graph", pio_info->pexo_fname);
fpg = fopen(filename, "w");
sprintf(filename, "%s.coords", pio_info->pexo_fname);
fpc = fopen(filename, "w");
}
if (nvtxs > 0) {
/* Allocate space for vertex weights and coordinates. */
x = (float *) malloc(nvtxs * sizeof(double));
if (ndim > 1) y = (float *) malloc(nvtxs * sizeof(double));
if (ndim > 2) z = (float *) malloc(nvtxs * sizeof(double));
vwgts = (float *) malloc(vwgt_dim*nvtxs * sizeof(float));
if (!x || (ndim > 1 && !y) || (ndim > 2 && !z) || !vwgts) {
Gen_Error(0, "fatal: Error allocating memory.");
return 0;
}
/* Generate random triangles and vertex weights. */
srand(0);
diff = 1.0/50.0;
switch (ndim) {
case 1:
for (i = 0; i < nvtxs; i+=3) {
x[i] = ((float) rand())/RAND_MAX;
x[i+1] = x[i] - diff;
x[i+2] = x[i] + diff;
if (fpc != NULL) fprintf(fpc, "%e\n%e\n%e\n", x[i],x[i+1],x[i+2]);
}
break;
case 2:
for (i = 0; i < nvtxs; i+=3) {
x[i] = ((float) rand())/RAND_MAX;
y[i] = ((float) rand())/RAND_MAX;
x[i+1] = x[i] - diff;
y[i+1] = y[i];
x[i+2] = x[i];
y[i+2] = y[i] + diff;
if (fpc != NULL) fprintf(fpc, "%e %e\n%e %e\n%e %e\n",
x[i], y[i],x[i+1], y[i+1],x[i+2], y[i+2]);
}
break;
case 3:
for (i = 0; i < nvtxs; i+=3) {
x[i] = ((float) rand())/RAND_MAX;
y[i] = ((float) rand())/RAND_MAX;
z[i] = ((float) rand())/RAND_MAX;
x[i+1] = x[i] - diff;
y[i+1] = y[i];
z[i+1] = z[i];
x[i+2] = x[i];
y[i+2] = y[i] + diff;
z[i+2] = z[i];
if (fpc != NULL) fprintf(fpc, "%e %e %e\n%e %e %e\n%e %e %e\n",
x[i], y[i], z[i],x[i+1], y[i+1], z[i+1],x[i+2], y[i+2], z[i+2]);
}
break;
}
if (pio_info->init_vwgt_dim == 0) {
for (i = 0; i < nvtxs; i++) {
/* Unit weights if no weights were requested. */
vwgts[i] = 1.0;
}
}
else{
memset(vwgts, 0, nvtxs * sizeof(float));
for (i=0,w=0; i < ntri; i++) {
for (j = 0; j < vwgt_dim; j++) {
/* Each point of triangle gets the same weight. */
/* Only assign one of the weight dimensions a weight>0. */
/* Modify to get more complicated test cases. */
if (j == i%vwgt_dim){
wgt = ((float) rand())/RAND_MAX;
vwgts[w+j] = wgt;
w += vwgt_dim;
vwgts[w+j] = wgt;
w += vwgt_dim;
vwgts[w+j] = wgt;
w += vwgt_dim;
}
}
}
}
/* Each vertex has two neighbors */
ewgt_dim = 0;
start = (int *) malloc((nvtxs+1) * sizeof(int));
adj = (int *)malloc(nvtxs*2*sizeof(int));
ewgts = NULL;
for (i = 0; i <= nvtxs; i++) {
start[i] = i*2;
}
for (i=0, w=0, j=1; i < ntri; i++,j+=3){
/* j is first vertex (1-based) in triangle */
adj[w++] = j+1; /* adjacencies of vertex j */
adj[w++] = j+2;
adj[w++] = j+2; /* adjacencies of vertex j+1 */
adj[w++] = j;
adj[w++] = j; /* adjacencies of vertex j+2 */
adj[w++] = j+1;
}
if (fpg != NULL) {
if (vwgt_dim==1)
fprintf(fpg, "%d %d 010\n", nvtxs, nvtxs);
else
fprintf(fpg, "%d %d 010 %d\n", nvtxs, nvtxs, vwgt_dim);
for (i = 0, w=0; i < nvtxs; i++, w += 2) {
for (j = 0; j < vwgt_dim; j++)
fprintf(fpg, "%e ", vwgts[i*vwgt_dim+j]);
fprintf(fpg, "%d %d",adj[w],adj[w+1]);
fprintf(fpg, "\n");
}
}
}
if (fpg != NULL) fclose(fpg);
if (fpc != NULL) fclose(fpc);
}
/* Distribute graph */
if (!chaco_dist_graph(MPI_COMM_WORLD, pio_info, 0, &gnvtxs, &nvtxs,
&start, &adj, &vwgt_dim, &vwgts, &ewgt_dim, &ewgts,
&ndim, &x, &y, &z, &assignments)) {
Gen_Error(0, "fatal: Error returned from chaco_dist_graph");
return 0;
}
if (!chaco_setup_mesh_struct(Proc, Num_Proc, prob, mesh, gnvtxs, nvtxs,
start, adj, vwgt_dim, vwgts, ewgt_dim, ewgts,
ndim, x, y, z, assignments, 1, no_geom)) {
Gen_Error(0, "fatal: Error returned from chaco_setup_mesh_struct");
return 0;
}
safe_free((void **)(void *) &adj);
safe_free((void **)(void *) &vwgts);
safe_free((void **)(void *) &ewgts);
safe_free((void **)(void *) &start);
safe_free((void **)(void *) &x);
safe_free((void **)(void *) &y);
safe_free((void **)(void *) &z);
safe_free((void **)(void *) &assignments);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
static int input_assign_normal(
ZOLTAN_FILE* finassign, /* input assignment file */
char *inassignname, /* name of input assignment file */
int nvtxs, /* number of vertices to output */
short *assignment) /* values to be printed */
{
const char *yo = "input_assign_normal";
int flag; /* logical conditional */
int end_flag; /* return flag from read_int() */
int i, j; /* loop counter */
DEBUG_TRACE_START(0, yo);
/* Get the assignment vector one line at a time, checking as you go. */
/* First read past any comments at top. */
end_flag = 1;
while (end_flag == 1) {
assignment[0] = read_int(finassign, &end_flag);
}
if (assignment[0] < 0) {
printf("ERROR: Entry %d in assignment file `%s' less than zero (%d)\n",
1, inassignname, assignment[0]);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
if (end_flag == -1) {
printf("ERROR: No values found in assignment file `%s'\n", inassignname);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
flag = 0;
if (assignment[0] > nvtxs)
flag = assignment[1];
for (i = 1; i < nvtxs; i++) {
j = ZOLTAN_FILE_scanf(finassign, "%hd", &(assignment[i]));
if (j != 1) {
printf("ERROR: Too few values in assignment file `%s'.\n", inassignname);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
if (assignment[i] < 0) {
printf("ERROR: Entry %d in assignment file `%s' less than zero (%d)\n",
i+1, inassignname, assignment[i]);
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (1);
}
if (assignment[i] > nvtxs) { /* warn since probably an error */
if (assignment[i] > flag)
flag = assignment[i];
}
}
if (flag && Debug_Chaco_Input) {
printf("WARNING: Possible error in assignment file `%s'\n", inassignname);
printf(" More assignment sets (%d) than vertices (%d)\n", flag, nvtxs);
}
/* Check for spurious extra stuff in file. */
flag = FALSE;
end_flag = 0;
while (!flag && end_flag != -1) {
read_int(finassign, &end_flag);
if (!end_flag)
flag = TRUE;
}
if (flag && Debug_Chaco_Input) {
printf("WARNING: Possible error in assignment file `%s'\n", inassignname);
printf(" Numerical data found after expected end of file\n");
}
ZOLTAN_FILE_close(finassign);
DEBUG_TRACE_END(0, yo);
return (0);
}
int migrate_elements(
int Proc,
MESH_INFO_PTR mesh,
struct Zoltan_Struct *zz,
int num_gid_entries,
int num_lid_entries,
int num_imp,
ZOLTAN_ID_PTR imp_gids,
ZOLTAN_ID_PTR imp_lids,
int *imp_procs,
int *imp_to_part,
int num_exp,
ZOLTAN_ID_PTR exp_gids,
ZOLTAN_ID_PTR exp_lids,
int *exp_procs,
int *exp_to_part)
{
/* Local declarations. */
char *yo = "migrate_elements";
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
/*
* register migration functions
*/
if (!Test.Null_Lists) {
/* If not passing NULL lists, let Help_Migrate call the
* pre-processing and post-processing routines.
*/
if (Zoltan_Set_Fn(zz, ZOLTAN_PRE_MIGRATE_PP_FN_TYPE,
(void (*)()) migrate_pre_process,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n");
return 0;
}
if (Zoltan_Set_Fn(zz, ZOLTAN_POST_MIGRATE_PP_FN_TYPE,
(void (*)()) migrate_post_process,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n");
return 0;
}
}
if (Test.Multi_Callbacks) {
if (Zoltan_Set_Fn(zz, ZOLTAN_OBJ_SIZE_MULTI_FN_TYPE,
(void (*)()) migrate_elem_size_multi,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n");
return 0;
}
if (Zoltan_Set_Fn(zz, ZOLTAN_PACK_OBJ_MULTI_FN_TYPE,
(void (*)()) migrate_pack_elem_multi,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n");
return 0;
}
if (Zoltan_Set_Fn(zz, ZOLTAN_UNPACK_OBJ_MULTI_FN_TYPE,
(void (*)()) migrate_unpack_elem_multi,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n");
return 0;
}
}
else {
if (Zoltan_Set_Fn(zz, ZOLTAN_OBJ_SIZE_FN_TYPE,
(void (*)()) migrate_elem_size,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n");
return 0;
}
if (Zoltan_Set_Fn(zz, ZOLTAN_PACK_OBJ_FN_TYPE,
(void (*)()) migrate_pack_elem,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n");
return 0;
}
if (Zoltan_Set_Fn(zz, ZOLTAN_UNPACK_OBJ_FN_TYPE,
(void (*)()) migrate_unpack_elem,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Set_Fn()\n");
return 0;
}
}
if (Test.Null_Lists == NONE) {
if (Zoltan_Migrate(zz, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part)
== ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Migrate()\n");
return 0;
}
}
else {
/* Call Zoltan_Help_Migrate with empty import lists. */
/* Have to "manually" call migrate_pre_process and migrate_post_process. */
int ierr = 0;
migrate_pre_process((void *) mesh, 1, 1,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part,
&ierr);
if (Test.Null_Lists == IMPORT_LISTS) {
if (Zoltan_Migrate(zz,
-1, NULL, NULL, NULL, NULL,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part)
== ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Migrate()\n");
return 0;
}
}
else {
if (Zoltan_Migrate(zz,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
-1, NULL, NULL, NULL, NULL)
== ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Zoltan_Migrate()\n");
return 0;
}
}
migrate_post_process((void *) mesh, 1, 1,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part,
&ierr);
}
DEBUG_TRACE_END(Proc, yo);
return 1;
}
int chaco_setup_mesh_struct(
int Proc,
int Num_Proc,
PROB_INFO_PTR prob, /* problem description */
MESH_INFO_PTR mesh, /* mesh information for the problem */
int gnvtxs, /* global number of vertices across all procs*/
int nvtxs, /* number of vertices in local graph */
int *start, /* start of edge list for each vertex */
int *adj, /* edge list data */
int vwgt_dim, /* # of weights per vertex */
float *vwgts, /* vertex weight list data */
int ewgt_dim, /* # of weights per edge */
float *ewgts, /* edge weight list data */
int ndim, /* dimension of the geometry */
float *x, /* x-coordinates of the vertices */
float *y, /* y-coordinates of the vertices */
float *z, /* z-coordinates of the vertices */
short *assignments, /* assignments from Chaco file; may be NULL */
int base, /* smallest vertex number to use;
base == 1 for Chaco;
may be 0 or 1 for HG files. */
int no_geom /* flag indicating whether coords are avail. */
)
{
const char *yo = "chaco_setup_mesh_struct";
int i;
DEBUG_TRACE_START(Proc, yo);
/* Initialize mesh structure for Chaco mesh. */
mesh->data_type = ZOLTAN_GRAPH;
mesh->vwgt_dim = vwgt_dim;
mesh->ewgt_dim = ewgt_dim;
mesh->num_elems = nvtxs;
mesh->elem_array_len = mesh->num_elems + 5;
mesh->num_dims = ndim;
mesh->num_el_blks = 1;
mesh->eb_etypes = (int *) malloc (5 * mesh->num_el_blks * sizeof(int));
if (!mesh->eb_etypes) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_ids = mesh->eb_etypes + mesh->num_el_blks;
mesh->eb_cnts = mesh->eb_ids + mesh->num_el_blks;
mesh->eb_nnodes = mesh->eb_cnts + mesh->num_el_blks;
mesh->eb_nattrs = mesh->eb_nnodes + mesh->num_el_blks;
mesh->eb_names = (char **) malloc (mesh->num_el_blks * sizeof(char *));
if (!mesh->eb_names) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_etypes[0] = -1;
mesh->eb_ids[0] = 1;
mesh->eb_cnts[0] = nvtxs;
mesh->eb_nattrs[0] = 0;
mesh->hindex = (int *) malloc(sizeof(int));
mesh->hindex[0] = 0;
/*
* Each element has one set of coordinates (i.e., node) if a coords file
* was provided; zero otherwise.
*/
MPI_Bcast( &no_geom, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (no_geom)
mesh->eb_nnodes[0] = 0;
else
mesh->eb_nnodes[0] = 1;
/* allocate space for name */
mesh->eb_names[0] = (char *) malloc((MAX_STR_LENGTH+1) * sizeof(char));
if (!mesh->eb_names[0]) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
strcpy(mesh->eb_names[0], "chaco");
/* allocate the element structure array */
mesh->elements = (ELEM_INFO_PTR) malloc (mesh->elem_array_len
* sizeof(ELEM_INFO));
if (!(mesh->elements)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/*
* intialize all of the element structs as unused by
* setting the globalID to -1
*/
for (i = 0; i < mesh->elem_array_len; i++)
initialize_element(&(mesh->elements[i]));
/*
* now fill the element structure array with the
* information from the Chaco file
*/
if (!chaco_fill_elements(Proc, Num_Proc, prob, mesh, gnvtxs, nvtxs,
start, adj, vwgt_dim, vwgts, ewgt_dim, ewgts,
ndim, x, y, z, assignments, 1)) {
Gen_Error(0, "fatal: Error returned from chaco_fill_elements");
return 0;
}
DEBUG_TRACE_END(Proc, yo);
return 1;
}
static int read_elem_info(int pexoid, int Proc, PROB_INFO_PTR prob,
MESH_INFO_PTR mesh)
{
/* Local declarations. */
char *yo = "read_elem_info";
int iblk, ielem, inode, lnode, cnode, iplace, len;
int max_nsur = 0;
int i;
int *nmap, *emap, *connect;
int **sur_elem, *nsurnd;
ELEM_INFO_PTR elements = mesh->elements;
double tmp0, tmp1, tmp2;
float *xptr = NULL, *yptr = NULL, *zptr = NULL;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
/* allocate memory for the global number maps */
nmap = (int *) malloc ((mesh->num_nodes + mesh->num_elems) * sizeof(int));
if (!nmap) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
emap = nmap + mesh->num_nodes;
/*
* get the global maps
*/
if (ex_get_elem_num_map(pexoid, emap) < 0) {
Gen_Error(0, "fatal: Error returned from ex_get_elem_num_map");
return 0;
}
if (ex_get_node_num_map(pexoid, nmap) < 0) {
Gen_Error(0, "fatal: Error returned from ex_get_node_num_map");
return 0;
}
/* allocate memory for the coordinates */
xptr = (float *) malloc (mesh->num_dims * mesh->num_nodes * sizeof(float));
if (!xptr) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
switch (mesh->num_dims) {
case 3:
zptr = xptr + 2 * mesh->num_nodes;
/* FALLTHRU */
case 2:
yptr = xptr + mesh->num_nodes;
}
if (ex_get_coord(pexoid, xptr, yptr, zptr) < 0) {
Gen_Error(0, "fatal: Error returned from ex_get_coord");
return 0;
}
/*
* figure out which element block needs
* the most space for its connect table
*/
len = 0;
for (iblk = 0; iblk < mesh->num_el_blks; iblk++)
if ((iplace = mesh->eb_cnts[iblk] * mesh->eb_nnodes[iblk]) > len)
len = iplace;
connect = (int *) malloc (len * sizeof(int));
if (!connect) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/***************************************************************************/
/* Fill the Connect table, Coordinates, Global Ids for each element */
/***************************************************************************/
iplace = 0;
for (iblk = 0; iblk < mesh->num_el_blks; iblk++) {
if (mesh->eb_cnts[iblk] > 0) {
if (ex_get_elem_conn(pexoid, mesh->eb_ids[iblk], connect) < 0) {
Gen_Error(0, "fatal: Error returned from ex_get_elem_conn");
return 0;
}
cnode = 0;
for (ielem = 0; ielem < mesh->eb_cnts[iblk]; ielem++) {
/* set some fields in the element structure */
elements[iplace].border = 0;
elements[iplace].globalID = emap[iplace];
elements[iplace].elem_blk = iblk;
elements[iplace].my_part = Proc;
elements[iplace].perm_value = -1;
elements[iplace].invperm_value = -1;
elements[iplace].nadj = 0;
elements[iplace].adj_len = 0;
/* first weights are all 1 for now */
elements[iplace].cpu_wgt[0] = 1.0;
if (MAX_CPU_WGTS>1){
/* second weights will be set later */
elements[iplace].cpu_wgt[1] = 0.0;
/* make artificial data for more multi-weights */
for (i = 2; i < MAX_CPU_WGTS; i++)
elements[iplace].cpu_wgt[i] = elements[iplace].my_part +
0.5*((elements[iplace].globalID)%i);
}
elements[iplace].mem_wgt = 1.0;
/* allocate space for the connect list and the coordinates */
elements[iplace].connect = (int *) malloc(mesh->eb_nnodes[iblk] *
sizeof(int));
if (!(elements[iplace].connect)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
elements[iplace].coord = (float **) malloc(mesh->eb_nnodes[iblk] *
sizeof(float *));
if (!(elements[iplace].coord)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/* save the connect table as local numbers for the moment */
tmp0 = tmp1 = tmp2 = 0.;
for (inode = 0; inode < mesh->eb_nnodes[iblk]; inode++) {
lnode = connect[cnode] - 1;
elements[iplace].connect[inode] = lnode;
cnode++;
elements[iplace].coord[inode] = (float *) calloc(mesh->num_dims,
sizeof(float));
if (!(elements[iplace].coord[inode])) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
switch (mesh->num_dims) {
case 3:
elements[iplace].coord[inode][2] = zptr[lnode];
tmp2 += zptr[lnode];
/* FALLTHRU */
case 2:
elements[iplace].coord[inode][1] = yptr[lnode];
tmp1 += yptr[lnode];
/* FALLTHRU */
case 1:
elements[iplace].coord[inode][0] = xptr[lnode];
tmp0 += xptr[lnode];
}
} /* End: "for (inode = 0; inode < mesh->eb_nnodes[iblk]; inode++)" */
elements[iplace].avg_coord[0] = tmp0 / mesh->eb_nnodes[iblk];
elements[iplace].avg_coord[1] = tmp1 / mesh->eb_nnodes[iblk];
elements[iplace].avg_coord[2] = tmp2 / mesh->eb_nnodes[iblk];
iplace++;
} /* End: "for (ielem = 0; ielem < mesh->eb_cnts[iblk]; ielem++)" */
} /* End: "if (mesh->eb_cnts[iblk] > 0)" */
} /* End: "for (iblk = 0; iblk < mesh->num_el_blks; iblk++)" */
/* free some memory */
free(connect);
free(xptr);
/*************************************************************************/
/* Find the adjacency list for each element */
/* Part one: find the surrounding elements for each node */
/*************************************************************************/
sur_elem = (int **) malloc(mesh->num_nodes * sizeof(int *));
if (!sur_elem) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
nsurnd = (int *) malloc(mesh->num_nodes * sizeof(int));
if (!nsurnd) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (!find_surnd_elem(mesh, sur_elem, nsurnd, &max_nsur)) {
Gen_Error(0, "fatal: Error returned from find_surnd_elems");
return 0;
}
/*************************************************************************/
/* Part two: Find the adjacencies on this processor */
/* and get the edge weights */
/*************************************************************************/
if (!find_adjacency(Proc, mesh, sur_elem, nsurnd, max_nsur)) {
Gen_Error(0, "fatal: Error returned from find_adjacency");
return 0;
}
/*
* convert the node numbers in the connect lists to Global IDs
* since they will be much easier to work with
*/
for (ielem = 0; ielem < mesh->num_elems; ielem++) {
iblk = elements[ielem].elem_blk;
for (inode = 0; inode < mesh->eb_nnodes[iblk]; inode++) {
elements[ielem].connect[inode] = nmap[elements[ielem].connect[inode]];
}
}
/*
* let cpu_wgt[1] be the number of sides (surfaces) not
* connected to other elements. (useful test case for multi-weight
* load balancing).
*/
if (MAX_CPU_WGTS>1){
for (ielem = 0; ielem < mesh->num_elems; ielem++) {
elements[ielem].cpu_wgt[1] = elements[ielem].adj_len - elements[ielem].nadj;
/* EBEB Strangely, nadj is often one less than expected so subtract one from the wgt to compensate. */
if (elements[ielem].cpu_wgt[1] >= 1.0)
elements[ielem].cpu_wgt[1] -= 1.0;
}
}
for (inode = 0; inode < mesh->num_nodes; inode++) free(sur_elem[inode]);
free(sur_elem);
free(nsurnd);
free(nmap);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
int read_chaco_file(int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
{
/* Local declarations. */
const char *yo = "read_chaco_mesh";
char cmesg[256];
char chaco_fname[FILENAME_MAX + 8];
int nvtxs, gnvtxs;
int vwgt_dim=0, ewgt_dim=0;
int ndim = 0;
int *start = NULL, *adj = NULL;
int no_geom = FALSE;
float *ewgts = NULL, *vwgts = NULL;
float *x = NULL, *y = NULL, *z = NULL;
short *assignments = NULL;
ZOLTAN_FILE *fp = NULL;
int file_error;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
/* Set Debug_Chaco_Input */
if (Debug_Driver > 2) Debug_Chaco_Input = 1;
if (Proc == 0) {
/* Open and read the Chaco graph file. */
sprintf(chaco_fname, "%s.graph", pio_info->pexo_fname);
fp = ZOLTAN_FILE_open(chaco_fname, "r", pio_info->file_comp);
file_error = (fp == NULL);
}
MPI_Bcast(&file_error, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (file_error) {
sprintf(cmesg, "fatal: Could not open Chaco graph file %s",
chaco_fname);
Gen_Error(0, cmesg);
return 0;
}
if (Proc == 0) {
/* read the array in on processor 0 */
if (chaco_input_graph(fp, chaco_fname, &start, &adj, &nvtxs,
&vwgt_dim, &vwgts, &ewgt_dim, &ewgts) != 0) {
Gen_Error(0, "fatal: Error returned from chaco_input_graph");
return 0;
}
/* Read Chaco geometry file, if provided. */
sprintf(chaco_fname, "%s.coords", pio_info->pexo_fname);
fp = ZOLTAN_FILE_open(chaco_fname, "r", pio_info->file_comp);
if (fp == NULL) {
no_geom = TRUE;
sprintf(cmesg, "warning: Could not open Chaco geometry file %s; "
"no geometry data will be read",
chaco_fname);
Gen_Error(0, cmesg);
}
else {
/* read the coordinates in on processor 0 */
if (chaco_input_geom(fp, chaco_fname, nvtxs, &ndim, &x, &y, &z) != 0) {
Gen_Error(0, "fatal: Error returned from chaco_input_geom");
return 0;
}
}
#ifdef MESS_UP_POINTS
/* Try to create a geometry that isn't nicely symmetric */
{
int i, j;
double min[3], max[3], a[3], b[3];
min[0] = max[0] = x[0];
if (ndim > 1){
min[1] = max[1] = y[0];
if (ndim > 2)
min[2] = max[2] = z[0];
}
for (i=1; i<nvtxs; i++) {
if (x[i] < min[0]) min[0] = x[i];
else if (x[i] > max[0]) max[0] = x[i];
if (ndim > 1){
if (y[i] < min[1]) min[1] = y[i];
else if (y[i] > max[1]) max[1] = y[i];
if (ndim > 2){
if (z[i] < min[2]) min[2] = z[i];
else if (z[i] > max[2]) max[2] = z[i];
}
}
}
for (i=0; i<ndim; i++) /* point inside but near edge of geometry */
a[i] = ((max[i] - min[i]) * .1) + min[i];
for (i=0; i<nvtxs; i++) { /* move 2/3 of points much closer to "a" */
if (i%3 == 0) continue;
b[0] = x[i];
if (ndim > 1){
b[1] = y[i];
if (ndim > 2){
b[2] = z[i];
}
}
for (j=0; j<ndim; j++){
b[j] = a[j] + ((b[j] - a[j])*.1);
}
x[i] = b[0];
if (ndim > 1){
y[i] = b[1];
if (ndim > 2){
z[i] = b[2];
}
}
}
}
#endif
/* Read Chaco assignment file, if requested */
if (pio_info->init_dist_type == INITIAL_FILE) {
sprintf(chaco_fname, "%s.assign", pio_info->pexo_fname);
if (pio_info->file_comp == GZIP)
sprintf(chaco_fname, "%s.gz", chaco_fname);
fp = ZOLTAN_FILE_open(chaco_fname, "r", pio_info->file_comp);
if (fp == NULL) {
sprintf(cmesg, "Error: Could not open Chaco assignment file %s; "
"initial distribution cannot be read",
chaco_fname);
Gen_Error(0, cmesg);
return 0;
}
else {
/* read the coordinates in on processor 0 */
assignments = (short *) malloc(nvtxs * sizeof(short));
if (!assignments) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (chaco_input_assign(fp, chaco_fname, nvtxs, assignments) != 0) {
Gen_Error(0, "fatal: Error returned from chaco_input_assign");
return 0;
}
}
}
}
/* Distribute graph */
if (!chaco_dist_graph(MPI_COMM_WORLD, pio_info, 0, &gnvtxs, &nvtxs,
&start, &adj, &vwgt_dim, &vwgts, &ewgt_dim, &ewgts,
&ndim, &x, &y, &z, &assignments) != 0) {
Gen_Error(0, "fatal: Error returned from chaco_dist_graph");
return 0;
}
if (!chaco_setup_mesh_struct(Proc, Num_Proc, prob, mesh, gnvtxs, nvtxs,
start, adj, vwgt_dim, vwgts, ewgt_dim, ewgts,
ndim, x, y, z, assignments, 1, no_geom)) {
Gen_Error(0, "fatal: Error returned from chaco_setup_mesh_struct");
return 0;
}
safe_free((void **)(void *) &adj);
safe_free((void **)(void *) &vwgts);
safe_free((void **)(void *) &ewgts);
safe_free((void **)(void *) &start);
safe_free((void **)(void *) &x);
safe_free((void **)(void *) &y);
safe_free((void **)(void *) &z);
safe_free((void **)(void *) &assignments);
DEBUG_TRACE_END(Proc, yo);
return 1;
}
int read_exoII_file(int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
{
#ifndef ZOLTAN_NEMESIS
Gen_Error(0, "Fatal: Nemesis requested but not linked with driver.");
return 0;
#else /* ZOLTAN_NEMESIS */
/* Local declarations. */
char *yo = "read_exoII_mesh";
char par_nem_fname[FILENAME_MAX+1], title[MAX_LINE_LENGTH+1];
char cmesg[256];
float ver;
int i, pexoid, cpu_ws = 0, io_ws = 0;
int *nnodes = NULL, *etypes = NULL;
#ifdef DEBUG_EXO
int j, k, elem;
#endif
FILE *fdtmp;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
/* since this is a test driver, set error reporting in exodus */
ex_opts(EX_VERBOSE | EX_DEBUG);
/* generate the parallel filename for this processor */
gen_par_filename(pio_info->pexo_fname, par_nem_fname, pio_info, Proc,
Num_Proc);
/*
* check whether parallel file exists. do the check with fopen
* as ex_open coredumps on the paragon when files do not exist.
*/
if ((fdtmp = fopen(par_nem_fname, "r")) == NULL) {
sprintf(cmesg,"fatal: parallel Exodus II file %s does not exist",
par_nem_fname);
Gen_Error(0, cmesg);
return 0;
}
else
fclose(fdtmp);
/*
* now open the existing parallel file using Exodus calls.
*/
if ((pexoid = ex_open(par_nem_fname, EX_READ, &cpu_ws, &io_ws,
&ver)) < 0) {
sprintf(cmesg,"fatal: could not open parallel Exodus II file %s",
par_nem_fname);
Gen_Error(0, cmesg);
return 0;
}
/* and get initial information */
if (ex_get_init(pexoid, title, &(mesh->num_dims),
&(mesh->num_nodes), &(mesh->num_elems),
&(mesh->num_el_blks), &(mesh->num_node_sets),
&(mesh->num_side_sets)) < 0) {
Gen_Error(0, "fatal: Error returned from ex_get_init");
return 0;
}
/* alocate some memory for the element blocks */
mesh->data_type = MESH;
mesh->vwgt_dim = 1; /* One weight for now. */
mesh->ewgt_dim = 1; /* One weight for now. */
mesh->eb_etypes = (int *) malloc (5 * mesh->num_el_blks * sizeof(int));
if (!mesh->eb_etypes) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->eb_ids = mesh->eb_etypes + mesh->num_el_blks;
mesh->eb_cnts = mesh->eb_ids + mesh->num_el_blks;
mesh->eb_nnodes = mesh->eb_cnts + mesh->num_el_blks;
mesh->eb_nattrs = mesh->eb_nnodes + mesh->num_el_blks;
mesh->eb_names = (char **) malloc (mesh->num_el_blks * sizeof(char *));
if (!mesh->eb_names) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
mesh->hindex = (int *) malloc(sizeof(int));
mesh->hindex[0] = 0;
if (ex_get_elem_blk_ids(pexoid, mesh->eb_ids) < 0) {
Gen_Error(0, "fatal: Error returned from ex_get_elem_blk_ids");
return 0;
}
/* allocate temporary storage for items needing global reduction. */
/* nemesis does not store most element block info about blocks for */
/* which the processor owns no elements. */
/* we, however, use this information in migration, so we need to */
/* accumulate it for all element blocks. kdd 2/2001 */
if (mesh->num_el_blks > 0) {
nnodes = (int *) malloc(2 * mesh->num_el_blks * sizeof(int));
if (!nnodes) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
etypes = nnodes + mesh->num_el_blks;
}
/* get the element block information */
for (i = 0; i < mesh->num_el_blks; i++) {
/* allocate space for name */
mesh->eb_names[i] = (char *) malloc((MAX_STR_LENGTH+1) * sizeof(char));
if (!mesh->eb_names[i]) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
if (ex_get_elem_block(pexoid, mesh->eb_ids[i], mesh->eb_names[i],
&(mesh->eb_cnts[i]), &(nnodes[i]),
&(mesh->eb_nattrs[i])) < 0) {
Gen_Error(0, "fatal: Error returned from ex_get_elem_block");
return 0;
}
if (mesh->eb_cnts[i] > 0) {
if ((etypes[i] = (int) get_elem_type(mesh->eb_names[i],
nnodes[i],
mesh->num_dims)) == E_TYPE_ERROR) {
Gen_Error(0, "fatal: could not get element type");
return 0;
}
}
else etypes[i] = (int) NULL_EL;
}
/* Perform reduction on necessary fields of element blocks. kdd 2/2001 */
MPI_Allreduce(nnodes, mesh->eb_nnodes, mesh->num_el_blks, MPI_INT, MPI_MAX,
MPI_COMM_WORLD);
MPI_Allreduce(etypes, mesh->eb_etypes, mesh->num_el_blks, MPI_INT, MPI_MIN,
MPI_COMM_WORLD);
for (i = 0; i < mesh->num_el_blks; i++) {
strcpy(mesh->eb_names[i], get_elem_name(mesh->eb_etypes[i]));
}
free(nnodes);
/*
* allocate memory for the elements
* allocate a little extra for element migration latter
*/
mesh->elem_array_len = mesh->num_elems + 5;
mesh->elements = (ELEM_INFO_PTR) malloc (mesh->elem_array_len
* sizeof(ELEM_INFO));
if (!(mesh->elements)) {
Gen_Error(0, "fatal: insufficient memory");
return 0;
}
/*
* intialize all of the element structs as unused by
* setting the globalID to -1
*/
for (i = 0; i < mesh->elem_array_len; i++)
initialize_element(&(mesh->elements[i]));
/* read the information for the individual elements */
if (!read_elem_info(pexoid, Proc, prob, mesh)) {
Gen_Error(0, "fatal: Error returned from read_elem_info");
return 0;
}
/* read the communication information */
if (!read_comm_map_info(pexoid, Proc, prob, mesh)) {
Gen_Error(0, "fatal: Error returned from read_comm_map_info");
return 0;
}
/* Close the parallel file */
if(ex_close (pexoid) < 0) {
Gen_Error(0, "fatal: Error returned from ex_close");
return 0;
}
/* print out the distributed mesh */
if (Debug_Driver > 3)
print_distributed_mesh(Proc, Num_Proc, mesh);
DEBUG_TRACE_END(Proc, yo);
return 1;
#endif /* ZOLTAN_NEMESIS */
}
int migrate_elements(
int Proc,
MESH_INFO_PTR mesh,
Zoltan &zz,
int num_gid_entries,
int num_lid_entries,
int num_imp,
ZOLTAN_ID_PTR imp_gids,
ZOLTAN_ID_PTR imp_lids,
int *imp_procs,
int *imp_to_part,
int num_exp,
ZOLTAN_ID_PTR exp_gids,
ZOLTAN_ID_PTR exp_lids,
int *exp_procs,
int *exp_to_part)
{
/* Local declarations. */
const char *yo = "migrate_elements";
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
/*
* register migration functions
*/
if (!Test.Null_Lists) {
/* If not passing NULL lists, let Help_Migrate call the
* pre-processing and post-processing routines.
*/
if (zz.Set_Pre_Migrate_PP_Fn(migrate_pre_process,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Set_Pre_Migrate_PP_Fn()\n");
return 0;
}
if (zz.Set_Post_Migrate_PP_Fn(migrate_post_process,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Set_Post_Migrate_PP_Fn()\n");
return 0;
}
}
if (Test.Multi_Callbacks) {
if (zz.Set_Obj_Size_Multi_Fn(migrate_elem_size_multi,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Set_Obj_Size_Multi_Fn()\n");
return 0;
}
if (zz.Set_Pack_Obj_Multi_Fn(migrate_pack_elem_multi,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Set_Pack_Obj_Multi_Fn()\n");
return 0;
}
if (zz.Set_Unpack_Obj_Multi_Fn(migrate_unpack_elem_multi,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Set_Unpack_Obj_Multi_Fn()\n");
return 0;
}
}
else {
if (zz.Set_Obj_Size_Fn(migrate_elem_size,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Set_Obj_Size_Fn()\n");
return 0;
}
if (zz.Set_Pack_Obj_Fn(migrate_pack_elem,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Set_Pack_Obj_Fn()\n");
return 0;
}
if (zz.Set_Unpack_Obj_Fn(migrate_unpack_elem,
(void *) mesh) == ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Set_Unpack_Obj_Fn()\n");
return 0;
}
}
if (Test.Null_Lists == NONE) {
if (zz.Migrate(num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part)
== ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Migrate()\n");
return 0;
}
}
else {
/* Call zz.Help_Migrate with empty import lists. */
/* Have to "manually" call migrate_pre_process and migrate_post_process. */
int ierr = 0;
migrate_pre_process((void *) mesh, 1, 1,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part,
&ierr);
if (Test.Null_Lists == IMPORT_LISTS) {
if (zz.Migrate(-1, NULL, NULL, NULL, NULL,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part)
== ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Migrate()\n");
return 0;
}
}
else {
if (zz.Migrate(num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
-1, NULL, NULL, NULL, NULL)
== ZOLTAN_FATAL) {
Gen_Error(0, "fatal: error returned from Migrate()\n");
return 0;
}
}
migrate_post_process((void *) mesh, 1, 1,
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part,
&ierr);
}
DEBUG_TRACE_END(Proc, yo);
return 1;
}
/*--------------------------------------------------------------------------*/
int output_gnu(char *cmd_file,
char *tag,
int Proc,
int Num_Proc,
PROB_INFO_PTR prob,
PARIO_INFO_PTR pio_info,
MESH_INFO_PTR mesh)
/*
* For 2D problems, output files that can be read by gnuplot for looking at
* results.
* We'll do 3D problems later.
*
* One gnuplot file is written for each processor.
* For Chaco input files, the file written contains coordinates of owned
* nodes and all nodes on that processor connected to the owned nodes. When
* drawn "with linespoints", the subdomains are drawn, but lines connecting the
* subdomains are not drawn.
*
* For Nemesis input files, the file written contains the coordinates of
* each node of owned elements. When drawn "with lines", the element outlines
* for each owned element are drawn.
*
* In addition, processor 0 writes a gnuplot command file telling gnuplot how
* to process the individual coordinate files written. This file can be used
* with the gnuplot "load" command to simplify generation of the gnuplot.
*/
{
/* Local declarations. */
char *yo = "output_gnu";
char par_out_fname[FILENAME_MAX+1], ctemp[FILENAME_MAX+1];
ELEM_INFO *current_elem, *nbor_elem;
int nbor, num_nodes;
char *datastyle;
int i, j;
FILE *fp;
/***************************** BEGIN EXECUTION ******************************/
DEBUG_TRACE_START(Proc, yo);
if (mesh->num_dims > 2) {
Gen_Error(0, "warning: cannot generate gnuplot data for 3D problems.");
DEBUG_TRACE_END(Proc, yo);
return 0;
}
if (mesh->eb_nnodes[0] == 0) {
/* No coordinate information is available. */
Gen_Error(0, "warning: cannot generate gnuplot data when no coordinate"
" input is given.");
DEBUG_TRACE_END(Proc, yo);
return 0;
}
/* generate the parallel filename for this processor */
strcpy(ctemp, pio_info->pexo_fname);
strcat(ctemp, ".");
strcat(ctemp, tag);
strcat(ctemp, ".gnu");
gen_par_filename(ctemp, par_out_fname, pio_info, Proc, Num_Proc);
fp = fopen(par_out_fname, "w");
if (pio_info->file_type == CHACO_FILE) {
/*
* For each node of Chaco graph, print the coordinates of the node.
* Then, for each neighboring node on the processor, print the neighbor's
* coordinates.
*/
datastyle = "linespoints";
for (i = 0; i < mesh->elem_array_len; i++) {
current_elem = &(mesh->elements[i]);
if (current_elem->globalID >= 0) {
/* Include the point itself, so that even if there are no edges,
* the point will appear. */
fprintf(fp, "\n%e %e\n",
current_elem->coord[0][0], current_elem->coord[0][1]);
for (j = 0; j < current_elem->nadj; j++) {
if (current_elem->adj_proc[j] == Proc) {
/* Plot the edge. Need to include current point and nbor point
* for each edge. */
fprintf(fp, "\n%e %e\n",
current_elem->coord[0][0], current_elem->coord[0][1]);
nbor = current_elem->adj[j];
nbor_elem = &(mesh->elements[nbor]);
fprintf(fp, "%e %e\n",
nbor_elem->coord[0][0], nbor_elem->coord[0][1]);
}
}
}
}
}
else { /* Nemesis input file */
/*
* For each element of Nemesis input file, print the coordinates of its
* nodes. No need to follow neighbors, as decomposition is by elements.
*/
datastyle = "lines";
for (i = 0; i < mesh->elem_array_len; i++) {
current_elem = &(mesh->elements[i]);
if (current_elem->globalID >= 0) {
num_nodes = mesh->eb_nnodes[current_elem->elem_blk];
for (j = 0; j < num_nodes; j++) {
fprintf(fp, "%e %e\n",
current_elem->coord[j][0], current_elem->coord[j][1]);
}
fprintf(fp, "\n");
}
}
}
fclose(fp);
if (Proc == 0) {
/* Write gnu master file with gnu commands for plotting */
strcpy(ctemp, pio_info->pexo_fname);
strcat(ctemp, ".");
strcat(ctemp, tag);
strcat(ctemp, ".gnuload");
fp = fopen(ctemp, "w");
fprintf(fp, "set nokey\n");
fprintf(fp, "set nolabel\n");
fprintf(fp, "set noxzeroaxis\n");
fprintf(fp, "set noyzeroaxis\n");
fprintf(fp, "set noxtics\n");
fprintf(fp, "set noytics\n");
fprintf(fp, "set data style %s\n", datastyle);
fprintf(fp, "plot ");
strcpy(ctemp, pio_info->pexo_fname);
strcat(ctemp, ".");
strcat(ctemp, tag);
strcat(ctemp, ".gnu");
for (i = 0; i < Num_Proc; i++) {
gen_par_filename(ctemp, par_out_fname, pio_info, i, Num_Proc);
fprintf(fp, "\"%s\"", par_out_fname);
if (i != Num_Proc-1) {
fprintf(fp, ",\\\n");
}
}
fprintf(fp, "\n");
fclose(fp);
}
DEBUG_TRACE_END(Proc, yo);
return 1;
}