int MESH_Renumber_GlobalIDs(Mesh_ptr mesh, MType mtype, int method,
int *preassigned_gids, MSTK_Comm comm) {
if (preassigned_gids) {
MSTK_Report("MESH_Renumber_GlobalIDs",
"Renumbering according to pre-assigned numbering not yet implemented",
MSTK_ERROR);
} else {
if (MESH_Num_Vertices(mesh) && (mtype == MVERTEX || mtype == MALLTYPE))
MESH_Renumber_EntityGlobalIDs(mesh, MVERTEX, method, NULL, comm);
if (MESH_Num_Edges(mesh) && (mtype == MEDGE || mtype == MALLTYPE))
MESH_Renumber_EntityGlobalIDs(mesh, MEDGE, method, NULL, comm);
if (MESH_Num_Faces(mesh) && (mtype == MFACE || mtype == MALLTYPE))
MESH_Renumber_EntityGlobalIDs(mesh, MFACE, method, NULL, comm);
if (MESH_Num_Regions(mesh) && (mtype == MREGION || mtype == MALLTYPE))
MESH_Renumber_EntityGlobalIDs(mesh, MREGION, method, NULL, comm);
// Ghost entity management in the mesh relies on ghost lists
// being sorted according to their global ID
MESH_Sort_GhostLists(mesh, compareGlobalID);
MESH_Sort_GlobalIDSearch_Lists(mesh);
}
return 1;
}
int MESH_ConcatSubMesh_Face(Mesh_ptr mesh, int num, Mesh_ptr *submeshes) {
int nfv, nfe, i, j, k, ival;
MVertex_ptr mv, new_mv, sub_mv;
MEdge_ptr me, new_me, sub_me;
MFace_ptr new_mf, sub_mf;
List_ptr mfverts, mfedges;
int add_face, idx, global_id, iloc, *loc;
double coor[3], rval;
void *pval;
Mesh_ptr submesh;
List_ptr parbndry_verts = List_New(10);
List_ptr parbndry_edges = List_New(10);
MEdge_ptr *fedges = (MEdge_ptr *) malloc(MAXPV2*sizeof(MEdge_ptr));
int *fedirs = (int *) malloc(MAXPV2*sizeof(int));
MAttrib_ptr parbndryatt = MAttrib_New(mesh, "on_parbndry", INT, MVERTEX);
/* collect edges and vertices on the partition boundary */
int num_parbndry_edges = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (ME_PType(me) != PINTERIOR) {
List_Add(parbndry_edges,me);
num_parbndry_edges++;
}
int num_parbndry_verts = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MV_PType(mv) != PINTERIOR) {
List_Add(parbndry_verts,mv);
MEnt_Set_AttVal(mv, parbndryatt, 1, 0.0, NULL);
num_parbndry_verts++;
}
/* sort based on global ID */
List_Sort(parbndry_edges,num_parbndry_edges,sizeof(MEdge_ptr),compareGlobalID);
List_Sort(parbndry_verts,num_parbndry_verts,sizeof(MVertex_ptr),compareGlobalID);
int *parbndry_vert_gids = (int *) malloc(num_parbndry_verts*sizeof(int));
int *parbndry_edge_gids = (int *) malloc(num_parbndry_edges*sizeof(int));
/* store them in array for binary search */
for (i = 0; i < num_parbndry_edges; i++) {
me = List_Entry(parbndry_edges,i);
parbndry_edge_gids[i] = ME_GlobalID(me);
}
for (i = 0; i < num_parbndry_verts; i++) {
mv = List_Entry(parbndry_verts,i);
parbndry_vert_gids[i] = MV_GlobalID(mv);
}
/* Make list of new edges and vertices which will be updated
with each mesh that is concatenated */
int max_vnew = 0, max_enew = 0;
for (i = 0; i < num; i++) {
max_vnew += MESH_Num_Vertices(submeshes[i]);
max_enew += MESH_Num_Edges(submeshes[i]);
}
int num_new_verts = 0, num_new_edges = 0;
int *new_vert_gids = (int *) malloc(max_vnew*sizeof(int));
int *new_edge_gids = (int *) malloc(max_enew*sizeof(int));
List_ptr new_verts = List_New(max_vnew);
List_ptr new_edges = List_New(max_enew);
/* Now process each mesh and add a layer of ghost elements from
each of them to the main partition */
for (i = 0; i < num; i++) {
submesh = submeshes[i];
MAttrib_ptr vidatt = MAttrib_New(submesh, "tempvid", POINTER, MVERTEX);
MAttrib_ptr eidatt = MAttrib_New(submesh, "tempeid", POINTER, MEDGE);
idx = 0;
while ((sub_mf = MESH_Next_Face(submesh, &idx))) {
add_face = 0;
/* Find matching vertices between the submesh and main mesh */
mfverts = MF_Vertices(sub_mf,1,0);
nfv = List_Num_Entries(mfverts);
for (j = 0; j < nfv; j++) {
sub_mv = List_Entry(mfverts,j);
/* Does the vertex have a known counterpart on the partition
* boundary of the main mesh? */
MEnt_Get_AttVal(sub_mv, vidatt, &ival, &rval, &mv);
if (mv) {
int on_parbndry=0;
MEnt_Get_AttVal(mv, parbndryatt, &on_parbndry, &rval, &pval);
if (on_parbndry)
add_face = 1;
} else {
/* Does the global ID of this vertex of the sub mesh face
* match the global ID of a partition boundary vertex in
* the main mesh? */
global_id = MV_GlobalID(sub_mv);
loc = (int *) bsearch(&global_id, parbndry_vert_gids, num_parbndry_verts, sizeof(int),
compareINT);
if (loc) { /* found a match */
add_face = 1;
iloc = loc - parbndry_vert_gids;
mv = List_Entry(parbndry_verts,iloc);
/* here set the ghost vertex property, only necessary when the input submeshes are not consistent */
if (MV_PType(mv) == PGHOST && MV_PType(sub_mv) != PGHOST) {
MV_Set_GEntDim(mv,MV_GEntDim(sub_mv));
MV_Set_GEntID(mv,MV_GEntID(sub_mv));
}
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, mv);
}
}
}
List_Delete(mfverts);
/* Find matching edges between the submesh and main mesh */
mfedges = MF_Edges(sub_mf,1,0);
nfe = List_Num_Entries(mfedges);
for (j = 0; j < nfe; j++) {
sub_me = List_Entry(mfedges,j);
/* Does the edge have a known counterpart on the partition
* boundary of the main mesh */
MEnt_Get_AttVal(sub_me, eidatt, &ival, &rval, &me);
if (!me) {
/* Does the global ID of this edge of the sub mesh face
* match the global ID of a partition boundary edge in the
* main mesh? */
global_id = ME_GlobalID(sub_me);
loc = (int *) bsearch(&global_id, parbndry_edge_gids, num_parbndry_edges, sizeof(int),
compareINT);
if (loc) {
iloc = loc - parbndry_edge_gids;
me = List_Entry(parbndry_edges,iloc);
/* here set the ghost edge property, only necessary when the input submeshes are not consistent */
if (ME_PType(me) == PGHOST && ME_PType(sub_me) != PGHOST) {
ME_Set_GEntDim(me,ME_GEntDim(sub_me));
ME_Set_GEntID(me,ME_GEntID(sub_me));
}
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, me);
}
}
}
if (!add_face) {
List_Delete(mfedges);
continue;
}
new_mf = MF_New(mesh); /* add face */
MF_Set_GEntDim(new_mf,MF_GEntDim(sub_mf));
MF_Set_GEntID(new_mf,MF_GEntID(sub_mf));
MF_Set_PType(new_mf,PGHOST);
MF_Set_MasterParID(new_mf,MF_MasterParID(sub_mf));
MF_Set_GlobalID(new_mf,MF_GlobalID(sub_mf));
nfe = List_Num_Entries(mfedges);
for (j = 0; j < nfe; j++) {
sub_me = List_Entry(mfedges,j);
global_id = ME_GlobalID(sub_me);
fedirs[j] = MF_EdgeDir_i(sub_mf,j) == 1 ? 1 : 0;
new_me = NULL;
MEnt_Get_AttVal(sub_me, eidatt, &ival, &rval, &new_me);
if (!new_me) {
/* search in the ghost layer if another edge with
* this global ID has been added */
loc = (int *) bsearch(&global_id, new_edge_gids, num_new_edges,
sizeof(int), compareINT);
if (loc) {
iloc = loc - new_edge_gids;
new_me = List_Entry(new_edges, iloc);
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, new_me);
}
}
if (new_me) {
if (MV_GlobalID(ME_Vertex(new_me,0)) != MV_GlobalID(ME_Vertex(sub_me,0)))
fedirs[j] = 1 - fedirs[j]; /* if the edge dir is not the same, reverse the edge dir */
} else { /* add a new edge to main mesh */
new_me = ME_New(mesh);
ME_Set_GEntDim(new_me,ME_GEntDim(sub_me));
ME_Set_GEntID(new_me,ME_GEntID(sub_me));
ME_Set_PType(new_me,PGHOST);
ME_Set_MasterParID(new_me,ME_MasterParID(sub_me));
ME_Set_GlobalID(new_me,ME_GlobalID(sub_me));
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, new_me);
List_Add(new_edges, new_me);
for (k = 0; k < 2; k++) {
sub_mv = ME_Vertex(sub_me,k);
global_id = MV_GlobalID(sub_mv);
new_mv = NULL;
MEnt_Get_AttVal(sub_mv, vidatt, &ival, &rval, &new_mv);
if (!new_mv) {
/* search in the ghost layer if another vertex with
* this global ID has been added */
loc = (int *) bsearch(&global_id, new_vert_gids, num_new_verts,
sizeof(int), compareINT);
if (loc) {
iloc = loc - new_vert_gids;
new_mv = List_Entry(new_verts, iloc);
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, new_mv);
}
}
if (!new_mv) { /* add a new vertex to main mesh */
new_mv = MV_New(mesh);
MV_Set_GEntDim(new_mv,MV_GEntDim(sub_mv));
MV_Set_GEntID(new_mv,MV_GEntID(sub_mv));
MV_Set_PType(new_mv,PGHOST);
MV_Set_MasterParID(new_mv,MV_MasterParID(sub_mv));
MV_Set_GlobalID(new_mv,MV_GlobalID(sub_mv));
MV_Coords(sub_mv,coor);
MV_Set_Coords(new_mv,coor);
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, new_mv);
List_Add(new_verts, new_mv);
}
ME_Set_Vertex(new_me,k,new_mv); /* set edge-vertex */
}
}
fedges[j] = new_me;
}
MF_Set_Edges(new_mf,nfe,fedges,fedirs); /* set face-edge */
List_Delete(mfedges);
}
idx = 0;
while ((sub_mv = MESH_Next_Vertex(submesh, &idx)))
MEnt_Rem_AttVal(sub_mv, vidatt);
MAttrib_Delete(vidatt);
idx = 0;
while ((sub_me = MESH_Next_Edge(submesh, &idx)))
MEnt_Rem_AttVal(sub_me, eidatt);
MAttrib_Delete(eidatt);
/* Sort the added entity lists by GlobalID */
num_new_edges = List_Num_Entries(new_edges);
List_Sort(new_edges, num_new_edges, sizeof(MEdge_ptr), compareGlobalID);
for (j = 0; j < num_new_edges; j++)
new_edge_gids[j] = ME_GlobalID(List_Entry(new_edges, j));
num_new_verts = List_Num_Entries(new_verts);
List_Sort(new_verts, num_new_verts, sizeof(MVertex_ptr), compareGlobalID);
for (j = 0; j < num_new_verts; j++)
new_vert_gids[j] = MV_GlobalID(List_Entry(new_verts, j));
}
idx = 0;
while ((mv = List_Next_Entry(parbndry_verts, &idx)))
MEnt_Rem_AttVal(mv, parbndryatt);
MAttrib_Delete(parbndryatt);
List_Delete(parbndry_edges);
List_Delete(parbndry_verts);
List_Delete(new_edges);
List_Delete(new_verts);
free(parbndry_vert_gids);
free(parbndry_edge_gids);
free(new_vert_gids);
free(new_edge_gids);
free(fedges);
free(fedirs);
return 1;
}
void MESH_Renumber(Mesh_ptr mesh, int renum_type, MType mtype) {
MVertex_ptr mv, v0=NULL;
MEdge_ptr me, e0=NULL;
MFace_ptr mf, f0=NULL;
MRegion_ptr mr, r0=NULL;
int idx, idx2, idx3;
int i, j;
int done;
MAttrib_ptr vidatt;
List_ptr vlist;
double xyz[3];
double rval;
int bandwidth, maxbandwidth1, maxbandwidth2;
double avebandwidth1, avebandwidth2;
void *pval;
if (renum_type == 0) {
if (mtype == MVERTEX || mtype == MALLTYPE) {
int nv = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MV_Set_ID(mv,++nv);
}
if (mtype == MEDGE || mtype == MALLTYPE) {
int ne = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
ME_Set_ID(me,++ne);
}
if (mtype == MFACE || mtype == MALLTYPE) {
int nf = 0;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MF_Set_ID(mf,++nf);
}
if (mtype == MREGION || mtype == MALLTYPE) {
int nr = 0;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MR_Set_ID(mr,++nr);
}
}
else if (renum_type == 1) {
double minx, miny, minz;
int minid, maxid;
int *nadj, *newmap, *adj, *offset, nconn;
int nalloc, depth, maxwidth;
#ifdef MSTK_USE_MARKERS
int mkid = MSTK_GetMarker();
#else
MAttrib_ptr mkatt = MAttrib_New(mesh, "mkatt", INT, MALLTYPE);
#endif
if (mtype == MVERTEX || mtype == MALLTYPE) {
int nv = MESH_Num_Vertices(mesh);
/* Compute a graph of vertex connections across elements (faces
for surface meshes, regions for solid meshes */
/* Start with the vertex in the lower leftmost corner */
minx = miny = minz = 1.0e+12;
v0 = NULL;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MV_Coords(mv,xyz);
if (xyz[0] <= minx && xyz[1] <= miny && xyz[2] <= minz) {
minx = xyz[0];
miny = xyz[1];
minz = xyz[2];
v0 = mv;
}
}
nadj = (int *) malloc(nv*sizeof(int));
nalloc = nv*5;
adj = (int *) malloc(nalloc*sizeof(int));
if (!MESH_Num_Regions(mesh)) {
int nentries = 0;
i = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
List_ptr vfaces, adjvlist;
MFace_ptr vf;
MVertex_ptr adjv;
adjvlist = List_New(0);
vfaces = MV_Faces(mv);
idx2 = 0;
while ((vf = List_Next_Entry(vfaces,&idx2))) {
List_ptr fverts = MF_Vertices(vf,1,0);
idx3 = 0;
while ((adjv = List_Next_Entry(fverts,&idx3))) {
if (adjv != mv) {
int vmarked;
#ifdef MSTK_USE_MARKERS
vmarked = MEnt_IsMarked(adjv,mkid);
#else
MEnt_Get_AttVal(adjv, mkatt, &vmarked, &rval, &pval);
#endif
if (!vmarked) {
List_Add(adjvlist,adjv);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjv,mkid);
#else
MEnt_Set_AttVal(adjv, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(fverts);
}
List_Delete(vfaces);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjvlist,mkid);
#endif
nadj[i] = List_Num_Entries(adjvlist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjv = List_Next_Entry(adjvlist,&idx2)))
adj[nentries++] = MV_ID(adjv)-1;
List_Delete(adjvlist);
i++;
}
}
else {
int nentries = 0;
i = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
List_ptr vregions, adjvlist;
MRegion_ptr vr;
MVertex_ptr adjv;
adjvlist = List_New(0);
vregions = MV_Regions(mv);
idx2 = 0;
while ((vr = List_Next_Entry(vregions,&idx2))) {
List_ptr rverts = MR_Vertices(vr);
idx3 = 0;
while ((adjv = List_Next_Entry(rverts,&idx3))) {
if (adjv != mv) {
int vmarked;
#ifdef MSTK_USE_MARKERS
vmarked = MEnt_IsMarked(adjv,mkid);
#else
MEnt_Get_AttVal(adjv, mkatt, &vmarked, &rval, &pval);
#endif
if (!vmarked) {
List_Add(adjvlist,adjv);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjv,mkid);
#else
MEnt_Set_AttVal(adjv, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(rverts);
}
List_Delete(vregions);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjvlist,mkid);
#endif
nadj[i] = List_Num_Entries(adjvlist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjv = List_Next_Entry(adjvlist,&idx2)))
adj[nentries++] = MV_ID(adjv)-1;
List_Delete(adjvlist);
i++;
}
}
/* Compute offsets into adj array */
offset = (int *) malloc(nv*sizeof(int));
offset[0] = 0;
for (i = 1; i < nv; i++)
offset[i] = offset[i-1] + nadj[i-1];
/* Compute maximum bandwidth before renumbering */
maxbandwidth1 = 0;
avebandwidth1 = 0;
for (i = 0; i < nv; i++) {
int off = offset[i];
int curid = i;
for (j = 0; j < nadj[i]; j++) {
int adjid = adj[off+j];
int diff = abs(adjid-curid);
maxbandwidth1 = (diff > maxbandwidth1) ? diff : maxbandwidth1;
avebandwidth1 += diff;
nconn++;
}
}
nconn = offset[nv-1]+nadj[nv-1];
avebandwidth1 /= nconn;
fprintf(stderr,
"Ave vertex ID difference on elements before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max vertex ID difference on elements before renumbering: %-d\n",
maxbandwidth1);
fprintf(stderr,"\n");
newmap = (int *) malloc(nv*sizeof(int));
Graph_Renumber_GPS(nv, MV_ID(v0)-1, nadj, adj, newmap, &depth, &maxwidth);
/* Compute bandwidth after renumbering */
maxbandwidth2 = 0;
avebandwidth2 = 0;
for (i = 0; i < nv; i++) {
int off = offset[i];
int curid = newmap[i];
for (j = 0; j < nadj[i]; j++) {
int adjid = newmap[adj[off+j]];
int diff = abs(adjid-curid);
maxbandwidth2 = (diff > maxbandwidth2) ? diff : maxbandwidth2;
avebandwidth2 += diff;
nconn++;
}
}
nconn = offset[nv-1]+nadj[nv-1];
avebandwidth2 /= nconn;
if (maxbandwidth2 < maxbandwidth1 && avebandwidth2 < avebandwidth1) {
/* Renumber */
idx = 0; i = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MV_Set_ID(mv,newmap[i]+1);
i++;
}
fprintf(stderr,
"Ave vertex ID difference on elements after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max vertex ID difference on elements after renumbering: %-d\n",
maxbandwidth2);
}
else {
nv = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MV_Set_ID(mv,++nv);
fprintf(stderr,"Bandwidth did not improve. Keeping old numbering with gaps eliminated\n");
}
fprintf(stderr,"\n\n\n");
free(nadj);
free(adj);
free(offset);
free(newmap);
}
/* Reorder edges according to a breadth first algorithm applied to
edges (differs from RCM in that it does not add adjacent nodes
in ascending order of their valence) */
if (mtype == MEDGE || mtype == MALLTYPE) {
int ne = MESH_Num_Edges(mesh);
MEdge_ptr ve;
List_ptr elist;
/************************* renumbering code ****************************/
ne = MESH_Num_Edges(mesh);
if (mtype == MALLTYPE) {
/* RCM algorithm already applied on the vertices. Use an edge
connected to the starting vertex as the first edge */
List_ptr vedges = MV_Edges(v0);
e0 = List_Entry(vedges,0);
List_Delete(vedges);
}
else {
/* Find the edge whose mid point is a minimum point */
minx = miny = minz = 1.0e+12;
e0 = NULL;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
double exyz[2][3];
MV_Coords(ME_Vertex(me,0),exyz[0]);
MV_Coords(ME_Vertex(me,1),exyz[1]);
xyz[0] = (exyz[0][0]+exyz[1][0])/2.0;
xyz[1] = (exyz[0][1]+exyz[1][1])/2.0;
xyz[2] = (exyz[0][2]+exyz[1][2])/2.0;
if (xyz[0] < minx && xyz[1] < miny && xyz[2] < minz) {
minx = xyz[0];
miny = xyz[1];
minz = xyz[2];
e0 = me;
}
}
}
nadj = (int *) malloc(ne*sizeof(int));
nalloc = ne*5;
adj = (int *) malloc(nalloc*sizeof(int));
if (!MESH_Num_Regions(mesh)) {
int nentries = 0;
i = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
List_ptr efaces, adjelist;
MFace_ptr ef;
MEdge_ptr adje;
adjelist = List_New(0);
efaces = ME_Faces(me);
idx2 = 0;
while ((ef = List_Next_Entry(efaces,&idx2))) {
List_ptr fedges = MF_Edges(ef,1,0);
idx3 = 0;
while ((adje = List_Next_Entry(fedges,&idx3))) {
if (adje != me) {
int emarked;
#ifdef MSTK_USE_MARKERS
emarked = MEnt_IsMarked(adje,mkid);
#else
MEnt_Get_AttVal(adje, mkatt, &emarked, &rval, &pval);
#endif
if (!emarked) {
List_Add(adjelist,adje);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adje,mkid);
#else
MEnt_Set_AttVal(adje, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(fedges);
}
List_Delete(efaces);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjelist,mkid);
#endif
nadj[i] = List_Num_Entries(adjelist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adje = List_Next_Entry(adjelist,&idx2)))
adj[nentries++] = ME_ID(adje)-1;
List_Delete(adjelist);
i++;
}
}
else {
int nentries = 0;
i = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
List_ptr eregions, adjelist;
MRegion_ptr er;
MEdge_ptr adje;
adjelist = List_New(0);
eregions = ME_Regions(me);
idx2 = 0;
while ((er = List_Next_Entry(eregions,&idx2))) {
List_ptr redges = MR_Edges(er);
idx3 = 0;
while ((adje = List_Next_Entry(redges,&idx3))) {
if (adje != me) {
int emarked;
#ifdef MSTK_USE_MARKERS
emarked = MEnt_IsMarked(adje,mkid);
#else
MEnt_Get_AttVal(adje, mkatt, &emarked, &rval, &pval);
#endif
if (!emarked) {
List_Add(adjelist,adje);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adje,mkid);
#else
MEnt_Set_AttVal(adje, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(redges);
}
List_Delete(eregions);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjelist,mkid);
#endif
nadj[i] = List_Num_Entries(adjelist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adje = List_Next_Entry(adjelist,&idx2)))
adj[nentries++] = ME_ID(adje)-1;
List_Delete(adjelist);
i++;
}
}
/* Compute offsets into adj array */
offset = (int *) malloc(ne*sizeof(int));
offset[0] = 0;
for (i = 1; i < ne; i++)
offset[i] = offset[i-1] + nadj[i-1];
/* Compute maximum bandwidth before renumbering */
maxbandwidth1 = 0;
avebandwidth1 = 0;
for (i = 0; i < ne; i++) {
int off = offset[i];
int curid = i;
for (j = 0; j < nadj[i]; j++) {
int adjid = adj[off+j];
int diff = abs(adjid-curid);
maxbandwidth1 = (diff > maxbandwidth1) ? diff : maxbandwidth1;
avebandwidth1 += diff;
nconn++;
}
}
nconn = offset[ne-1]+nadj[ne-1];
avebandwidth1 /= nconn;
fprintf(stderr,
"Ave edge ID difference on elements before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max edge ID difference on elements before renumbering: %-d\n",
maxbandwidth1);
fprintf(stderr,"\n");
/* Call Graph Renumbering algorithm */
newmap = (int *) malloc(ne*sizeof(int));
Graph_Renumber_GPS(ne, ME_ID(e0)-1, nadj, adj, newmap, &depth, &maxwidth);
/* Compute bandwidth after renumbering */
maxbandwidth2 = 0;
avebandwidth2 = 0;
for (i = 0; i < ne; i++) {
int off = offset[i];
int curid = newmap[i];
for (j = 0; j < nadj[i]; j++) {
int adjid = newmap[adj[off+j]];
int diff = abs(adjid-curid);
maxbandwidth2 = (diff > maxbandwidth2) ? diff : maxbandwidth2;
avebandwidth2 += diff;
nconn++;
}
}
nconn = offset[ne-1]+nadj[ne-1];
avebandwidth2 /= nconn;
if (maxbandwidth2 < maxbandwidth1 && avebandwidth2 < avebandwidth1) {
/* Renumber */
idx = 0; i = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
ME_Set_ID(me,newmap[i]+1);
i++;
}
fprintf(stderr,
"Ave edge ID difference on elements after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max edge ID difference on elements after renumbering: %-d\n",
maxbandwidth2);
}
else {
ne = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
ME_Set_ID(me,++ne);
fprintf(stderr,"Bandwidth did not improve. Keeping old numbering with gaps eliminated\n");
}
fprintf(stderr,"\n\n\n");
free(nadj);
free(adj);
free(offset);
free(newmap);
}
/* Reorder faces according to a breadth first algorithm applied to
edges (differs from RCM in that it does not add adjacent graph nodes
in ascending order of their valence) */
if (mtype == MFACE || mtype == MALLTYPE) {
int nf = MESH_Num_Faces(mesh);
if (mtype == MALLTYPE) {
/* RCM algorithm already applied on the vertices. Use an edge
connected to the starting vertex as the first edge */
List_ptr vfaces = MV_Faces(v0);
f0 = List_Entry(vfaces,0);
List_Delete(vfaces);
}
else {
/* Find the face whose mid point is a minimum point */
minx = miny = minz = 1.0e+12;
f0 = NULL;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
double fxyz[MAXPV2][3];
int nfv;
MF_Coords(mf,&nfv,fxyz);
xyz[0] = fxyz[0][0];
xyz[1] = fxyz[0][1];
xyz[2] = fxyz[0][2];
for (i = 1; i < nfv; i++) {
xyz[0] += fxyz[i][0];
xyz[1] += fxyz[i][1];
xyz[2] += fxyz[i][2];
}
xyz[0] /= nfv; xyz[1] /= nfv; xyz[2] /= nfv;
if (xyz[0] < minx && xyz[1] < miny && xyz[2] < minz) {
minx = xyz[0];
miny = xyz[1];
minz = xyz[2];
f0 = mf;
}
}
}
nadj = (int *) malloc(nf*sizeof(int));
nalloc = nf*5;
adj = (int *) malloc(nalloc*sizeof(int));
if (!MESH_Num_Regions(mesh)) {
int nentries = 0;
i = 0;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
List_ptr vfaces, fverts, adjflist;
MFace_ptr vf, adjf;
MVertex_ptr fv;
adjflist = List_New(0);
fverts = MF_Vertices(mf,1,0);
idx2 = 0;
while ((fv = List_Next_Entry(fverts,&idx2))) {
List_ptr vfaces = MV_Faces(fv);
idx3 = 0;
while ((adjf = List_Next_Entry(vfaces,&idx3))) {
if (adjf != mf) {
int fmarked;
#ifdef MSTK_USE_MARKERS
fmarked = MEnt_IsMarked(adjf,mkid);
#else
MEnt_Get_AttVal(adjf, mkatt, &fmarked, &rval, &pval);
#endif
if (fmarked) {
List_Add(adjflist,adjf);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjf,mkid);
#else
MEnt_Set_AttVal(adjf, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(vfaces);
}
List_Delete(fverts);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjflist,mkid);
#endif
nadj[i] = List_Num_Entries(adjflist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjf = List_Next_Entry(adjflist,&idx2)))
adj[nentries++] = MF_ID(adjf)-1;
List_Delete(adjflist);
i++;
}
}
else {
int nentries = 0;
i = 0;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
List_ptr fregions, adjflist;
MRegion_ptr fr;
MFace_ptr adjf;
adjflist = List_New(0);
fregions = MF_Regions(mf);
idx2 = 0;
while ((fr = List_Next_Entry(fregions,&idx2))) {
List_ptr rfaces = MR_Faces(fr);
idx3 = 0;
while ((adjf = List_Next_Entry(rfaces,&idx3))) {
if (adjf != mf) {
int fmarked;
#ifdef MSTK_USE_MARKERS
fmarked = MEnt_IsMarked(adjf,mkid);
#else
MEnt_Get_AttVal(adjf, mkatt, &fmarked, &rval, &pval);
#endif
if (fmarked) {
List_Add(adjflist,adjf);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjf,mkid);
#else
MEnt_Set_AttVal(adjf, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(rfaces);
}
List_Delete(fregions);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjflist,mkid);
#endif
nadj[i] = List_Num_Entries(adjflist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjf = List_Next_Entry(adjflist,&idx2)))
adj[nentries++] = MF_ID(adjf)-1;
List_Delete(adjflist);
i++;
}
}
/* Compute offsets into adj array */
offset = (int *) malloc(nf*sizeof(int));
offset[0] = 0;
for (i = 1; i < nf; i++)
offset[i] = offset[i-1] + nadj[i-1];
/* Compute maximum bandwidth before renumbering */
maxbandwidth1 = 0;
avebandwidth1 = 0;
for (i = 0; i < nf; i++) {
int off = offset[i];
int curid = i;
for (j = 0; j < nadj[i]; j++) {
int adjid = adj[off+j];
int diff = abs(adjid-curid);
maxbandwidth1 = (diff > maxbandwidth1) ? diff : maxbandwidth1;
avebandwidth1 += diff;
nconn++;
}
}
nconn = offset[nf-1]+nadj[nf-1];
avebandwidth1 /= nconn;
if (MESH_Num_Regions(mesh)) {
fprintf(stderr,
"Ave face ID difference on elements before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max face ID difference on elements before renumbering: %-d\n",
maxbandwidth1);
}
else {
fprintf(stderr,
"Ave face ID difference before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max face ID difference before renumbering: %-d\n",
maxbandwidth1);
}
fprintf(stderr,"\n");
/* Call Graph Renumbering algorithm */
newmap = (int *) malloc(nf*sizeof(int));
Graph_Renumber_GPS(nf, MF_ID(f0)-1, nadj, adj, newmap, &depth, &maxwidth);
/* Compute bandwidth after renumbering */
maxbandwidth2 = 0;
avebandwidth2 = 0;
for (i = 0; i < nf; i++) {
int off = offset[i];
int curid = newmap[i];
for (j = 0; j < nadj[i]; j++) {
int adjid = newmap[adj[off+j]];
int diff = abs(adjid-curid);
maxbandwidth2 = (diff > maxbandwidth2) ? diff : maxbandwidth2;
avebandwidth2 += diff;
nconn++;
}
}
nconn = offset[nf-1]+nadj[nf-1];
avebandwidth2 /= nconn;
if (maxbandwidth2 < maxbandwidth1 && avebandwidth2 < avebandwidth1) {
/* Renumber */
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
MF_Set_ID(mf,newmap[i]+1);
i++;
}
if (MESH_Num_Regions(mesh)) {
fprintf(stderr,
"Ave face ID difference on elements after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max face ID difference on elements after renumbering: %-d\n",
maxbandwidth2);
}
else {
fprintf(stderr,
"Ave face ID difference after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max face ID difference after renumbering: %-d\n",
maxbandwidth2);
}
}
else {
nf = 0;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MF_Set_ID(mf,++nf);
fprintf(stderr,"Bandwidth did not improve. Keeping old numbering with gaps eliminated\n");
}
fprintf(stderr,"\n\n\n");
free(nadj);
free(adj);
free(offset);
free(newmap);
}
if (mtype == MREGION || mtype == MALLTYPE) {
int nr = MESH_Num_Regions(mesh);
if (nr) {
if (mtype == MALLTYPE) {
/* Renumbering algorithm already applied on the vertices. Use
a region connected to the starting vertex as the first
region */
List_ptr vregions = MV_Regions(v0);
r0 = List_Entry(vregions,0);
List_Delete(vregions);
}
else {
/* Find the region whose center point is a minimum point */
minx = miny = minz = 1.0e+12;
r0 = NULL;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
double rxyz[MAXPV3][3];
int nrv;
MR_Coords(mr,&nrv,rxyz);
xyz[0] = rxyz[0][0];
xyz[1] = rxyz[0][1];
xyz[2] = rxyz[0][2];
for (i = 1; i < nrv; i++) {
xyz[0] += rxyz[i][0];
xyz[1] += rxyz[i][1];
xyz[2] += rxyz[i][2];
}
xyz[0] /= nrv; xyz[1] /= nrv; xyz[2] /= nrv;
if (xyz[0] < minx && xyz[1] < miny && xyz[2] < minz) {
minx = xyz[0];
miny = xyz[1];
minz = xyz[2];
r0 = mr;
}
}
}
nadj = (int *) malloc(nr*sizeof(int));
nalloc = nr*5;
adj = (int *) malloc(nalloc*sizeof(int));
int nentries = 0;
i = 0;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
List_ptr vregions, rverts, adjrlist;
MRegion_ptr vr, adjr;
MVertex_ptr rv;
adjrlist = List_New(0);
rverts = MR_Vertices(mr);
idx2 = 0;
while ((rv = List_Next_Entry(rverts,&idx2))) {
List_ptr vregions = MV_Regions(rv);
idx3 = 0;
while ((adjr = List_Next_Entry(vregions,&idx3))) {
if (adjr != mr) {
int rmarked;
#ifdef MSTK_USE_MARKERS
rmarked = MEnt_IsMarked(adjr,mkid);
#else
MEnt_Get_AttVal(adjr, mkatt, &rmarked, &rval, &pval);
#endif
List_Add(adjrlist,adjr);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjr,mkid);
#else
MEnt_Set_AttVal(adjr, mkatt, 1, 0.0, NULL);
#endif
}
}
List_Delete(vregions);
}
List_Delete(rverts);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjrlist,mkid);
#endif
nadj[i] = List_Num_Entries(adjrlist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjr = List_Next_Entry(adjrlist,&idx2)))
adj[nentries++] = MR_ID(adjr)-1;
List_Delete(adjrlist);
i++;
}
/* Compute offsets into adj array */
offset = (int *) malloc(nr*sizeof(int));
offset[0] = 0;
for (i = 1; i < nr; i++)
offset[i] = offset[i-1] + nadj[i-1];
/* Compute maximum bandwidth before renumbering */
maxbandwidth1 = 0;
avebandwidth1 = 0;
for (i = 0; i < nr; i++) {
int off = offset[i];
int curid = i;
for (j = 0; j < nadj[i]; j++) {
int adjid = adj[off+j];
int diff = abs(adjid-curid);
maxbandwidth1 = (diff > maxbandwidth1) ? diff : maxbandwidth1;
avebandwidth1 += diff;
nconn++;
}
}
nconn = offset[nr-1]+nadj[nr-1];
avebandwidth1 /= nconn;
fprintf(stderr,
"Ave region ID difference before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max region ID difference before renumbering: %-d\n",
maxbandwidth1);
fprintf(stderr,"\n");
/* Call Graph Renumbering algorithm */
newmap = (int *) malloc(nr*sizeof(int));
Graph_Renumber_GPS(nr, MR_ID(r0)-1, nadj, adj, newmap, &depth, &maxwidth);
/* Compute bandwidth after renumbering */
maxbandwidth2 = 0;
avebandwidth2 = 0;
for (i = 0; i < nr; i++) {
int off = offset[i];
int curid = newmap[i];
for (j = 0; j < nadj[i]; j++) {
int adjid = newmap[adj[off+j]];
int diff = abs(adjid-curid);
maxbandwidth2 = (diff > maxbandwidth2) ? diff : maxbandwidth2;
avebandwidth2 += diff;
nconn++;
}
}
nconn = offset[nr-1]+nadj[nr-1];
avebandwidth2 /= nconn;
if (maxbandwidth2 < maxbandwidth1 && avebandwidth2 < avebandwidth1) {
/* Renumber */
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
MR_Set_ID(mr,newmap[i]+1);
i++;
}
fprintf(stderr,
"Ave region ID difference after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max region ID difference after renumbering: %-d\n",
maxbandwidth2);
}
else {
nr = 0;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MR_Set_ID(mr,++nr);
fprintf(stderr,"Bandwidth did not improve. Keeping old numbering with gaps eliminated\n");
}
fprintf(stderr,"\n\n\n");
free(nadj);
free(adj);
free(offset);
free(newmap);
}
}
#ifdef MSTK_USE_MARKERS
MSTK_FreeMarker(mkid);
#endif
}
vidatt = MAttrib_New(mesh,"vidrcm",INT,MVERTEX);
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MEnt_Set_AttVal(mv,vidatt,MV_ID(mv),0.0,NULL);
}
/* We have to reset the max IDs stored in the mesh so that we can correctly
assign IDs to new entities */
MESH_Reset_Cached_MaxIDs(mesh);
return;
}
int main(int argc, char *argv[]) {
char infname[256], outfname[256], attfname[256], setfname[256];
Mesh_ptr mesh;
int len, ok=0;
int build_classfn=1, partition=0, weave=0, use_geometry=0, parallel_check=0;
int num_ghost_layers=0, partmethod=0, attfile_given=0, setfile_given=0;
MshFmt inmeshformat, outmeshformat;
if (argc < 3) {
fprintf(stderr,"\n");
fprintf(stderr,"usage: mpirun -n NP %s <--partition=y|1|n|0> <--partition-method=0|1|2> <--parallel-check=y|1|n|0> <--attfile=attfilename> <--setfile=setfilename> infilename outfilename\n\n",progname);
fprintf(stderr,"partition-method = 0, METIS\n");
fprintf(stderr," = 1, ZOLTAN with GRAPH partioning\n");
fprintf(stderr," = 2, ZOLTAN with RCB partitioning\n");
fprintf(stderr,"Choose 2 if you want to avoid partitioning models\n");
fprintf(stderr,"with high aspect ratio along the short directions\n");
fprintf(stderr,"\n");
fprintf(stderr,"infilename = Input Exodus II File\n");
fprintf(stderr,"outfilename = Output Exodus II file name\n");
fprintf(stderr,"attfilename = Auxiliary file with real valued attributes\n");
fprintf(stderr,"setfilename = Auxiliary file with entity set specifications\n");
fprintf(stderr,"\n");
exit(-1);
}
#ifdef MSTK_HAVE_MPI
MPI_Init(&argc,&argv);
#endif
MSTK_Init();
int rank=0, numprocs=1;
#ifdef MSTK_HAVE_MPI
MSTK_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&numprocs);
#else
MSTK_Comm comm = NULL;
#endif
if (argc > 3) {
int i;
for (i = 1; i < argc-2; i++) {
if (strncmp(argv[i],"--partition=",12) == 0) {
if (strncmp(argv[i]+12,"y",1) == 0 ||
strncmp(argv[i]+12,"1",1) == 0)
partition = 1;
else if (strncmp(argv[i]+12,"n",1) == 0 ||
strncmp(argv[i]+12,"0",1) == 0)
partition = 0;
else
MSTK_Report(progname,
"--partition option should be y, n, 1 or 0",
MSTK_FATAL);
}
else if (strncmp(argv[i],"--partition-method",18) == 0 ||
strncmp(argv[i],"--partition_method",18) == 0) {
sscanf(argv[i]+19,"%d",&partmethod);
partition = 1;
}
else if (strncmp(argv[i],"--parallel-check",16) == 0 ||
strncmp(argv[i],"--parallel_check",16) == 0) {
if (strncmp(argv[i]+17,"y",1) == 0 ||
strncmp(argv[i]+17,"1",1) == 1)
parallel_check = 1;
else if (strncmp(argv[i]+17,"n",13) == 0 ||
strncmp(argv[i]+17,"0",13) == 0)
parallel_check = 0;
}
else if (strncmp(argv[i],"--attfile",9) == 0) {
sscanf(argv[i]+10,"%s",attfname);
attfile_given=1;
}
else if (strncmp(argv[i],"--setfile",9) == 0) {
sscanf(argv[i]+10,"%s",setfname);
setfile_given=1;
}
else
fprintf(stderr,"Unrecognized option...Ignoring\n");
}
/* If running on multiple processors, but partition is 0, assume
that you are partitioning a serial mesh */
if (numprocs > 1 && partition == 0) {
MSTK_Report(progname,
"Running on multiple processors but partition option not specified",
MSTK_WARN);
MSTK_Report(progname,
"Output mesh will be partitioned into as many parts as processors",
MSTK_WARN);
partition = 1;
}
}
strcpy(infname,argv[argc-2]);
strcpy(outfname,argv[argc-1]);
len = strlen(infname);
if (len > 4 && strncmp(&(infname[len-4]),".exo",4) == 0)
inmeshformat = EXODUSII;
else
MSTK_Report(progname,"Input file must have .exo extension", MSTK_FATAL);
len = strlen(outfname);
if (len > 4 && strncmp(&(outfname[len-4]),".exo",4) == 0)
outmeshformat = EXODUSII;
else if (len > 4 && strncmp(&(outfname[len-4]),".par",4) == 0)
outmeshformat = EXODUSII;
else
MSTK_Report(progname,"Output file must have .exo or .par extension", MSTK_FATAL);
/* Import the Exodus II mesh */
mesh = MESH_New(F1);
#ifdef MSTK_HAVE_MPI
if (rank == 0) {
#endif
int opts[5]={0,0,0,0,0};
ok = 0;
fprintf(stderr,"Importing mesh from ExodusII file...");
opts[0] = 0; /* don't partition while importing - do it later */
opts[1] = 0;
opts[2] = 0; /* no ghost layers */
opts[3] = 0;
ok = MESH_ImportFromFile(mesh,infname,"exodusii",opts,comm);
if (ok)
fprintf(stderr,"done\n\n");
else {
MSTK_Report(progname,"Failed\n",MSTK_FATAL);
}
/* Read in the attributes only on rank 0 and attach it to the mesh */
if (attfile_given)
import_attributes(mesh, attfname);
/* Import element set definitions and create in the mesh */
if (setfile_given)
import_sets(mesh, setfname);
#ifdef MSTK_HAVE_MPI
}
#endif
#ifdef MSTK_HAVE_MPI
if (partition) {
/* Need to make sure that we have a mesh only on processor 0 */
/* For now we cannot repartition */
int prepartitioned = 0, mesh_present = 0;
int dim = 3;
if (rank > 0) {
if (mesh && MESH_Num_Vertices(mesh) != 0)
mesh_present = 1;
}
MPI_Reduce(&mesh_present,&prepartitioned,1,MPI_INT,MPI_MAX,0,comm);
MPI_Bcast(&prepartitioned,1,MPI_INT,0,comm);
if (!prepartitioned) {
Mesh_ptr mesh0=NULL;
if (rank == 0) {
fprintf(stderr,"Partitioning mesh into %d parts...",numprocs);
if (MESH_Num_Regions(mesh))
dim = 3;
else if (MESH_Num_Faces(mesh))
dim = 2;
else {
fprintf(stderr,"Partitioning possible only for 2D or 3D meshes\n");
exit(-1);
}
mesh0 = mesh;
mesh = NULL;
}
int ring = 0; /* No ghost ring of elements */
int with_attr = 1; /* Do allow exchange of attributes */
int del_inmesh = 1; /* Delete input mesh after partitioning */
ok = MSTK_Mesh_Distribute(mesh0, &mesh, &dim, ring, with_attr,
partmethod, del_inmesh, comm);
if (rank == 0) {
if (ok)
fprintf(stderr,"done\n");
else {
fprintf(stderr,"failed\n");
exit(-1);
}
}
/* if (rank == 0)
MESH_Delete(mesh0);
*/
}
}
#endif /* MSTK_HAVE_MPI */
#ifdef MSTK_HAVE_MPI
if (numprocs > 1 && parallel_check == 1) {
/* Do a parallel consistency check too */
ok = ok && MESH_Parallel_Check(mesh,comm);
}
#endif
fprintf(stderr,"\nExporting mesh to ExodusII format...");
ok = MESH_ExportToFile(mesh,outfname,"exodusii",-1,NULL,NULL,comm);
if (ok)
fprintf(stderr,"Done\n");
else {
fprintf(stderr,"Failed\n");
exit(-1);
}
MESH_Delete(mesh);
// while (1);
#ifdef MSTK_HAVE_MPI
MPI_Finalize();
#endif
return 1;
}
int MESH_WriteToFile(Mesh_ptr mesh, const char *filename, RepType rtype, MSTK_Comm comm) {
FILE *fp;
char mesg[80], attname[256];
int i, j, k, idx;
int gdim, gid;
int mvid, mvid0, mvid1, mvid2, mrid2, meid, mfid, mrid;
int nav, nar, nfe, nfv, nrf, nrv, dir=0;
int nv, ne, nf, nr;
int natt, ncomp, ival, nent;
double xyz[3], rval, rdummy, *rval_arr;
void *pval, *pdummy;
MVertex_ptr mv, mv0, mv1, mv2;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr, mr2;
List_ptr adjverts, mfedges, mfverts, mrfaces, mrverts, adjregs;
RepType reptype;
MAttrib_ptr attrib, vidatt, eidatt, fidatt, ridatt;
MType attentdim;
MAttType atttype;
char modfilename[256];
strcpy(modfilename, filename);
int rank = 0, numprocs = 1;
#ifdef MSTK_HAVE_MPI
if (comm) {
MPI_Comm_size((MPI_Comm)comm, &numprocs);
MPI_Comm_rank((MPI_Comm)comm, &rank);
}
if (numprocs > 1) {
int ndigits = 0;
int div = 1;
while (numprocs/div) {div *= 10; ndigits++;}
sprintf(modfilename,"%s.%d.%0*d",filename,numprocs,ndigits,rank);
}
#endif
if (!(fp = fopen(modfilename,"w"))) {
sprintf(mesg,"Cannot open file %-s for writing",modfilename);
MSTK_Report("MESH_WriteToFile",mesg,MSTK_ERROR);
return 0;
}
if (rtype != UNKNOWN_REP) {
reptype = rtype;
}
else {
reptype = MESH_RepType(mesh);
}
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
fprintf(fp,"MSTK %-2.1lf\n",MSTK_FILE_VER);
fprintf(fp,"%s %d %d %d %d\n",
MESH_rtype_str[reptype],
nv,
(reptype >= R1 && reptype <= R4)?0:ne,
(reptype >= R1 && reptype <= R2 && nr)?0:nf,
nr);
vidatt = MAttrib_New(mesh,"vidatt",INT,MVERTEX);
eidatt = MAttrib_New(mesh,"eidatt",INT,MEDGE);
fidatt = MAttrib_New(mesh,"fidatt",INT,MFACE);
ridatt = MAttrib_New(mesh,"ridatt",INT,MREGION);
idx = 0; i = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MEnt_Set_AttVal(mv,vidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
MEnt_Set_AttVal(me,eidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MEnt_Set_AttVal(mf,fidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MEnt_Set_AttVal(mr,ridatt,++i,0.0,NULL);
fprintf(fp,"vertices\n");
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MV_Coords(mv,xyz);
gdim = MV_GEntDim(mv);
gid = MV_GEntID(mv);
fprintf(fp,"%24.16lf %24.16lf %24.16lf %d %d\n",
xyz[0],xyz[1],xyz[2],gdim,gid);
}
if (reptype == R2 || reptype == R4) {
fprintf(fp,"adjvertices\n");
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
nav = MV_Num_AdjVertices(mv);
fprintf(fp,"%d ",nav);
adjverts = MV_AdjVertices(mv);
for (j = 0; j < nav; j++) {
mv2 = List_Entry(adjverts,j);
MEnt_Get_AttVal(mv2,vidatt,&mvid2,&rval,&pval);
fprintf(fp,"%d ",mvid2);
}
fprintf(fp,"\n");
List_Delete(adjverts);
}
}
if (reptype <= F4 && ne) {
fprintf(fp,"edges\n");
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
mv0 = ME_Vertex(me,0);
MEnt_Get_AttVal(mv0,vidatt,&mvid0,&rval,&pval);
mv1 = ME_Vertex(me,1);
MEnt_Get_AttVal(mv1,vidatt,&mvid1,&rval,&pval);
gdim = ME_GEntDim(me);
gid = ME_GEntID(me);
fprintf(fp,"%d %d \t%d %d\n",mvid0,mvid1,gdim,gid);
}
}
if (reptype <= F4) {
/* For full representations, always write out faces in terms of edges */
fprintf(fp,"faces edge\n");
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
nfe = MF_Num_Edges(mf);
fprintf(fp,"%d ",nfe);
mfedges = MF_Edges(mf,1,0);
for (j = 0; j < nfe; j++) {
me = List_Entry(mfedges,j);
dir = MF_EdgeDir_i(mf,j);
MEnt_Get_AttVal(me,eidatt,&meid,&rval,&pval);
if (dir != 1) meid = -meid;
fprintf(fp,"%d ",meid);
}
List_Delete(mfedges);
gdim = MF_GEntDim(mf);
/*
gent = MF_GEntity(mf);
gid = gent ? -99 : 0;
*/
gid = MF_GEntID(mf);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
else {
/* For reduced representations, R3 and R4 always write out faces
in terms of vertices. For reduced representations, R1 and R2
write out faces in terms of vertices only when there are no
regions (i.e. faces are the highest level mesh entities) */
if ((reptype > R2) || (nr == 0)) {
fprintf(fp,"faces vertex\n");
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
nfv = MF_Num_Edges(mf);
fprintf(fp,"%d ",nfv);
mfverts = MF_Vertices(mf,1,0);
for (j = 0; j < nfv; j++) {
mv = List_Entry(mfverts,j);
MEnt_Get_AttVal(mv,vidatt,&mvid,&rval,&pval);
fprintf(fp,"%d ",mvid);
}
List_Delete(mfverts);
gdim = MF_GEntDim(mf);
gid = MF_GEntID(mf);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
}
if (nr) {
if (reptype <= F4 || reptype >= R2) {
fprintf(fp,"regions face\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nrf = MR_Num_Faces(mr);
fprintf(fp,"%d ",nrf);
mrfaces = MR_Faces(mr);
for (j = 0; j < nrf; j++) {
mf = List_Entry(mrfaces,j);
dir = MR_FaceDir_i(mr,j);
MEnt_Get_AttVal(mf,fidatt,&mfid,&rval,&pval);
if (dir != 1) mfid = -mfid;
fprintf(fp,"%d ",mfid);
}
List_Delete(mrfaces);
gdim = MF_GEntDim(mr);
gid = MR_GEntID(mr);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
else {
fprintf(fp,"regions vertex\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nrv = MR_Num_Vertices(mr);
fprintf(fp,"%d ",nrv);
mrverts = MR_Vertices(mr);
for (j = 0; j < nrv; j++) {
mv = List_Entry(mrverts,j);
MEnt_Get_AttVal(mv,vidatt,&mvid,&rval,&pval);
fprintf(fp,"%d ",mvid);
}
List_Delete(mrverts);
gdim = MR_GEntDim(mr);
gid = MR_GEntID(mr);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
if (reptype == R2 || reptype == R4) {
fprintf(fp,"adjregions\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nar = MR_Num_Faces(mr);
fprintf(fp,"%d ",nar);
adjregs = MR_AdjRegions(mr);
for (j = 0; j < nar; j++) {
mr2 = List_Entry(adjregs,j);
if ((long) mr2 == -1)
fprintf(fp,"%d ",0);
else {
MEnt_Get_AttVal(mr2,ridatt,&mrid2,&rval,&pval);
fprintf(fp,"%d ",mrid2);
}
}
fprintf(fp,"\n");
List_Delete(adjregs);
}
}
}
/* Write out attributes if there are more than the 4 that we created
in this routine */
if ((natt = MESH_Num_Attribs(mesh)) > 4) {
fprintf(fp,"attributes\n");
for (i = 0; i < natt; i++) {
attrib = MESH_Attrib(mesh,i);
/* Don't write out attribs we created for the internal use of
this routine */
if (attrib == vidatt || attrib == eidatt || attrib == fidatt ||
attrib == ridatt) continue;
MAttrib_Get_Name(attrib,attname);
atttype = MAttrib_Get_Type(attrib);
if (atttype == POINTER) continue; /* cannot write it out */
ncomp = MAttrib_Get_NumComps(attrib);
attentdim = MAttrib_Get_EntDim(attrib);
/* First count how many entities actually have the attribute assigned */
nent = 0;
switch(attentdim) {
case MVERTEX:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) nent++;
break;
case MEDGE:
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) nent++;
break;
case MFACE:
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) nent++;
break;
case MREGION:
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) nent++;
break;
case MALLTYPE:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) nent++;
break;
default:
break;
} /* switch (attentdim) */
/* No point in writing out attribute if no entity uses it! Or is there? */
if (!nent) continue;
fprintf(fp,"%-s\n",attname);
switch(atttype) {
case INT:
fprintf(fp,"INT\n");
break;
case DOUBLE:
fprintf(fp,"DOUBLE\n");
break;
case VECTOR:
fprintf(fp,"VECTOR\n");
break;
case TENSOR:
fprintf(fp,"TENSOR\n");
break;
default:
MSTK_Report("MESH_WriteToFile",
"Unrecognizable or unprintable attribute type\n",MSTK_WARN);
continue;
}
fprintf(fp,"%-d\n",ncomp);
switch(attentdim) {
case MVERTEX:
fprintf(fp,"MVERTEX\n");
break;
case MEDGE:
fprintf(fp,"MEDGE\n");
break;
case MFACE:
fprintf(fp,"MFACE\n");
break;
case MREGION:
fprintf(fp,"MREGION\n");
break;
case MALLTYPE:
fprintf(fp,"MALLTYPE\n");
break;
default:
MSTK_Report("Mesh_WriteToFile","Unrecognized entity type",MSTK_WARN);
break;
}
fprintf(fp,"%-d\n",nent);
switch(attentdim) {
case MVERTEX:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mv,vidatt,&mvid,&rdummy,&pdummy);
fprintf(fp,"0 %-d ",mvid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MEDGE:
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(me,eidatt,&meid,&rdummy,&pdummy);
fprintf(fp,"1 %-d ",meid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MFACE:
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mf,fidatt,&mfid,&rdummy,&pdummy);
fprintf(fp,"2 %-d ",mfid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MREGION:
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mr,ridatt,&mrid,&rdummy,&pdummy);
fprintf(fp,"3 %-d ",mrid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MALLTYPE:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mv,vidatt,&mvid,&rdummy,&pdummy);
fprintf(fp,"0 %-d ",mvid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(me,eidatt,&meid,&rdummy,&pdummy);
fprintf(fp,"1 %-d ",meid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mf,fidatt,&mfid,&rdummy,&pdummy);
fprintf(fp,"2 %-d ",mfid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mr,ridatt,&mrid,&rdummy,&pdummy);
fprintf(fp,"3 %-d ",mrid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
default:
break;
} /* switch (attentdim) */
} /* for (i = 0; i < natt) */
} /* if (Mesh_Num_Attribs(mesh)) */
idx = 0; i = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MEnt_Rem_AttVal(mv,vidatt);
idx = 0; i = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
MEnt_Rem_AttVal(me,eidatt);
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MEnt_Rem_AttVal(mf,fidatt);
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MEnt_Rem_AttVal(mr,ridatt);
MAttrib_Delete(vidatt);
MAttrib_Delete(eidatt);
MAttrib_Delete(fidatt);
MAttrib_Delete(ridatt);
fclose(fp);
return 1;
}
int MESH_PartitionWithZoltan(Mesh_ptr mesh, int nparts, int **part, int noptions,
char **options, MSTK_Comm comm) {
MEdge_ptr fedge;
MFace_ptr mf, oppf, rface;
MRegion_ptr mr, oppr;
List_ptr fedges, efaces, rfaces, fregions;
int i, j, k, id;
int nv, ne, nf, nr=0, nfe, nef, nfr, nrf, idx, idx2;
int numflag, nedgecut, ipos;
int wtflag;
int rc;
float ver;
struct Zoltan_Struct *zz;
GRAPH_DATA graph;
int changes, numGidEntries, numLidEntries, numImport, numExport;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int rank;
MPI_Comm_rank(comm,&rank);
rc = Zoltan_Initialize(0, NULL, &ver);
if (rc != ZOLTAN_OK){
MSTK_Report("MESH_PartitionWithZoltan","Could not initialize Zoltan",MSTK_FATAL);
MPI_Finalize();
exit(0);
}
/******************************************************************
** Create a Zoltan library structure for this instance of partition
********************************************************************/
zz = Zoltan_Create(comm);
/*****************************************************************
** Figure out partitioning method
*****************************************************************/
char partition_method_str[32];
strcpy(partition_method_str,"RCB"); /* Default - Recursive Coordinate Bisection */
if (noptions) {
for (i = 0; i < noptions; i++) {
if (strncmp(options[i],"LB_PARTITION",12) == 0) {
char *result = NULL, instring[256];
strcpy(instring,options[i]);
result = strtok(instring,"=");
result = strtok(NULL," ");
strcpy(partition_method_str,result);
}
}
}
if (rank == 0) {
char mesg[256];
sprintf(mesg,"Using partitioning method %s for ZOLTAN\n",partition_method_str);
MSTK_Report("MESH_PartitionWithZoltan",mesg,MSTK_MESG);
}
/* General parameters for Zoltan */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", partition_method_str);
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
graph.numMyNodes = 0;
graph.numAllNbors = 0;
graph.nodeGID = NULL;
graph.nodeCoords = NULL;
graph.nborIndex = NULL;
graph.nborGID = NULL;
graph.nborProc = NULL;
if (strcmp(partition_method_str,"RCB") == 0) {
if (rank == 0) {
nr = MESH_Num_Regions(mesh);
nf = MESH_Num_Faces(mesh);
if (!nf && !nr)
MSTK_Report("MESH_PartitionWithZoltan","Cannot partition wire meshes",
MSTK_FATAL);
if (nr == 0) { /* Surface or planar mesh */
int ndim = 2; /* assume mesh is planar */
idx = 0;
MVertex_ptr mv;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
double vxyz[3];
MV_Coords(mv,vxyz);
if (vxyz[2] != 0.0) {
ndim = 3; /* non-planar or planar with non-zero z */
break;
}
}
NDIM_4_ZOLTAN = ndim-1; /* ignore last dimension to avoid partitioning in that dimension */
graph.numMyNodes = nf;
graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nf);
graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nf);
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
double fxyz[MAXPV2][3], cen[3];
int nfv;
MF_Coords(mf,&nfv,fxyz);
cen[0] = cen[1] = cen[2] = 0.0;
for (j = 0; j < nfv; j++)
for (k = 0; k < NDIM_4_ZOLTAN; k++)
cen[k] += fxyz[j][k];
for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nfv;
id = MF_ID(mf);
graph.nodeGID[id-1] = id;
memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double));
}
}
else { /* Volume mesh */
int ndim = 3;
NDIM_4_ZOLTAN = ndim-1; /* ignore last dimension to avoid partitioning in that dimension */
graph.numMyNodes = nr;
graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nr);
graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nr);
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
double rxyz[MAXPV3][3], cen[3];
int nrv;
MR_Coords(mr,&nrv,rxyz);
cen[0] = cen[1] = cen[2] = 0.0;
for (j = 0; j < nrv; j++)
for (k = 0; k < NDIM_4_ZOLTAN; k++)
cen[k] += rxyz[j][k];
for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nrv;
for (k = 0; k < NDIM_4_ZOLTAN; k++)
if (fabs(cen[k]) < 1.0e-10) cen[k] = 0.0;
id = MR_ID(mr);
graph.nodeGID[id-1] = id;
memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double));
}
}
}
MPI_Bcast(&NDIM_4_ZOLTAN,1,MPI_INT,0,comm);
/* Set some default values */
Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS","1");
// Zoltan_Set_Param(zz, "AVERAGE_CUTS", "1");
if (noptions > 1) {
for (i = 1; i < noptions; i++) {
char *paramstr = NULL, *valuestr = NULL, instring[256];
strcpy(instring,options[i]);
paramstr = strtok(instring,"=");
valuestr = strtok(NULL," ");
Zoltan_Set_Param(zz,paramstr,valuestr);
}
}
/* Query functions - defined in simpleQueries.h */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph);
Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph);
Zoltan_Set_Num_Geom_Fn(zz, get_num_dimensions_reduced, &graph); /* reduced dimensions */
Zoltan_Set_Geom_Multi_Fn(zz, get_element_centers_reduced, &graph); /* reduced dimension centers */
}
else if (strcmp(partition_method_str,"GRAPH") == 0) {
if(rank == 0) {
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
ipos = 0;
/* build nodes and neighbors list, similar as in partition with metis
Assign processor 0 the whole mesh, assign other processors a NULL mesh */
if (nr == 0) {
if (nf == 0) {
MSTK_Report("MESH_PartitionWithZoltan",
"Cannot partition wire meshes with Zoltan",MSTK_FATAL);
exit(-1);
}
graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nf);
graph.nborIndex = (int *)malloc(sizeof(int) * (nf + 1));
graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*ne);
graph.nborProc = (int *)malloc(sizeof(int) * 2*ne);
graph.nborIndex[0] = 0;
/* Surface mesh */
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
graph.nodeGID[i] = MF_ID(mf);
fedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(fedges);
idx2 = 0;
while ((fedge = List_Next_Entry(fedges,&idx2))) {
efaces = ME_Faces(fedge);
nef = List_Num_Entries(efaces);
if (nef == 1) {
continue; /* boundary edge; nothing to do */
} else {
int j;
for (j = 0; j < nef; j++) {
oppf = List_Entry(efaces,j);
if (oppf == mf) {
graph.nborGID[ipos] = MF_ID(oppf);
/* initially set all nodes on processor 0 */
graph.nborProc[ipos] = 0;
ipos++;
}
}
}
List_Delete(efaces);
}
List_Delete(fedges);
i++;
graph.nborIndex[i] = ipos;
}
graph.numMyNodes = i;
graph.numAllNbors = ipos;
}
else {
graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nr);
graph.nborIndex = (int *)malloc(sizeof(int) * (nr + 1));
graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*nf);
graph.nborProc = (int *)malloc(sizeof(int) * 2*nf);
graph.nborIndex[0] = 0;
/* Volume mesh */
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
graph.nodeGID[i] = MR_ID(mr);
rfaces = MR_Faces(mr);
nrf = List_Num_Entries(rfaces);
idx2 = 0;
while ((rface = List_Next_Entry(rfaces,&idx2))) {
fregions = MF_Regions(rface);
nfr = List_Num_Entries(fregions);
if (nfr > 1) {
oppr = List_Entry(fregions,0);
if (oppr == mr)
oppr = List_Entry(fregions,1);
graph.nborGID[ipos] = MR_ID(oppr);
/* initially set all nodes on processor 0 */
graph.nborProc[ipos] = 0;
ipos++;
}
List_Delete(fregions);
}
List_Delete(rfaces);
i++;
graph.nborIndex[i] = ipos;
}
graph.numMyNodes = i;
graph.numAllNbors = ipos;
}
}
/* Graph parameters */
/* Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); */
Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */
/* Query functions - defined in simpleQueries.h */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph);
Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &graph);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &graph);
}
/* Partition the graph */
/******************************************************************
** Zoltan can now partition the graph.
** We assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of nodes to be sent to me */
&importGlobalGids, /* Global IDs of nodes to be sent to me */
&importLocalGids, /* Local IDs of nodes to be sent to me */
&importProcs, /* Process rank for source of each incoming node */
&importToPart, /* New partition for each incoming node */
&numExport, /* Number of nodes I must send to other processes*/
&exportGlobalGids, /* Global IDs of the nodes I must send */
&exportLocalGids, /* Local IDs of the nodes I must send */
&exportProcs, /* Process to which I send each of the nodes */
&exportToPart); /* Partition to which each node will belong */
if (rc != ZOLTAN_OK){
if (rank == 0)
MSTK_Report("MESH_PartitionWithZoltan","Could not partition mesh with ZOLTAN",
MSTK_ERROR);
Zoltan_Destroy(&zz);
MPI_Finalize();
return 0;
}
if(rank == 0) {
*part = (int *) calloc(graph.numMyNodes,sizeof(int));
for ( i = 0; i < numExport; i++ ) {
(*part)[exportGlobalGids[i]-1] = exportToPart[i];
}
if (graph.nodeGID) free(graph.nodeGID);
if (graph.nodeCoords) free(graph.nodeCoords);
if (graph.nborIndex) free(graph.nborIndex);
if (graph.nborGID) free(graph.nborGID);
if (graph.nborProc) free(graph.nborProc);
}
else {
*part = NULL;
}
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart);
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart);
Zoltan_Destroy(&zz);
return 1;
}
int MESH_PartitionWithMetis(Mesh_ptr mesh, int nparts, int **part) {
MEdge_ptr fedge;
MFace_ptr mf, oppf, rface;
MRegion_ptr mr, oppr;
List_ptr fedges, efaces, rfaces, fregions;
int i, ncells, ipos;
int nv, ne, nf, nr, nfe, nef, nfr, nrf, idx, idx2;
#ifdef METIS_5
idx_t ngraphvtx, numflag, nedgecut, numparts, ncons;
idx_t wtflag, metisopts[METIS_NOPTIONS];
idx_t *vsize, *idxpart;
idx_t *xadj, *adjncy, *vwgt, *adjwgt;
real_t *tpwgts, *ubvec;
#else
idxtype ngraphvtx, numflag, nedgecut, numparts;
idxtype wtflag, metisopts[5] = {0,0,0,0,0};
idxtype *xadj, *adjncy, *vwgt, *adjwgt, *idxpart;
#endif
/* First build a nodal graph of the mesh in the format required by
metis */
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
ipos = 0;
if (nr == 0) {
if (nf == 0) {
fprintf(stderr,"Cannot partition wire meshes\n");
exit(-1);
}
#ifdef METIS_5
xadj = (idx_t *) malloc((nf+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*ne*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nf+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*ne*sizeof(idxtype));
#endif
ncells = nf;
/* Surface mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mf = MESH_Next_Face(mesh,&idx))) {
fedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(fedges);
idx2 = 0;
while ((fedge = List_Next_Entry(fedges,&idx2))) {
efaces = ME_Faces(fedge);
nef = List_Num_Entries(efaces);
if (nef == 1) {
continue; /* boundary edge; nothing to do */
}
else {
int j;
for (j = 0; j < nef; j++) {
oppf = List_Entry(efaces,j);
if (oppf != mf) {
adjncy[ipos] = MF_ID(oppf)-1;
ipos++;
}
}
}
List_Delete(efaces);
}
List_Delete(fedges);
i++;
xadj[i] = ipos;
}
}
else {
#ifdef METIS_5
xadj = (idx_t *) malloc((nr+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*nf*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nr+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*nf*sizeof(idxtype));
#endif
ncells = nr;
/* Volume mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mr = MESH_Next_Region(mesh,&idx))) {
rfaces = MR_Faces(mr);
nrf = List_Num_Entries(rfaces);
idx2 = 0;
while ((rface = List_Next_Entry(rfaces,&idx2))) {
fregions = MF_Regions(rface);
nfr = List_Num_Entries(fregions);
if (nfr > 1) {
oppr = List_Entry(fregions,0);
if (oppr == mr)
oppr = List_Entry(fregions,1);
adjncy[ipos] = MR_ID(oppr)-1;
ipos++;
}
List_Delete(fregions);
}
List_Delete(rfaces);
i++;
xadj[i] = ipos;
}
}
/* Partition the graph */
wtflag = 0; /* No weights are specified */
vwgt = adjwgt = NULL;
numflag = 0; /* C style numbering of elements (nodes of the dual graph) */
ngraphvtx = ncells; /* we want the variable to be of type idxtype or idx_t */
numparts = nparts; /* we want the variable to be of type idxtype or idx_t */
#ifdef METIS_5
idxpart = (idx_t *) malloc(ncells*sizeof(idx_t));
ncons = 1; /* Number of constraints */
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
METIS_SetDefaultOptions(metisopts);
metisopts[METIS_OPTION_NUMBERING] = 0;
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,
idxpart);
else
METIS_PartGraphKway(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,idxpart);
#else
idxpart = (idxtype *) malloc(ncells*sizeof(idxtype));
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,
&numflag,&numparts,metisopts,&nedgecut,idxpart);
else
METIS_PartGraphKway(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,&numflag,
&numparts,metisopts,&nedgecut,idxpart);
#endif
free(xadj);
free(adjncy);
*part = (int *) malloc(ncells*sizeof(int));
for (i = 0; i < ncells; i++)
(*part)[i] = (int) idxpart[i];
free(idxpart);
return 1;
}
int MESH_AssignGlobalIDs_Vertex(Mesh_ptr submesh, int have_GIDs, MSTK_Comm comm) {
int i, j, nv, nbv, ne, nf, nr, mesh_info[10];
MVertex_ptr mv;
List_ptr boundary_verts;
RepType rtype;
int index_nbv, max_nbv, iloc, num_ghost_verts, global_id;
int *global_mesh_info, *vertex_ov_label, *vertex_ov_global_id, *id_on_ov_list;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
for (i = 0; i < 10; i++) mesh_info[i] = 0;
rtype = MESH_RepType(submesh);
nv = MESH_Num_Vertices(submesh);
ne = MESH_Num_Edges(submesh);
nf = MESH_Num_Faces(submesh);
nr = MESH_Num_Regions(submesh);
mesh_info[0] = rtype;
mesh_info[1] = nv;
mesh_info[2] = ne;
mesh_info[3] = nf;
mesh_info[4] = nr;
/* calculate number of boundary vertices */
nbv = 0; boundary_verts = List_New(10);
if (nr) {
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (vertex_on_boundary3D(mv)) {
MV_Flag_OnParBoundary(mv);
List_Add(boundary_verts,mv);
nbv++;
}
}
}
else {
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (vertex_on_boundary2D(mv)) {
MV_Flag_OnParBoundary(mv);
List_Add(boundary_verts,mv);
nbv++;
}
}
}
mesh_info[5] = nbv;
/*
gather submeshes information
right now we only need nv and nbv, and later num_ghost_verts, but we gather all mesh_info
*/
global_mesh_info = (int *)malloc(10*num*sizeof(int));
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* get largest number of boundary vertices of all the processors */
max_nbv = 0;
for(i = 0; i < num; i++)
if(max_nbv < global_mesh_info[10*i+5])
max_nbv = global_mesh_info[10*i+5];
if (have_GIDs) {
int *list_boundary_vertex_gid = (int *)malloc(max_nbv*sizeof(int));
int *recv_list_vertex_gid = (int *)malloc(num*max_nbv*sizeof(int));
/* sort boundary vertices based on Global ID, for binary search */
List_Sort(boundary_verts,nbv,sizeof(MVertex_ptr),compareGlobalID);
/* only global ids are sent */
index_nbv = 0;
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
list_boundary_vertex_gid[index_nbv] = MV_GlobalID(mv);
index_nbv++;
}
MPI_Allgather(list_boundary_vertex_gid,max_nbv,MPI_INT,recv_list_vertex_gid,max_nbv,MPI_INT,comm);
/* indicate if a vertex is overlapped */
vertex_ov_label = (int *)malloc(num*max_nbv*sizeof(int));
/*
store the local boundary id on ov processor
it is used to assign global id of local ghost vertices
no need to store master partition id, MV_MasterParID(mv) is already assigned
*/
id_on_ov_list = (int *)malloc(max_nbv*sizeof(int));
for (i = 0; i < num*max_nbv; i++)
vertex_ov_label[i] = 0;
num_ghost_verts = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
int gid = MV_GlobalID(mv);
/* check which previous processor has a vertex with same Global ID*/
for(j = 0; j < rank; j++) {
/* since each processor has sorted the boundary vertices, use binary search */
int *loc = (int *)bsearch(&gid,
&recv_list_vertex_gid[max_nbv*j],
global_mesh_info[10*j+5],
sizeof(int),
compareINT);
/* if found the vertex on previous processors */
if(loc) {
/* here the location iloc is relative to the beginning of the jth processor */
iloc = (int)(loc - &recv_list_vertex_gid[max_nbv*j]);
MV_Set_PType(mv,PGHOST);
MV_Set_MasterParID(mv,j);
num_ghost_verts++;
/* label the original vertex as overlapped */
vertex_ov_label[max_nbv*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
/* if found on processor j, no need to test for j+1,j+2...*/
break;
}
}
}
}
free(list_boundary_vertex_gid);
free(recv_list_vertex_gid);
}
else {
double coor[3];
int *list_boundary_vertex = (int *)malloc(max_nbv*sizeof(int));
double *list_boundary_coor = (double *)malloc(3*max_nbv*sizeof(double));
int *recv_list_vertex = (int *)malloc(num*max_nbv*sizeof(int));
double *recv_list_coor = (double *)malloc(3*num*max_nbv*sizeof(double));
/* sort boundary vertices based on coordinate value, for binary search */
List_Sort(boundary_verts,nbv,sizeof(MVertex_ptr),compareVertexCoor);
/* only local id and coordinate values are sent */
index_nbv = 0;
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
list_boundary_vertex[index_nbv] = MV_ID(mv);
MV_Coords(mv,coor);
list_boundary_coor[index_nbv*3] = coor[0];
list_boundary_coor[index_nbv*3+1] = coor[1];
list_boundary_coor[index_nbv*3+2] = coor[2];
index_nbv++;
}
MPI_Allgather(list_boundary_vertex,max_nbv,MPI_INT,recv_list_vertex,max_nbv,MPI_INT,comm);
MPI_Allgather(list_boundary_coor,3*max_nbv,MPI_DOUBLE,recv_list_coor,3*max_nbv,MPI_DOUBLE,comm);
/* indicate if a vertex is overlapped */
vertex_ov_label = (int *)malloc(num*max_nbv*sizeof(int));
/*
store the local boundary id on ov processor
it is used to assign global id of local ghost vertices
no need to store master partition id, MV_MasterParID(mv) is already assigned
*/
id_on_ov_list = (int *)malloc(max_nbv*sizeof(int));
for (i = 0; i < num*max_nbv; i++)
vertex_ov_label[i] = 0;
num_ghost_verts = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
MV_Coords(mv,coor);
/* check which previous processor has the same coordinate vertex */
for(j = 0; j < rank; j++) {
/* since each processor has sorted the boundary vertices, use binary search */
double *loc = (double *)bsearch(&coor,
&recv_list_coor[3*max_nbv*j],
global_mesh_info[10*j+5],
3*sizeof(double),
compareCoorDouble);
/* if found the vertex on previous processors */
if(loc) {
/* here the location iloc is relative to the beginning of the jth processor */
iloc = (int)(loc - &recv_list_coor[3*max_nbv*j])/3;
MV_Set_PType(mv,PGHOST);
MV_Set_MasterParID(mv,j);
num_ghost_verts++;
/* label the original vertex as overlapped */
vertex_ov_label[max_nbv*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
/* if found on processor j, no need to test for j+1,j+2...*/
break;
}
}
}
}
free(list_boundary_coor);
free(recv_list_coor);
free(list_boundary_vertex);
free(recv_list_vertex);
}
/* num of ghost verts */
mesh_info[9] = num_ghost_verts;
/* update ghost verts number */
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* since this is a OR reduction, we can use MPI_IN_PLACE, send buffer same as recv buffer */
MPI_Allreduce(MPI_IN_PLACE,vertex_ov_label,num*max_nbv,MPI_INT,MPI_LOR,comm);
/* calculate starting global id number for vertices*/
if (!have_GIDs) {
global_id = 1;
for(i = 0; i < rank; i++)
global_id = global_id + global_mesh_info[10*i+1] - global_mesh_info[10*i+9];
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (MV_PType(mv) == PGHOST)
continue;
MV_Set_GlobalID(mv,global_id++);
MV_Set_MasterParID(mv,rank);
}
}
/* store overlapped vertices IDs and broadast */
vertex_ov_global_id = (int *)malloc(num*max_nbv*sizeof(int));
for(i = 0; i < num*max_nbv; i++)
vertex_ov_global_id[i] = 0;
for(i = 0; i < nbv; i++) {
if(vertex_ov_label[rank*max_nbv+i]) {
mv = List_Entry(boundary_verts,i);
MV_Set_PType(mv,POVERLAP);
vertex_ov_global_id[rank*max_nbv+i] = MV_GlobalID(mv);
}
}
MPI_Allreduce(MPI_IN_PLACE,vertex_ov_global_id,num*max_nbv,MPI_INT,MPI_MAX,comm);
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
if(MV_PType(mv) == PGHOST)
MV_Set_GlobalID(mv,vertex_ov_global_id[MV_MasterParID(mv)*max_nbv+id_on_ov_list[i]]);
}
List_Delete(boundary_verts);
free(global_mesh_info);
free(vertex_ov_label);
free(vertex_ov_global_id);
free(id_on_ov_list);
return 1;
}
int MESH_Send_Vertices(Mesh_ptr mesh, int torank, MSTK_Comm comm,
int *numreq, int *maxreq, MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
int i, j, nv;
MVertex_ptr mv;
double coor[3];
MPI_Request mpirequest;
if (requests == NULL)
MSTK_Report("MESH_Surf_SendMesh","MPI requests array is NULL",MSTK_FATAL);
if (*maxreq == 0) {
*maxreq = 25;
*requests = (MPI_Request *) malloc(*maxreq*sizeof(MPI_Request));
*numreq = 0;
}
else if (*maxreq < (*numreq) + 11) {
*maxreq = 2*(*maxreq) + 11;
*requests = (MPI_Request *) realloc(*requests,*maxreq*sizeof(MPI_Request));
}
/* Now send out detailed vertex info */
nv = MESH_Num_Vertices(mesh);
int *list_vertex = (int *) malloc(3*nv*sizeof(int));
/* Store the 3 auxilliary data fields - would be nice if we didn't
* have to send the data in such an error prone way (with bit
* shifting) that requires knowledge of the internal structure of
* MEntity */
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(mesh,i);
list_vertex[3*i] = (MV_GEntID(mv)<<3) | (MV_GEntDim(mv));
list_vertex[3*i+1] = (MV_MasterParID(mv) <<3) | MV_OnParBoundary(mv)<<2 | (MV_PType(mv));
list_vertex[3*i+2] = MV_GlobalID(mv);
}
/* send vertices */
MPI_Isend(list_vertex,3*nv,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;;
int nptrs = 1;
if (*maxptrs2free == 0) {
*maxptrs2free = 25;
*ptrs2free = (void **) malloc(*maxptrs2free*sizeof(void *));
*numptrs2free = 0;
}
else if (*maxptrs2free < (*numptrs2free) + nptrs) {
*maxptrs2free = 2*(*maxptrs2free) + nptrs;
*ptrs2free = (void **) realloc(*ptrs2free,(*maxptrs2free)*sizeof(void *));
}
(*ptrs2free)[(*numptrs2free)++] = list_vertex;
return 1;
}
int MESH_Send_Attribute(Mesh_ptr mesh, MAttrib_ptr attrib, int torank,
MSTK_Comm comm, int *numreq, int *maxreq,
MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
int j, k;
int num, ncomp, ival;
double rval;
void *pval;
double *rval_arr;
int *list_info;
MType mtype;
MAttType att_type;
MEntity_ptr ment;
MPI_Request mpirequest;
if (requests == NULL)
MSTK_Report("MSTK_SendMSet","Invalid MPI request buffer",MSTK_FATAL);
if (*maxreq == 0) {
*maxreq = 25;
*requests = (MPI_Request *) malloc(*maxreq*sizeof(MPI_Request));
*numreq = 0;
}
else if (*maxreq < (*numreq) + 2) {
*maxreq *= 2;
*requests = (MPI_Request *) realloc(*requests,*maxreq*sizeof(MPI_Request));
}
/* get attribute properties */
att_type = MAttrib_Get_Type(attrib);
ncomp = MAttrib_Get_NumComps(attrib);
mtype = MAttrib_Get_EntDim(attrib);
/* attribute entity type, used before ghost list established, so no ghost */
switch (mtype) {
case MVERTEX:
num = MESH_Num_Vertices(mesh);
break;
case MEDGE:
num = MESH_Num_Edges(mesh);
break;
case MFACE:
num = MESH_Num_Faces(mesh);
break;
case MREGION:
num = MESH_Num_Regions(mesh);
break;
default:
num = 0;
#ifdef DEBUG2
MSTK_Report("MESH_SendAttr()","Cannot send attributes on entity type MALLTYPE",MSTK_WARN);
#endif
return 0;
}
/* attribute index and global id */
list_info = (int *) malloc(num*sizeof(int));
/* attribute values */
int *list_value_int = NULL;
double *list_value_double = NULL;
if (att_type == INT)
list_value_int = (int *) malloc(num*ncomp*sizeof(int));
else
list_value_double = (double *) malloc(num*ncomp*sizeof(double));
/* collect data */
for(j = 0; j < num; j++) {
switch (mtype) {
case MVERTEX:
ment = MESH_Vertex(mesh,j);
break;
case MEDGE:
ment = MESH_Edge(mesh,j);
break;
case MFACE:
ment = MESH_Face(mesh,j);
break;
case MREGION:
ment = MESH_Region(mesh,j);
break;
default:
MSTK_Report("MESH_SendAttr()","Invalid entity type",MSTK_WARN);
return 0;
}
MEnt_Get_AttVal(ment,attrib,&ival,&rval,&pval);
list_info[j] = MEnt_GlobalID(ment);
if (att_type == INT)
list_value_int[j] = ival;
else {
if(ncomp == 1)
list_value_double[j] = rval;
if(ncomp > 1) {
rval_arr = (double *)pval;
for(k = 0; k < ncomp; k++)
list_value_double[ncomp*j+k] = rval_arr[k];
}
}
}
/* send entity global IDs */
MPI_Isend(list_info,num,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
/* send values */
if (att_type == INT) {
MPI_Isend(list_value_int,num*ncomp,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
}
else {
MPI_Isend(list_value_double,num*ncomp,MPI_DOUBLE,torank,torank,comm,
&mpirequest);
(*requests)[*numreq] = mpirequest;
}
(*numreq)++;
/* track the buffers used for sending so that they can be released later */
if (*maxptrs2free == 0) {
*maxptrs2free = 25;
*ptrs2free = (void **) malloc(*maxptrs2free*sizeof(void *));
*numptrs2free = 0;
}
else if (*maxptrs2free < (*numptrs2free) + 2) {
*maxptrs2free = 2*(*maxptrs2free) + 2;
*ptrs2free = (void **) realloc(*ptrs2free,(*maxptrs2free)*sizeof(void *));
}
(*ptrs2free)[(*numptrs2free)++] = list_info;
if (att_type == INT)
(*ptrs2free)[(*numptrs2free)++] = list_value_int;
else
(*ptrs2free)[(*numptrs2free)++] = list_value_double;
return 1;
}