int MESH_Renumber_GlobalIDs(Mesh_ptr mesh, MType mtype, int method,
int *preassigned_gids, MSTK_Comm comm) {
if (preassigned_gids) {
MSTK_Report("MESH_Renumber_GlobalIDs",
"Renumbering according to pre-assigned numbering not yet implemented",
MSTK_ERROR);
} else {
if (MESH_Num_Vertices(mesh) && (mtype == MVERTEX || mtype == MALLTYPE))
MESH_Renumber_EntityGlobalIDs(mesh, MVERTEX, method, NULL, comm);
if (MESH_Num_Edges(mesh) && (mtype == MEDGE || mtype == MALLTYPE))
MESH_Renumber_EntityGlobalIDs(mesh, MEDGE, method, NULL, comm);
if (MESH_Num_Faces(mesh) && (mtype == MFACE || mtype == MALLTYPE))
MESH_Renumber_EntityGlobalIDs(mesh, MFACE, method, NULL, comm);
if (MESH_Num_Regions(mesh) && (mtype == MREGION || mtype == MALLTYPE))
MESH_Renumber_EntityGlobalIDs(mesh, MREGION, method, NULL, comm);
// Ghost entity management in the mesh relies on ghost lists
// being sorted according to their global ID
MESH_Sort_GhostLists(mesh, compareGlobalID);
MESH_Sort_GlobalIDSearch_Lists(mesh);
}
return 1;
}
int MESH_ConcatSubMesh_Face(Mesh_ptr mesh, int num, Mesh_ptr *submeshes) {
int nfv, nfe, i, j, k, ival;
MVertex_ptr mv, new_mv, sub_mv;
MEdge_ptr me, new_me, sub_me;
MFace_ptr new_mf, sub_mf;
List_ptr mfverts, mfedges;
int add_face, idx, global_id, iloc, *loc;
double coor[3], rval;
void *pval;
Mesh_ptr submesh;
List_ptr parbndry_verts = List_New(10);
List_ptr parbndry_edges = List_New(10);
MEdge_ptr *fedges = (MEdge_ptr *) malloc(MAXPV2*sizeof(MEdge_ptr));
int *fedirs = (int *) malloc(MAXPV2*sizeof(int));
MAttrib_ptr parbndryatt = MAttrib_New(mesh, "on_parbndry", INT, MVERTEX);
/* collect edges and vertices on the partition boundary */
int num_parbndry_edges = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (ME_PType(me) != PINTERIOR) {
List_Add(parbndry_edges,me);
num_parbndry_edges++;
}
int num_parbndry_verts = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MV_PType(mv) != PINTERIOR) {
List_Add(parbndry_verts,mv);
MEnt_Set_AttVal(mv, parbndryatt, 1, 0.0, NULL);
num_parbndry_verts++;
}
/* sort based on global ID */
List_Sort(parbndry_edges,num_parbndry_edges,sizeof(MEdge_ptr),compareGlobalID);
List_Sort(parbndry_verts,num_parbndry_verts,sizeof(MVertex_ptr),compareGlobalID);
int *parbndry_vert_gids = (int *) malloc(num_parbndry_verts*sizeof(int));
int *parbndry_edge_gids = (int *) malloc(num_parbndry_edges*sizeof(int));
/* store them in array for binary search */
for (i = 0; i < num_parbndry_edges; i++) {
me = List_Entry(parbndry_edges,i);
parbndry_edge_gids[i] = ME_GlobalID(me);
}
for (i = 0; i < num_parbndry_verts; i++) {
mv = List_Entry(parbndry_verts,i);
parbndry_vert_gids[i] = MV_GlobalID(mv);
}
/* Make list of new edges and vertices which will be updated
with each mesh that is concatenated */
int max_vnew = 0, max_enew = 0;
for (i = 0; i < num; i++) {
max_vnew += MESH_Num_Vertices(submeshes[i]);
max_enew += MESH_Num_Edges(submeshes[i]);
}
int num_new_verts = 0, num_new_edges = 0;
int *new_vert_gids = (int *) malloc(max_vnew*sizeof(int));
int *new_edge_gids = (int *) malloc(max_enew*sizeof(int));
List_ptr new_verts = List_New(max_vnew);
List_ptr new_edges = List_New(max_enew);
/* Now process each mesh and add a layer of ghost elements from
each of them to the main partition */
for (i = 0; i < num; i++) {
submesh = submeshes[i];
MAttrib_ptr vidatt = MAttrib_New(submesh, "tempvid", POINTER, MVERTEX);
MAttrib_ptr eidatt = MAttrib_New(submesh, "tempeid", POINTER, MEDGE);
idx = 0;
while ((sub_mf = MESH_Next_Face(submesh, &idx))) {
add_face = 0;
/* Find matching vertices between the submesh and main mesh */
mfverts = MF_Vertices(sub_mf,1,0);
nfv = List_Num_Entries(mfverts);
for (j = 0; j < nfv; j++) {
sub_mv = List_Entry(mfverts,j);
/* Does the vertex have a known counterpart on the partition
* boundary of the main mesh? */
MEnt_Get_AttVal(sub_mv, vidatt, &ival, &rval, &mv);
if (mv) {
int on_parbndry=0;
MEnt_Get_AttVal(mv, parbndryatt, &on_parbndry, &rval, &pval);
if (on_parbndry)
add_face = 1;
} else {
/* Does the global ID of this vertex of the sub mesh face
* match the global ID of a partition boundary vertex in
* the main mesh? */
global_id = MV_GlobalID(sub_mv);
loc = (int *) bsearch(&global_id, parbndry_vert_gids, num_parbndry_verts, sizeof(int),
compareINT);
if (loc) { /* found a match */
add_face = 1;
iloc = loc - parbndry_vert_gids;
mv = List_Entry(parbndry_verts,iloc);
/* here set the ghost vertex property, only necessary when the input submeshes are not consistent */
if (MV_PType(mv) == PGHOST && MV_PType(sub_mv) != PGHOST) {
MV_Set_GEntDim(mv,MV_GEntDim(sub_mv));
MV_Set_GEntID(mv,MV_GEntID(sub_mv));
}
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, mv);
}
}
}
List_Delete(mfverts);
/* Find matching edges between the submesh and main mesh */
mfedges = MF_Edges(sub_mf,1,0);
nfe = List_Num_Entries(mfedges);
for (j = 0; j < nfe; j++) {
sub_me = List_Entry(mfedges,j);
/* Does the edge have a known counterpart on the partition
* boundary of the main mesh */
MEnt_Get_AttVal(sub_me, eidatt, &ival, &rval, &me);
if (!me) {
/* Does the global ID of this edge of the sub mesh face
* match the global ID of a partition boundary edge in the
* main mesh? */
global_id = ME_GlobalID(sub_me);
loc = (int *) bsearch(&global_id, parbndry_edge_gids, num_parbndry_edges, sizeof(int),
compareINT);
if (loc) {
iloc = loc - parbndry_edge_gids;
me = List_Entry(parbndry_edges,iloc);
/* here set the ghost edge property, only necessary when the input submeshes are not consistent */
if (ME_PType(me) == PGHOST && ME_PType(sub_me) != PGHOST) {
ME_Set_GEntDim(me,ME_GEntDim(sub_me));
ME_Set_GEntID(me,ME_GEntID(sub_me));
}
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, me);
}
}
}
if (!add_face) {
List_Delete(mfedges);
continue;
}
new_mf = MF_New(mesh); /* add face */
MF_Set_GEntDim(new_mf,MF_GEntDim(sub_mf));
MF_Set_GEntID(new_mf,MF_GEntID(sub_mf));
MF_Set_PType(new_mf,PGHOST);
MF_Set_MasterParID(new_mf,MF_MasterParID(sub_mf));
MF_Set_GlobalID(new_mf,MF_GlobalID(sub_mf));
nfe = List_Num_Entries(mfedges);
for (j = 0; j < nfe; j++) {
sub_me = List_Entry(mfedges,j);
global_id = ME_GlobalID(sub_me);
fedirs[j] = MF_EdgeDir_i(sub_mf,j) == 1 ? 1 : 0;
new_me = NULL;
MEnt_Get_AttVal(sub_me, eidatt, &ival, &rval, &new_me);
if (!new_me) {
/* search in the ghost layer if another edge with
* this global ID has been added */
loc = (int *) bsearch(&global_id, new_edge_gids, num_new_edges,
sizeof(int), compareINT);
if (loc) {
iloc = loc - new_edge_gids;
new_me = List_Entry(new_edges, iloc);
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, new_me);
}
}
if (new_me) {
if (MV_GlobalID(ME_Vertex(new_me,0)) != MV_GlobalID(ME_Vertex(sub_me,0)))
fedirs[j] = 1 - fedirs[j]; /* if the edge dir is not the same, reverse the edge dir */
} else { /* add a new edge to main mesh */
new_me = ME_New(mesh);
ME_Set_GEntDim(new_me,ME_GEntDim(sub_me));
ME_Set_GEntID(new_me,ME_GEntID(sub_me));
ME_Set_PType(new_me,PGHOST);
ME_Set_MasterParID(new_me,ME_MasterParID(sub_me));
ME_Set_GlobalID(new_me,ME_GlobalID(sub_me));
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, new_me);
List_Add(new_edges, new_me);
for (k = 0; k < 2; k++) {
sub_mv = ME_Vertex(sub_me,k);
global_id = MV_GlobalID(sub_mv);
new_mv = NULL;
MEnt_Get_AttVal(sub_mv, vidatt, &ival, &rval, &new_mv);
if (!new_mv) {
/* search in the ghost layer if another vertex with
* this global ID has been added */
loc = (int *) bsearch(&global_id, new_vert_gids, num_new_verts,
sizeof(int), compareINT);
if (loc) {
iloc = loc - new_vert_gids;
new_mv = List_Entry(new_verts, iloc);
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, new_mv);
}
}
if (!new_mv) { /* add a new vertex to main mesh */
new_mv = MV_New(mesh);
MV_Set_GEntDim(new_mv,MV_GEntDim(sub_mv));
MV_Set_GEntID(new_mv,MV_GEntID(sub_mv));
MV_Set_PType(new_mv,PGHOST);
MV_Set_MasterParID(new_mv,MV_MasterParID(sub_mv));
MV_Set_GlobalID(new_mv,MV_GlobalID(sub_mv));
MV_Coords(sub_mv,coor);
MV_Set_Coords(new_mv,coor);
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, new_mv);
List_Add(new_verts, new_mv);
}
ME_Set_Vertex(new_me,k,new_mv); /* set edge-vertex */
}
}
fedges[j] = new_me;
}
MF_Set_Edges(new_mf,nfe,fedges,fedirs); /* set face-edge */
List_Delete(mfedges);
}
idx = 0;
while ((sub_mv = MESH_Next_Vertex(submesh, &idx)))
MEnt_Rem_AttVal(sub_mv, vidatt);
MAttrib_Delete(vidatt);
idx = 0;
while ((sub_me = MESH_Next_Edge(submesh, &idx)))
MEnt_Rem_AttVal(sub_me, eidatt);
MAttrib_Delete(eidatt);
/* Sort the added entity lists by GlobalID */
num_new_edges = List_Num_Entries(new_edges);
List_Sort(new_edges, num_new_edges, sizeof(MEdge_ptr), compareGlobalID);
for (j = 0; j < num_new_edges; j++)
new_edge_gids[j] = ME_GlobalID(List_Entry(new_edges, j));
num_new_verts = List_Num_Entries(new_verts);
List_Sort(new_verts, num_new_verts, sizeof(MVertex_ptr), compareGlobalID);
for (j = 0; j < num_new_verts; j++)
new_vert_gids[j] = MV_GlobalID(List_Entry(new_verts, j));
}
idx = 0;
while ((mv = List_Next_Entry(parbndry_verts, &idx)))
MEnt_Rem_AttVal(mv, parbndryatt);
MAttrib_Delete(parbndryatt);
List_Delete(parbndry_edges);
List_Delete(parbndry_verts);
List_Delete(new_edges);
List_Delete(new_verts);
free(parbndry_vert_gids);
free(parbndry_edge_gids);
free(new_vert_gids);
free(new_edge_gids);
free(fedges);
free(fedirs);
return 1;
}
void MESH_Renumber(Mesh_ptr mesh, int renum_type, MType mtype) {
MVertex_ptr mv, v0=NULL;
MEdge_ptr me, e0=NULL;
MFace_ptr mf, f0=NULL;
MRegion_ptr mr, r0=NULL;
int idx, idx2, idx3;
int i, j;
int done;
MAttrib_ptr vidatt;
List_ptr vlist;
double xyz[3];
double rval;
int bandwidth, maxbandwidth1, maxbandwidth2;
double avebandwidth1, avebandwidth2;
void *pval;
if (renum_type == 0) {
if (mtype == MVERTEX || mtype == MALLTYPE) {
int nv = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MV_Set_ID(mv,++nv);
}
if (mtype == MEDGE || mtype == MALLTYPE) {
int ne = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
ME_Set_ID(me,++ne);
}
if (mtype == MFACE || mtype == MALLTYPE) {
int nf = 0;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MF_Set_ID(mf,++nf);
}
if (mtype == MREGION || mtype == MALLTYPE) {
int nr = 0;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MR_Set_ID(mr,++nr);
}
}
else if (renum_type == 1) {
double minx, miny, minz;
int minid, maxid;
int *nadj, *newmap, *adj, *offset, nconn;
int nalloc, depth, maxwidth;
#ifdef MSTK_USE_MARKERS
int mkid = MSTK_GetMarker();
#else
MAttrib_ptr mkatt = MAttrib_New(mesh, "mkatt", INT, MALLTYPE);
#endif
if (mtype == MVERTEX || mtype == MALLTYPE) {
int nv = MESH_Num_Vertices(mesh);
/* Compute a graph of vertex connections across elements (faces
for surface meshes, regions for solid meshes */
/* Start with the vertex in the lower leftmost corner */
minx = miny = minz = 1.0e+12;
v0 = NULL;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MV_Coords(mv,xyz);
if (xyz[0] <= minx && xyz[1] <= miny && xyz[2] <= minz) {
minx = xyz[0];
miny = xyz[1];
minz = xyz[2];
v0 = mv;
}
}
nadj = (int *) malloc(nv*sizeof(int));
nalloc = nv*5;
adj = (int *) malloc(nalloc*sizeof(int));
if (!MESH_Num_Regions(mesh)) {
int nentries = 0;
i = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
List_ptr vfaces, adjvlist;
MFace_ptr vf;
MVertex_ptr adjv;
adjvlist = List_New(0);
vfaces = MV_Faces(mv);
idx2 = 0;
while ((vf = List_Next_Entry(vfaces,&idx2))) {
List_ptr fverts = MF_Vertices(vf,1,0);
idx3 = 0;
while ((adjv = List_Next_Entry(fverts,&idx3))) {
if (adjv != mv) {
int vmarked;
#ifdef MSTK_USE_MARKERS
vmarked = MEnt_IsMarked(adjv,mkid);
#else
MEnt_Get_AttVal(adjv, mkatt, &vmarked, &rval, &pval);
#endif
if (!vmarked) {
List_Add(adjvlist,adjv);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjv,mkid);
#else
MEnt_Set_AttVal(adjv, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(fverts);
}
List_Delete(vfaces);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjvlist,mkid);
#endif
nadj[i] = List_Num_Entries(adjvlist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjv = List_Next_Entry(adjvlist,&idx2)))
adj[nentries++] = MV_ID(adjv)-1;
List_Delete(adjvlist);
i++;
}
}
else {
int nentries = 0;
i = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
List_ptr vregions, adjvlist;
MRegion_ptr vr;
MVertex_ptr adjv;
adjvlist = List_New(0);
vregions = MV_Regions(mv);
idx2 = 0;
while ((vr = List_Next_Entry(vregions,&idx2))) {
List_ptr rverts = MR_Vertices(vr);
idx3 = 0;
while ((adjv = List_Next_Entry(rverts,&idx3))) {
if (adjv != mv) {
int vmarked;
#ifdef MSTK_USE_MARKERS
vmarked = MEnt_IsMarked(adjv,mkid);
#else
MEnt_Get_AttVal(adjv, mkatt, &vmarked, &rval, &pval);
#endif
if (!vmarked) {
List_Add(adjvlist,adjv);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjv,mkid);
#else
MEnt_Set_AttVal(adjv, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(rverts);
}
List_Delete(vregions);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjvlist,mkid);
#endif
nadj[i] = List_Num_Entries(adjvlist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjv = List_Next_Entry(adjvlist,&idx2)))
adj[nentries++] = MV_ID(adjv)-1;
List_Delete(adjvlist);
i++;
}
}
/* Compute offsets into adj array */
offset = (int *) malloc(nv*sizeof(int));
offset[0] = 0;
for (i = 1; i < nv; i++)
offset[i] = offset[i-1] + nadj[i-1];
/* Compute maximum bandwidth before renumbering */
maxbandwidth1 = 0;
avebandwidth1 = 0;
for (i = 0; i < nv; i++) {
int off = offset[i];
int curid = i;
for (j = 0; j < nadj[i]; j++) {
int adjid = adj[off+j];
int diff = abs(adjid-curid);
maxbandwidth1 = (diff > maxbandwidth1) ? diff : maxbandwidth1;
avebandwidth1 += diff;
nconn++;
}
}
nconn = offset[nv-1]+nadj[nv-1];
avebandwidth1 /= nconn;
fprintf(stderr,
"Ave vertex ID difference on elements before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max vertex ID difference on elements before renumbering: %-d\n",
maxbandwidth1);
fprintf(stderr,"\n");
newmap = (int *) malloc(nv*sizeof(int));
Graph_Renumber_GPS(nv, MV_ID(v0)-1, nadj, adj, newmap, &depth, &maxwidth);
/* Compute bandwidth after renumbering */
maxbandwidth2 = 0;
avebandwidth2 = 0;
for (i = 0; i < nv; i++) {
int off = offset[i];
int curid = newmap[i];
for (j = 0; j < nadj[i]; j++) {
int adjid = newmap[adj[off+j]];
int diff = abs(adjid-curid);
maxbandwidth2 = (diff > maxbandwidth2) ? diff : maxbandwidth2;
avebandwidth2 += diff;
nconn++;
}
}
nconn = offset[nv-1]+nadj[nv-1];
avebandwidth2 /= nconn;
if (maxbandwidth2 < maxbandwidth1 && avebandwidth2 < avebandwidth1) {
/* Renumber */
idx = 0; i = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MV_Set_ID(mv,newmap[i]+1);
i++;
}
fprintf(stderr,
"Ave vertex ID difference on elements after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max vertex ID difference on elements after renumbering: %-d\n",
maxbandwidth2);
}
else {
nv = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MV_Set_ID(mv,++nv);
fprintf(stderr,"Bandwidth did not improve. Keeping old numbering with gaps eliminated\n");
}
fprintf(stderr,"\n\n\n");
free(nadj);
free(adj);
free(offset);
free(newmap);
}
/* Reorder edges according to a breadth first algorithm applied to
edges (differs from RCM in that it does not add adjacent nodes
in ascending order of their valence) */
if (mtype == MEDGE || mtype == MALLTYPE) {
int ne = MESH_Num_Edges(mesh);
MEdge_ptr ve;
List_ptr elist;
/************************* renumbering code ****************************/
ne = MESH_Num_Edges(mesh);
if (mtype == MALLTYPE) {
/* RCM algorithm already applied on the vertices. Use an edge
connected to the starting vertex as the first edge */
List_ptr vedges = MV_Edges(v0);
e0 = List_Entry(vedges,0);
List_Delete(vedges);
}
else {
/* Find the edge whose mid point is a minimum point */
minx = miny = minz = 1.0e+12;
e0 = NULL;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
double exyz[2][3];
MV_Coords(ME_Vertex(me,0),exyz[0]);
MV_Coords(ME_Vertex(me,1),exyz[1]);
xyz[0] = (exyz[0][0]+exyz[1][0])/2.0;
xyz[1] = (exyz[0][1]+exyz[1][1])/2.0;
xyz[2] = (exyz[0][2]+exyz[1][2])/2.0;
if (xyz[0] < minx && xyz[1] < miny && xyz[2] < minz) {
minx = xyz[0];
miny = xyz[1];
minz = xyz[2];
e0 = me;
}
}
}
nadj = (int *) malloc(ne*sizeof(int));
nalloc = ne*5;
adj = (int *) malloc(nalloc*sizeof(int));
if (!MESH_Num_Regions(mesh)) {
int nentries = 0;
i = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
List_ptr efaces, adjelist;
MFace_ptr ef;
MEdge_ptr adje;
adjelist = List_New(0);
efaces = ME_Faces(me);
idx2 = 0;
while ((ef = List_Next_Entry(efaces,&idx2))) {
List_ptr fedges = MF_Edges(ef,1,0);
idx3 = 0;
while ((adje = List_Next_Entry(fedges,&idx3))) {
if (adje != me) {
int emarked;
#ifdef MSTK_USE_MARKERS
emarked = MEnt_IsMarked(adje,mkid);
#else
MEnt_Get_AttVal(adje, mkatt, &emarked, &rval, &pval);
#endif
if (!emarked) {
List_Add(adjelist,adje);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adje,mkid);
#else
MEnt_Set_AttVal(adje, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(fedges);
}
List_Delete(efaces);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjelist,mkid);
#endif
nadj[i] = List_Num_Entries(adjelist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adje = List_Next_Entry(adjelist,&idx2)))
adj[nentries++] = ME_ID(adje)-1;
List_Delete(adjelist);
i++;
}
}
else {
int nentries = 0;
i = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
List_ptr eregions, adjelist;
MRegion_ptr er;
MEdge_ptr adje;
adjelist = List_New(0);
eregions = ME_Regions(me);
idx2 = 0;
while ((er = List_Next_Entry(eregions,&idx2))) {
List_ptr redges = MR_Edges(er);
idx3 = 0;
while ((adje = List_Next_Entry(redges,&idx3))) {
if (adje != me) {
int emarked;
#ifdef MSTK_USE_MARKERS
emarked = MEnt_IsMarked(adje,mkid);
#else
MEnt_Get_AttVal(adje, mkatt, &emarked, &rval, &pval);
#endif
if (!emarked) {
List_Add(adjelist,adje);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adje,mkid);
#else
MEnt_Set_AttVal(adje, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(redges);
}
List_Delete(eregions);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjelist,mkid);
#endif
nadj[i] = List_Num_Entries(adjelist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adje = List_Next_Entry(adjelist,&idx2)))
adj[nentries++] = ME_ID(adje)-1;
List_Delete(adjelist);
i++;
}
}
/* Compute offsets into adj array */
offset = (int *) malloc(ne*sizeof(int));
offset[0] = 0;
for (i = 1; i < ne; i++)
offset[i] = offset[i-1] + nadj[i-1];
/* Compute maximum bandwidth before renumbering */
maxbandwidth1 = 0;
avebandwidth1 = 0;
for (i = 0; i < ne; i++) {
int off = offset[i];
int curid = i;
for (j = 0; j < nadj[i]; j++) {
int adjid = adj[off+j];
int diff = abs(adjid-curid);
maxbandwidth1 = (diff > maxbandwidth1) ? diff : maxbandwidth1;
avebandwidth1 += diff;
nconn++;
}
}
nconn = offset[ne-1]+nadj[ne-1];
avebandwidth1 /= nconn;
fprintf(stderr,
"Ave edge ID difference on elements before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max edge ID difference on elements before renumbering: %-d\n",
maxbandwidth1);
fprintf(stderr,"\n");
/* Call Graph Renumbering algorithm */
newmap = (int *) malloc(ne*sizeof(int));
Graph_Renumber_GPS(ne, ME_ID(e0)-1, nadj, adj, newmap, &depth, &maxwidth);
/* Compute bandwidth after renumbering */
maxbandwidth2 = 0;
avebandwidth2 = 0;
for (i = 0; i < ne; i++) {
int off = offset[i];
int curid = newmap[i];
for (j = 0; j < nadj[i]; j++) {
int adjid = newmap[adj[off+j]];
int diff = abs(adjid-curid);
maxbandwidth2 = (diff > maxbandwidth2) ? diff : maxbandwidth2;
avebandwidth2 += diff;
nconn++;
}
}
nconn = offset[ne-1]+nadj[ne-1];
avebandwidth2 /= nconn;
if (maxbandwidth2 < maxbandwidth1 && avebandwidth2 < avebandwidth1) {
/* Renumber */
idx = 0; i = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
ME_Set_ID(me,newmap[i]+1);
i++;
}
fprintf(stderr,
"Ave edge ID difference on elements after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max edge ID difference on elements after renumbering: %-d\n",
maxbandwidth2);
}
else {
ne = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
ME_Set_ID(me,++ne);
fprintf(stderr,"Bandwidth did not improve. Keeping old numbering with gaps eliminated\n");
}
fprintf(stderr,"\n\n\n");
free(nadj);
free(adj);
free(offset);
free(newmap);
}
/* Reorder faces according to a breadth first algorithm applied to
edges (differs from RCM in that it does not add adjacent graph nodes
in ascending order of their valence) */
if (mtype == MFACE || mtype == MALLTYPE) {
int nf = MESH_Num_Faces(mesh);
if (mtype == MALLTYPE) {
/* RCM algorithm already applied on the vertices. Use an edge
connected to the starting vertex as the first edge */
List_ptr vfaces = MV_Faces(v0);
f0 = List_Entry(vfaces,0);
List_Delete(vfaces);
}
else {
/* Find the face whose mid point is a minimum point */
minx = miny = minz = 1.0e+12;
f0 = NULL;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
double fxyz[MAXPV2][3];
int nfv;
MF_Coords(mf,&nfv,fxyz);
xyz[0] = fxyz[0][0];
xyz[1] = fxyz[0][1];
xyz[2] = fxyz[0][2];
for (i = 1; i < nfv; i++) {
xyz[0] += fxyz[i][0];
xyz[1] += fxyz[i][1];
xyz[2] += fxyz[i][2];
}
xyz[0] /= nfv; xyz[1] /= nfv; xyz[2] /= nfv;
if (xyz[0] < minx && xyz[1] < miny && xyz[2] < minz) {
minx = xyz[0];
miny = xyz[1];
minz = xyz[2];
f0 = mf;
}
}
}
nadj = (int *) malloc(nf*sizeof(int));
nalloc = nf*5;
adj = (int *) malloc(nalloc*sizeof(int));
if (!MESH_Num_Regions(mesh)) {
int nentries = 0;
i = 0;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
List_ptr vfaces, fverts, adjflist;
MFace_ptr vf, adjf;
MVertex_ptr fv;
adjflist = List_New(0);
fverts = MF_Vertices(mf,1,0);
idx2 = 0;
while ((fv = List_Next_Entry(fverts,&idx2))) {
List_ptr vfaces = MV_Faces(fv);
idx3 = 0;
while ((adjf = List_Next_Entry(vfaces,&idx3))) {
if (adjf != mf) {
int fmarked;
#ifdef MSTK_USE_MARKERS
fmarked = MEnt_IsMarked(adjf,mkid);
#else
MEnt_Get_AttVal(adjf, mkatt, &fmarked, &rval, &pval);
#endif
if (fmarked) {
List_Add(adjflist,adjf);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjf,mkid);
#else
MEnt_Set_AttVal(adjf, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(vfaces);
}
List_Delete(fverts);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjflist,mkid);
#endif
nadj[i] = List_Num_Entries(adjflist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjf = List_Next_Entry(adjflist,&idx2)))
adj[nentries++] = MF_ID(adjf)-1;
List_Delete(adjflist);
i++;
}
}
else {
int nentries = 0;
i = 0;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
List_ptr fregions, adjflist;
MRegion_ptr fr;
MFace_ptr adjf;
adjflist = List_New(0);
fregions = MF_Regions(mf);
idx2 = 0;
while ((fr = List_Next_Entry(fregions,&idx2))) {
List_ptr rfaces = MR_Faces(fr);
idx3 = 0;
while ((adjf = List_Next_Entry(rfaces,&idx3))) {
if (adjf != mf) {
int fmarked;
#ifdef MSTK_USE_MARKERS
fmarked = MEnt_IsMarked(adjf,mkid);
#else
MEnt_Get_AttVal(adjf, mkatt, &fmarked, &rval, &pval);
#endif
if (fmarked) {
List_Add(adjflist,adjf);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjf,mkid);
#else
MEnt_Set_AttVal(adjf, mkatt, 1, 0.0, NULL);
#endif
}
}
}
List_Delete(rfaces);
}
List_Delete(fregions);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjflist,mkid);
#endif
nadj[i] = List_Num_Entries(adjflist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjf = List_Next_Entry(adjflist,&idx2)))
adj[nentries++] = MF_ID(adjf)-1;
List_Delete(adjflist);
i++;
}
}
/* Compute offsets into adj array */
offset = (int *) malloc(nf*sizeof(int));
offset[0] = 0;
for (i = 1; i < nf; i++)
offset[i] = offset[i-1] + nadj[i-1];
/* Compute maximum bandwidth before renumbering */
maxbandwidth1 = 0;
avebandwidth1 = 0;
for (i = 0; i < nf; i++) {
int off = offset[i];
int curid = i;
for (j = 0; j < nadj[i]; j++) {
int adjid = adj[off+j];
int diff = abs(adjid-curid);
maxbandwidth1 = (diff > maxbandwidth1) ? diff : maxbandwidth1;
avebandwidth1 += diff;
nconn++;
}
}
nconn = offset[nf-1]+nadj[nf-1];
avebandwidth1 /= nconn;
if (MESH_Num_Regions(mesh)) {
fprintf(stderr,
"Ave face ID difference on elements before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max face ID difference on elements before renumbering: %-d\n",
maxbandwidth1);
}
else {
fprintf(stderr,
"Ave face ID difference before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max face ID difference before renumbering: %-d\n",
maxbandwidth1);
}
fprintf(stderr,"\n");
/* Call Graph Renumbering algorithm */
newmap = (int *) malloc(nf*sizeof(int));
Graph_Renumber_GPS(nf, MF_ID(f0)-1, nadj, adj, newmap, &depth, &maxwidth);
/* Compute bandwidth after renumbering */
maxbandwidth2 = 0;
avebandwidth2 = 0;
for (i = 0; i < nf; i++) {
int off = offset[i];
int curid = newmap[i];
for (j = 0; j < nadj[i]; j++) {
int adjid = newmap[adj[off+j]];
int diff = abs(adjid-curid);
maxbandwidth2 = (diff > maxbandwidth2) ? diff : maxbandwidth2;
avebandwidth2 += diff;
nconn++;
}
}
nconn = offset[nf-1]+nadj[nf-1];
avebandwidth2 /= nconn;
if (maxbandwidth2 < maxbandwidth1 && avebandwidth2 < avebandwidth1) {
/* Renumber */
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
MF_Set_ID(mf,newmap[i]+1);
i++;
}
if (MESH_Num_Regions(mesh)) {
fprintf(stderr,
"Ave face ID difference on elements after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max face ID difference on elements after renumbering: %-d\n",
maxbandwidth2);
}
else {
fprintf(stderr,
"Ave face ID difference after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max face ID difference after renumbering: %-d\n",
maxbandwidth2);
}
}
else {
nf = 0;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MF_Set_ID(mf,++nf);
fprintf(stderr,"Bandwidth did not improve. Keeping old numbering with gaps eliminated\n");
}
fprintf(stderr,"\n\n\n");
free(nadj);
free(adj);
free(offset);
free(newmap);
}
if (mtype == MREGION || mtype == MALLTYPE) {
int nr = MESH_Num_Regions(mesh);
if (nr) {
if (mtype == MALLTYPE) {
/* Renumbering algorithm already applied on the vertices. Use
a region connected to the starting vertex as the first
region */
List_ptr vregions = MV_Regions(v0);
r0 = List_Entry(vregions,0);
List_Delete(vregions);
}
else {
/* Find the region whose center point is a minimum point */
minx = miny = minz = 1.0e+12;
r0 = NULL;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
double rxyz[MAXPV3][3];
int nrv;
MR_Coords(mr,&nrv,rxyz);
xyz[0] = rxyz[0][0];
xyz[1] = rxyz[0][1];
xyz[2] = rxyz[0][2];
for (i = 1; i < nrv; i++) {
xyz[0] += rxyz[i][0];
xyz[1] += rxyz[i][1];
xyz[2] += rxyz[i][2];
}
xyz[0] /= nrv; xyz[1] /= nrv; xyz[2] /= nrv;
if (xyz[0] < minx && xyz[1] < miny && xyz[2] < minz) {
minx = xyz[0];
miny = xyz[1];
minz = xyz[2];
r0 = mr;
}
}
}
nadj = (int *) malloc(nr*sizeof(int));
nalloc = nr*5;
adj = (int *) malloc(nalloc*sizeof(int));
int nentries = 0;
i = 0;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
List_ptr vregions, rverts, adjrlist;
MRegion_ptr vr, adjr;
MVertex_ptr rv;
adjrlist = List_New(0);
rverts = MR_Vertices(mr);
idx2 = 0;
while ((rv = List_Next_Entry(rverts,&idx2))) {
List_ptr vregions = MV_Regions(rv);
idx3 = 0;
while ((adjr = List_Next_Entry(vregions,&idx3))) {
if (adjr != mr) {
int rmarked;
#ifdef MSTK_USE_MARKERS
rmarked = MEnt_IsMarked(adjr,mkid);
#else
MEnt_Get_AttVal(adjr, mkatt, &rmarked, &rval, &pval);
#endif
List_Add(adjrlist,adjr);
#ifdef MSTK_USE_MARKERS
MEnt_Mark(adjr,mkid);
#else
MEnt_Set_AttVal(adjr, mkatt, 1, 0.0, NULL);
#endif
}
}
List_Delete(vregions);
}
List_Delete(rverts);
#ifdef MSTK_USE_MARKERS
List_Unmark(adjrlist,mkid);
#endif
nadj[i] = List_Num_Entries(adjrlist);
if (nentries+nadj[i] > nalloc) {
nalloc *= 2;
adj = (int *) realloc(adj,nalloc*sizeof(int));
}
idx2 = 0;
while ((adjr = List_Next_Entry(adjrlist,&idx2)))
adj[nentries++] = MR_ID(adjr)-1;
List_Delete(adjrlist);
i++;
}
/* Compute offsets into adj array */
offset = (int *) malloc(nr*sizeof(int));
offset[0] = 0;
for (i = 1; i < nr; i++)
offset[i] = offset[i-1] + nadj[i-1];
/* Compute maximum bandwidth before renumbering */
maxbandwidth1 = 0;
avebandwidth1 = 0;
for (i = 0; i < nr; i++) {
int off = offset[i];
int curid = i;
for (j = 0; j < nadj[i]; j++) {
int adjid = adj[off+j];
int diff = abs(adjid-curid);
maxbandwidth1 = (diff > maxbandwidth1) ? diff : maxbandwidth1;
avebandwidth1 += diff;
nconn++;
}
}
nconn = offset[nr-1]+nadj[nr-1];
avebandwidth1 /= nconn;
fprintf(stderr,
"Ave region ID difference before renumbering: %-lf\n",
avebandwidth1);
fprintf(stderr,
"Max region ID difference before renumbering: %-d\n",
maxbandwidth1);
fprintf(stderr,"\n");
/* Call Graph Renumbering algorithm */
newmap = (int *) malloc(nr*sizeof(int));
Graph_Renumber_GPS(nr, MR_ID(r0)-1, nadj, adj, newmap, &depth, &maxwidth);
/* Compute bandwidth after renumbering */
maxbandwidth2 = 0;
avebandwidth2 = 0;
for (i = 0; i < nr; i++) {
int off = offset[i];
int curid = newmap[i];
for (j = 0; j < nadj[i]; j++) {
int adjid = newmap[adj[off+j]];
int diff = abs(adjid-curid);
maxbandwidth2 = (diff > maxbandwidth2) ? diff : maxbandwidth2;
avebandwidth2 += diff;
nconn++;
}
}
nconn = offset[nr-1]+nadj[nr-1];
avebandwidth2 /= nconn;
if (maxbandwidth2 < maxbandwidth1 && avebandwidth2 < avebandwidth1) {
/* Renumber */
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
MR_Set_ID(mr,newmap[i]+1);
i++;
}
fprintf(stderr,
"Ave region ID difference after renumbering: %-lf\n",
avebandwidth2);
fprintf(stderr,
"Max region ID difference after renumbering: %-d\n",
maxbandwidth2);
}
else {
nr = 0;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MR_Set_ID(mr,++nr);
fprintf(stderr,"Bandwidth did not improve. Keeping old numbering with gaps eliminated\n");
}
fprintf(stderr,"\n\n\n");
free(nadj);
free(adj);
free(offset);
free(newmap);
}
}
#ifdef MSTK_USE_MARKERS
MSTK_FreeMarker(mkid);
#endif
}
vidatt = MAttrib_New(mesh,"vidrcm",INT,MVERTEX);
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MEnt_Set_AttVal(mv,vidatt,MV_ID(mv),0.0,NULL);
}
/* We have to reset the max IDs stored in the mesh so that we can correctly
assign IDs to new entities */
MESH_Reset_Cached_MaxIDs(mesh);
return;
}
int MESH_PartitionWithZoltan(Mesh_ptr mesh, int nparts, int **part, int noptions,
char **options, MSTK_Comm comm) {
MEdge_ptr fedge;
MFace_ptr mf, oppf, rface;
MRegion_ptr mr, oppr;
List_ptr fedges, efaces, rfaces, fregions;
int i, j, k, id;
int nv, ne, nf, nr=0, nfe, nef, nfr, nrf, idx, idx2;
int numflag, nedgecut, ipos;
int wtflag;
int rc;
float ver;
struct Zoltan_Struct *zz;
GRAPH_DATA graph;
int changes, numGidEntries, numLidEntries, numImport, numExport;
ZOLTAN_ID_PTR importGlobalGids, importLocalGids, exportGlobalGids, exportLocalGids;
int *importProcs, *importToPart, *exportProcs, *exportToPart;
int rank;
MPI_Comm_rank(comm,&rank);
rc = Zoltan_Initialize(0, NULL, &ver);
if (rc != ZOLTAN_OK){
MSTK_Report("MESH_PartitionWithZoltan","Could not initialize Zoltan",MSTK_FATAL);
MPI_Finalize();
exit(0);
}
/******************************************************************
** Create a Zoltan library structure for this instance of partition
********************************************************************/
zz = Zoltan_Create(comm);
/*****************************************************************
** Figure out partitioning method
*****************************************************************/
char partition_method_str[32];
strcpy(partition_method_str,"RCB"); /* Default - Recursive Coordinate Bisection */
if (noptions) {
for (i = 0; i < noptions; i++) {
if (strncmp(options[i],"LB_PARTITION",12) == 0) {
char *result = NULL, instring[256];
strcpy(instring,options[i]);
result = strtok(instring,"=");
result = strtok(NULL," ");
strcpy(partition_method_str,result);
}
}
}
if (rank == 0) {
char mesg[256];
sprintf(mesg,"Using partitioning method %s for ZOLTAN\n",partition_method_str);
MSTK_Report("MESH_PartitionWithZoltan",mesg,MSTK_MESG);
}
/* General parameters for Zoltan */
Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0");
Zoltan_Set_Param(zz, "LB_METHOD", partition_method_str);
Zoltan_Set_Param(zz, "LB_APPROACH", "PARTITION");
Zoltan_Set_Param(zz, "NUM_GID_ENTRIES", "1");
Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "1");
Zoltan_Set_Param(zz, "RETURN_LISTS", "ALL");
graph.numMyNodes = 0;
graph.numAllNbors = 0;
graph.nodeGID = NULL;
graph.nodeCoords = NULL;
graph.nborIndex = NULL;
graph.nborGID = NULL;
graph.nborProc = NULL;
if (strcmp(partition_method_str,"RCB") == 0) {
if (rank == 0) {
nr = MESH_Num_Regions(mesh);
nf = MESH_Num_Faces(mesh);
if (!nf && !nr)
MSTK_Report("MESH_PartitionWithZoltan","Cannot partition wire meshes",
MSTK_FATAL);
if (nr == 0) { /* Surface or planar mesh */
int ndim = 2; /* assume mesh is planar */
idx = 0;
MVertex_ptr mv;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
double vxyz[3];
MV_Coords(mv,vxyz);
if (vxyz[2] != 0.0) {
ndim = 3; /* non-planar or planar with non-zero z */
break;
}
}
NDIM_4_ZOLTAN = ndim-1; /* ignore last dimension to avoid partitioning in that dimension */
graph.numMyNodes = nf;
graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nf);
graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nf);
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
double fxyz[MAXPV2][3], cen[3];
int nfv;
MF_Coords(mf,&nfv,fxyz);
cen[0] = cen[1] = cen[2] = 0.0;
for (j = 0; j < nfv; j++)
for (k = 0; k < NDIM_4_ZOLTAN; k++)
cen[k] += fxyz[j][k];
for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nfv;
id = MF_ID(mf);
graph.nodeGID[id-1] = id;
memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double));
}
}
else { /* Volume mesh */
int ndim = 3;
NDIM_4_ZOLTAN = ndim-1; /* ignore last dimension to avoid partitioning in that dimension */
graph.numMyNodes = nr;
graph.nodeGID = (ZOLTAN_ID_TYPE *) malloc(sizeof(ZOLTAN_ID_TYPE) * nr);
graph.nodeCoords = (double *) malloc(sizeof(double) * NDIM_4_ZOLTAN * nr);
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
double rxyz[MAXPV3][3], cen[3];
int nrv;
MR_Coords(mr,&nrv,rxyz);
cen[0] = cen[1] = cen[2] = 0.0;
for (j = 0; j < nrv; j++)
for (k = 0; k < NDIM_4_ZOLTAN; k++)
cen[k] += rxyz[j][k];
for (k = 0; k < NDIM_4_ZOLTAN; k++) cen[k] /= nrv;
for (k = 0; k < NDIM_4_ZOLTAN; k++)
if (fabs(cen[k]) < 1.0e-10) cen[k] = 0.0;
id = MR_ID(mr);
graph.nodeGID[id-1] = id;
memcpy(&(graph.nodeCoords[NDIM_4_ZOLTAN*(id-1)]),cen,NDIM_4_ZOLTAN*sizeof(double));
}
}
}
MPI_Bcast(&NDIM_4_ZOLTAN,1,MPI_INT,0,comm);
/* Set some default values */
Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS","1");
// Zoltan_Set_Param(zz, "AVERAGE_CUTS", "1");
if (noptions > 1) {
for (i = 1; i < noptions; i++) {
char *paramstr = NULL, *valuestr = NULL, instring[256];
strcpy(instring,options[i]);
paramstr = strtok(instring,"=");
valuestr = strtok(NULL," ");
Zoltan_Set_Param(zz,paramstr,valuestr);
}
}
/* Query functions - defined in simpleQueries.h */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph);
Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph);
Zoltan_Set_Num_Geom_Fn(zz, get_num_dimensions_reduced, &graph); /* reduced dimensions */
Zoltan_Set_Geom_Multi_Fn(zz, get_element_centers_reduced, &graph); /* reduced dimension centers */
}
else if (strcmp(partition_method_str,"GRAPH") == 0) {
if(rank == 0) {
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
ipos = 0;
/* build nodes and neighbors list, similar as in partition with metis
Assign processor 0 the whole mesh, assign other processors a NULL mesh */
if (nr == 0) {
if (nf == 0) {
MSTK_Report("MESH_PartitionWithZoltan",
"Cannot partition wire meshes with Zoltan",MSTK_FATAL);
exit(-1);
}
graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nf);
graph.nborIndex = (int *)malloc(sizeof(int) * (nf + 1));
graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*ne);
graph.nborProc = (int *)malloc(sizeof(int) * 2*ne);
graph.nborIndex[0] = 0;
/* Surface mesh */
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
graph.nodeGID[i] = MF_ID(mf);
fedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(fedges);
idx2 = 0;
while ((fedge = List_Next_Entry(fedges,&idx2))) {
efaces = ME_Faces(fedge);
nef = List_Num_Entries(efaces);
if (nef == 1) {
continue; /* boundary edge; nothing to do */
} else {
int j;
for (j = 0; j < nef; j++) {
oppf = List_Entry(efaces,j);
if (oppf == mf) {
graph.nborGID[ipos] = MF_ID(oppf);
/* initially set all nodes on processor 0 */
graph.nborProc[ipos] = 0;
ipos++;
}
}
}
List_Delete(efaces);
}
List_Delete(fedges);
i++;
graph.nborIndex[i] = ipos;
}
graph.numMyNodes = i;
graph.numAllNbors = ipos;
}
else {
graph.nodeGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * nr);
graph.nborIndex = (int *)malloc(sizeof(int) * (nr + 1));
graph.nborGID = (ZOLTAN_ID_TYPE *)malloc(sizeof(ZOLTAN_ID_TYPE) * 2*nf);
graph.nborProc = (int *)malloc(sizeof(int) * 2*nf);
graph.nborIndex[0] = 0;
/* Volume mesh */
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
graph.nodeGID[i] = MR_ID(mr);
rfaces = MR_Faces(mr);
nrf = List_Num_Entries(rfaces);
idx2 = 0;
while ((rface = List_Next_Entry(rfaces,&idx2))) {
fregions = MF_Regions(rface);
nfr = List_Num_Entries(fregions);
if (nfr > 1) {
oppr = List_Entry(fregions,0);
if (oppr == mr)
oppr = List_Entry(fregions,1);
graph.nborGID[ipos] = MR_ID(oppr);
/* initially set all nodes on processor 0 */
graph.nborProc[ipos] = 0;
ipos++;
}
List_Delete(fregions);
}
List_Delete(rfaces);
i++;
graph.nborIndex[i] = ipos;
}
graph.numMyNodes = i;
graph.numAllNbors = ipos;
}
}
/* Graph parameters */
/* Zoltan_Set_Param(zz, "CHECK_GRAPH", "2"); */
Zoltan_Set_Param(zz, "PHG_EDGE_SIZE_THRESHOLD", ".35"); /* 0-remove all, 1-remove none */
/* Query functions - defined in simpleQueries.h */
Zoltan_Set_Num_Obj_Fn(zz, get_number_of_nodes, &graph);
Zoltan_Set_Obj_List_Fn(zz, get_node_list, &graph);
Zoltan_Set_Num_Edges_Multi_Fn(zz, get_num_edges_list, &graph);
Zoltan_Set_Edge_List_Multi_Fn(zz, get_edge_list, &graph);
}
/* Partition the graph */
/******************************************************************
** Zoltan can now partition the graph.
** We assume the number of partitions is
** equal to the number of processes. Process rank 0 will own
** partition 0, process rank 1 will own partition 1, and so on.
******************************************************************/
rc = Zoltan_LB_Partition(zz, /* input (all remaining fields are output) */
&changes, /* 1 if partitioning was changed, 0 otherwise */
&numGidEntries, /* Number of integers used for a global ID */
&numLidEntries, /* Number of integers used for a local ID */
&numImport, /* Number of nodes to be sent to me */
&importGlobalGids, /* Global IDs of nodes to be sent to me */
&importLocalGids, /* Local IDs of nodes to be sent to me */
&importProcs, /* Process rank for source of each incoming node */
&importToPart, /* New partition for each incoming node */
&numExport, /* Number of nodes I must send to other processes*/
&exportGlobalGids, /* Global IDs of the nodes I must send */
&exportLocalGids, /* Local IDs of the nodes I must send */
&exportProcs, /* Process to which I send each of the nodes */
&exportToPart); /* Partition to which each node will belong */
if (rc != ZOLTAN_OK){
if (rank == 0)
MSTK_Report("MESH_PartitionWithZoltan","Could not partition mesh with ZOLTAN",
MSTK_ERROR);
Zoltan_Destroy(&zz);
MPI_Finalize();
return 0;
}
if(rank == 0) {
*part = (int *) calloc(graph.numMyNodes,sizeof(int));
for ( i = 0; i < numExport; i++ ) {
(*part)[exportGlobalGids[i]-1] = exportToPart[i];
}
if (graph.nodeGID) free(graph.nodeGID);
if (graph.nodeCoords) free(graph.nodeCoords);
if (graph.nborIndex) free(graph.nborIndex);
if (graph.nborGID) free(graph.nborGID);
if (graph.nborProc) free(graph.nborProc);
}
else {
*part = NULL;
}
Zoltan_LB_Free_Part(&exportGlobalGids, &exportLocalGids, &exportProcs, &exportToPart);
Zoltan_LB_Free_Part(&importGlobalGids, &importLocalGids, &importProcs, &importToPart);
Zoltan_Destroy(&zz);
return 1;
}
int MESH_WriteToFile(Mesh_ptr mesh, const char *filename, RepType rtype, MSTK_Comm comm) {
FILE *fp;
char mesg[80], attname[256];
int i, j, k, idx;
int gdim, gid;
int mvid, mvid0, mvid1, mvid2, mrid2, meid, mfid, mrid;
int nav, nar, nfe, nfv, nrf, nrv, dir=0;
int nv, ne, nf, nr;
int natt, ncomp, ival, nent;
double xyz[3], rval, rdummy, *rval_arr;
void *pval, *pdummy;
MVertex_ptr mv, mv0, mv1, mv2;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr, mr2;
List_ptr adjverts, mfedges, mfverts, mrfaces, mrverts, adjregs;
RepType reptype;
MAttrib_ptr attrib, vidatt, eidatt, fidatt, ridatt;
MType attentdim;
MAttType atttype;
char modfilename[256];
strcpy(modfilename, filename);
int rank = 0, numprocs = 1;
#ifdef MSTK_HAVE_MPI
if (comm) {
MPI_Comm_size((MPI_Comm)comm, &numprocs);
MPI_Comm_rank((MPI_Comm)comm, &rank);
}
if (numprocs > 1) {
int ndigits = 0;
int div = 1;
while (numprocs/div) {div *= 10; ndigits++;}
sprintf(modfilename,"%s.%d.%0*d",filename,numprocs,ndigits,rank);
}
#endif
if (!(fp = fopen(modfilename,"w"))) {
sprintf(mesg,"Cannot open file %-s for writing",modfilename);
MSTK_Report("MESH_WriteToFile",mesg,MSTK_ERROR);
return 0;
}
if (rtype != UNKNOWN_REP) {
reptype = rtype;
}
else {
reptype = MESH_RepType(mesh);
}
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
fprintf(fp,"MSTK %-2.1lf\n",MSTK_FILE_VER);
fprintf(fp,"%s %d %d %d %d\n",
MESH_rtype_str[reptype],
nv,
(reptype >= R1 && reptype <= R4)?0:ne,
(reptype >= R1 && reptype <= R2 && nr)?0:nf,
nr);
vidatt = MAttrib_New(mesh,"vidatt",INT,MVERTEX);
eidatt = MAttrib_New(mesh,"eidatt",INT,MEDGE);
fidatt = MAttrib_New(mesh,"fidatt",INT,MFACE);
ridatt = MAttrib_New(mesh,"ridatt",INT,MREGION);
idx = 0; i = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MEnt_Set_AttVal(mv,vidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
MEnt_Set_AttVal(me,eidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MEnt_Set_AttVal(mf,fidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MEnt_Set_AttVal(mr,ridatt,++i,0.0,NULL);
fprintf(fp,"vertices\n");
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MV_Coords(mv,xyz);
gdim = MV_GEntDim(mv);
gid = MV_GEntID(mv);
fprintf(fp,"%24.16lf %24.16lf %24.16lf %d %d\n",
xyz[0],xyz[1],xyz[2],gdim,gid);
}
if (reptype == R2 || reptype == R4) {
fprintf(fp,"adjvertices\n");
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
nav = MV_Num_AdjVertices(mv);
fprintf(fp,"%d ",nav);
adjverts = MV_AdjVertices(mv);
for (j = 0; j < nav; j++) {
mv2 = List_Entry(adjverts,j);
MEnt_Get_AttVal(mv2,vidatt,&mvid2,&rval,&pval);
fprintf(fp,"%d ",mvid2);
}
fprintf(fp,"\n");
List_Delete(adjverts);
}
}
if (reptype <= F4 && ne) {
fprintf(fp,"edges\n");
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
mv0 = ME_Vertex(me,0);
MEnt_Get_AttVal(mv0,vidatt,&mvid0,&rval,&pval);
mv1 = ME_Vertex(me,1);
MEnt_Get_AttVal(mv1,vidatt,&mvid1,&rval,&pval);
gdim = ME_GEntDim(me);
gid = ME_GEntID(me);
fprintf(fp,"%d %d \t%d %d\n",mvid0,mvid1,gdim,gid);
}
}
if (reptype <= F4) {
/* For full representations, always write out faces in terms of edges */
fprintf(fp,"faces edge\n");
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
nfe = MF_Num_Edges(mf);
fprintf(fp,"%d ",nfe);
mfedges = MF_Edges(mf,1,0);
for (j = 0; j < nfe; j++) {
me = List_Entry(mfedges,j);
dir = MF_EdgeDir_i(mf,j);
MEnt_Get_AttVal(me,eidatt,&meid,&rval,&pval);
if (dir != 1) meid = -meid;
fprintf(fp,"%d ",meid);
}
List_Delete(mfedges);
gdim = MF_GEntDim(mf);
/*
gent = MF_GEntity(mf);
gid = gent ? -99 : 0;
*/
gid = MF_GEntID(mf);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
else {
/* For reduced representations, R3 and R4 always write out faces
in terms of vertices. For reduced representations, R1 and R2
write out faces in terms of vertices only when there are no
regions (i.e. faces are the highest level mesh entities) */
if ((reptype > R2) || (nr == 0)) {
fprintf(fp,"faces vertex\n");
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
nfv = MF_Num_Edges(mf);
fprintf(fp,"%d ",nfv);
mfverts = MF_Vertices(mf,1,0);
for (j = 0; j < nfv; j++) {
mv = List_Entry(mfverts,j);
MEnt_Get_AttVal(mv,vidatt,&mvid,&rval,&pval);
fprintf(fp,"%d ",mvid);
}
List_Delete(mfverts);
gdim = MF_GEntDim(mf);
gid = MF_GEntID(mf);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
}
if (nr) {
if (reptype <= F4 || reptype >= R2) {
fprintf(fp,"regions face\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nrf = MR_Num_Faces(mr);
fprintf(fp,"%d ",nrf);
mrfaces = MR_Faces(mr);
for (j = 0; j < nrf; j++) {
mf = List_Entry(mrfaces,j);
dir = MR_FaceDir_i(mr,j);
MEnt_Get_AttVal(mf,fidatt,&mfid,&rval,&pval);
if (dir != 1) mfid = -mfid;
fprintf(fp,"%d ",mfid);
}
List_Delete(mrfaces);
gdim = MF_GEntDim(mr);
gid = MR_GEntID(mr);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
else {
fprintf(fp,"regions vertex\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nrv = MR_Num_Vertices(mr);
fprintf(fp,"%d ",nrv);
mrverts = MR_Vertices(mr);
for (j = 0; j < nrv; j++) {
mv = List_Entry(mrverts,j);
MEnt_Get_AttVal(mv,vidatt,&mvid,&rval,&pval);
fprintf(fp,"%d ",mvid);
}
List_Delete(mrverts);
gdim = MR_GEntDim(mr);
gid = MR_GEntID(mr);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
if (reptype == R2 || reptype == R4) {
fprintf(fp,"adjregions\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nar = MR_Num_Faces(mr);
fprintf(fp,"%d ",nar);
adjregs = MR_AdjRegions(mr);
for (j = 0; j < nar; j++) {
mr2 = List_Entry(adjregs,j);
if ((long) mr2 == -1)
fprintf(fp,"%d ",0);
else {
MEnt_Get_AttVal(mr2,ridatt,&mrid2,&rval,&pval);
fprintf(fp,"%d ",mrid2);
}
}
fprintf(fp,"\n");
List_Delete(adjregs);
}
}
}
/* Write out attributes if there are more than the 4 that we created
in this routine */
if ((natt = MESH_Num_Attribs(mesh)) > 4) {
fprintf(fp,"attributes\n");
for (i = 0; i < natt; i++) {
attrib = MESH_Attrib(mesh,i);
/* Don't write out attribs we created for the internal use of
this routine */
if (attrib == vidatt || attrib == eidatt || attrib == fidatt ||
attrib == ridatt) continue;
MAttrib_Get_Name(attrib,attname);
atttype = MAttrib_Get_Type(attrib);
if (atttype == POINTER) continue; /* cannot write it out */
ncomp = MAttrib_Get_NumComps(attrib);
attentdim = MAttrib_Get_EntDim(attrib);
/* First count how many entities actually have the attribute assigned */
nent = 0;
switch(attentdim) {
case MVERTEX:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) nent++;
break;
case MEDGE:
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) nent++;
break;
case MFACE:
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) nent++;
break;
case MREGION:
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) nent++;
break;
case MALLTYPE:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) nent++;
break;
default:
break;
} /* switch (attentdim) */
/* No point in writing out attribute if no entity uses it! Or is there? */
if (!nent) continue;
fprintf(fp,"%-s\n",attname);
switch(atttype) {
case INT:
fprintf(fp,"INT\n");
break;
case DOUBLE:
fprintf(fp,"DOUBLE\n");
break;
case VECTOR:
fprintf(fp,"VECTOR\n");
break;
case TENSOR:
fprintf(fp,"TENSOR\n");
break;
default:
MSTK_Report("MESH_WriteToFile",
"Unrecognizable or unprintable attribute type\n",MSTK_WARN);
continue;
}
fprintf(fp,"%-d\n",ncomp);
switch(attentdim) {
case MVERTEX:
fprintf(fp,"MVERTEX\n");
break;
case MEDGE:
fprintf(fp,"MEDGE\n");
break;
case MFACE:
fprintf(fp,"MFACE\n");
break;
case MREGION:
fprintf(fp,"MREGION\n");
break;
case MALLTYPE:
fprintf(fp,"MALLTYPE\n");
break;
default:
MSTK_Report("Mesh_WriteToFile","Unrecognized entity type",MSTK_WARN);
break;
}
fprintf(fp,"%-d\n",nent);
switch(attentdim) {
case MVERTEX:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mv,vidatt,&mvid,&rdummy,&pdummy);
fprintf(fp,"0 %-d ",mvid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MEDGE:
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(me,eidatt,&meid,&rdummy,&pdummy);
fprintf(fp,"1 %-d ",meid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MFACE:
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mf,fidatt,&mfid,&rdummy,&pdummy);
fprintf(fp,"2 %-d ",mfid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MREGION:
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mr,ridatt,&mrid,&rdummy,&pdummy);
fprintf(fp,"3 %-d ",mrid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MALLTYPE:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mv,vidatt,&mvid,&rdummy,&pdummy);
fprintf(fp,"0 %-d ",mvid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(me,eidatt,&meid,&rdummy,&pdummy);
fprintf(fp,"1 %-d ",meid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mf,fidatt,&mfid,&rdummy,&pdummy);
fprintf(fp,"2 %-d ",mfid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mr,ridatt,&mrid,&rdummy,&pdummy);
fprintf(fp,"3 %-d ",mrid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
default:
break;
} /* switch (attentdim) */
} /* for (i = 0; i < natt) */
} /* if (Mesh_Num_Attribs(mesh)) */
idx = 0; i = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MEnt_Rem_AttVal(mv,vidatt);
idx = 0; i = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
MEnt_Rem_AttVal(me,eidatt);
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MEnt_Rem_AttVal(mf,fidatt);
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MEnt_Rem_AttVal(mr,ridatt);
MAttrib_Delete(vidatt);
MAttrib_Delete(eidatt);
MAttrib_Delete(fidatt);
MAttrib_Delete(ridatt);
fclose(fp);
return 1;
}
int MESH_PartitionWithMetis(Mesh_ptr mesh, int nparts, int **part) {
MEdge_ptr fedge;
MFace_ptr mf, oppf, rface;
MRegion_ptr mr, oppr;
List_ptr fedges, efaces, rfaces, fregions;
int i, ncells, ipos;
int nv, ne, nf, nr, nfe, nef, nfr, nrf, idx, idx2;
#ifdef METIS_5
idx_t ngraphvtx, numflag, nedgecut, numparts, ncons;
idx_t wtflag, metisopts[METIS_NOPTIONS];
idx_t *vsize, *idxpart;
idx_t *xadj, *adjncy, *vwgt, *adjwgt;
real_t *tpwgts, *ubvec;
#else
idxtype ngraphvtx, numflag, nedgecut, numparts;
idxtype wtflag, metisopts[5] = {0,0,0,0,0};
idxtype *xadj, *adjncy, *vwgt, *adjwgt, *idxpart;
#endif
/* First build a nodal graph of the mesh in the format required by
metis */
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
ipos = 0;
if (nr == 0) {
if (nf == 0) {
fprintf(stderr,"Cannot partition wire meshes\n");
exit(-1);
}
#ifdef METIS_5
xadj = (idx_t *) malloc((nf+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*ne*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nf+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*ne*sizeof(idxtype));
#endif
ncells = nf;
/* Surface mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mf = MESH_Next_Face(mesh,&idx))) {
fedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(fedges);
idx2 = 0;
while ((fedge = List_Next_Entry(fedges,&idx2))) {
efaces = ME_Faces(fedge);
nef = List_Num_Entries(efaces);
if (nef == 1) {
continue; /* boundary edge; nothing to do */
}
else {
int j;
for (j = 0; j < nef; j++) {
oppf = List_Entry(efaces,j);
if (oppf != mf) {
adjncy[ipos] = MF_ID(oppf)-1;
ipos++;
}
}
}
List_Delete(efaces);
}
List_Delete(fedges);
i++;
xadj[i] = ipos;
}
}
else {
#ifdef METIS_5
xadj = (idx_t *) malloc((nr+1)*sizeof(idx_t));
adjncy = (idx_t *) malloc(2*nf*sizeof(idx_t));
#else
xadj = (idxtype *) malloc((nr+1)*sizeof(idxtype));
adjncy = (idxtype *) malloc(2*nf*sizeof(idxtype));
#endif
ncells = nr;
/* Volume mesh */
idx = 0; i = 0;
xadj[i] = ipos;
while ((mr = MESH_Next_Region(mesh,&idx))) {
rfaces = MR_Faces(mr);
nrf = List_Num_Entries(rfaces);
idx2 = 0;
while ((rface = List_Next_Entry(rfaces,&idx2))) {
fregions = MF_Regions(rface);
nfr = List_Num_Entries(fregions);
if (nfr > 1) {
oppr = List_Entry(fregions,0);
if (oppr == mr)
oppr = List_Entry(fregions,1);
adjncy[ipos] = MR_ID(oppr)-1;
ipos++;
}
List_Delete(fregions);
}
List_Delete(rfaces);
i++;
xadj[i] = ipos;
}
}
/* Partition the graph */
wtflag = 0; /* No weights are specified */
vwgt = adjwgt = NULL;
numflag = 0; /* C style numbering of elements (nodes of the dual graph) */
ngraphvtx = ncells; /* we want the variable to be of type idxtype or idx_t */
numparts = nparts; /* we want the variable to be of type idxtype or idx_t */
#ifdef METIS_5
idxpart = (idx_t *) malloc(ncells*sizeof(idx_t));
ncons = 1; /* Number of constraints */
vsize = NULL;
tpwgts = NULL;
ubvec = NULL;
METIS_SetDefaultOptions(metisopts);
metisopts[METIS_OPTION_NUMBERING] = 0;
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,
idxpart);
else
METIS_PartGraphKway(&ngraphvtx,&ncons,xadj,adjncy,vwgt,vsize,adjwgt,
&numparts,tpwgts,ubvec,metisopts,&nedgecut,idxpart);
#else
idxpart = (idxtype *) malloc(ncells*sizeof(idxtype));
if (nparts <= 8)
METIS_PartGraphRecursive(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,
&numflag,&numparts,metisopts,&nedgecut,idxpart);
else
METIS_PartGraphKway(&ngraphvtx,xadj,adjncy,vwgt,adjwgt,&wtflag,&numflag,
&numparts,metisopts,&nedgecut,idxpart);
#endif
free(xadj);
free(adjncy);
*part = (int *) malloc(ncells*sizeof(int));
for (i = 0; i < ncells; i++)
(*part)[i] = (int) idxpart[i];
free(idxpart);
return 1;
}
int MESH_AssignGlobalIDs_Edge(Mesh_ptr submesh, MSTK_Comm comm) {
int i, j, k, nbe, noe, nge, ne, mesh_info[10], global_id;
MVertex_ptr mv;
MEdge_ptr me;
List_ptr boundary_edges;
int *loc, edge_id[2], max_nbe, index_nbe, iloc, is_boundary;
int *global_mesh_info, *list_edge, *recv_list_edge, *edge_ov_label, *id_on_ov_list;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
for (i = 0; i < 10; i++) mesh_info[i] = 0;
ne = MESH_Num_Edges(submesh);
mesh_info[2] = ne;
/*
collect 'boundary' edges
if endpoints are either GHOST or OVERLAP, then it is a boundary edge
*/
nbe = 0; boundary_edges = List_New(10);
for(i = 0; i < ne; i++) {
is_boundary = 1;
me = MESH_Edge(submesh,i);
for( k = 0; k < 2; k++) {
mv = ME_Vertex(me,k);
if(MV_PType(mv)!=PGHOST && MV_PType(mv)!=POVERLAP)
is_boundary = 0;
}
if(is_boundary) {
ME_Flag_OnParBoundary(me);
List_Add(boundary_edges,me);
nbe++;
}
}
mesh_info[6] = nbe;
List_Sort(boundary_edges,nbe,sizeof(MEdge_ptr),compareEdgeID);
global_mesh_info = (int *)malloc(10*num*sizeof(int));
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
max_nbe = 0;
for(i = 0; i < num; i++)
if(max_nbe < global_mesh_info[10*i+6])
max_nbe = global_mesh_info[10*i+6];
list_edge = (int *)malloc(max_nbe*2*sizeof(int));
recv_list_edge = (int *)malloc(num*max_nbe*2*sizeof(int));
/* indicate if a edge is overlapped */
edge_ov_label = (int *)malloc(num*max_nbe*sizeof(int));
for (i = 0; i < num*max_nbe; i++)
edge_ov_label[i] = 0;
id_on_ov_list = (int *)malloc(max_nbe*sizeof(int));
/* pack edge information to send */
index_nbe = 0;
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
list_edge[index_nbe] = MV_GlobalID(ME_Vertex(me,0));
list_edge[index_nbe+1] = MV_GlobalID(ME_Vertex(me,1));
index_nbe += 2;
}
MPI_Allgather(list_edge,2*max_nbe,MPI_INT,recv_list_edge,2*max_nbe,MPI_INT,comm);
nge = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
if(ME_GlobalID(me) > 0) continue; /* if already assigned */
edge_id[0] = MV_GlobalID(ME_Vertex(me,0));
edge_id[1] = MV_GlobalID(ME_Vertex(me,1));
for(j = 0; j < rank; j++) {
loc = (int *)bsearch(&edge_id,
&recv_list_edge[2*max_nbe*j],
global_mesh_info[10*j+6],
2*sizeof(int),
compareEdgeINT);
if(loc) {
iloc = (int)(loc - &recv_list_edge[2*max_nbe*j])/2;
ME_Set_PType(me,PGHOST);
ME_Set_MasterParID(me,j);
edge_ov_label[max_nbe*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
nge++;
break;
}
}
}
}
/* num of ghost verts */
mesh_info[9] = nge;
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* since this is a OR reduction, we can use MPI_IN_PLACE, send buffer same as recv buffer */
MPI_Allreduce(MPI_IN_PLACE,edge_ov_label,num*max_nbe,MPI_INT,MPI_LOR,comm);
/* Assign global ID for non ghost edge */
global_id = 1;
for(i = 0; i < rank; i++)
global_id = global_id + global_mesh_info[10*i+2] - global_mesh_info[10*i+9];
for(i = 0; i < ne; i++) {
me = MESH_Edge(submesh,i);
if (ME_PType(me) == PGHOST) continue;
if (!ME_GlobalID(me)) ME_Set_GlobalID(me,global_id++);
ME_Set_MasterParID(me,rank);
}
/* label OVERLAP edge */
noe = 0;
for(i = 0; i < nbe; i++)
if(edge_ov_label[rank*max_nbe+i]) {
me = List_Entry(boundary_edges,i);
ME_Set_PType(me,POVERLAP);
noe++;
}
/* this time only global id are sent */
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
list_edge[i] = ME_GlobalID(me);
}
MPI_Allgather(list_edge,max_nbe,MPI_INT,recv_list_edge,max_nbe,MPI_INT,comm);
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
if(ME_PType(me)==PGHOST) {
int gid = recv_list_edge[ME_MasterParID(me)*max_nbe+id_on_ov_list[i]];
#ifdef DEBUG
if (ME_GlobalID(me) && ME_GlobalID(me) != gid)
MSTK_Report("MESH_Assign_GlobalIDs_Edge",
"Ghost edge already has different global ID",
MSTK_WARN);
#endif
ME_Set_GlobalID(me, gid);
}
}
List_Delete(boundary_edges);
free(global_mesh_info);
free(edge_ov_label);
free(id_on_ov_list);
free(list_edge);
free(recv_list_edge);
return 1;
}
int MESH_AssignGlobalIDs_Vertex(Mesh_ptr submesh, int have_GIDs, MSTK_Comm comm) {
int i, j, nv, nbv, ne, nf, nr, mesh_info[10];
MVertex_ptr mv;
List_ptr boundary_verts;
RepType rtype;
int index_nbv, max_nbv, iloc, num_ghost_verts, global_id;
int *global_mesh_info, *vertex_ov_label, *vertex_ov_global_id, *id_on_ov_list;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
for (i = 0; i < 10; i++) mesh_info[i] = 0;
rtype = MESH_RepType(submesh);
nv = MESH_Num_Vertices(submesh);
ne = MESH_Num_Edges(submesh);
nf = MESH_Num_Faces(submesh);
nr = MESH_Num_Regions(submesh);
mesh_info[0] = rtype;
mesh_info[1] = nv;
mesh_info[2] = ne;
mesh_info[3] = nf;
mesh_info[4] = nr;
/* calculate number of boundary vertices */
nbv = 0; boundary_verts = List_New(10);
if (nr) {
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (vertex_on_boundary3D(mv)) {
MV_Flag_OnParBoundary(mv);
List_Add(boundary_verts,mv);
nbv++;
}
}
}
else {
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (vertex_on_boundary2D(mv)) {
MV_Flag_OnParBoundary(mv);
List_Add(boundary_verts,mv);
nbv++;
}
}
}
mesh_info[5] = nbv;
/*
gather submeshes information
right now we only need nv and nbv, and later num_ghost_verts, but we gather all mesh_info
*/
global_mesh_info = (int *)malloc(10*num*sizeof(int));
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* get largest number of boundary vertices of all the processors */
max_nbv = 0;
for(i = 0; i < num; i++)
if(max_nbv < global_mesh_info[10*i+5])
max_nbv = global_mesh_info[10*i+5];
if (have_GIDs) {
int *list_boundary_vertex_gid = (int *)malloc(max_nbv*sizeof(int));
int *recv_list_vertex_gid = (int *)malloc(num*max_nbv*sizeof(int));
/* sort boundary vertices based on Global ID, for binary search */
List_Sort(boundary_verts,nbv,sizeof(MVertex_ptr),compareGlobalID);
/* only global ids are sent */
index_nbv = 0;
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
list_boundary_vertex_gid[index_nbv] = MV_GlobalID(mv);
index_nbv++;
}
MPI_Allgather(list_boundary_vertex_gid,max_nbv,MPI_INT,recv_list_vertex_gid,max_nbv,MPI_INT,comm);
/* indicate if a vertex is overlapped */
vertex_ov_label = (int *)malloc(num*max_nbv*sizeof(int));
/*
store the local boundary id on ov processor
it is used to assign global id of local ghost vertices
no need to store master partition id, MV_MasterParID(mv) is already assigned
*/
id_on_ov_list = (int *)malloc(max_nbv*sizeof(int));
for (i = 0; i < num*max_nbv; i++)
vertex_ov_label[i] = 0;
num_ghost_verts = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
int gid = MV_GlobalID(mv);
/* check which previous processor has a vertex with same Global ID*/
for(j = 0; j < rank; j++) {
/* since each processor has sorted the boundary vertices, use binary search */
int *loc = (int *)bsearch(&gid,
&recv_list_vertex_gid[max_nbv*j],
global_mesh_info[10*j+5],
sizeof(int),
compareINT);
/* if found the vertex on previous processors */
if(loc) {
/* here the location iloc is relative to the beginning of the jth processor */
iloc = (int)(loc - &recv_list_vertex_gid[max_nbv*j]);
MV_Set_PType(mv,PGHOST);
MV_Set_MasterParID(mv,j);
num_ghost_verts++;
/* label the original vertex as overlapped */
vertex_ov_label[max_nbv*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
/* if found on processor j, no need to test for j+1,j+2...*/
break;
}
}
}
}
free(list_boundary_vertex_gid);
free(recv_list_vertex_gid);
}
else {
double coor[3];
int *list_boundary_vertex = (int *)malloc(max_nbv*sizeof(int));
double *list_boundary_coor = (double *)malloc(3*max_nbv*sizeof(double));
int *recv_list_vertex = (int *)malloc(num*max_nbv*sizeof(int));
double *recv_list_coor = (double *)malloc(3*num*max_nbv*sizeof(double));
/* sort boundary vertices based on coordinate value, for binary search */
List_Sort(boundary_verts,nbv,sizeof(MVertex_ptr),compareVertexCoor);
/* only local id and coordinate values are sent */
index_nbv = 0;
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
list_boundary_vertex[index_nbv] = MV_ID(mv);
MV_Coords(mv,coor);
list_boundary_coor[index_nbv*3] = coor[0];
list_boundary_coor[index_nbv*3+1] = coor[1];
list_boundary_coor[index_nbv*3+2] = coor[2];
index_nbv++;
}
MPI_Allgather(list_boundary_vertex,max_nbv,MPI_INT,recv_list_vertex,max_nbv,MPI_INT,comm);
MPI_Allgather(list_boundary_coor,3*max_nbv,MPI_DOUBLE,recv_list_coor,3*max_nbv,MPI_DOUBLE,comm);
/* indicate if a vertex is overlapped */
vertex_ov_label = (int *)malloc(num*max_nbv*sizeof(int));
/*
store the local boundary id on ov processor
it is used to assign global id of local ghost vertices
no need to store master partition id, MV_MasterParID(mv) is already assigned
*/
id_on_ov_list = (int *)malloc(max_nbv*sizeof(int));
for (i = 0; i < num*max_nbv; i++)
vertex_ov_label[i] = 0;
num_ghost_verts = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
MV_Coords(mv,coor);
/* check which previous processor has the same coordinate vertex */
for(j = 0; j < rank; j++) {
/* since each processor has sorted the boundary vertices, use binary search */
double *loc = (double *)bsearch(&coor,
&recv_list_coor[3*max_nbv*j],
global_mesh_info[10*j+5],
3*sizeof(double),
compareCoorDouble);
/* if found the vertex on previous processors */
if(loc) {
/* here the location iloc is relative to the beginning of the jth processor */
iloc = (int)(loc - &recv_list_coor[3*max_nbv*j])/3;
MV_Set_PType(mv,PGHOST);
MV_Set_MasterParID(mv,j);
num_ghost_verts++;
/* label the original vertex as overlapped */
vertex_ov_label[max_nbv*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
/* if found on processor j, no need to test for j+1,j+2...*/
break;
}
}
}
}
free(list_boundary_coor);
free(recv_list_coor);
free(list_boundary_vertex);
free(recv_list_vertex);
}
/* num of ghost verts */
mesh_info[9] = num_ghost_verts;
/* update ghost verts number */
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* since this is a OR reduction, we can use MPI_IN_PLACE, send buffer same as recv buffer */
MPI_Allreduce(MPI_IN_PLACE,vertex_ov_label,num*max_nbv,MPI_INT,MPI_LOR,comm);
/* calculate starting global id number for vertices*/
if (!have_GIDs) {
global_id = 1;
for(i = 0; i < rank; i++)
global_id = global_id + global_mesh_info[10*i+1] - global_mesh_info[10*i+9];
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (MV_PType(mv) == PGHOST)
continue;
MV_Set_GlobalID(mv,global_id++);
MV_Set_MasterParID(mv,rank);
}
}
/* store overlapped vertices IDs and broadast */
vertex_ov_global_id = (int *)malloc(num*max_nbv*sizeof(int));
for(i = 0; i < num*max_nbv; i++)
vertex_ov_global_id[i] = 0;
for(i = 0; i < nbv; i++) {
if(vertex_ov_label[rank*max_nbv+i]) {
mv = List_Entry(boundary_verts,i);
MV_Set_PType(mv,POVERLAP);
vertex_ov_global_id[rank*max_nbv+i] = MV_GlobalID(mv);
}
}
MPI_Allreduce(MPI_IN_PLACE,vertex_ov_global_id,num*max_nbv,MPI_INT,MPI_MAX,comm);
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
if(MV_PType(mv) == PGHOST)
MV_Set_GlobalID(mv,vertex_ov_global_id[MV_MasterParID(mv)*max_nbv+id_on_ov_list[i]]);
}
List_Delete(boundary_verts);
free(global_mesh_info);
free(vertex_ov_label);
free(vertex_ov_global_id);
free(id_on_ov_list);
return 1;
}
int MESH_Send_Attribute(Mesh_ptr mesh, MAttrib_ptr attrib, int torank,
MSTK_Comm comm, int *numreq, int *maxreq,
MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
int j, k;
int num, ncomp, ival;
double rval;
void *pval;
double *rval_arr;
int *list_info;
MType mtype;
MAttType att_type;
MEntity_ptr ment;
MPI_Request mpirequest;
if (requests == NULL)
MSTK_Report("MSTK_SendMSet","Invalid MPI request buffer",MSTK_FATAL);
if (*maxreq == 0) {
*maxreq = 25;
*requests = (MPI_Request *) malloc(*maxreq*sizeof(MPI_Request));
*numreq = 0;
}
else if (*maxreq < (*numreq) + 2) {
*maxreq *= 2;
*requests = (MPI_Request *) realloc(*requests,*maxreq*sizeof(MPI_Request));
}
/* get attribute properties */
att_type = MAttrib_Get_Type(attrib);
ncomp = MAttrib_Get_NumComps(attrib);
mtype = MAttrib_Get_EntDim(attrib);
/* attribute entity type, used before ghost list established, so no ghost */
switch (mtype) {
case MVERTEX:
num = MESH_Num_Vertices(mesh);
break;
case MEDGE:
num = MESH_Num_Edges(mesh);
break;
case MFACE:
num = MESH_Num_Faces(mesh);
break;
case MREGION:
num = MESH_Num_Regions(mesh);
break;
default:
num = 0;
#ifdef DEBUG2
MSTK_Report("MESH_SendAttr()","Cannot send attributes on entity type MALLTYPE",MSTK_WARN);
#endif
return 0;
}
/* attribute index and global id */
list_info = (int *) malloc(num*sizeof(int));
/* attribute values */
int *list_value_int = NULL;
double *list_value_double = NULL;
if (att_type == INT)
list_value_int = (int *) malloc(num*ncomp*sizeof(int));
else
list_value_double = (double *) malloc(num*ncomp*sizeof(double));
/* collect data */
for(j = 0; j < num; j++) {
switch (mtype) {
case MVERTEX:
ment = MESH_Vertex(mesh,j);
break;
case MEDGE:
ment = MESH_Edge(mesh,j);
break;
case MFACE:
ment = MESH_Face(mesh,j);
break;
case MREGION:
ment = MESH_Region(mesh,j);
break;
default:
MSTK_Report("MESH_SendAttr()","Invalid entity type",MSTK_WARN);
return 0;
}
MEnt_Get_AttVal(ment,attrib,&ival,&rval,&pval);
list_info[j] = MEnt_GlobalID(ment);
if (att_type == INT)
list_value_int[j] = ival;
else {
if(ncomp == 1)
list_value_double[j] = rval;
if(ncomp > 1) {
rval_arr = (double *)pval;
for(k = 0; k < ncomp; k++)
list_value_double[ncomp*j+k] = rval_arr[k];
}
}
}
/* send entity global IDs */
MPI_Isend(list_info,num,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
/* send values */
if (att_type == INT) {
MPI_Isend(list_value_int,num*ncomp,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
}
else {
MPI_Isend(list_value_double,num*ncomp,MPI_DOUBLE,torank,torank,comm,
&mpirequest);
(*requests)[*numreq] = mpirequest;
}
(*numreq)++;
/* track the buffers used for sending so that they can be released later */
if (*maxptrs2free == 0) {
*maxptrs2free = 25;
*ptrs2free = (void **) malloc(*maxptrs2free*sizeof(void *));
*numptrs2free = 0;
}
else if (*maxptrs2free < (*numptrs2free) + 2) {
*maxptrs2free = 2*(*maxptrs2free) + 2;
*ptrs2free = (void **) realloc(*ptrs2free,(*maxptrs2free)*sizeof(void *));
}
(*ptrs2free)[(*numptrs2free)++] = list_info;
if (att_type == INT)
(*ptrs2free)[(*numptrs2free)++] = list_value_int;
else
(*ptrs2free)[(*numptrs2free)++] = list_value_double;
return 1;
}
int MESH_Send_NonVertexEntities_FN(Mesh_ptr mesh, int torank, MSTK_Comm comm,
int *numreq, int *maxreq, MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
int i, j, nv, ne, nf, nr;
int nevs, nfes, nrfs, nfe, nrv, nrf, dir;
int maxnfe, maxnrf;
int *mesh_info;
int *list_edge=NULL, *list_face=NULL, *list_region=NULL;
MVertex_ptr mv;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
List_ptr mfedges, mrfaces, mrverts;
RepType rtype;
double coor[3];
MPI_Request mpirequest;
if (requests == NULL)
MSTK_Report("MESH_Surf_SendMesh","MPI requests array is NULL",MSTK_FATAL);
if (*maxreq == 0) {
*maxreq = 25;
*requests = (MPI_Request *) malloc(*maxreq*sizeof(MPI_Request));
*numreq = 0;
}
else if (*maxreq < (*numreq) + 13) {
*maxreq = 2*(*maxreq) + 11;
*requests = (MPI_Request *) realloc(*requests,*maxreq*sizeof(MPI_Request));
}
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
/* some other known quantitites - 5 items per edge (2 for verts
and 3 for extra data), maxnfe+4 items per face (1 for number of
edges, maxnfe for edge indices, anad 3 for extra data),
maxnrf+4 items per region (1 for number of faces, maxnrf for
face indices and 3 for extra data */
maxnfe = 0;
for (i = 0; i < nf; i++) {
mf = MESH_Face(mesh,i);
nfe = MF_Num_Edges(mf);
if (nfe > maxnfe)
maxnfe = nfe;
}
maxnrf = 0;
for (i = 0; i < nr; i++) {
mr = MESH_Region(mesh,i);
nrf = MR_Num_Faces(mr);
if (nrf > maxnrf)
maxnrf = nrf;
}
// The amount of extra info we are sending and their meaning is obviously
// known on the receiving side too. So nevs, nfes and nrfs can be
// calculated without us sending it
nevs = (2+3)*ne;
nfes = (1 + maxnfe + 3)*nf;
nrfs = (1 + maxnrf + 3)*nr;
/* Reserve nevs spots for each edge */
list_edge = (int *) malloc(5*ne*sizeof(int));
nevs = 0;
/* Store the vertex ids, then the 3 auxilliary data fields */
for(i = 0; i < ne; i++) {
me = MESH_Edge(mesh,i);
list_edge[nevs] = MV_ID(ME_Vertex(me,0));
list_edge[nevs+1] = MV_ID(ME_Vertex(me,1));
list_edge[nevs+2] = (ME_GEntID(me)<<3) | (ME_GEntDim(me));
list_edge[nevs+3] = (ME_MasterParID(me) <<3) | (ME_OnParBoundary(me)<<2) | (ME_PType(me));
list_edge[nevs+4] = ME_GlobalID(me);
nevs += 5;
}
/* send detailed edge info */
MPI_Isend(list_edge,nevs,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
/* Reserve nfes spots for each face */
list_face = (int *) malloc(nfes*sizeof(int));
nfes = 0;
/* first store nfe, then the edge ids, then the 3 auxilliary data fields */
for(i = 0; i < nf; i++) {
mf = MESH_Face(mesh,i);
mfedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(mfedges);
list_face[nfes] = nfe;
for(j = 0; j < nfe; j++) {
dir = MF_EdgeDir_i(mf,j) ? 1 : -1;
list_face[nfes+j+1] = dir*ME_ID(List_Entry(mfedges,j));
}
list_face[nfes+nfe+1] = (MF_GEntID(mf)<<3) | (MF_GEntDim(mf));
list_face[nfes+nfe+2] = (MF_MasterParID(mf)<<3) | (MF_OnParBoundary(mf)<<2) | (MF_PType(mf));
list_face[nfes+nfe+3] = MF_GlobalID(mf);
nfes += (nfe + 4);
List_Delete(mfedges);
}
/* send detailed face info */
MPI_Isend(list_face,nfes,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
if (nr) {
list_region = (int *) malloc(nrfs*sizeof(int));
nrfs = 0;
/* first store nrf, then the face ids, then the 3 auxilliary data fields */
for(i = 0; i < nr; i++) {
mr = MESH_Region(mesh,i);
mrfaces = MR_Faces(mr);
nrf = List_Num_Entries(mrfaces);
list_region[nrfs] = nrf;
for(j = 0; j < nrf; j++) {
dir = MR_FaceDir_i(mr,j) == 1 ? 1 : -1;
list_region[nrfs+j+1] = dir*MF_ID(List_Entry(mrfaces,j));
}
list_region[nrfs+nrf+1] = (MR_GEntID(mr)<<3) | (MR_GEntDim(mr));
list_region[nrfs+nrf+2] = (MR_MasterParID(mr)<<3) | (MR_PType(mr)); /* MR_PType is 2 bits; 3 bit is 0 */
list_region[nrfs+nrf+3] = MR_GlobalID(mr);
nrfs += (nrf + 4);
List_Delete(mrfaces);
}
/* send detailed region info */
MPI_Isend(list_region,nrfs,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
}
/* collect allocated memory so it can be freed in a higher level
routine after MPI_Waitall or MPI_Test has ensured that the send
has been completed */
if (ptrs2free == NULL)
MSTK_Report("MESH_Surf_SendMesh_FN","ptrs2free array is NULL",MSTK_FATAL);
int nptrs = 3;
if (*maxptrs2free == 0) {
*maxptrs2free = 25;
*ptrs2free = (void **) malloc(*maxptrs2free*sizeof(void *));
*numptrs2free = 0;
}
else if (*maxptrs2free < (*numptrs2free) + nptrs) {
*maxptrs2free = 2*(*maxptrs2free) + nptrs;
*ptrs2free = (void **) realloc(*ptrs2free,(*maxptrs2free)*sizeof(void *));
}
if (ne)
(*ptrs2free)[(*numptrs2free)++] = list_edge;
if (nf)
(*ptrs2free)[(*numptrs2free)++] = list_face;
if (nr)
(*ptrs2free)[(*numptrs2free)++] = list_region;
return 1;
}
