int MESH_Update_ParallelAdj(Mesh_ptr mesh, MSTK_Comm comm) {
int i, idx, nv, ne, nf, nr, local_ov_num[4];
MVertex_ptr mv;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
MType mtype;
int myprtn, numprtns;
MPI_Comm_rank(comm,&myprtn);
MPI_Comm_size(comm,&numprtns);
if (numprtns == 1) return 1;
/* set ghost adjacencies */
idx = 0;
while ((mv = MESH_Next_GhostVertex(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MV_MasterParID(mv),MVERTEX);
idx = 0;
while ((me = MESH_Next_GhostEdge(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,ME_MasterParID(me),MEDGE);
idx = 0;
while ((mf = MESH_Next_GhostFace(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MF_MasterParID(mf),MFACE);
idx = 0;
while ((mr = MESH_Next_GhostRegion(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MR_MasterParID(mr),MREGION);
/* derive which processors this processor has overlaps with */
int *local_par_adj = (int *) malloc(numprtns*sizeof(int));
int *global_par_adj = (int *) malloc(numprtns*numprtns*sizeof(int));
for (i = 0; i < numprtns; i++) {
local_par_adj[i] = 0;
for (mtype = MVERTEX; mtype <= MREGION; mtype++) {
int j = MESH_Has_Ghosts_From_Prtn(mesh,i,mtype);
local_par_adj[i] |= j<<(2*mtype);
}
}
/* At this point, it is assumed that this processor ('prtn') has
knowledge of all the processors that it has ghost entities from
and what type of entities they are. We do an MPI_Allgather so
that the processor can find out the reverse info, i.e., which
processors are expecting ghost entities from this processor and
what type of entities. This info then goes in as the overlap
entity info for this processor */
for (i = 0; i < numprtns*numprtns; i++) global_par_adj[i] = 0;
MPI_Allgather(local_par_adj,numprtns,MPI_INT,global_par_adj,numprtns,MPI_INT,comm);
/* Now set overlap adjacency flags */
unsigned int ovnum = 0;
unsigned int *prtnums = (unsigned int *) malloc(numprtns*sizeof(unsigned int));
for (i = 0; i < numprtns; i++) {
for (mtype = MVERTEX; mtype <= MREGION; mtype++) {
int j = global_par_adj[i*numprtns + myprtn] & 1<<(2*mtype);
if (j)
MESH_Flag_Has_Overlaps_On_Prtn(mesh,i,mtype);
}
}
/* Right now the model we use is that every partition sends ALL its
overlap entity data to any partition that asks for it */
/* So, if a processor 'i' has ghosts from partition 'j', it needs to
know the total number of overlap entities on partition 'j' in
order to allocate sufficient receive buffers */
int *global_ov_num = (int *) malloc(4*numprtns*sizeof(int));
/* local overlap entity numbers */
local_ov_num[0] = MESH_Num_OverlapVertices(mesh);
local_ov_num[1] = MESH_Num_OverlapEdges(mesh);
local_ov_num[2] = MESH_Num_OverlapFaces(mesh);
local_ov_num[3] = MESH_Num_OverlapRegions(mesh);
MPI_Allgather(local_ov_num,4,MPI_INT,global_ov_num,4,MPI_INT,comm);
/* Set how many entities a partition can expect to receive from
another partititon whether it is used on this partition or not */
MESH_Init_Par_Recv_Info(mesh);
for(i = 0; i < numprtns; i++) {
if (MESH_Has_Ghosts_From_Prtn(mesh,i,MANYTYPE)) {
for (mtype = MVERTEX; mtype <= MREGION; mtype++)
MESH_Set_Num_Recv_From_Prtn(mesh,i,mtype,global_ov_num[4*i+mtype]);
}
}
free(global_ov_num);
free(local_par_adj);
free(global_par_adj);
free(prtnums);
MESH_Mark_ParallelAdj_Current(mesh);
return 1;
}
int MESH_AssignGlobalIDs_Vertex(Mesh_ptr submesh, int have_GIDs, MSTK_Comm comm) {
int i, j, nv, nbv, ne, nf, nr, mesh_info[10];
MVertex_ptr mv;
List_ptr boundary_verts;
RepType rtype;
int index_nbv, max_nbv, iloc, num_ghost_verts, global_id;
int *global_mesh_info, *vertex_ov_label, *vertex_ov_global_id, *id_on_ov_list;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
for (i = 0; i < 10; i++) mesh_info[i] = 0;
rtype = MESH_RepType(submesh);
nv = MESH_Num_Vertices(submesh);
ne = MESH_Num_Edges(submesh);
nf = MESH_Num_Faces(submesh);
nr = MESH_Num_Regions(submesh);
mesh_info[0] = rtype;
mesh_info[1] = nv;
mesh_info[2] = ne;
mesh_info[3] = nf;
mesh_info[4] = nr;
/* calculate number of boundary vertices */
nbv = 0; boundary_verts = List_New(10);
if (nr) {
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (vertex_on_boundary3D(mv)) {
MV_Flag_OnParBoundary(mv);
List_Add(boundary_verts,mv);
nbv++;
}
}
}
else {
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (vertex_on_boundary2D(mv)) {
MV_Flag_OnParBoundary(mv);
List_Add(boundary_verts,mv);
nbv++;
}
}
}
mesh_info[5] = nbv;
/*
gather submeshes information
right now we only need nv and nbv, and later num_ghost_verts, but we gather all mesh_info
*/
global_mesh_info = (int *)malloc(10*num*sizeof(int));
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* get largest number of boundary vertices of all the processors */
max_nbv = 0;
for(i = 0; i < num; i++)
if(max_nbv < global_mesh_info[10*i+5])
max_nbv = global_mesh_info[10*i+5];
if (have_GIDs) {
int *list_boundary_vertex_gid = (int *)malloc(max_nbv*sizeof(int));
int *recv_list_vertex_gid = (int *)malloc(num*max_nbv*sizeof(int));
/* sort boundary vertices based on Global ID, for binary search */
List_Sort(boundary_verts,nbv,sizeof(MVertex_ptr),compareGlobalID);
/* only global ids are sent */
index_nbv = 0;
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
list_boundary_vertex_gid[index_nbv] = MV_GlobalID(mv);
index_nbv++;
}
MPI_Allgather(list_boundary_vertex_gid,max_nbv,MPI_INT,recv_list_vertex_gid,max_nbv,MPI_INT,comm);
/* indicate if a vertex is overlapped */
vertex_ov_label = (int *)malloc(num*max_nbv*sizeof(int));
/*
store the local boundary id on ov processor
it is used to assign global id of local ghost vertices
no need to store master partition id, MV_MasterParID(mv) is already assigned
*/
id_on_ov_list = (int *)malloc(max_nbv*sizeof(int));
for (i = 0; i < num*max_nbv; i++)
vertex_ov_label[i] = 0;
num_ghost_verts = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
int gid = MV_GlobalID(mv);
/* check which previous processor has a vertex with same Global ID*/
for(j = 0; j < rank; j++) {
/* since each processor has sorted the boundary vertices, use binary search */
int *loc = (int *)bsearch(&gid,
&recv_list_vertex_gid[max_nbv*j],
global_mesh_info[10*j+5],
sizeof(int),
compareINT);
/* if found the vertex on previous processors */
if(loc) {
/* here the location iloc is relative to the beginning of the jth processor */
iloc = (int)(loc - &recv_list_vertex_gid[max_nbv*j]);
MV_Set_PType(mv,PGHOST);
MV_Set_MasterParID(mv,j);
num_ghost_verts++;
/* label the original vertex as overlapped */
vertex_ov_label[max_nbv*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
/* if found on processor j, no need to test for j+1,j+2...*/
break;
}
}
}
}
free(list_boundary_vertex_gid);
free(recv_list_vertex_gid);
}
else {
double coor[3];
int *list_boundary_vertex = (int *)malloc(max_nbv*sizeof(int));
double *list_boundary_coor = (double *)malloc(3*max_nbv*sizeof(double));
int *recv_list_vertex = (int *)malloc(num*max_nbv*sizeof(int));
double *recv_list_coor = (double *)malloc(3*num*max_nbv*sizeof(double));
/* sort boundary vertices based on coordinate value, for binary search */
List_Sort(boundary_verts,nbv,sizeof(MVertex_ptr),compareVertexCoor);
/* only local id and coordinate values are sent */
index_nbv = 0;
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
list_boundary_vertex[index_nbv] = MV_ID(mv);
MV_Coords(mv,coor);
list_boundary_coor[index_nbv*3] = coor[0];
list_boundary_coor[index_nbv*3+1] = coor[1];
list_boundary_coor[index_nbv*3+2] = coor[2];
index_nbv++;
}
MPI_Allgather(list_boundary_vertex,max_nbv,MPI_INT,recv_list_vertex,max_nbv,MPI_INT,comm);
MPI_Allgather(list_boundary_coor,3*max_nbv,MPI_DOUBLE,recv_list_coor,3*max_nbv,MPI_DOUBLE,comm);
/* indicate if a vertex is overlapped */
vertex_ov_label = (int *)malloc(num*max_nbv*sizeof(int));
/*
store the local boundary id on ov processor
it is used to assign global id of local ghost vertices
no need to store master partition id, MV_MasterParID(mv) is already assigned
*/
id_on_ov_list = (int *)malloc(max_nbv*sizeof(int));
for (i = 0; i < num*max_nbv; i++)
vertex_ov_label[i] = 0;
num_ghost_verts = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
MV_Coords(mv,coor);
/* check which previous processor has the same coordinate vertex */
for(j = 0; j < rank; j++) {
/* since each processor has sorted the boundary vertices, use binary search */
double *loc = (double *)bsearch(&coor,
&recv_list_coor[3*max_nbv*j],
global_mesh_info[10*j+5],
3*sizeof(double),
compareCoorDouble);
/* if found the vertex on previous processors */
if(loc) {
/* here the location iloc is relative to the beginning of the jth processor */
iloc = (int)(loc - &recv_list_coor[3*max_nbv*j])/3;
MV_Set_PType(mv,PGHOST);
MV_Set_MasterParID(mv,j);
num_ghost_verts++;
/* label the original vertex as overlapped */
vertex_ov_label[max_nbv*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
/* if found on processor j, no need to test for j+1,j+2...*/
break;
}
}
}
}
free(list_boundary_coor);
free(recv_list_coor);
free(list_boundary_vertex);
free(recv_list_vertex);
}
/* num of ghost verts */
mesh_info[9] = num_ghost_verts;
/* update ghost verts number */
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* since this is a OR reduction, we can use MPI_IN_PLACE, send buffer same as recv buffer */
MPI_Allreduce(MPI_IN_PLACE,vertex_ov_label,num*max_nbv,MPI_INT,MPI_LOR,comm);
/* calculate starting global id number for vertices*/
if (!have_GIDs) {
global_id = 1;
for(i = 0; i < rank; i++)
global_id = global_id + global_mesh_info[10*i+1] - global_mesh_info[10*i+9];
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (MV_PType(mv) == PGHOST)
continue;
MV_Set_GlobalID(mv,global_id++);
MV_Set_MasterParID(mv,rank);
}
}
/* store overlapped vertices IDs and broadast */
vertex_ov_global_id = (int *)malloc(num*max_nbv*sizeof(int));
for(i = 0; i < num*max_nbv; i++)
vertex_ov_global_id[i] = 0;
for(i = 0; i < nbv; i++) {
if(vertex_ov_label[rank*max_nbv+i]) {
mv = List_Entry(boundary_verts,i);
MV_Set_PType(mv,POVERLAP);
vertex_ov_global_id[rank*max_nbv+i] = MV_GlobalID(mv);
}
}
MPI_Allreduce(MPI_IN_PLACE,vertex_ov_global_id,num*max_nbv,MPI_INT,MPI_MAX,comm);
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
if(MV_PType(mv) == PGHOST)
MV_Set_GlobalID(mv,vertex_ov_global_id[MV_MasterParID(mv)*max_nbv+id_on_ov_list[i]]);
}
List_Delete(boundary_verts);
free(global_mesh_info);
free(vertex_ov_label);
free(vertex_ov_global_id);
free(id_on_ov_list);
return 1;
}
int MESH_ConcatSubMesh_Face(Mesh_ptr mesh, int num, Mesh_ptr *submeshes) {
int nfv, nfe, i, j, k, ival;
MVertex_ptr mv, new_mv, sub_mv;
MEdge_ptr me, new_me, sub_me;
MFace_ptr new_mf, sub_mf;
List_ptr mfverts, mfedges;
int add_face, idx, global_id, iloc, *loc;
double coor[3], rval;
void *pval;
Mesh_ptr submesh;
List_ptr parbndry_verts = List_New(10);
List_ptr parbndry_edges = List_New(10);
MEdge_ptr *fedges = (MEdge_ptr *) malloc(MAXPV2*sizeof(MEdge_ptr));
int *fedirs = (int *) malloc(MAXPV2*sizeof(int));
MAttrib_ptr parbndryatt = MAttrib_New(mesh, "on_parbndry", INT, MVERTEX);
/* collect edges and vertices on the partition boundary */
int num_parbndry_edges = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (ME_PType(me) != PINTERIOR) {
List_Add(parbndry_edges,me);
num_parbndry_edges++;
}
int num_parbndry_verts = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MV_PType(mv) != PINTERIOR) {
List_Add(parbndry_verts,mv);
MEnt_Set_AttVal(mv, parbndryatt, 1, 0.0, NULL);
num_parbndry_verts++;
}
/* sort based on global ID */
List_Sort(parbndry_edges,num_parbndry_edges,sizeof(MEdge_ptr),compareGlobalID);
List_Sort(parbndry_verts,num_parbndry_verts,sizeof(MVertex_ptr),compareGlobalID);
int *parbndry_vert_gids = (int *) malloc(num_parbndry_verts*sizeof(int));
int *parbndry_edge_gids = (int *) malloc(num_parbndry_edges*sizeof(int));
/* store them in array for binary search */
for (i = 0; i < num_parbndry_edges; i++) {
me = List_Entry(parbndry_edges,i);
parbndry_edge_gids[i] = ME_GlobalID(me);
}
for (i = 0; i < num_parbndry_verts; i++) {
mv = List_Entry(parbndry_verts,i);
parbndry_vert_gids[i] = MV_GlobalID(mv);
}
/* Make list of new edges and vertices which will be updated
with each mesh that is concatenated */
int max_vnew = 0, max_enew = 0;
for (i = 0; i < num; i++) {
max_vnew += MESH_Num_Vertices(submeshes[i]);
max_enew += MESH_Num_Edges(submeshes[i]);
}
int num_new_verts = 0, num_new_edges = 0;
int *new_vert_gids = (int *) malloc(max_vnew*sizeof(int));
int *new_edge_gids = (int *) malloc(max_enew*sizeof(int));
List_ptr new_verts = List_New(max_vnew);
List_ptr new_edges = List_New(max_enew);
/* Now process each mesh and add a layer of ghost elements from
each of them to the main partition */
for (i = 0; i < num; i++) {
submesh = submeshes[i];
MAttrib_ptr vidatt = MAttrib_New(submesh, "tempvid", POINTER, MVERTEX);
MAttrib_ptr eidatt = MAttrib_New(submesh, "tempeid", POINTER, MEDGE);
idx = 0;
while ((sub_mf = MESH_Next_Face(submesh, &idx))) {
add_face = 0;
/* Find matching vertices between the submesh and main mesh */
mfverts = MF_Vertices(sub_mf,1,0);
nfv = List_Num_Entries(mfverts);
for (j = 0; j < nfv; j++) {
sub_mv = List_Entry(mfverts,j);
/* Does the vertex have a known counterpart on the partition
* boundary of the main mesh? */
MEnt_Get_AttVal(sub_mv, vidatt, &ival, &rval, &mv);
if (mv) {
int on_parbndry=0;
MEnt_Get_AttVal(mv, parbndryatt, &on_parbndry, &rval, &pval);
if (on_parbndry)
add_face = 1;
} else {
/* Does the global ID of this vertex of the sub mesh face
* match the global ID of a partition boundary vertex in
* the main mesh? */
global_id = MV_GlobalID(sub_mv);
loc = (int *) bsearch(&global_id, parbndry_vert_gids, num_parbndry_verts, sizeof(int),
compareINT);
if (loc) { /* found a match */
add_face = 1;
iloc = loc - parbndry_vert_gids;
mv = List_Entry(parbndry_verts,iloc);
/* here set the ghost vertex property, only necessary when the input submeshes are not consistent */
if (MV_PType(mv) == PGHOST && MV_PType(sub_mv) != PGHOST) {
MV_Set_GEntDim(mv,MV_GEntDim(sub_mv));
MV_Set_GEntID(mv,MV_GEntID(sub_mv));
}
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, mv);
}
}
}
List_Delete(mfverts);
/* Find matching edges between the submesh and main mesh */
mfedges = MF_Edges(sub_mf,1,0);
nfe = List_Num_Entries(mfedges);
for (j = 0; j < nfe; j++) {
sub_me = List_Entry(mfedges,j);
/* Does the edge have a known counterpart on the partition
* boundary of the main mesh */
MEnt_Get_AttVal(sub_me, eidatt, &ival, &rval, &me);
if (!me) {
/* Does the global ID of this edge of the sub mesh face
* match the global ID of a partition boundary edge in the
* main mesh? */
global_id = ME_GlobalID(sub_me);
loc = (int *) bsearch(&global_id, parbndry_edge_gids, num_parbndry_edges, sizeof(int),
compareINT);
if (loc) {
iloc = loc - parbndry_edge_gids;
me = List_Entry(parbndry_edges,iloc);
/* here set the ghost edge property, only necessary when the input submeshes are not consistent */
if (ME_PType(me) == PGHOST && ME_PType(sub_me) != PGHOST) {
ME_Set_GEntDim(me,ME_GEntDim(sub_me));
ME_Set_GEntID(me,ME_GEntID(sub_me));
}
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, me);
}
}
}
if (!add_face) {
List_Delete(mfedges);
continue;
}
new_mf = MF_New(mesh); /* add face */
MF_Set_GEntDim(new_mf,MF_GEntDim(sub_mf));
MF_Set_GEntID(new_mf,MF_GEntID(sub_mf));
MF_Set_PType(new_mf,PGHOST);
MF_Set_MasterParID(new_mf,MF_MasterParID(sub_mf));
MF_Set_GlobalID(new_mf,MF_GlobalID(sub_mf));
nfe = List_Num_Entries(mfedges);
for (j = 0; j < nfe; j++) {
sub_me = List_Entry(mfedges,j);
global_id = ME_GlobalID(sub_me);
fedirs[j] = MF_EdgeDir_i(sub_mf,j) == 1 ? 1 : 0;
new_me = NULL;
MEnt_Get_AttVal(sub_me, eidatt, &ival, &rval, &new_me);
if (!new_me) {
/* search in the ghost layer if another edge with
* this global ID has been added */
loc = (int *) bsearch(&global_id, new_edge_gids, num_new_edges,
sizeof(int), compareINT);
if (loc) {
iloc = loc - new_edge_gids;
new_me = List_Entry(new_edges, iloc);
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, new_me);
}
}
if (new_me) {
if (MV_GlobalID(ME_Vertex(new_me,0)) != MV_GlobalID(ME_Vertex(sub_me,0)))
fedirs[j] = 1 - fedirs[j]; /* if the edge dir is not the same, reverse the edge dir */
} else { /* add a new edge to main mesh */
new_me = ME_New(mesh);
ME_Set_GEntDim(new_me,ME_GEntDim(sub_me));
ME_Set_GEntID(new_me,ME_GEntID(sub_me));
ME_Set_PType(new_me,PGHOST);
ME_Set_MasterParID(new_me,ME_MasterParID(sub_me));
ME_Set_GlobalID(new_me,ME_GlobalID(sub_me));
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, new_me);
List_Add(new_edges, new_me);
for (k = 0; k < 2; k++) {
sub_mv = ME_Vertex(sub_me,k);
global_id = MV_GlobalID(sub_mv);
new_mv = NULL;
MEnt_Get_AttVal(sub_mv, vidatt, &ival, &rval, &new_mv);
if (!new_mv) {
/* search in the ghost layer if another vertex with
* this global ID has been added */
loc = (int *) bsearch(&global_id, new_vert_gids, num_new_verts,
sizeof(int), compareINT);
if (loc) {
iloc = loc - new_vert_gids;
new_mv = List_Entry(new_verts, iloc);
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, new_mv);
}
}
if (!new_mv) { /* add a new vertex to main mesh */
new_mv = MV_New(mesh);
MV_Set_GEntDim(new_mv,MV_GEntDim(sub_mv));
MV_Set_GEntID(new_mv,MV_GEntID(sub_mv));
MV_Set_PType(new_mv,PGHOST);
MV_Set_MasterParID(new_mv,MV_MasterParID(sub_mv));
MV_Set_GlobalID(new_mv,MV_GlobalID(sub_mv));
MV_Coords(sub_mv,coor);
MV_Set_Coords(new_mv,coor);
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, new_mv);
List_Add(new_verts, new_mv);
}
ME_Set_Vertex(new_me,k,new_mv); /* set edge-vertex */
}
}
fedges[j] = new_me;
}
MF_Set_Edges(new_mf,nfe,fedges,fedirs); /* set face-edge */
List_Delete(mfedges);
}
idx = 0;
while ((sub_mv = MESH_Next_Vertex(submesh, &idx)))
MEnt_Rem_AttVal(sub_mv, vidatt);
MAttrib_Delete(vidatt);
idx = 0;
while ((sub_me = MESH_Next_Edge(submesh, &idx)))
MEnt_Rem_AttVal(sub_me, eidatt);
MAttrib_Delete(eidatt);
/* Sort the added entity lists by GlobalID */
num_new_edges = List_Num_Entries(new_edges);
List_Sort(new_edges, num_new_edges, sizeof(MEdge_ptr), compareGlobalID);
for (j = 0; j < num_new_edges; j++)
new_edge_gids[j] = ME_GlobalID(List_Entry(new_edges, j));
num_new_verts = List_Num_Entries(new_verts);
List_Sort(new_verts, num_new_verts, sizeof(MVertex_ptr), compareGlobalID);
for (j = 0; j < num_new_verts; j++)
new_vert_gids[j] = MV_GlobalID(List_Entry(new_verts, j));
}
idx = 0;
while ((mv = List_Next_Entry(parbndry_verts, &idx)))
MEnt_Rem_AttVal(mv, parbndryatt);
MAttrib_Delete(parbndryatt);
List_Delete(parbndry_edges);
List_Delete(parbndry_verts);
List_Delete(new_edges);
List_Delete(new_verts);
free(parbndry_vert_gids);
free(parbndry_edge_gids);
free(new_vert_gids);
free(new_edge_gids);
free(fedges);
free(fedirs);
return 1;
}
int MESH_Send_Vertices(Mesh_ptr mesh, int torank, MSTK_Comm comm,
int *numreq, int *maxreq, MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
int i, j, nv;
MVertex_ptr mv;
double coor[3];
MPI_Request mpirequest;
if (requests == NULL)
MSTK_Report("MESH_Surf_SendMesh","MPI requests array is NULL",MSTK_FATAL);
if (*maxreq == 0) {
*maxreq = 25;
*requests = (MPI_Request *) malloc(*maxreq*sizeof(MPI_Request));
*numreq = 0;
}
else if (*maxreq < (*numreq) + 11) {
*maxreq = 2*(*maxreq) + 11;
*requests = (MPI_Request *) realloc(*requests,*maxreq*sizeof(MPI_Request));
}
/* Now send out detailed vertex info */
nv = MESH_Num_Vertices(mesh);
int *list_vertex = (int *) malloc(3*nv*sizeof(int));
/* Store the 3 auxilliary data fields - would be nice if we didn't
* have to send the data in such an error prone way (with bit
* shifting) that requires knowledge of the internal structure of
* MEntity */
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(mesh,i);
list_vertex[3*i] = (MV_GEntID(mv)<<3) | (MV_GEntDim(mv));
list_vertex[3*i+1] = (MV_MasterParID(mv) <<3) | MV_OnParBoundary(mv)<<2 | (MV_PType(mv));
list_vertex[3*i+2] = MV_GlobalID(mv);
}
/* send vertices */
MPI_Isend(list_vertex,3*nv,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;;
int nptrs = 1;
if (*maxptrs2free == 0) {
*maxptrs2free = 25;
*ptrs2free = (void **) malloc(*maxptrs2free*sizeof(void *));
*numptrs2free = 0;
}
else if (*maxptrs2free < (*numptrs2free) + nptrs) {
*maxptrs2free = 2*(*maxptrs2free) + nptrs;
*ptrs2free = (void **) realloc(*ptrs2free,(*maxptrs2free)*sizeof(void *));
}
(*ptrs2free)[(*numptrs2free)++] = list_vertex;
return 1;
}