int MESH_Update_ParallelAdj(Mesh_ptr mesh, MSTK_Comm comm) {
int i, idx, nv, ne, nf, nr, local_ov_num[4];
MVertex_ptr mv;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
MType mtype;
int myprtn, numprtns;
MPI_Comm_rank(comm,&myprtn);
MPI_Comm_size(comm,&numprtns);
if (numprtns == 1) return 1;
/* set ghost adjacencies */
idx = 0;
while ((mv = MESH_Next_GhostVertex(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MV_MasterParID(mv),MVERTEX);
idx = 0;
while ((me = MESH_Next_GhostEdge(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,ME_MasterParID(me),MEDGE);
idx = 0;
while ((mf = MESH_Next_GhostFace(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MF_MasterParID(mf),MFACE);
idx = 0;
while ((mr = MESH_Next_GhostRegion(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MR_MasterParID(mr),MREGION);
/* derive which processors this processor has overlaps with */
int *local_par_adj = (int *) malloc(numprtns*sizeof(int));
int *global_par_adj = (int *) malloc(numprtns*numprtns*sizeof(int));
for (i = 0; i < numprtns; i++) {
local_par_adj[i] = 0;
for (mtype = MVERTEX; mtype <= MREGION; mtype++) {
int j = MESH_Has_Ghosts_From_Prtn(mesh,i,mtype);
local_par_adj[i] |= j<<(2*mtype);
}
}
/* At this point, it is assumed that this processor ('prtn') has
knowledge of all the processors that it has ghost entities from
and what type of entities they are. We do an MPI_Allgather so
that the processor can find out the reverse info, i.e., which
processors are expecting ghost entities from this processor and
what type of entities. This info then goes in as the overlap
entity info for this processor */
for (i = 0; i < numprtns*numprtns; i++) global_par_adj[i] = 0;
MPI_Allgather(local_par_adj,numprtns,MPI_INT,global_par_adj,numprtns,MPI_INT,comm);
/* Now set overlap adjacency flags */
unsigned int ovnum = 0;
unsigned int *prtnums = (unsigned int *) malloc(numprtns*sizeof(unsigned int));
for (i = 0; i < numprtns; i++) {
for (mtype = MVERTEX; mtype <= MREGION; mtype++) {
int j = global_par_adj[i*numprtns + myprtn] & 1<<(2*mtype);
if (j)
MESH_Flag_Has_Overlaps_On_Prtn(mesh,i,mtype);
}
}
/* Right now the model we use is that every partition sends ALL its
overlap entity data to any partition that asks for it */
/* So, if a processor 'i' has ghosts from partition 'j', it needs to
know the total number of overlap entities on partition 'j' in
order to allocate sufficient receive buffers */
int *global_ov_num = (int *) malloc(4*numprtns*sizeof(int));
/* local overlap entity numbers */
local_ov_num[0] = MESH_Num_OverlapVertices(mesh);
local_ov_num[1] = MESH_Num_OverlapEdges(mesh);
local_ov_num[2] = MESH_Num_OverlapFaces(mesh);
local_ov_num[3] = MESH_Num_OverlapRegions(mesh);
MPI_Allgather(local_ov_num,4,MPI_INT,global_ov_num,4,MPI_INT,comm);
/* Set how many entities a partition can expect to receive from
another partititon whether it is used on this partition or not */
MESH_Init_Par_Recv_Info(mesh);
for(i = 0; i < numprtns; i++) {
if (MESH_Has_Ghosts_From_Prtn(mesh,i,MANYTYPE)) {
for (mtype = MVERTEX; mtype <= MREGION; mtype++)
MESH_Set_Num_Recv_From_Prtn(mesh,i,mtype,global_ov_num[4*i+mtype]);
}
}
free(global_ov_num);
free(local_par_adj);
free(global_par_adj);
free(prtnums);
MESH_Mark_ParallelAdj_Current(mesh);
return 1;
}
int MESH_AssignGlobalIDs_Edge(Mesh_ptr submesh, MSTK_Comm comm) {
int i, j, k, nbe, noe, nge, ne, mesh_info[10], global_id;
MVertex_ptr mv;
MEdge_ptr me;
List_ptr boundary_edges;
int *loc, edge_id[2], max_nbe, index_nbe, iloc, is_boundary;
int *global_mesh_info, *list_edge, *recv_list_edge, *edge_ov_label, *id_on_ov_list;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
for (i = 0; i < 10; i++) mesh_info[i] = 0;
ne = MESH_Num_Edges(submesh);
mesh_info[2] = ne;
/*
collect 'boundary' edges
if endpoints are either GHOST or OVERLAP, then it is a boundary edge
*/
nbe = 0; boundary_edges = List_New(10);
for(i = 0; i < ne; i++) {
is_boundary = 1;
me = MESH_Edge(submesh,i);
for( k = 0; k < 2; k++) {
mv = ME_Vertex(me,k);
if(MV_PType(mv)!=PGHOST && MV_PType(mv)!=POVERLAP)
is_boundary = 0;
}
if(is_boundary) {
ME_Flag_OnParBoundary(me);
List_Add(boundary_edges,me);
nbe++;
}
}
mesh_info[6] = nbe;
List_Sort(boundary_edges,nbe,sizeof(MEdge_ptr),compareEdgeID);
global_mesh_info = (int *)malloc(10*num*sizeof(int));
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
max_nbe = 0;
for(i = 0; i < num; i++)
if(max_nbe < global_mesh_info[10*i+6])
max_nbe = global_mesh_info[10*i+6];
list_edge = (int *)malloc(max_nbe*2*sizeof(int));
recv_list_edge = (int *)malloc(num*max_nbe*2*sizeof(int));
/* indicate if a edge is overlapped */
edge_ov_label = (int *)malloc(num*max_nbe*sizeof(int));
for (i = 0; i < num*max_nbe; i++)
edge_ov_label[i] = 0;
id_on_ov_list = (int *)malloc(max_nbe*sizeof(int));
/* pack edge information to send */
index_nbe = 0;
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
list_edge[index_nbe] = MV_GlobalID(ME_Vertex(me,0));
list_edge[index_nbe+1] = MV_GlobalID(ME_Vertex(me,1));
index_nbe += 2;
}
MPI_Allgather(list_edge,2*max_nbe,MPI_INT,recv_list_edge,2*max_nbe,MPI_INT,comm);
nge = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
if(ME_GlobalID(me) > 0) continue; /* if already assigned */
edge_id[0] = MV_GlobalID(ME_Vertex(me,0));
edge_id[1] = MV_GlobalID(ME_Vertex(me,1));
for(j = 0; j < rank; j++) {
loc = (int *)bsearch(&edge_id,
&recv_list_edge[2*max_nbe*j],
global_mesh_info[10*j+6],
2*sizeof(int),
compareEdgeINT);
if(loc) {
iloc = (int)(loc - &recv_list_edge[2*max_nbe*j])/2;
ME_Set_PType(me,PGHOST);
ME_Set_MasterParID(me,j);
edge_ov_label[max_nbe*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
nge++;
break;
}
}
}
}
/* num of ghost verts */
mesh_info[9] = nge;
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* since this is a OR reduction, we can use MPI_IN_PLACE, send buffer same as recv buffer */
MPI_Allreduce(MPI_IN_PLACE,edge_ov_label,num*max_nbe,MPI_INT,MPI_LOR,comm);
/* Assign global ID for non ghost edge */
global_id = 1;
for(i = 0; i < rank; i++)
global_id = global_id + global_mesh_info[10*i+2] - global_mesh_info[10*i+9];
for(i = 0; i < ne; i++) {
me = MESH_Edge(submesh,i);
if (ME_PType(me) == PGHOST) continue;
if (!ME_GlobalID(me)) ME_Set_GlobalID(me,global_id++);
ME_Set_MasterParID(me,rank);
}
/* label OVERLAP edge */
noe = 0;
for(i = 0; i < nbe; i++)
if(edge_ov_label[rank*max_nbe+i]) {
me = List_Entry(boundary_edges,i);
ME_Set_PType(me,POVERLAP);
noe++;
}
/* this time only global id are sent */
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
list_edge[i] = ME_GlobalID(me);
}
MPI_Allgather(list_edge,max_nbe,MPI_INT,recv_list_edge,max_nbe,MPI_INT,comm);
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
if(ME_PType(me)==PGHOST) {
int gid = recv_list_edge[ME_MasterParID(me)*max_nbe+id_on_ov_list[i]];
#ifdef DEBUG
if (ME_GlobalID(me) && ME_GlobalID(me) != gid)
MSTK_Report("MESH_Assign_GlobalIDs_Edge",
"Ghost edge already has different global ID",
MSTK_WARN);
#endif
ME_Set_GlobalID(me, gid);
}
}
List_Delete(boundary_edges);
free(global_mesh_info);
free(edge_ov_label);
free(id_on_ov_list);
free(list_edge);
free(recv_list_edge);
return 1;
}
int MESH_ConcatSubMesh_Face(Mesh_ptr mesh, int num, Mesh_ptr *submeshes) {
int nfv, nfe, i, j, k, ival;
MVertex_ptr mv, new_mv, sub_mv;
MEdge_ptr me, new_me, sub_me;
MFace_ptr new_mf, sub_mf;
List_ptr mfverts, mfedges;
int add_face, idx, global_id, iloc, *loc;
double coor[3], rval;
void *pval;
Mesh_ptr submesh;
List_ptr parbndry_verts = List_New(10);
List_ptr parbndry_edges = List_New(10);
MEdge_ptr *fedges = (MEdge_ptr *) malloc(MAXPV2*sizeof(MEdge_ptr));
int *fedirs = (int *) malloc(MAXPV2*sizeof(int));
MAttrib_ptr parbndryatt = MAttrib_New(mesh, "on_parbndry", INT, MVERTEX);
/* collect edges and vertices on the partition boundary */
int num_parbndry_edges = 0;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (ME_PType(me) != PINTERIOR) {
List_Add(parbndry_edges,me);
num_parbndry_edges++;
}
int num_parbndry_verts = 0;
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MV_PType(mv) != PINTERIOR) {
List_Add(parbndry_verts,mv);
MEnt_Set_AttVal(mv, parbndryatt, 1, 0.0, NULL);
num_parbndry_verts++;
}
/* sort based on global ID */
List_Sort(parbndry_edges,num_parbndry_edges,sizeof(MEdge_ptr),compareGlobalID);
List_Sort(parbndry_verts,num_parbndry_verts,sizeof(MVertex_ptr),compareGlobalID);
int *parbndry_vert_gids = (int *) malloc(num_parbndry_verts*sizeof(int));
int *parbndry_edge_gids = (int *) malloc(num_parbndry_edges*sizeof(int));
/* store them in array for binary search */
for (i = 0; i < num_parbndry_edges; i++) {
me = List_Entry(parbndry_edges,i);
parbndry_edge_gids[i] = ME_GlobalID(me);
}
for (i = 0; i < num_parbndry_verts; i++) {
mv = List_Entry(parbndry_verts,i);
parbndry_vert_gids[i] = MV_GlobalID(mv);
}
/* Make list of new edges and vertices which will be updated
with each mesh that is concatenated */
int max_vnew = 0, max_enew = 0;
for (i = 0; i < num; i++) {
max_vnew += MESH_Num_Vertices(submeshes[i]);
max_enew += MESH_Num_Edges(submeshes[i]);
}
int num_new_verts = 0, num_new_edges = 0;
int *new_vert_gids = (int *) malloc(max_vnew*sizeof(int));
int *new_edge_gids = (int *) malloc(max_enew*sizeof(int));
List_ptr new_verts = List_New(max_vnew);
List_ptr new_edges = List_New(max_enew);
/* Now process each mesh and add a layer of ghost elements from
each of them to the main partition */
for (i = 0; i < num; i++) {
submesh = submeshes[i];
MAttrib_ptr vidatt = MAttrib_New(submesh, "tempvid", POINTER, MVERTEX);
MAttrib_ptr eidatt = MAttrib_New(submesh, "tempeid", POINTER, MEDGE);
idx = 0;
while ((sub_mf = MESH_Next_Face(submesh, &idx))) {
add_face = 0;
/* Find matching vertices between the submesh and main mesh */
mfverts = MF_Vertices(sub_mf,1,0);
nfv = List_Num_Entries(mfverts);
for (j = 0; j < nfv; j++) {
sub_mv = List_Entry(mfverts,j);
/* Does the vertex have a known counterpart on the partition
* boundary of the main mesh? */
MEnt_Get_AttVal(sub_mv, vidatt, &ival, &rval, &mv);
if (mv) {
int on_parbndry=0;
MEnt_Get_AttVal(mv, parbndryatt, &on_parbndry, &rval, &pval);
if (on_parbndry)
add_face = 1;
} else {
/* Does the global ID of this vertex of the sub mesh face
* match the global ID of a partition boundary vertex in
* the main mesh? */
global_id = MV_GlobalID(sub_mv);
loc = (int *) bsearch(&global_id, parbndry_vert_gids, num_parbndry_verts, sizeof(int),
compareINT);
if (loc) { /* found a match */
add_face = 1;
iloc = loc - parbndry_vert_gids;
mv = List_Entry(parbndry_verts,iloc);
/* here set the ghost vertex property, only necessary when the input submeshes are not consistent */
if (MV_PType(mv) == PGHOST && MV_PType(sub_mv) != PGHOST) {
MV_Set_GEntDim(mv,MV_GEntDim(sub_mv));
MV_Set_GEntID(mv,MV_GEntID(sub_mv));
}
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, mv);
}
}
}
List_Delete(mfverts);
/* Find matching edges between the submesh and main mesh */
mfedges = MF_Edges(sub_mf,1,0);
nfe = List_Num_Entries(mfedges);
for (j = 0; j < nfe; j++) {
sub_me = List_Entry(mfedges,j);
/* Does the edge have a known counterpart on the partition
* boundary of the main mesh */
MEnt_Get_AttVal(sub_me, eidatt, &ival, &rval, &me);
if (!me) {
/* Does the global ID of this edge of the sub mesh face
* match the global ID of a partition boundary edge in the
* main mesh? */
global_id = ME_GlobalID(sub_me);
loc = (int *) bsearch(&global_id, parbndry_edge_gids, num_parbndry_edges, sizeof(int),
compareINT);
if (loc) {
iloc = loc - parbndry_edge_gids;
me = List_Entry(parbndry_edges,iloc);
/* here set the ghost edge property, only necessary when the input submeshes are not consistent */
if (ME_PType(me) == PGHOST && ME_PType(sub_me) != PGHOST) {
ME_Set_GEntDim(me,ME_GEntDim(sub_me));
ME_Set_GEntID(me,ME_GEntID(sub_me));
}
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, me);
}
}
}
if (!add_face) {
List_Delete(mfedges);
continue;
}
new_mf = MF_New(mesh); /* add face */
MF_Set_GEntDim(new_mf,MF_GEntDim(sub_mf));
MF_Set_GEntID(new_mf,MF_GEntID(sub_mf));
MF_Set_PType(new_mf,PGHOST);
MF_Set_MasterParID(new_mf,MF_MasterParID(sub_mf));
MF_Set_GlobalID(new_mf,MF_GlobalID(sub_mf));
nfe = List_Num_Entries(mfedges);
for (j = 0; j < nfe; j++) {
sub_me = List_Entry(mfedges,j);
global_id = ME_GlobalID(sub_me);
fedirs[j] = MF_EdgeDir_i(sub_mf,j) == 1 ? 1 : 0;
new_me = NULL;
MEnt_Get_AttVal(sub_me, eidatt, &ival, &rval, &new_me);
if (!new_me) {
/* search in the ghost layer if another edge with
* this global ID has been added */
loc = (int *) bsearch(&global_id, new_edge_gids, num_new_edges,
sizeof(int), compareINT);
if (loc) {
iloc = loc - new_edge_gids;
new_me = List_Entry(new_edges, iloc);
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, new_me);
}
}
if (new_me) {
if (MV_GlobalID(ME_Vertex(new_me,0)) != MV_GlobalID(ME_Vertex(sub_me,0)))
fedirs[j] = 1 - fedirs[j]; /* if the edge dir is not the same, reverse the edge dir */
} else { /* add a new edge to main mesh */
new_me = ME_New(mesh);
ME_Set_GEntDim(new_me,ME_GEntDim(sub_me));
ME_Set_GEntID(new_me,ME_GEntID(sub_me));
ME_Set_PType(new_me,PGHOST);
ME_Set_MasterParID(new_me,ME_MasterParID(sub_me));
ME_Set_GlobalID(new_me,ME_GlobalID(sub_me));
MEnt_Set_AttVal(sub_me, eidatt, 0, 0.0, new_me);
List_Add(new_edges, new_me);
for (k = 0; k < 2; k++) {
sub_mv = ME_Vertex(sub_me,k);
global_id = MV_GlobalID(sub_mv);
new_mv = NULL;
MEnt_Get_AttVal(sub_mv, vidatt, &ival, &rval, &new_mv);
if (!new_mv) {
/* search in the ghost layer if another vertex with
* this global ID has been added */
loc = (int *) bsearch(&global_id, new_vert_gids, num_new_verts,
sizeof(int), compareINT);
if (loc) {
iloc = loc - new_vert_gids;
new_mv = List_Entry(new_verts, iloc);
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, new_mv);
}
}
if (!new_mv) { /* add a new vertex to main mesh */
new_mv = MV_New(mesh);
MV_Set_GEntDim(new_mv,MV_GEntDim(sub_mv));
MV_Set_GEntID(new_mv,MV_GEntID(sub_mv));
MV_Set_PType(new_mv,PGHOST);
MV_Set_MasterParID(new_mv,MV_MasterParID(sub_mv));
MV_Set_GlobalID(new_mv,MV_GlobalID(sub_mv));
MV_Coords(sub_mv,coor);
MV_Set_Coords(new_mv,coor);
MEnt_Set_AttVal(sub_mv, vidatt, 0, 0.0, new_mv);
List_Add(new_verts, new_mv);
}
ME_Set_Vertex(new_me,k,new_mv); /* set edge-vertex */
}
}
fedges[j] = new_me;
}
MF_Set_Edges(new_mf,nfe,fedges,fedirs); /* set face-edge */
List_Delete(mfedges);
}
idx = 0;
while ((sub_mv = MESH_Next_Vertex(submesh, &idx)))
MEnt_Rem_AttVal(sub_mv, vidatt);
MAttrib_Delete(vidatt);
idx = 0;
while ((sub_me = MESH_Next_Edge(submesh, &idx)))
MEnt_Rem_AttVal(sub_me, eidatt);
MAttrib_Delete(eidatt);
/* Sort the added entity lists by GlobalID */
num_new_edges = List_Num_Entries(new_edges);
List_Sort(new_edges, num_new_edges, sizeof(MEdge_ptr), compareGlobalID);
for (j = 0; j < num_new_edges; j++)
new_edge_gids[j] = ME_GlobalID(List_Entry(new_edges, j));
num_new_verts = List_Num_Entries(new_verts);
List_Sort(new_verts, num_new_verts, sizeof(MVertex_ptr), compareGlobalID);
for (j = 0; j < num_new_verts; j++)
new_vert_gids[j] = MV_GlobalID(List_Entry(new_verts, j));
}
idx = 0;
while ((mv = List_Next_Entry(parbndry_verts, &idx)))
MEnt_Rem_AttVal(mv, parbndryatt);
MAttrib_Delete(parbndryatt);
List_Delete(parbndry_edges);
List_Delete(parbndry_verts);
List_Delete(new_edges);
List_Delete(new_verts);
free(parbndry_vert_gids);
free(parbndry_edge_gids);
free(new_vert_gids);
free(new_edge_gids);
free(fedges);
free(fedirs);
return 1;
}
int MESH_Send_NonVertexEntities_FN(Mesh_ptr mesh, int torank, MSTK_Comm comm,
int *numreq, int *maxreq, MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
int i, j, nv, ne, nf, nr;
int nevs, nfes, nrfs, nfe, nrv, nrf, dir;
int maxnfe, maxnrf;
int *mesh_info;
int *list_edge=NULL, *list_face=NULL, *list_region=NULL;
MVertex_ptr mv;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
List_ptr mfedges, mrfaces, mrverts;
RepType rtype;
double coor[3];
MPI_Request mpirequest;
if (requests == NULL)
MSTK_Report("MESH_Surf_SendMesh","MPI requests array is NULL",MSTK_FATAL);
if (*maxreq == 0) {
*maxreq = 25;
*requests = (MPI_Request *) malloc(*maxreq*sizeof(MPI_Request));
*numreq = 0;
}
else if (*maxreq < (*numreq) + 13) {
*maxreq = 2*(*maxreq) + 11;
*requests = (MPI_Request *) realloc(*requests,*maxreq*sizeof(MPI_Request));
}
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
/* some other known quantitites - 5 items per edge (2 for verts
and 3 for extra data), maxnfe+4 items per face (1 for number of
edges, maxnfe for edge indices, anad 3 for extra data),
maxnrf+4 items per region (1 for number of faces, maxnrf for
face indices and 3 for extra data */
maxnfe = 0;
for (i = 0; i < nf; i++) {
mf = MESH_Face(mesh,i);
nfe = MF_Num_Edges(mf);
if (nfe > maxnfe)
maxnfe = nfe;
}
maxnrf = 0;
for (i = 0; i < nr; i++) {
mr = MESH_Region(mesh,i);
nrf = MR_Num_Faces(mr);
if (nrf > maxnrf)
maxnrf = nrf;
}
// The amount of extra info we are sending and their meaning is obviously
// known on the receiving side too. So nevs, nfes and nrfs can be
// calculated without us sending it
nevs = (2+3)*ne;
nfes = (1 + maxnfe + 3)*nf;
nrfs = (1 + maxnrf + 3)*nr;
/* Reserve nevs spots for each edge */
list_edge = (int *) malloc(5*ne*sizeof(int));
nevs = 0;
/* Store the vertex ids, then the 3 auxilliary data fields */
for(i = 0; i < ne; i++) {
me = MESH_Edge(mesh,i);
list_edge[nevs] = MV_ID(ME_Vertex(me,0));
list_edge[nevs+1] = MV_ID(ME_Vertex(me,1));
list_edge[nevs+2] = (ME_GEntID(me)<<3) | (ME_GEntDim(me));
list_edge[nevs+3] = (ME_MasterParID(me) <<3) | (ME_OnParBoundary(me)<<2) | (ME_PType(me));
list_edge[nevs+4] = ME_GlobalID(me);
nevs += 5;
}
/* send detailed edge info */
MPI_Isend(list_edge,nevs,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
/* Reserve nfes spots for each face */
list_face = (int *) malloc(nfes*sizeof(int));
nfes = 0;
/* first store nfe, then the edge ids, then the 3 auxilliary data fields */
for(i = 0; i < nf; i++) {
mf = MESH_Face(mesh,i);
mfedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(mfedges);
list_face[nfes] = nfe;
for(j = 0; j < nfe; j++) {
dir = MF_EdgeDir_i(mf,j) ? 1 : -1;
list_face[nfes+j+1] = dir*ME_ID(List_Entry(mfedges,j));
}
list_face[nfes+nfe+1] = (MF_GEntID(mf)<<3) | (MF_GEntDim(mf));
list_face[nfes+nfe+2] = (MF_MasterParID(mf)<<3) | (MF_OnParBoundary(mf)<<2) | (MF_PType(mf));
list_face[nfes+nfe+3] = MF_GlobalID(mf);
nfes += (nfe + 4);
List_Delete(mfedges);
}
/* send detailed face info */
MPI_Isend(list_face,nfes,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
if (nr) {
list_region = (int *) malloc(nrfs*sizeof(int));
nrfs = 0;
/* first store nrf, then the face ids, then the 3 auxilliary data fields */
for(i = 0; i < nr; i++) {
mr = MESH_Region(mesh,i);
mrfaces = MR_Faces(mr);
nrf = List_Num_Entries(mrfaces);
list_region[nrfs] = nrf;
for(j = 0; j < nrf; j++) {
dir = MR_FaceDir_i(mr,j) == 1 ? 1 : -1;
list_region[nrfs+j+1] = dir*MF_ID(List_Entry(mrfaces,j));
}
list_region[nrfs+nrf+1] = (MR_GEntID(mr)<<3) | (MR_GEntDim(mr));
list_region[nrfs+nrf+2] = (MR_MasterParID(mr)<<3) | (MR_PType(mr)); /* MR_PType is 2 bits; 3 bit is 0 */
list_region[nrfs+nrf+3] = MR_GlobalID(mr);
nrfs += (nrf + 4);
List_Delete(mrfaces);
}
/* send detailed region info */
MPI_Isend(list_region,nrfs,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
}
/* collect allocated memory so it can be freed in a higher level
routine after MPI_Waitall or MPI_Test has ensured that the send
has been completed */
if (ptrs2free == NULL)
MSTK_Report("MESH_Surf_SendMesh_FN","ptrs2free array is NULL",MSTK_FATAL);
int nptrs = 3;
if (*maxptrs2free == 0) {
*maxptrs2free = 25;
*ptrs2free = (void **) malloc(*maxptrs2free*sizeof(void *));
*numptrs2free = 0;
}
else if (*maxptrs2free < (*numptrs2free) + nptrs) {
*maxptrs2free = 2*(*maxptrs2free) + nptrs;
*ptrs2free = (void **) realloc(*ptrs2free,(*maxptrs2free)*sizeof(void *));
}
if (ne)
(*ptrs2free)[(*numptrs2free)++] = list_edge;
if (nf)
(*ptrs2free)[(*numptrs2free)++] = list_face;
if (nr)
(*ptrs2free)[(*numptrs2free)++] = list_region;
return 1;
}
