int MESH_Send_Attribute(Mesh_ptr mesh, MAttrib_ptr attrib, int torank,
MSTK_Comm comm, int *numreq, int *maxreq,
MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
int j, k;
int num, ncomp, ival;
double rval;
void *pval;
double *rval_arr;
int *list_info;
MType mtype;
MAttType att_type;
MEntity_ptr ment;
MPI_Request mpirequest;
if (requests == NULL)
MSTK_Report("MSTK_SendMSet","Invalid MPI request buffer",MSTK_FATAL);
if (*maxreq == 0) {
*maxreq = 25;
*requests = (MPI_Request *) malloc(*maxreq*sizeof(MPI_Request));
*numreq = 0;
}
else if (*maxreq < (*numreq) + 2) {
*maxreq *= 2;
*requests = (MPI_Request *) realloc(*requests,*maxreq*sizeof(MPI_Request));
}
/* get attribute properties */
att_type = MAttrib_Get_Type(attrib);
ncomp = MAttrib_Get_NumComps(attrib);
mtype = MAttrib_Get_EntDim(attrib);
/* attribute entity type, used before ghost list established, so no ghost */
switch (mtype) {
case MVERTEX:
num = MESH_Num_Vertices(mesh);
break;
case MEDGE:
num = MESH_Num_Edges(mesh);
break;
case MFACE:
num = MESH_Num_Faces(mesh);
break;
case MREGION:
num = MESH_Num_Regions(mesh);
break;
default:
num = 0;
#ifdef DEBUG2
MSTK_Report("MESH_SendAttr()","Cannot send attributes on entity type MALLTYPE",MSTK_WARN);
#endif
return 0;
}
/* attribute index and global id */
list_info = (int *) malloc(num*sizeof(int));
/* attribute values */
int *list_value_int = NULL;
double *list_value_double = NULL;
if (att_type == INT)
list_value_int = (int *) malloc(num*ncomp*sizeof(int));
else
list_value_double = (double *) malloc(num*ncomp*sizeof(double));
/* collect data */
for(j = 0; j < num; j++) {
switch (mtype) {
case MVERTEX:
ment = MESH_Vertex(mesh,j);
break;
case MEDGE:
ment = MESH_Edge(mesh,j);
break;
case MFACE:
ment = MESH_Face(mesh,j);
break;
case MREGION:
ment = MESH_Region(mesh,j);
break;
default:
MSTK_Report("MESH_SendAttr()","Invalid entity type",MSTK_WARN);
return 0;
}
MEnt_Get_AttVal(ment,attrib,&ival,&rval,&pval);
list_info[j] = MEnt_GlobalID(ment);
if (att_type == INT)
list_value_int[j] = ival;
else {
if(ncomp == 1)
list_value_double[j] = rval;
if(ncomp > 1) {
rval_arr = (double *)pval;
for(k = 0; k < ncomp; k++)
list_value_double[ncomp*j+k] = rval_arr[k];
}
}
}
/* send entity global IDs */
MPI_Isend(list_info,num,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
/* send values */
if (att_type == INT) {
MPI_Isend(list_value_int,num*ncomp,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
}
else {
MPI_Isend(list_value_double,num*ncomp,MPI_DOUBLE,torank,torank,comm,
&mpirequest);
(*requests)[*numreq] = mpirequest;
}
(*numreq)++;
/* track the buffers used for sending so that they can be released later */
if (*maxptrs2free == 0) {
*maxptrs2free = 25;
*ptrs2free = (void **) malloc(*maxptrs2free*sizeof(void *));
*numptrs2free = 0;
}
else if (*maxptrs2free < (*numptrs2free) + 2) {
*maxptrs2free = 2*(*maxptrs2free) + 2;
*ptrs2free = (void **) realloc(*ptrs2free,(*maxptrs2free)*sizeof(void *));
}
(*ptrs2free)[(*numptrs2free)++] = list_info;
if (att_type == INT)
(*ptrs2free)[(*numptrs2free)++] = list_value_int;
else
(*ptrs2free)[(*numptrs2free)++] = list_value_double;
return 1;
}
int MESH_AssignGlobalIDs_Edge(Mesh_ptr submesh, MSTK_Comm comm) {
int i, j, k, nbe, noe, nge, ne, mesh_info[10], global_id;
MVertex_ptr mv;
MEdge_ptr me;
List_ptr boundary_edges;
int *loc, edge_id[2], max_nbe, index_nbe, iloc, is_boundary;
int *global_mesh_info, *list_edge, *recv_list_edge, *edge_ov_label, *id_on_ov_list;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
for (i = 0; i < 10; i++) mesh_info[i] = 0;
ne = MESH_Num_Edges(submesh);
mesh_info[2] = ne;
/*
collect 'boundary' edges
if endpoints are either GHOST or OVERLAP, then it is a boundary edge
*/
nbe = 0; boundary_edges = List_New(10);
for(i = 0; i < ne; i++) {
is_boundary = 1;
me = MESH_Edge(submesh,i);
for( k = 0; k < 2; k++) {
mv = ME_Vertex(me,k);
if(MV_PType(mv)!=PGHOST && MV_PType(mv)!=POVERLAP)
is_boundary = 0;
}
if(is_boundary) {
ME_Flag_OnParBoundary(me);
List_Add(boundary_edges,me);
nbe++;
}
}
mesh_info[6] = nbe;
List_Sort(boundary_edges,nbe,sizeof(MEdge_ptr),compareEdgeID);
global_mesh_info = (int *)malloc(10*num*sizeof(int));
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
max_nbe = 0;
for(i = 0; i < num; i++)
if(max_nbe < global_mesh_info[10*i+6])
max_nbe = global_mesh_info[10*i+6];
list_edge = (int *)malloc(max_nbe*2*sizeof(int));
recv_list_edge = (int *)malloc(num*max_nbe*2*sizeof(int));
/* indicate if a edge is overlapped */
edge_ov_label = (int *)malloc(num*max_nbe*sizeof(int));
for (i = 0; i < num*max_nbe; i++)
edge_ov_label[i] = 0;
id_on_ov_list = (int *)malloc(max_nbe*sizeof(int));
/* pack edge information to send */
index_nbe = 0;
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
list_edge[index_nbe] = MV_GlobalID(ME_Vertex(me,0));
list_edge[index_nbe+1] = MV_GlobalID(ME_Vertex(me,1));
index_nbe += 2;
}
MPI_Allgather(list_edge,2*max_nbe,MPI_INT,recv_list_edge,2*max_nbe,MPI_INT,comm);
nge = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
if(ME_GlobalID(me) > 0) continue; /* if already assigned */
edge_id[0] = MV_GlobalID(ME_Vertex(me,0));
edge_id[1] = MV_GlobalID(ME_Vertex(me,1));
for(j = 0; j < rank; j++) {
loc = (int *)bsearch(&edge_id,
&recv_list_edge[2*max_nbe*j],
global_mesh_info[10*j+6],
2*sizeof(int),
compareEdgeINT);
if(loc) {
iloc = (int)(loc - &recv_list_edge[2*max_nbe*j])/2;
ME_Set_PType(me,PGHOST);
ME_Set_MasterParID(me,j);
edge_ov_label[max_nbe*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
nge++;
break;
}
}
}
}
/* num of ghost verts */
mesh_info[9] = nge;
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* since this is a OR reduction, we can use MPI_IN_PLACE, send buffer same as recv buffer */
MPI_Allreduce(MPI_IN_PLACE,edge_ov_label,num*max_nbe,MPI_INT,MPI_LOR,comm);
/* Assign global ID for non ghost edge */
global_id = 1;
for(i = 0; i < rank; i++)
global_id = global_id + global_mesh_info[10*i+2] - global_mesh_info[10*i+9];
for(i = 0; i < ne; i++) {
me = MESH_Edge(submesh,i);
if (ME_PType(me) == PGHOST) continue;
if (!ME_GlobalID(me)) ME_Set_GlobalID(me,global_id++);
ME_Set_MasterParID(me,rank);
}
/* label OVERLAP edge */
noe = 0;
for(i = 0; i < nbe; i++)
if(edge_ov_label[rank*max_nbe+i]) {
me = List_Entry(boundary_edges,i);
ME_Set_PType(me,POVERLAP);
noe++;
}
/* this time only global id are sent */
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
list_edge[i] = ME_GlobalID(me);
}
MPI_Allgather(list_edge,max_nbe,MPI_INT,recv_list_edge,max_nbe,MPI_INT,comm);
for(i = 0; i < nbe; i++) {
me = List_Entry(boundary_edges,i);
if(ME_PType(me)==PGHOST) {
int gid = recv_list_edge[ME_MasterParID(me)*max_nbe+id_on_ov_list[i]];
#ifdef DEBUG
if (ME_GlobalID(me) && ME_GlobalID(me) != gid)
MSTK_Report("MESH_Assign_GlobalIDs_Edge",
"Ghost edge already has different global ID",
MSTK_WARN);
#endif
ME_Set_GlobalID(me, gid);
}
}
List_Delete(boundary_edges);
free(global_mesh_info);
free(edge_ov_label);
free(id_on_ov_list);
free(list_edge);
free(recv_list_edge);
return 1;
}
int MESH_Send_NonVertexEntities_FN(Mesh_ptr mesh, int torank, MSTK_Comm comm,
int *numreq, int *maxreq, MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
int i, j, nv, ne, nf, nr;
int nevs, nfes, nrfs, nfe, nrv, nrf, dir;
int maxnfe, maxnrf;
int *mesh_info;
int *list_edge=NULL, *list_face=NULL, *list_region=NULL;
MVertex_ptr mv;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
List_ptr mfedges, mrfaces, mrverts;
RepType rtype;
double coor[3];
MPI_Request mpirequest;
if (requests == NULL)
MSTK_Report("MESH_Surf_SendMesh","MPI requests array is NULL",MSTK_FATAL);
if (*maxreq == 0) {
*maxreq = 25;
*requests = (MPI_Request *) malloc(*maxreq*sizeof(MPI_Request));
*numreq = 0;
}
else if (*maxreq < (*numreq) + 13) {
*maxreq = 2*(*maxreq) + 11;
*requests = (MPI_Request *) realloc(*requests,*maxreq*sizeof(MPI_Request));
}
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
/* some other known quantitites - 5 items per edge (2 for verts
and 3 for extra data), maxnfe+4 items per face (1 for number of
edges, maxnfe for edge indices, anad 3 for extra data),
maxnrf+4 items per region (1 for number of faces, maxnrf for
face indices and 3 for extra data */
maxnfe = 0;
for (i = 0; i < nf; i++) {
mf = MESH_Face(mesh,i);
nfe = MF_Num_Edges(mf);
if (nfe > maxnfe)
maxnfe = nfe;
}
maxnrf = 0;
for (i = 0; i < nr; i++) {
mr = MESH_Region(mesh,i);
nrf = MR_Num_Faces(mr);
if (nrf > maxnrf)
maxnrf = nrf;
}
// The amount of extra info we are sending and their meaning is obviously
// known on the receiving side too. So nevs, nfes and nrfs can be
// calculated without us sending it
nevs = (2+3)*ne;
nfes = (1 + maxnfe + 3)*nf;
nrfs = (1 + maxnrf + 3)*nr;
/* Reserve nevs spots for each edge */
list_edge = (int *) malloc(5*ne*sizeof(int));
nevs = 0;
/* Store the vertex ids, then the 3 auxilliary data fields */
for(i = 0; i < ne; i++) {
me = MESH_Edge(mesh,i);
list_edge[nevs] = MV_ID(ME_Vertex(me,0));
list_edge[nevs+1] = MV_ID(ME_Vertex(me,1));
list_edge[nevs+2] = (ME_GEntID(me)<<3) | (ME_GEntDim(me));
list_edge[nevs+3] = (ME_MasterParID(me) <<3) | (ME_OnParBoundary(me)<<2) | (ME_PType(me));
list_edge[nevs+4] = ME_GlobalID(me);
nevs += 5;
}
/* send detailed edge info */
MPI_Isend(list_edge,nevs,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
/* Reserve nfes spots for each face */
list_face = (int *) malloc(nfes*sizeof(int));
nfes = 0;
/* first store nfe, then the edge ids, then the 3 auxilliary data fields */
for(i = 0; i < nf; i++) {
mf = MESH_Face(mesh,i);
mfedges = MF_Edges(mf,1,0);
nfe = List_Num_Entries(mfedges);
list_face[nfes] = nfe;
for(j = 0; j < nfe; j++) {
dir = MF_EdgeDir_i(mf,j) ? 1 : -1;
list_face[nfes+j+1] = dir*ME_ID(List_Entry(mfedges,j));
}
list_face[nfes+nfe+1] = (MF_GEntID(mf)<<3) | (MF_GEntDim(mf));
list_face[nfes+nfe+2] = (MF_MasterParID(mf)<<3) | (MF_OnParBoundary(mf)<<2) | (MF_PType(mf));
list_face[nfes+nfe+3] = MF_GlobalID(mf);
nfes += (nfe + 4);
List_Delete(mfedges);
}
/* send detailed face info */
MPI_Isend(list_face,nfes,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
if (nr) {
list_region = (int *) malloc(nrfs*sizeof(int));
nrfs = 0;
/* first store nrf, then the face ids, then the 3 auxilliary data fields */
for(i = 0; i < nr; i++) {
mr = MESH_Region(mesh,i);
mrfaces = MR_Faces(mr);
nrf = List_Num_Entries(mrfaces);
list_region[nrfs] = nrf;
for(j = 0; j < nrf; j++) {
dir = MR_FaceDir_i(mr,j) == 1 ? 1 : -1;
list_region[nrfs+j+1] = dir*MF_ID(List_Entry(mrfaces,j));
}
list_region[nrfs+nrf+1] = (MR_GEntID(mr)<<3) | (MR_GEntDim(mr));
list_region[nrfs+nrf+2] = (MR_MasterParID(mr)<<3) | (MR_PType(mr)); /* MR_PType is 2 bits; 3 bit is 0 */
list_region[nrfs+nrf+3] = MR_GlobalID(mr);
nrfs += (nrf + 4);
List_Delete(mrfaces);
}
/* send detailed region info */
MPI_Isend(list_region,nrfs,MPI_INT,torank,torank,comm,&mpirequest);
(*requests)[*numreq] = mpirequest;
(*numreq)++;
}
/* collect allocated memory so it can be freed in a higher level
routine after MPI_Waitall or MPI_Test has ensured that the send
has been completed */
if (ptrs2free == NULL)
MSTK_Report("MESH_Surf_SendMesh_FN","ptrs2free array is NULL",MSTK_FATAL);
int nptrs = 3;
if (*maxptrs2free == 0) {
*maxptrs2free = 25;
*ptrs2free = (void **) malloc(*maxptrs2free*sizeof(void *));
*numptrs2free = 0;
}
else if (*maxptrs2free < (*numptrs2free) + nptrs) {
*maxptrs2free = 2*(*maxptrs2free) + nptrs;
*ptrs2free = (void **) realloc(*ptrs2free,(*maxptrs2free)*sizeof(void *));
}
if (ne)
(*ptrs2free)[(*numptrs2free)++] = list_edge;
if (nf)
(*ptrs2free)[(*numptrs2free)++] = list_face;
if (nr)
(*ptrs2free)[(*numptrs2free)++] = list_region;
return 1;
}
