int MESH_Get_OverlapAdj_From_GhostAdj(Mesh_ptr mesh, MSTK_Comm comm) {
int i, idx, nv, ne, nf, nr, local_ov_num[4];
MVertex_ptr mv;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
MType mtype;
int myprtn, numprtns;
MPI_Comm_rank(comm,&myprtn);
MPI_Comm_size(comm,&numprtns);
if (numprtns == 1) return 1;
/* derive which processors this processor has overlaps with */
int *local_par_adj = (int *) malloc(numprtns*sizeof(int));
int *global_par_adj = (int *) malloc(numprtns*numprtns*sizeof(int));
for (i = 0; i < numprtns; i++) {
local_par_adj[i] = 0;
for (mtype = MVERTEX; mtype <= MREGION; mtype++) {
int j = MESH_Has_Ghosts_From_Prtn(mesh,i,mtype);
local_par_adj[i] |= j<<(2*mtype);
}
}
/* At this point, it is assumed that this processor ('prtn') has
knowledge of all the processors that it has ghost entities from
and what type of entities they are. We do an MPI_Allgather so
that the processor can find out the reverse info, i.e., which
processors are expecting ghost entities from this processor and
what type of entities. This info then goes in as the overlap
entity info for this processor */
for (i = 0; i < numprtns*numprtns; i++) global_par_adj[i] = 0;
MPI_Allgather(local_par_adj,numprtns,MPI_INT,global_par_adj,numprtns,MPI_INT,comm);
/* Now set overlap adjacency flags */
unsigned int ovnum = 0;
unsigned int *prtnums = (unsigned int *) malloc(numprtns*sizeof(unsigned int));
for (i = 0; i < numprtns; i++) {
for (mtype = MVERTEX; mtype <= MREGION; mtype++) {
int j = global_par_adj[i*numprtns + myprtn] & 1<<(2*mtype);
if (j)
MESH_Flag_Has_Overlaps_On_Prtn(mesh,i,mtype);
}
}
free(local_par_adj);
free(global_par_adj);
free(prtnums);
/* DON'T MARK PARALLEL ADJACENCY INFO AS CURRENT AS YET BECAUSE THE ACTUAL
NUMBER OF ENTITIES TO BE SENT AND RECEIVED IS NOT FINALIZED */
/* MESH_Mark_ParallelAdj_Current(mesh); */
return 1;
}
int MESH_Update_ParallelAdj(Mesh_ptr mesh, MSTK_Comm comm) {
int i, idx, nv, ne, nf, nr, local_ov_num[4];
MVertex_ptr mv;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
MType mtype;
int myprtn, numprtns;
MPI_Comm_rank(comm,&myprtn);
MPI_Comm_size(comm,&numprtns);
if (numprtns == 1) return 1;
/* set ghost adjacencies */
idx = 0;
while ((mv = MESH_Next_GhostVertex(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MV_MasterParID(mv),MVERTEX);
idx = 0;
while ((me = MESH_Next_GhostEdge(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,ME_MasterParID(me),MEDGE);
idx = 0;
while ((mf = MESH_Next_GhostFace(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MF_MasterParID(mf),MFACE);
idx = 0;
while ((mr = MESH_Next_GhostRegion(mesh,&idx)))
MESH_Flag_Has_Ghosts_From_Prtn(mesh,MR_MasterParID(mr),MREGION);
/* derive which processors this processor has overlaps with */
int *local_par_adj = (int *) malloc(numprtns*sizeof(int));
int *global_par_adj = (int *) malloc(numprtns*numprtns*sizeof(int));
for (i = 0; i < numprtns; i++) {
local_par_adj[i] = 0;
for (mtype = MVERTEX; mtype <= MREGION; mtype++) {
int j = MESH_Has_Ghosts_From_Prtn(mesh,i,mtype);
local_par_adj[i] |= j<<(2*mtype);
}
}
/* At this point, it is assumed that this processor ('prtn') has
knowledge of all the processors that it has ghost entities from
and what type of entities they are. We do an MPI_Allgather so
that the processor can find out the reverse info, i.e., which
processors are expecting ghost entities from this processor and
what type of entities. This info then goes in as the overlap
entity info for this processor */
for (i = 0; i < numprtns*numprtns; i++) global_par_adj[i] = 0;
MPI_Allgather(local_par_adj,numprtns,MPI_INT,global_par_adj,numprtns,MPI_INT,comm);
/* Now set overlap adjacency flags */
unsigned int ovnum = 0;
unsigned int *prtnums = (unsigned int *) malloc(numprtns*sizeof(unsigned int));
for (i = 0; i < numprtns; i++) {
for (mtype = MVERTEX; mtype <= MREGION; mtype++) {
int j = global_par_adj[i*numprtns + myprtn] & 1<<(2*mtype);
if (j)
MESH_Flag_Has_Overlaps_On_Prtn(mesh,i,mtype);
}
}
/* Right now the model we use is that every partition sends ALL its
overlap entity data to any partition that asks for it */
/* So, if a processor 'i' has ghosts from partition 'j', it needs to
know the total number of overlap entities on partition 'j' in
order to allocate sufficient receive buffers */
int *global_ov_num = (int *) malloc(4*numprtns*sizeof(int));
/* local overlap entity numbers */
local_ov_num[0] = MESH_Num_OverlapVertices(mesh);
local_ov_num[1] = MESH_Num_OverlapEdges(mesh);
local_ov_num[2] = MESH_Num_OverlapFaces(mesh);
local_ov_num[3] = MESH_Num_OverlapRegions(mesh);
MPI_Allgather(local_ov_num,4,MPI_INT,global_ov_num,4,MPI_INT,comm);
/* Set how many entities a partition can expect to receive from
another partititon whether it is used on this partition or not */
MESH_Init_Par_Recv_Info(mesh);
for(i = 0; i < numprtns; i++) {
if (MESH_Has_Ghosts_From_Prtn(mesh,i,MANYTYPE)) {
for (mtype = MVERTEX; mtype <= MREGION; mtype++)
MESH_Set_Num_Recv_From_Prtn(mesh,i,mtype,global_ov_num[4*i+mtype]);
}
}
free(global_ov_num);
free(local_par_adj);
free(global_par_adj);
free(prtnums);
MESH_Mark_ParallelAdj_Current(mesh);
return 1;
}
/*
Send 1-ring Faces to neighbor processors, and receive them
First update the parallel adjancy information,
*/
int MESH_Parallel_AddGhost_Face(Mesh_ptr submesh, MSTK_Comm comm) {
int i, num_recv_procs, index_recv_mesh;
Mesh_ptr send_mesh;
Mesh_ptr *recv_meshes;
int with_attr = 0;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
/* build the 1-ring layer send mesh */
send_mesh = MESH_New(MESH_RepType(submesh));
MESH_BuildSubMesh(submesh,2,send_mesh);
/*
first update parallel adjancy information
any two processor that has vertex connection now has all connection
*/
for (i = 0; i < num; i++) {
if(i == rank) continue;
if( MESH_Has_Ghosts_From_Prtn(submesh,i,MVERTEX) ) {
MESH_Flag_Has_Ghosts_From_Prtn(submesh,i,MALLTYPE);
MESH_Flag_Has_Overlaps_On_Prtn(submesh,i,MALLTYPE);
}
if( MESH_Has_Overlaps_On_Prtn(submesh,i,MVERTEX) ) {
MESH_Flag_Has_Overlaps_On_Prtn(submesh,i,MALLTYPE);
MESH_Flag_Has_Ghosts_From_Prtn(submesh,i,MALLTYPE);
}
}
MESH_Update_ParallelAdj(submesh, comm);
/* allocate meshes to receive from other processors */
num_recv_procs = MESH_Num_GhostPrtns(submesh);
recv_meshes = (Mesh_ptr*)malloc(num_recv_procs*sizeof(Mesh_ptr));
for(i = 0; i < num_recv_procs; i++)
recv_meshes[i] = MESH_New(MESH_RepType(submesh));
/* printf(" number of recv_procs %d,on rank %d\n", num_recv_procs, rank); */
int numreq = 0;
int maxreq = 25; /* should be 17*(num-1) but use small number for testing
realloc of the request array */
MPI_Request *requests = (MPI_Request *) malloc(maxreq*sizeof(MPI_Request));
int numptrs2free = 0;
int maxptrs2free = 25; /* should be about 12*(num-1) to avoid realloc */
void ** ptrs2free = (void **) malloc(maxptrs2free*sizeof(void *));
index_recv_mesh = 0;
for (i = 0; i < num; i++) {
if (i < rank) {
if (MESH_Has_Ghosts_From_Prtn(submesh,i,MFACE))
MESH_RecvMesh(recv_meshes[index_recv_mesh++],i,with_attr,comm);
if (MESH_Has_Overlaps_On_Prtn(submesh,i,MFACE))
MESH_SendMesh(send_mesh,i,with_attr,comm,
&numreq,&maxreq,&requests,&numptrs2free,&maxptrs2free,
&ptrs2free);
}
if (i > rank) {
if (MESH_Has_Overlaps_On_Prtn(submesh,i,MFACE))
MESH_SendMesh(send_mesh,i,with_attr,comm,
&numreq,&maxreq,&requests,&numptrs2free,&maxptrs2free,
&ptrs2free);
if (MESH_Has_Ghosts_From_Prtn(submesh,i,MFACE))
MESH_RecvMesh(recv_meshes[index_recv_mesh++],i,with_attr,comm);
}
}
if (MPI_Waitall(numreq,requests,MPI_STATUSES_IGNORE) != MPI_SUCCESS)
MSTK_Report("MSTK_Mesh_Distribute","Problem distributing mesh",MSTK_FATAL);
if (numreq) free(requests);
/* free all the memory allocated in sendmesh routines */
int p;
for (p = 0; p < numptrs2free; ++p) free(ptrs2free[p]);
if (numptrs2free) free(ptrs2free);
/* install the recv_meshes */
MESH_ConcatSubMesh(submesh, 2, num_recv_procs, recv_meshes);
/* delete recvmeshes */
for (i = 0; i < num_recv_procs; i++)
MESH_Delete(recv_meshes[i]);
free(recv_meshes);
MESH_Delete(send_mesh);
return 1;
}
