int MESH_AssignGlobalIDs_Face(Mesh_ptr submesh, MSTK_Comm comm) {
int i, nf, global_id, mesh_info[10];
MFace_ptr mf;
RepType rtype;
int *global_mesh_info;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
for (i = 0; i < 10; i++) mesh_info[i] = 0;
rtype = MESH_RepType(submesh);
nf = MESH_Num_Faces(submesh);
mesh_info[0] = rtype;
mesh_info[3] = nf;
global_mesh_info = (int *)malloc(10*num*sizeof(int));
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* calculate starting global id number for faces*/
global_id = 1;
for(i = 0; i < rank; i++)
global_id = global_id + global_mesh_info[10*i+3];
for(i = 0; i < nf; i++) {
mf = MESH_Face(submesh,i);
MF_Set_PType(mf,PINTERIOR);
if (!MF_GlobalID(mf)) MF_Set_GlobalID(mf,global_id++);
MF_Set_MasterParID(mf,rank);
}
free(global_mesh_info);
return 1;
}
int MESH_Send_NonVertexEntities(Mesh_ptr mesh, int fromrank, MSTK_Comm comm,
int *numreq, int *maxreq,
MPI_Request **requests,
int *numptrs2free, int *maxptrs2free,
void ***ptrs2free) {
RepType rtype = MESH_RepType(mesh);
return MESH_Send_NonVertexEntities_jmp[rtype](mesh,fromrank,comm,
numreq,maxreq,requests,
numptrs2free,maxptrs2free,
ptrs2free);
}
MRegion_ptr MR_GhostNew(Mesh_ptr mesh) {
MRegion_ptr r;
RepType RTYPE;
r = (MRegion_ptr) malloc(sizeof(MRegion));
MEnt_Init_CmnData((MEntity_ptr) r);
MEnt_Set_Mesh((MEntity_ptr) r,mesh);
MEnt_Set_Dim((MEntity_ptr) r,3);
MEnt_Set_GEntDim((MEntity_ptr) r,3);
MEnt_Set_GEntID((MEntity_ptr) r,0);
r->adj = (void *) NULL;
RTYPE = mesh ? MESH_RepType(mesh) : F1;
MR_Set_RepType(r,RTYPE);
if (mesh) {
MESH_Add_GhostRegion(mesh,r);
}
return r;
}
MEdge_ptr ME_GhostNew(Mesh_ptr mesh) {
MEdge_ptr e;
RepType RTYPE;
e = (MEdge_ptr) malloc(sizeof(MEdge));
MEnt_Init_CmnData((MEntity_ptr) e);
MEnt_Set_Mesh((MEntity_ptr) e,mesh);
MEnt_Set_Dim((MEntity_ptr) e,1);
MEnt_Set_GEntDim((MEntity_ptr) e,4); /* Nonsensical value as we don't
know what it is */
MEnt_Set_GEntID((MEntity_ptr) e,0);
e->adj = (void *) NULL;
e->vertex[0] = e->vertex[1] = (MVertex_ptr) NULL;
RTYPE = mesh ? MESH_RepType(mesh) : F1;
ME_Set_RepType(e,RTYPE);
if (mesh) {
MESH_Add_GhostEdge(mesh,e);
}
return e;
}
int MESH_ImportFromExodusII(Mesh_ptr mesh, const char *filename, int *parallel_opts,
MSTK_Comm comm) {
char mesg[256], funcname[32]="MESH_ImportFromExodusII";
char title[256], elem_type[256], sidesetname[256], nodesetname[256];
char matsetname[256];
char **elem_blknames;
int i, j, k, k1;
int comp_ws = sizeof(double), io_ws = 0;
int exoid=0, status;
int ndim, nnodes, nelems, nelblock, nnodesets, nsidesets;
int nedges, nedge_blk, nfaces, nface_blk, nelemsets;
int nedgesets, nfacesets, nnodemaps, nedgemaps, nfacemaps, nelemmaps;
int *elem_blk_ids, *connect, *node_map, *elem_map, *nnpe;
int nelnodes, neledges, nelfaces;
int nelem_i, natts;
int *sideset_ids, *ss_elem_list, *ss_side_list, *nodeset_ids, *ns_node_list;
int num_nodes_in_set, num_sides_in_set, num_df_in_set;
double *xvals, *yvals, *zvals, xyz[3];
float version;
int exo_nrf[3] = {4,5,6};
int exo_nrfverts[3][6] =
{{3,3,3,3,0,0},{4,4,4,3,3,0},{4,4,4,4,4,4}};
int exo_rfverts[3][6][4] =
{{{0,1,3,-1},{1,2,3,-1},{2,0,3,-1},{2,1,0,-1},{-1,-1,-1,-1},{-1,-1,-1,-1}},
{{0,1,4,3},{1,2,5,4},{2,0,3,5},{2,1,0,-1},{3,4,5,-1},{-1,-1,-1,-1}},
{{0,1,5,4},{1,2,6,5},{2,3,7,6},{3,0,4,7},{3,2,1,0},{4,5,6,7}}};
List_ptr fedges, rfaces;
MVertex_ptr mv, *fverts, *rverts;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
MAttrib_ptr nmapatt=NULL, elblockatt=NULL, nodesetatt=NULL, sidesetatt=NULL;
MSet_ptr faceset=NULL, nodeset=NULL, sideset=NULL, matset=NULL;
int distributed=0;
ex_init_params exopar;
FILE *fp;
char basename[256], modfilename[256], *ext;
#ifdef MSTK_HAVE_MPI
int rank=0, numprocs=1;
if (comm) {
MPI_Comm_size(comm,&numprocs);
MPI_Comm_rank(comm,&rank);
}
if (numprocs > 1 && parallel_opts) {
if (parallel_opts[0] == 1) { /* distribute the mesh */
int num_ghost_layers = parallel_opts[2];
int have_zoltan = 0, have_metis = 0;
#ifdef _MSTK_HAVE_ZOLTAN
have_zoltan = 1;
#endif
#ifdef _MSTK_HAVE_METIS
have_metis = 1;
#endif
int part_method = parallel_opts[3];
if (part_method == 0) {
if (!have_metis) {
MSTK_Report(funcname,"Metis not available. Trying Zoltan",MSTK_WARN);
part_method = 1;
if (!have_zoltan)
MSTK_Report(funcname,"No partitioner defined",MSTK_FATAL);
}
}
else if (part_method == 1 || part_method == 2) {
if (!have_zoltan) {
MSTK_Report(funcname,"Zoltan not available. Trying Metis",MSTK_WARN);
part_method = 0;
if (!have_metis)
MSTK_Report(funcname,"No partitioner defined",MSTK_FATAL);
}
}
if (parallel_opts[1] == 0) {
/* Read on processor 0 and distribute to all other processors */
Mesh_ptr globalmesh;
int topodim;
if (rank == 0) { /* Read only on processor 0 */
globalmesh = MESH_New(MESH_RepType(mesh));
int read_status = MESH_ReadExodusII_Serial(globalmesh,filename,rank);
if (!read_status) {
sprintf(mesg,"Could not read Exodus II file %s successfully\n",
filename);
MSTK_Report(funcname,mesg,MSTK_FATAL);
}
topodim = (MESH_Num_Regions(globalmesh) == 0) ? 2 : 3;
}
else
globalmesh = NULL;
int with_attr = 1;
int del_inmesh = 1;
int dist_status = MSTK_Mesh_Distribute(globalmesh, &mesh, &topodim,
num_ghost_layers,
with_attr, part_method,
del_inmesh, comm);
if (!dist_status)
MSTK_Report(funcname,
"Could not distribute meshes to other processors",
MSTK_FATAL);
}
else if (parallel_opts[1] == 1) {
}
else if (parallel_opts[1] == 2) {
}
else if (parallel_opts[1] == 3) {
}
int parallel_check = MESH_Parallel_Check(mesh,comm);
if (!parallel_check)
MSTK_Report(funcname,
"Parallel mesh checks failed",
MSTK_FATAL);
}
else { /* Read the mesh on rank 0 but don't distribute */
if (rank == 0)
return (MESH_ReadExodusII_Serial(mesh,filename,0));
}
} /* if numprocs > 1 */
else {
if (rank == 0)
return (MESH_ReadExodusII_Serial(mesh,filename,0));
else
return 1;
}
#else /* Serial process */
return (MESH_ReadExodusII_Serial(mesh,filename,0));
#endif
return 1;
}
MEdge_ptr MEs_Merge(MEdge_ptr e1, MEdge_ptr e2, int topoflag) {
RepType rtype = MESH_RepType(ME_Mesh(e1));
return (*MEs_Merge_jmp[rtype])(e1,e2,topoflag);
}
MVertex_ptr MVs_Merge(MVertex_ptr v1, MVertex_ptr v2, int topoflag) {
RepType rtype = MESH_RepType(MV_Mesh(v1));
return (*MVs_Merge_jmp[rtype])(v1,v2,topoflag);
}
/*
Send 1-ring Faces to neighbor processors, and receive them
First update the parallel adjancy information,
*/
int MESH_Parallel_AddGhost_Face(Mesh_ptr submesh, MSTK_Comm comm) {
int i, num_recv_procs, index_recv_mesh;
Mesh_ptr send_mesh;
Mesh_ptr *recv_meshes;
int with_attr = 0;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
/* build the 1-ring layer send mesh */
send_mesh = MESH_New(MESH_RepType(submesh));
MESH_BuildSubMesh(submesh,2,send_mesh);
/*
first update parallel adjancy information
any two processor that has vertex connection now has all connection
*/
for (i = 0; i < num; i++) {
if(i == rank) continue;
if( MESH_Has_Ghosts_From_Prtn(submesh,i,MVERTEX) ) {
MESH_Flag_Has_Ghosts_From_Prtn(submesh,i,MALLTYPE);
MESH_Flag_Has_Overlaps_On_Prtn(submesh,i,MALLTYPE);
}
if( MESH_Has_Overlaps_On_Prtn(submesh,i,MVERTEX) ) {
MESH_Flag_Has_Overlaps_On_Prtn(submesh,i,MALLTYPE);
MESH_Flag_Has_Ghosts_From_Prtn(submesh,i,MALLTYPE);
}
}
MESH_Update_ParallelAdj(submesh, comm);
/* allocate meshes to receive from other processors */
num_recv_procs = MESH_Num_GhostPrtns(submesh);
recv_meshes = (Mesh_ptr*)malloc(num_recv_procs*sizeof(Mesh_ptr));
for(i = 0; i < num_recv_procs; i++)
recv_meshes[i] = MESH_New(MESH_RepType(submesh));
/* printf(" number of recv_procs %d,on rank %d\n", num_recv_procs, rank); */
int numreq = 0;
int maxreq = 25; /* should be 17*(num-1) but use small number for testing
realloc of the request array */
MPI_Request *requests = (MPI_Request *) malloc(maxreq*sizeof(MPI_Request));
int numptrs2free = 0;
int maxptrs2free = 25; /* should be about 12*(num-1) to avoid realloc */
void ** ptrs2free = (void **) malloc(maxptrs2free*sizeof(void *));
index_recv_mesh = 0;
for (i = 0; i < num; i++) {
if (i < rank) {
if (MESH_Has_Ghosts_From_Prtn(submesh,i,MFACE))
MESH_RecvMesh(recv_meshes[index_recv_mesh++],i,with_attr,comm);
if (MESH_Has_Overlaps_On_Prtn(submesh,i,MFACE))
MESH_SendMesh(send_mesh,i,with_attr,comm,
&numreq,&maxreq,&requests,&numptrs2free,&maxptrs2free,
&ptrs2free);
}
if (i > rank) {
if (MESH_Has_Overlaps_On_Prtn(submesh,i,MFACE))
MESH_SendMesh(send_mesh,i,with_attr,comm,
&numreq,&maxreq,&requests,&numptrs2free,&maxptrs2free,
&ptrs2free);
if (MESH_Has_Ghosts_From_Prtn(submesh,i,MFACE))
MESH_RecvMesh(recv_meshes[index_recv_mesh++],i,with_attr,comm);
}
}
if (MPI_Waitall(numreq,requests,MPI_STATUSES_IGNORE) != MPI_SUCCESS)
MSTK_Report("MSTK_Mesh_Distribute","Problem distributing mesh",MSTK_FATAL);
if (numreq) free(requests);
/* free all the memory allocated in sendmesh routines */
int p;
for (p = 0; p < numptrs2free; ++p) free(ptrs2free[p]);
if (numptrs2free) free(ptrs2free);
/* install the recv_meshes */
MESH_ConcatSubMesh(submesh, 2, num_recv_procs, recv_meshes);
/* delete recvmeshes */
for (i = 0; i < num_recv_procs; i++)
MESH_Delete(recv_meshes[i]);
free(recv_meshes);
MESH_Delete(send_mesh);
return 1;
}
int MESH_WriteToFile(Mesh_ptr mesh, const char *filename, RepType rtype, MSTK_Comm comm) {
FILE *fp;
char mesg[80], attname[256];
int i, j, k, idx;
int gdim, gid;
int mvid, mvid0, mvid1, mvid2, mrid2, meid, mfid, mrid;
int nav, nar, nfe, nfv, nrf, nrv, dir=0;
int nv, ne, nf, nr;
int natt, ncomp, ival, nent;
double xyz[3], rval, rdummy, *rval_arr;
void *pval, *pdummy;
MVertex_ptr mv, mv0, mv1, mv2;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr, mr2;
List_ptr adjverts, mfedges, mfverts, mrfaces, mrverts, adjregs;
RepType reptype;
MAttrib_ptr attrib, vidatt, eidatt, fidatt, ridatt;
MType attentdim;
MAttType atttype;
char modfilename[256];
strcpy(modfilename, filename);
int rank = 0, numprocs = 1;
#ifdef MSTK_HAVE_MPI
if (comm) {
MPI_Comm_size((MPI_Comm)comm, &numprocs);
MPI_Comm_rank((MPI_Comm)comm, &rank);
}
if (numprocs > 1) {
int ndigits = 0;
int div = 1;
while (numprocs/div) {div *= 10; ndigits++;}
sprintf(modfilename,"%s.%d.%0*d",filename,numprocs,ndigits,rank);
}
#endif
if (!(fp = fopen(modfilename,"w"))) {
sprintf(mesg,"Cannot open file %-s for writing",modfilename);
MSTK_Report("MESH_WriteToFile",mesg,MSTK_ERROR);
return 0;
}
if (rtype != UNKNOWN_REP) {
reptype = rtype;
}
else {
reptype = MESH_RepType(mesh);
}
nv = MESH_Num_Vertices(mesh);
ne = MESH_Num_Edges(mesh);
nf = MESH_Num_Faces(mesh);
nr = MESH_Num_Regions(mesh);
fprintf(fp,"MSTK %-2.1lf\n",MSTK_FILE_VER);
fprintf(fp,"%s %d %d %d %d\n",
MESH_rtype_str[reptype],
nv,
(reptype >= R1 && reptype <= R4)?0:ne,
(reptype >= R1 && reptype <= R2 && nr)?0:nf,
nr);
vidatt = MAttrib_New(mesh,"vidatt",INT,MVERTEX);
eidatt = MAttrib_New(mesh,"eidatt",INT,MEDGE);
fidatt = MAttrib_New(mesh,"fidatt",INT,MFACE);
ridatt = MAttrib_New(mesh,"ridatt",INT,MREGION);
idx = 0; i = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MEnt_Set_AttVal(mv,vidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
MEnt_Set_AttVal(me,eidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MEnt_Set_AttVal(mf,fidatt,++i,0.0,NULL);
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MEnt_Set_AttVal(mr,ridatt,++i,0.0,NULL);
fprintf(fp,"vertices\n");
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
MV_Coords(mv,xyz);
gdim = MV_GEntDim(mv);
gid = MV_GEntID(mv);
fprintf(fp,"%24.16lf %24.16lf %24.16lf %d %d\n",
xyz[0],xyz[1],xyz[2],gdim,gid);
}
if (reptype == R2 || reptype == R4) {
fprintf(fp,"adjvertices\n");
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
nav = MV_Num_AdjVertices(mv);
fprintf(fp,"%d ",nav);
adjverts = MV_AdjVertices(mv);
for (j = 0; j < nav; j++) {
mv2 = List_Entry(adjverts,j);
MEnt_Get_AttVal(mv2,vidatt,&mvid2,&rval,&pval);
fprintf(fp,"%d ",mvid2);
}
fprintf(fp,"\n");
List_Delete(adjverts);
}
}
if (reptype <= F4 && ne) {
fprintf(fp,"edges\n");
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
mv0 = ME_Vertex(me,0);
MEnt_Get_AttVal(mv0,vidatt,&mvid0,&rval,&pval);
mv1 = ME_Vertex(me,1);
MEnt_Get_AttVal(mv1,vidatt,&mvid1,&rval,&pval);
gdim = ME_GEntDim(me);
gid = ME_GEntID(me);
fprintf(fp,"%d %d \t%d %d\n",mvid0,mvid1,gdim,gid);
}
}
if (reptype <= F4) {
/* For full representations, always write out faces in terms of edges */
fprintf(fp,"faces edge\n");
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
nfe = MF_Num_Edges(mf);
fprintf(fp,"%d ",nfe);
mfedges = MF_Edges(mf,1,0);
for (j = 0; j < nfe; j++) {
me = List_Entry(mfedges,j);
dir = MF_EdgeDir_i(mf,j);
MEnt_Get_AttVal(me,eidatt,&meid,&rval,&pval);
if (dir != 1) meid = -meid;
fprintf(fp,"%d ",meid);
}
List_Delete(mfedges);
gdim = MF_GEntDim(mf);
/*
gent = MF_GEntity(mf);
gid = gent ? -99 : 0;
*/
gid = MF_GEntID(mf);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
else {
/* For reduced representations, R3 and R4 always write out faces
in terms of vertices. For reduced representations, R1 and R2
write out faces in terms of vertices only when there are no
regions (i.e. faces are the highest level mesh entities) */
if ((reptype > R2) || (nr == 0)) {
fprintf(fp,"faces vertex\n");
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
nfv = MF_Num_Edges(mf);
fprintf(fp,"%d ",nfv);
mfverts = MF_Vertices(mf,1,0);
for (j = 0; j < nfv; j++) {
mv = List_Entry(mfverts,j);
MEnt_Get_AttVal(mv,vidatt,&mvid,&rval,&pval);
fprintf(fp,"%d ",mvid);
}
List_Delete(mfverts);
gdim = MF_GEntDim(mf);
gid = MF_GEntID(mf);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
}
if (nr) {
if (reptype <= F4 || reptype >= R2) {
fprintf(fp,"regions face\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nrf = MR_Num_Faces(mr);
fprintf(fp,"%d ",nrf);
mrfaces = MR_Faces(mr);
for (j = 0; j < nrf; j++) {
mf = List_Entry(mrfaces,j);
dir = MR_FaceDir_i(mr,j);
MEnt_Get_AttVal(mf,fidatt,&mfid,&rval,&pval);
if (dir != 1) mfid = -mfid;
fprintf(fp,"%d ",mfid);
}
List_Delete(mrfaces);
gdim = MF_GEntDim(mr);
gid = MR_GEntID(mr);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
else {
fprintf(fp,"regions vertex\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nrv = MR_Num_Vertices(mr);
fprintf(fp,"%d ",nrv);
mrverts = MR_Vertices(mr);
for (j = 0; j < nrv; j++) {
mv = List_Entry(mrverts,j);
MEnt_Get_AttVal(mv,vidatt,&mvid,&rval,&pval);
fprintf(fp,"%d ",mvid);
}
List_Delete(mrverts);
gdim = MR_GEntDim(mr);
gid = MR_GEntID(mr);
fprintf(fp,"\t%d %d\n",gdim,gid);
}
}
if (reptype == R2 || reptype == R4) {
fprintf(fp,"adjregions\n");
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
nar = MR_Num_Faces(mr);
fprintf(fp,"%d ",nar);
adjregs = MR_AdjRegions(mr);
for (j = 0; j < nar; j++) {
mr2 = List_Entry(adjregs,j);
if ((long) mr2 == -1)
fprintf(fp,"%d ",0);
else {
MEnt_Get_AttVal(mr2,ridatt,&mrid2,&rval,&pval);
fprintf(fp,"%d ",mrid2);
}
}
fprintf(fp,"\n");
List_Delete(adjregs);
}
}
}
/* Write out attributes if there are more than the 4 that we created
in this routine */
if ((natt = MESH_Num_Attribs(mesh)) > 4) {
fprintf(fp,"attributes\n");
for (i = 0; i < natt; i++) {
attrib = MESH_Attrib(mesh,i);
/* Don't write out attribs we created for the internal use of
this routine */
if (attrib == vidatt || attrib == eidatt || attrib == fidatt ||
attrib == ridatt) continue;
MAttrib_Get_Name(attrib,attname);
atttype = MAttrib_Get_Type(attrib);
if (atttype == POINTER) continue; /* cannot write it out */
ncomp = MAttrib_Get_NumComps(attrib);
attentdim = MAttrib_Get_EntDim(attrib);
/* First count how many entities actually have the attribute assigned */
nent = 0;
switch(attentdim) {
case MVERTEX:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) nent++;
break;
case MEDGE:
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) nent++;
break;
case MFACE:
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) nent++;
break;
case MREGION:
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) nent++;
break;
case MALLTYPE:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) nent++;
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) nent++;
break;
default:
break;
} /* switch (attentdim) */
/* No point in writing out attribute if no entity uses it! Or is there? */
if (!nent) continue;
fprintf(fp,"%-s\n",attname);
switch(atttype) {
case INT:
fprintf(fp,"INT\n");
break;
case DOUBLE:
fprintf(fp,"DOUBLE\n");
break;
case VECTOR:
fprintf(fp,"VECTOR\n");
break;
case TENSOR:
fprintf(fp,"TENSOR\n");
break;
default:
MSTK_Report("MESH_WriteToFile",
"Unrecognizable or unprintable attribute type\n",MSTK_WARN);
continue;
}
fprintf(fp,"%-d\n",ncomp);
switch(attentdim) {
case MVERTEX:
fprintf(fp,"MVERTEX\n");
break;
case MEDGE:
fprintf(fp,"MEDGE\n");
break;
case MFACE:
fprintf(fp,"MFACE\n");
break;
case MREGION:
fprintf(fp,"MREGION\n");
break;
case MALLTYPE:
fprintf(fp,"MALLTYPE\n");
break;
default:
MSTK_Report("Mesh_WriteToFile","Unrecognized entity type",MSTK_WARN);
break;
}
fprintf(fp,"%-d\n",nent);
switch(attentdim) {
case MVERTEX:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mv,vidatt,&mvid,&rdummy,&pdummy);
fprintf(fp,"0 %-d ",mvid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MEDGE:
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(me,eidatt,&meid,&rdummy,&pdummy);
fprintf(fp,"1 %-d ",meid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MFACE:
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mf,fidatt,&mfid,&rdummy,&pdummy);
fprintf(fp,"2 %-d ",mfid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MREGION:
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mr,ridatt,&mrid,&rdummy,&pdummy);
fprintf(fp,"3 %-d ",mrid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
case MALLTYPE:
idx = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx))) {
if (MEnt_Get_AttVal(mv,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mv,vidatt,&mvid,&rdummy,&pdummy);
fprintf(fp,"0 %-d ",mvid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((me = MESH_Next_Edge(mesh,&idx))) {
if (MEnt_Get_AttVal(me,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(me,eidatt,&meid,&rdummy,&pdummy);
fprintf(fp,"1 %-d ",meid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((mf = MESH_Next_Face(mesh,&idx))) {
if (MEnt_Get_AttVal(mf,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mf,fidatt,&mfid,&rdummy,&pdummy);
fprintf(fp,"2 %-d ",mfid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
idx = 0;
while ((mr = MESH_Next_Region(mesh,&idx))) {
if (MEnt_Get_AttVal(mr,attrib,&ival,&rval,&pval)) {
MEnt_Get_AttVal(mr,ridatt,&mrid,&rdummy,&pdummy);
fprintf(fp,"3 %-d ",mrid);
switch (atttype) {
case INT:
fprintf(fp," %-d",ival);
break;
case DOUBLE:
fprintf(fp," %-lf ",rval);
break;
case VECTOR: case TENSOR:
rval_arr = (double *) pval;
for (k = 0; k < ncomp; k++)
fprintf(fp," %-lf ",rval_arr[k]);
break;
default:
break;
}
fprintf(fp,"\n");
}
}
break;
default:
break;
} /* switch (attentdim) */
} /* for (i = 0; i < natt) */
} /* if (Mesh_Num_Attribs(mesh)) */
idx = 0; i = 0;
while ((mv = MESH_Next_Vertex(mesh,&idx)))
MEnt_Rem_AttVal(mv,vidatt);
idx = 0; i = 0;
while ((me = MESH_Next_Edge(mesh,&idx)))
MEnt_Rem_AttVal(me,eidatt);
idx = 0; i = 0;
while ((mf = MESH_Next_Face(mesh,&idx)))
MEnt_Rem_AttVal(mf,fidatt);
idx = 0; i = 0;
while ((mr = MESH_Next_Region(mesh,&idx)))
MEnt_Rem_AttVal(mr,ridatt);
MAttrib_Delete(vidatt);
MAttrib_Delete(eidatt);
MAttrib_Delete(fidatt);
MAttrib_Delete(ridatt);
fclose(fp);
return 1;
}
int MESH_AssignGlobalIDs_Vertex(Mesh_ptr submesh, int have_GIDs, MSTK_Comm comm) {
int i, j, nv, nbv, ne, nf, nr, mesh_info[10];
MVertex_ptr mv;
List_ptr boundary_verts;
RepType rtype;
int index_nbv, max_nbv, iloc, num_ghost_verts, global_id;
int *global_mesh_info, *vertex_ov_label, *vertex_ov_global_id, *id_on_ov_list;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
for (i = 0; i < 10; i++) mesh_info[i] = 0;
rtype = MESH_RepType(submesh);
nv = MESH_Num_Vertices(submesh);
ne = MESH_Num_Edges(submesh);
nf = MESH_Num_Faces(submesh);
nr = MESH_Num_Regions(submesh);
mesh_info[0] = rtype;
mesh_info[1] = nv;
mesh_info[2] = ne;
mesh_info[3] = nf;
mesh_info[4] = nr;
/* calculate number of boundary vertices */
nbv = 0; boundary_verts = List_New(10);
if (nr) {
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (vertex_on_boundary3D(mv)) {
MV_Flag_OnParBoundary(mv);
List_Add(boundary_verts,mv);
nbv++;
}
}
}
else {
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (vertex_on_boundary2D(mv)) {
MV_Flag_OnParBoundary(mv);
List_Add(boundary_verts,mv);
nbv++;
}
}
}
mesh_info[5] = nbv;
/*
gather submeshes information
right now we only need nv and nbv, and later num_ghost_verts, but we gather all mesh_info
*/
global_mesh_info = (int *)malloc(10*num*sizeof(int));
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* get largest number of boundary vertices of all the processors */
max_nbv = 0;
for(i = 0; i < num; i++)
if(max_nbv < global_mesh_info[10*i+5])
max_nbv = global_mesh_info[10*i+5];
if (have_GIDs) {
int *list_boundary_vertex_gid = (int *)malloc(max_nbv*sizeof(int));
int *recv_list_vertex_gid = (int *)malloc(num*max_nbv*sizeof(int));
/* sort boundary vertices based on Global ID, for binary search */
List_Sort(boundary_verts,nbv,sizeof(MVertex_ptr),compareGlobalID);
/* only global ids are sent */
index_nbv = 0;
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
list_boundary_vertex_gid[index_nbv] = MV_GlobalID(mv);
index_nbv++;
}
MPI_Allgather(list_boundary_vertex_gid,max_nbv,MPI_INT,recv_list_vertex_gid,max_nbv,MPI_INT,comm);
/* indicate if a vertex is overlapped */
vertex_ov_label = (int *)malloc(num*max_nbv*sizeof(int));
/*
store the local boundary id on ov processor
it is used to assign global id of local ghost vertices
no need to store master partition id, MV_MasterParID(mv) is already assigned
*/
id_on_ov_list = (int *)malloc(max_nbv*sizeof(int));
for (i = 0; i < num*max_nbv; i++)
vertex_ov_label[i] = 0;
num_ghost_verts = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
int gid = MV_GlobalID(mv);
/* check which previous processor has a vertex with same Global ID*/
for(j = 0; j < rank; j++) {
/* since each processor has sorted the boundary vertices, use binary search */
int *loc = (int *)bsearch(&gid,
&recv_list_vertex_gid[max_nbv*j],
global_mesh_info[10*j+5],
sizeof(int),
compareINT);
/* if found the vertex on previous processors */
if(loc) {
/* here the location iloc is relative to the beginning of the jth processor */
iloc = (int)(loc - &recv_list_vertex_gid[max_nbv*j]);
MV_Set_PType(mv,PGHOST);
MV_Set_MasterParID(mv,j);
num_ghost_verts++;
/* label the original vertex as overlapped */
vertex_ov_label[max_nbv*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
/* if found on processor j, no need to test for j+1,j+2...*/
break;
}
}
}
}
free(list_boundary_vertex_gid);
free(recv_list_vertex_gid);
}
else {
double coor[3];
int *list_boundary_vertex = (int *)malloc(max_nbv*sizeof(int));
double *list_boundary_coor = (double *)malloc(3*max_nbv*sizeof(double));
int *recv_list_vertex = (int *)malloc(num*max_nbv*sizeof(int));
double *recv_list_coor = (double *)malloc(3*num*max_nbv*sizeof(double));
/* sort boundary vertices based on coordinate value, for binary search */
List_Sort(boundary_verts,nbv,sizeof(MVertex_ptr),compareVertexCoor);
/* only local id and coordinate values are sent */
index_nbv = 0;
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
list_boundary_vertex[index_nbv] = MV_ID(mv);
MV_Coords(mv,coor);
list_boundary_coor[index_nbv*3] = coor[0];
list_boundary_coor[index_nbv*3+1] = coor[1];
list_boundary_coor[index_nbv*3+2] = coor[2];
index_nbv++;
}
MPI_Allgather(list_boundary_vertex,max_nbv,MPI_INT,recv_list_vertex,max_nbv,MPI_INT,comm);
MPI_Allgather(list_boundary_coor,3*max_nbv,MPI_DOUBLE,recv_list_coor,3*max_nbv,MPI_DOUBLE,comm);
/* indicate if a vertex is overlapped */
vertex_ov_label = (int *)malloc(num*max_nbv*sizeof(int));
/*
store the local boundary id on ov processor
it is used to assign global id of local ghost vertices
no need to store master partition id, MV_MasterParID(mv) is already assigned
*/
id_on_ov_list = (int *)malloc(max_nbv*sizeof(int));
for (i = 0; i < num*max_nbv; i++)
vertex_ov_label[i] = 0;
num_ghost_verts = 0;
/* for processor other than 0 */
if(rank > 0) {
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
MV_Coords(mv,coor);
/* check which previous processor has the same coordinate vertex */
for(j = 0; j < rank; j++) {
/* since each processor has sorted the boundary vertices, use binary search */
double *loc = (double *)bsearch(&coor,
&recv_list_coor[3*max_nbv*j],
global_mesh_info[10*j+5],
3*sizeof(double),
compareCoorDouble);
/* if found the vertex on previous processors */
if(loc) {
/* here the location iloc is relative to the beginning of the jth processor */
iloc = (int)(loc - &recv_list_coor[3*max_nbv*j])/3;
MV_Set_PType(mv,PGHOST);
MV_Set_MasterParID(mv,j);
num_ghost_verts++;
/* label the original vertex as overlapped */
vertex_ov_label[max_nbv*j+iloc] |= 1;
id_on_ov_list[i] = iloc;
/* if found on processor j, no need to test for j+1,j+2...*/
break;
}
}
}
}
free(list_boundary_coor);
free(recv_list_coor);
free(list_boundary_vertex);
free(recv_list_vertex);
}
/* num of ghost verts */
mesh_info[9] = num_ghost_verts;
/* update ghost verts number */
MPI_Allgather(mesh_info,10,MPI_INT,global_mesh_info,10,MPI_INT,comm);
/* since this is a OR reduction, we can use MPI_IN_PLACE, send buffer same as recv buffer */
MPI_Allreduce(MPI_IN_PLACE,vertex_ov_label,num*max_nbv,MPI_INT,MPI_LOR,comm);
/* calculate starting global id number for vertices*/
if (!have_GIDs) {
global_id = 1;
for(i = 0; i < rank; i++)
global_id = global_id + global_mesh_info[10*i+1] - global_mesh_info[10*i+9];
for(i = 0; i < nv; i++) {
mv = MESH_Vertex(submesh,i);
if (MV_PType(mv) == PGHOST)
continue;
MV_Set_GlobalID(mv,global_id++);
MV_Set_MasterParID(mv,rank);
}
}
/* store overlapped vertices IDs and broadast */
vertex_ov_global_id = (int *)malloc(num*max_nbv*sizeof(int));
for(i = 0; i < num*max_nbv; i++)
vertex_ov_global_id[i] = 0;
for(i = 0; i < nbv; i++) {
if(vertex_ov_label[rank*max_nbv+i]) {
mv = List_Entry(boundary_verts,i);
MV_Set_PType(mv,POVERLAP);
vertex_ov_global_id[rank*max_nbv+i] = MV_GlobalID(mv);
}
}
MPI_Allreduce(MPI_IN_PLACE,vertex_ov_global_id,num*max_nbv,MPI_INT,MPI_MAX,comm);
for(i = 0; i < nbv; i++) {
mv = List_Entry(boundary_verts,i);
if(MV_PType(mv) == PGHOST)
MV_Set_GlobalID(mv,vertex_ov_global_id[MV_MasterParID(mv)*max_nbv+id_on_ov_list[i]]);
}
List_Delete(boundary_verts);
free(global_mesh_info);
free(vertex_ov_label);
free(vertex_ov_global_id);
free(id_on_ov_list);
return 1;
}
