int MSTK_Mesh_Read_Distribute(Mesh_ptr *recv_mesh,
const char* global_mesh_name,
int *dim, int ring, int with_attr,
int method, MSTK_Comm comm) {
int i, ok;
int rank;
MPI_Comm_rank(comm,&rank);
Mesh_ptr mesh=NULL;
if(rank == 0) {
mesh = MESH_New(UNKNOWN_REP);
ok = MESH_InitFromFile(mesh,global_mesh_name,comm);
if (!ok) {
fprintf(stderr,"Cannot open input file %s\n\n\n",global_mesh_name);
exit(-1);
}
fprintf(stdout,"mstk mesh %s read in successfully\n",global_mesh_name);
*dim = MESH_Num_Regions(mesh) ? 3 : 2;
}
int del_inmesh = 1;
MSTK_Mesh_Distribute(mesh, recv_mesh, dim, ring, with_attr, method,
del_inmesh, comm);
return 1;
}
int MESH_ImportFromExodusII(Mesh_ptr mesh, const char *filename, int *parallel_opts,
MSTK_Comm comm) {
char mesg[256], funcname[32]="MESH_ImportFromExodusII";
char title[256], elem_type[256], sidesetname[256], nodesetname[256];
char matsetname[256];
char **elem_blknames;
int i, j, k, k1;
int comp_ws = sizeof(double), io_ws = 0;
int exoid=0, status;
int ndim, nnodes, nelems, nelblock, nnodesets, nsidesets;
int nedges, nedge_blk, nfaces, nface_blk, nelemsets;
int nedgesets, nfacesets, nnodemaps, nedgemaps, nfacemaps, nelemmaps;
int *elem_blk_ids, *connect, *node_map, *elem_map, *nnpe;
int nelnodes, neledges, nelfaces;
int nelem_i, natts;
int *sideset_ids, *ss_elem_list, *ss_side_list, *nodeset_ids, *ns_node_list;
int num_nodes_in_set, num_sides_in_set, num_df_in_set;
double *xvals, *yvals, *zvals, xyz[3];
float version;
int exo_nrf[3] = {4,5,6};
int exo_nrfverts[3][6] =
{{3,3,3,3,0,0},{4,4,4,3,3,0},{4,4,4,4,4,4}};
int exo_rfverts[3][6][4] =
{{{0,1,3,-1},{1,2,3,-1},{2,0,3,-1},{2,1,0,-1},{-1,-1,-1,-1},{-1,-1,-1,-1}},
{{0,1,4,3},{1,2,5,4},{2,0,3,5},{2,1,0,-1},{3,4,5,-1},{-1,-1,-1,-1}},
{{0,1,5,4},{1,2,6,5},{2,3,7,6},{3,0,4,7},{3,2,1,0},{4,5,6,7}}};
List_ptr fedges, rfaces;
MVertex_ptr mv, *fverts, *rverts;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
MAttrib_ptr nmapatt=NULL, elblockatt=NULL, nodesetatt=NULL, sidesetatt=NULL;
MSet_ptr faceset=NULL, nodeset=NULL, sideset=NULL, matset=NULL;
int distributed=0;
ex_init_params exopar;
FILE *fp;
char basename[256], modfilename[256], *ext;
#ifdef MSTK_HAVE_MPI
int rank=0, numprocs=1;
if (comm) {
MPI_Comm_size(comm,&numprocs);
MPI_Comm_rank(comm,&rank);
}
if (numprocs > 1 && parallel_opts) {
if (parallel_opts[0] == 1) { /* distribute the mesh */
int num_ghost_layers = parallel_opts[2];
int have_zoltan = 0, have_metis = 0;
#ifdef _MSTK_HAVE_ZOLTAN
have_zoltan = 1;
#endif
#ifdef _MSTK_HAVE_METIS
have_metis = 1;
#endif
int part_method = parallel_opts[3];
if (part_method == 0) {
if (!have_metis) {
MSTK_Report(funcname,"Metis not available. Trying Zoltan",MSTK_WARN);
part_method = 1;
if (!have_zoltan)
MSTK_Report(funcname,"No partitioner defined",MSTK_FATAL);
}
}
else if (part_method == 1 || part_method == 2) {
if (!have_zoltan) {
MSTK_Report(funcname,"Zoltan not available. Trying Metis",MSTK_WARN);
part_method = 0;
if (!have_metis)
MSTK_Report(funcname,"No partitioner defined",MSTK_FATAL);
}
}
if (parallel_opts[1] == 0) {
/* Read on processor 0 and distribute to all other processors */
Mesh_ptr globalmesh;
int topodim;
if (rank == 0) { /* Read only on processor 0 */
globalmesh = MESH_New(MESH_RepType(mesh));
int read_status = MESH_ReadExodusII_Serial(globalmesh,filename,rank);
if (!read_status) {
sprintf(mesg,"Could not read Exodus II file %s successfully\n",
filename);
MSTK_Report(funcname,mesg,MSTK_FATAL);
}
topodim = (MESH_Num_Regions(globalmesh) == 0) ? 2 : 3;
}
else
globalmesh = NULL;
int with_attr = 1;
int del_inmesh = 1;
int dist_status = MSTK_Mesh_Distribute(globalmesh, &mesh, &topodim,
num_ghost_layers,
with_attr, part_method,
del_inmesh, comm);
if (!dist_status)
MSTK_Report(funcname,
"Could not distribute meshes to other processors",
MSTK_FATAL);
}
else if (parallel_opts[1] == 1) {
}
else if (parallel_opts[1] == 2) {
}
else if (parallel_opts[1] == 3) {
}
int parallel_check = MESH_Parallel_Check(mesh,comm);
if (!parallel_check)
MSTK_Report(funcname,
"Parallel mesh checks failed",
MSTK_FATAL);
}
else { /* Read the mesh on rank 0 but don't distribute */
if (rank == 0)
return (MESH_ReadExodusII_Serial(mesh,filename,0));
}
} /* if numprocs > 1 */
else {
if (rank == 0)
return (MESH_ReadExodusII_Serial(mesh,filename,0));
else
return 1;
}
#else /* Serial process */
return (MESH_ReadExodusII_Serial(mesh,filename,0));
#endif
return 1;
}
int main(int argc, char *argv[]) {
char infname[256], outfname[256], attfname[256], setfname[256];
Mesh_ptr mesh;
int len, ok=0;
int build_classfn=1, partition=0, weave=0, use_geometry=0, parallel_check=0;
int num_ghost_layers=0, partmethod=0, attfile_given=0, setfile_given=0;
MshFmt inmeshformat, outmeshformat;
if (argc < 3) {
fprintf(stderr,"\n");
fprintf(stderr,"usage: mpirun -n NP %s <--partition=y|1|n|0> <--partition-method=0|1|2> <--parallel-check=y|1|n|0> <--attfile=attfilename> <--setfile=setfilename> infilename outfilename\n\n",progname);
fprintf(stderr,"partition-method = 0, METIS\n");
fprintf(stderr," = 1, ZOLTAN with GRAPH partioning\n");
fprintf(stderr," = 2, ZOLTAN with RCB partitioning\n");
fprintf(stderr,"Choose 2 if you want to avoid partitioning models\n");
fprintf(stderr,"with high aspect ratio along the short directions\n");
fprintf(stderr,"\n");
fprintf(stderr,"infilename = Input Exodus II File\n");
fprintf(stderr,"outfilename = Output Exodus II file name\n");
fprintf(stderr,"attfilename = Auxiliary file with real valued attributes\n");
fprintf(stderr,"setfilename = Auxiliary file with entity set specifications\n");
fprintf(stderr,"\n");
exit(-1);
}
#ifdef MSTK_HAVE_MPI
MPI_Init(&argc,&argv);
#endif
MSTK_Init();
int rank=0, numprocs=1;
#ifdef MSTK_HAVE_MPI
MSTK_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&numprocs);
#else
MSTK_Comm comm = NULL;
#endif
if (argc > 3) {
int i;
for (i = 1; i < argc-2; i++) {
if (strncmp(argv[i],"--partition=",12) == 0) {
if (strncmp(argv[i]+12,"y",1) == 0 ||
strncmp(argv[i]+12,"1",1) == 0)
partition = 1;
else if (strncmp(argv[i]+12,"n",1) == 0 ||
strncmp(argv[i]+12,"0",1) == 0)
partition = 0;
else
MSTK_Report(progname,
"--partition option should be y, n, 1 or 0",
MSTK_FATAL);
}
else if (strncmp(argv[i],"--partition-method",18) == 0 ||
strncmp(argv[i],"--partition_method",18) == 0) {
sscanf(argv[i]+19,"%d",&partmethod);
partition = 1;
}
else if (strncmp(argv[i],"--parallel-check",16) == 0 ||
strncmp(argv[i],"--parallel_check",16) == 0) {
if (strncmp(argv[i]+17,"y",1) == 0 ||
strncmp(argv[i]+17,"1",1) == 1)
parallel_check = 1;
else if (strncmp(argv[i]+17,"n",13) == 0 ||
strncmp(argv[i]+17,"0",13) == 0)
parallel_check = 0;
}
else if (strncmp(argv[i],"--attfile",9) == 0) {
sscanf(argv[i]+10,"%s",attfname);
attfile_given=1;
}
else if (strncmp(argv[i],"--setfile",9) == 0) {
sscanf(argv[i]+10,"%s",setfname);
setfile_given=1;
}
else
fprintf(stderr,"Unrecognized option...Ignoring\n");
}
/* If running on multiple processors, but partition is 0, assume
that you are partitioning a serial mesh */
if (numprocs > 1 && partition == 0) {
MSTK_Report(progname,
"Running on multiple processors but partition option not specified",
MSTK_WARN);
MSTK_Report(progname,
"Output mesh will be partitioned into as many parts as processors",
MSTK_WARN);
partition = 1;
}
}
strcpy(infname,argv[argc-2]);
strcpy(outfname,argv[argc-1]);
len = strlen(infname);
if (len > 4 && strncmp(&(infname[len-4]),".exo",4) == 0)
inmeshformat = EXODUSII;
else
MSTK_Report(progname,"Input file must have .exo extension", MSTK_FATAL);
len = strlen(outfname);
if (len > 4 && strncmp(&(outfname[len-4]),".exo",4) == 0)
outmeshformat = EXODUSII;
else if (len > 4 && strncmp(&(outfname[len-4]),".par",4) == 0)
outmeshformat = EXODUSII;
else
MSTK_Report(progname,"Output file must have .exo or .par extension", MSTK_FATAL);
/* Import the Exodus II mesh */
mesh = MESH_New(F1);
#ifdef MSTK_HAVE_MPI
if (rank == 0) {
#endif
int opts[5]={0,0,0,0,0};
ok = 0;
fprintf(stderr,"Importing mesh from ExodusII file...");
opts[0] = 0; /* don't partition while importing - do it later */
opts[1] = 0;
opts[2] = 0; /* no ghost layers */
opts[3] = 0;
ok = MESH_ImportFromFile(mesh,infname,"exodusii",opts,comm);
if (ok)
fprintf(stderr,"done\n\n");
else {
MSTK_Report(progname,"Failed\n",MSTK_FATAL);
}
/* Read in the attributes only on rank 0 and attach it to the mesh */
if (attfile_given)
import_attributes(mesh, attfname);
/* Import element set definitions and create in the mesh */
if (setfile_given)
import_sets(mesh, setfname);
#ifdef MSTK_HAVE_MPI
}
#endif
#ifdef MSTK_HAVE_MPI
if (partition) {
/* Need to make sure that we have a mesh only on processor 0 */
/* For now we cannot repartition */
int prepartitioned = 0, mesh_present = 0;
int dim = 3;
if (rank > 0) {
if (mesh && MESH_Num_Vertices(mesh) != 0)
mesh_present = 1;
}
MPI_Reduce(&mesh_present,&prepartitioned,1,MPI_INT,MPI_MAX,0,comm);
MPI_Bcast(&prepartitioned,1,MPI_INT,0,comm);
if (!prepartitioned) {
Mesh_ptr mesh0=NULL;
if (rank == 0) {
fprintf(stderr,"Partitioning mesh into %d parts...",numprocs);
if (MESH_Num_Regions(mesh))
dim = 3;
else if (MESH_Num_Faces(mesh))
dim = 2;
else {
fprintf(stderr,"Partitioning possible only for 2D or 3D meshes\n");
exit(-1);
}
mesh0 = mesh;
mesh = NULL;
}
int ring = 0; /* No ghost ring of elements */
int with_attr = 1; /* Do allow exchange of attributes */
int del_inmesh = 1; /* Delete input mesh after partitioning */
ok = MSTK_Mesh_Distribute(mesh0, &mesh, &dim, ring, with_attr,
partmethod, del_inmesh, comm);
if (rank == 0) {
if (ok)
fprintf(stderr,"done\n");
else {
fprintf(stderr,"failed\n");
exit(-1);
}
}
/* if (rank == 0)
MESH_Delete(mesh0);
*/
}
}
#endif /* MSTK_HAVE_MPI */
#ifdef MSTK_HAVE_MPI
if (numprocs > 1 && parallel_check == 1) {
/* Do a parallel consistency check too */
ok = ok && MESH_Parallel_Check(mesh,comm);
}
#endif
fprintf(stderr,"\nExporting mesh to ExodusII format...");
ok = MESH_ExportToFile(mesh,outfname,"exodusii",-1,NULL,NULL,comm);
if (ok)
fprintf(stderr,"Done\n");
else {
fprintf(stderr,"Failed\n");
exit(-1);
}
MESH_Delete(mesh);
// while (1);
#ifdef MSTK_HAVE_MPI
MPI_Finalize();
#endif
return 1;
}
