int MSTK_Mesh_Read_Distribute(Mesh_ptr *recv_mesh,
const char* global_mesh_name,
int *dim, int ring, int with_attr,
int method, MSTK_Comm comm) {
int i, ok;
int rank;
MPI_Comm_rank(comm,&rank);
Mesh_ptr mesh=NULL;
if(rank == 0) {
mesh = MESH_New(UNKNOWN_REP);
ok = MESH_InitFromFile(mesh,global_mesh_name,comm);
if (!ok) {
fprintf(stderr,"Cannot open input file %s\n\n\n",global_mesh_name);
exit(-1);
}
fprintf(stdout,"mstk mesh %s read in successfully\n",global_mesh_name);
*dim = MESH_Num_Regions(mesh) ? 3 : 2;
}
int del_inmesh = 1;
MSTK_Mesh_Distribute(mesh, recv_mesh, dim, ring, with_attr, method,
del_inmesh, comm);
return 1;
}
int MESH_ImportFromExodusII(Mesh_ptr mesh, const char *filename, int *parallel_opts,
MSTK_Comm comm) {
char mesg[256], funcname[32]="MESH_ImportFromExodusII";
char title[256], elem_type[256], sidesetname[256], nodesetname[256];
char matsetname[256];
char **elem_blknames;
int i, j, k, k1;
int comp_ws = sizeof(double), io_ws = 0;
int exoid=0, status;
int ndim, nnodes, nelems, nelblock, nnodesets, nsidesets;
int nedges, nedge_blk, nfaces, nface_blk, nelemsets;
int nedgesets, nfacesets, nnodemaps, nedgemaps, nfacemaps, nelemmaps;
int *elem_blk_ids, *connect, *node_map, *elem_map, *nnpe;
int nelnodes, neledges, nelfaces;
int nelem_i, natts;
int *sideset_ids, *ss_elem_list, *ss_side_list, *nodeset_ids, *ns_node_list;
int num_nodes_in_set, num_sides_in_set, num_df_in_set;
double *xvals, *yvals, *zvals, xyz[3];
float version;
int exo_nrf[3] = {4,5,6};
int exo_nrfverts[3][6] =
{{3,3,3,3,0,0},{4,4,4,3,3,0},{4,4,4,4,4,4}};
int exo_rfverts[3][6][4] =
{{{0,1,3,-1},{1,2,3,-1},{2,0,3,-1},{2,1,0,-1},{-1,-1,-1,-1},{-1,-1,-1,-1}},
{{0,1,4,3},{1,2,5,4},{2,0,3,5},{2,1,0,-1},{3,4,5,-1},{-1,-1,-1,-1}},
{{0,1,5,4},{1,2,6,5},{2,3,7,6},{3,0,4,7},{3,2,1,0},{4,5,6,7}}};
List_ptr fedges, rfaces;
MVertex_ptr mv, *fverts, *rverts;
MEdge_ptr me;
MFace_ptr mf;
MRegion_ptr mr;
MAttrib_ptr nmapatt=NULL, elblockatt=NULL, nodesetatt=NULL, sidesetatt=NULL;
MSet_ptr faceset=NULL, nodeset=NULL, sideset=NULL, matset=NULL;
int distributed=0;
ex_init_params exopar;
FILE *fp;
char basename[256], modfilename[256], *ext;
#ifdef MSTK_HAVE_MPI
int rank=0, numprocs=1;
if (comm) {
MPI_Comm_size(comm,&numprocs);
MPI_Comm_rank(comm,&rank);
}
if (numprocs > 1 && parallel_opts) {
if (parallel_opts[0] == 1) { /* distribute the mesh */
int num_ghost_layers = parallel_opts[2];
int have_zoltan = 0, have_metis = 0;
#ifdef _MSTK_HAVE_ZOLTAN
have_zoltan = 1;
#endif
#ifdef _MSTK_HAVE_METIS
have_metis = 1;
#endif
int part_method = parallel_opts[3];
if (part_method == 0) {
if (!have_metis) {
MSTK_Report(funcname,"Metis not available. Trying Zoltan",MSTK_WARN);
part_method = 1;
if (!have_zoltan)
MSTK_Report(funcname,"No partitioner defined",MSTK_FATAL);
}
}
else if (part_method == 1 || part_method == 2) {
if (!have_zoltan) {
MSTK_Report(funcname,"Zoltan not available. Trying Metis",MSTK_WARN);
part_method = 0;
if (!have_metis)
MSTK_Report(funcname,"No partitioner defined",MSTK_FATAL);
}
}
if (parallel_opts[1] == 0) {
/* Read on processor 0 and distribute to all other processors */
Mesh_ptr globalmesh;
int topodim;
if (rank == 0) { /* Read only on processor 0 */
globalmesh = MESH_New(MESH_RepType(mesh));
int read_status = MESH_ReadExodusII_Serial(globalmesh,filename,rank);
if (!read_status) {
sprintf(mesg,"Could not read Exodus II file %s successfully\n",
filename);
MSTK_Report(funcname,mesg,MSTK_FATAL);
}
topodim = (MESH_Num_Regions(globalmesh) == 0) ? 2 : 3;
}
else
globalmesh = NULL;
int with_attr = 1;
int del_inmesh = 1;
int dist_status = MSTK_Mesh_Distribute(globalmesh, &mesh, &topodim,
num_ghost_layers,
with_attr, part_method,
del_inmesh, comm);
if (!dist_status)
MSTK_Report(funcname,
"Could not distribute meshes to other processors",
MSTK_FATAL);
}
else if (parallel_opts[1] == 1) {
}
else if (parallel_opts[1] == 2) {
}
else if (parallel_opts[1] == 3) {
}
int parallel_check = MESH_Parallel_Check(mesh,comm);
if (!parallel_check)
MSTK_Report(funcname,
"Parallel mesh checks failed",
MSTK_FATAL);
}
else { /* Read the mesh on rank 0 but don't distribute */
if (rank == 0)
return (MESH_ReadExodusII_Serial(mesh,filename,0));
}
} /* if numprocs > 1 */
else {
if (rank == 0)
return (MESH_ReadExodusII_Serial(mesh,filename,0));
else
return 1;
}
#else /* Serial process */
return (MESH_ReadExodusII_Serial(mesh,filename,0));
#endif
return 1;
}
int main(int argc, char **argv) {
int len, ok;
int i, j, id, idx, ncells, file_found;
int num_parts, imethod;
int nproc, rank;
Mesh_ptr mesh;
char basename[256], meshname[256], gmvfilename[256], method[256];
if (argc == 1) {
fprintf(stderr,"Usage: %s --num=num_parts --method=Metis/Zoltan (default=Metis) --file=name.mstk\n",argv[0]);
exit(-1);
}
file_found = 0;
num_parts = 4;
strcpy(method,"Metis");
for (i = 1; i < argc; i++) {
if (strncmp(argv[i],"--file=",7) == 0) {
sscanf(argv[i]+7,"%s",meshname);
file_found = 1;
}
else if (strncmp(argv[i],"--num=",6) == 0)
sscanf(argv[i]+6,"%d",&num_parts);
else if (strncmp(argv[i],"--method=",9) == 0)
sscanf(argv[i]+9,"%s",method);
else if (strncmp(argv[i],"--help",6) == 0) {
fprintf(stderr,"Usage: %s --num=num_parts --method=Metis/Zoltan (default=Metis) --file=name.mstk\n",argv[0]);
exit(-1);
}
}
if (strncasecmp(method,"metis",5) == 0)
imethod = 0;
else if (strncasecmp(method,"zoltan",6) == 0)
imethod = 1;
else {
fprintf(stderr,"vizpart: Partitioning method not recognized\n");
exit(-1);
}
if (!file_found) {
fprintf(stderr,"Must specify input filename using --file argument\n");
exit(-1);
}
MPI_Init(&argc,&argv);
MSTK_Init();
MSTK_Comm comm = MPI_COMM_WORLD;
MPI_Comm_size(comm,&nproc);
MPI_Comm_rank(comm,&rank);
if (imethod == 1 && nproc != num_parts) {
fprintf(stderr,"Zoltan based partitioner: Number of processors must equal requested number of partition\n");
exit(-1);
}
strcpy(basename,meshname);
len = strlen(meshname);
if (len > 5 && strncmp(&(meshname[len-5]),".mstk",5) == 0)
basename[len-5] = '\0';
else
strcat(meshname,".mstk");
mesh = MESH_New(UNKNOWN_REP);
if (rank == 0) {
ok = MESH_InitFromFile(mesh,meshname,comm);
if (!ok) {
fprintf(stderr,"Cannot file input file %s\n\n\n",meshname);
exit(-1);
}
if ( (ncells = MESH_Num_Regions(mesh)) > 0) {
printf("3D mesh with %d regions\n", ncells);
}
else if ( (ncells = MESH_Num_Faces(mesh)) > 0)
printf("2D mesh with %d faces\n", ncells);
else {
fprintf(stderr,"Mesh is neither solid nor surface mesh. Exiting...\n");
exit(-1);
}
}
Mesh_ptr *submeshes = (Mesh_ptr*) malloc(num_parts*sizeof(Mesh_ptr));
int *part;
MESH_Get_Partitioning(mesh, imethod, &part, comm);
if(rank == 0) {
int del_inmesh = 0;
MESH_Partition(mesh, num_parts, part, submeshes);
idx = 0;
if(MESH_Num_Regions(mesh)) {
MAttrib_ptr region_part = MAttrib_New(mesh, "part", INT, MREGION);
MRegion_ptr mr;
while ((mr = MESH_Next_Region(mesh, &idx))) {
id = MR_ID(mr);
MEnt_Set_AttVal(mr,region_part,part[id-1],0,NULL);
}
}
else {
MAttrib_ptr face_part = MAttrib_New(mesh, "part", INT, MFACE);
MFace_ptr mf;
while ((mf = MESH_Next_Face(mesh, &idx))) {
id = MF_ID(mf);
MEnt_Set_AttVal(mf,face_part,part[idx-1],0,NULL);
}
}
sprintf(gmvfilename,"%s_part.gmv",basename);
MESH_ExportToGMV(mesh,gmvfilename,0,NULL,NULL,comm);
for( i = 0; i < num_parts; i++) {
sprintf(gmvfilename,"%s_part.%04d.%04d.gmv",basename,i,0);
MESH_ExportToGMV(submeshes[i],gmvfilename,0,NULL,NULL,comm);
}
for (i = 0; i < num_parts; i++)
MESH_Delete(submeshes[i]);
}
free(submeshes);
MESH_Delete(mesh);
free(part);
MPI_Finalize();
return 1;
}
int MSTK_Mesh_Distribute(Mesh_ptr parentmesh, Mesh_ptr *mysubmesh, int *dim,
int ring, int with_attr, PartitionMethod method,
int del_inmesh, MSTK_Comm comm) {
int i, a, m, n, recv_dim;
int *send_dim, *part=NULL;
int rank, numprocs, *toranks;
int DebugWait=0;
Mesh_ptr *submeshes=NULL;
MAttrib_ptr attrib;
MSet_ptr mset;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&numprocs);
recv_dim = rank+5; /* ??? */
while (DebugWait) ;
#ifdef DEBUG
double elapsed_time;
double t0 = MPI_Wtime();
#endif
send_dim = (int *) malloc(numprocs*sizeof(int));
for (i = 0; i < numprocs; i++) send_dim[i] = *dim;
MPI_Scatter(send_dim, 1, MPI_INT, &recv_dim, 1, MPI_INT, 0, comm);
free(send_dim);
if (rank != 0)
*dim = recv_dim;
/* PARTITIONING OF THE MESH GRAPH AND GETTING THE PARTITION
NUMBERS FOR EACH ELEMENT */
MESH_Get_Partitioning(parentmesh, method, &part, comm);
if (rank == 0) {
toranks = (int *) malloc(numprocs*sizeof(int));
for (i = 0; i < numprocs; i++) toranks[i] = i;
MESH_Partition_and_Send(parentmesh, numprocs, part, toranks, ring,
with_attr, del_inmesh, comm, mysubmesh);
free(toranks);
fprintf(stderr,"Finished partioning and sending on rank 0\n");
}
if (part) free(part);
/* RECEIVING THE INFORMATION ON OTHER PROCESSORS - THIS IS
STRAIGHTFORWARD AND USES BLOCKING MPI RECEIVES BECAUSE EACH
STEP NEEDS TO BE COMPLETED BEFORE THE NEXT */
if (rank > 0) { /* Receive the mesh from processor 0 */
int fromrank = 0;
int nv, ne, nf, nr;
RepType rtype;
if (!(*mysubmesh))
*mysubmesh = MESH_New(UNKNOWN_REP);
MESH_RecvMesh(*mysubmesh, fromrank, with_attr, comm);
fprintf(stderr,"Received mesh on rank %-d\n", rank);
}
/* FINISH UP */
/* Build the sorted lists of ghost and overlap entities */
MESH_Build_GhostLists(*mysubmesh,*dim);
/* Some additional parallel information update to indicate which
processors communicate with which others */
MESH_Set_Prtn(*mysubmesh,rank,numprocs);
MESH_Update_ParallelAdj(*mysubmesh,comm);
MESH_Disable_GlobalIDSearch(*mysubmesh);
#ifdef DEBUG
elapsed_time = MPI_Wtime() - t0;
fprintf(stderr,"Elapsed time after mesh distribution on processor %-d is %lf s\n",rank,elapsed_time);
#endif
/* Put a barrier so that distribution of meshes takes place one at a time
in a simulation that may have multiple mesh objects on each processor */
MPI_Barrier(comm);
return 1;
}
/*
Send 1-ring Faces to neighbor processors, and receive them
First update the parallel adjancy information,
*/
int MESH_Parallel_AddGhost_Face(Mesh_ptr submesh, MSTK_Comm comm) {
int i, num_recv_procs, index_recv_mesh;
Mesh_ptr send_mesh;
Mesh_ptr *recv_meshes;
int with_attr = 0;
int rank, num;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&num);
/* build the 1-ring layer send mesh */
send_mesh = MESH_New(MESH_RepType(submesh));
MESH_BuildSubMesh(submesh,2,send_mesh);
/*
first update parallel adjancy information
any two processor that has vertex connection now has all connection
*/
for (i = 0; i < num; i++) {
if(i == rank) continue;
if( MESH_Has_Ghosts_From_Prtn(submesh,i,MVERTEX) ) {
MESH_Flag_Has_Ghosts_From_Prtn(submesh,i,MALLTYPE);
MESH_Flag_Has_Overlaps_On_Prtn(submesh,i,MALLTYPE);
}
if( MESH_Has_Overlaps_On_Prtn(submesh,i,MVERTEX) ) {
MESH_Flag_Has_Overlaps_On_Prtn(submesh,i,MALLTYPE);
MESH_Flag_Has_Ghosts_From_Prtn(submesh,i,MALLTYPE);
}
}
MESH_Update_ParallelAdj(submesh, comm);
/* allocate meshes to receive from other processors */
num_recv_procs = MESH_Num_GhostPrtns(submesh);
recv_meshes = (Mesh_ptr*)malloc(num_recv_procs*sizeof(Mesh_ptr));
for(i = 0; i < num_recv_procs; i++)
recv_meshes[i] = MESH_New(MESH_RepType(submesh));
/* printf(" number of recv_procs %d,on rank %d\n", num_recv_procs, rank); */
int numreq = 0;
int maxreq = 25; /* should be 17*(num-1) but use small number for testing
realloc of the request array */
MPI_Request *requests = (MPI_Request *) malloc(maxreq*sizeof(MPI_Request));
int numptrs2free = 0;
int maxptrs2free = 25; /* should be about 12*(num-1) to avoid realloc */
void ** ptrs2free = (void **) malloc(maxptrs2free*sizeof(void *));
index_recv_mesh = 0;
for (i = 0; i < num; i++) {
if (i < rank) {
if (MESH_Has_Ghosts_From_Prtn(submesh,i,MFACE))
MESH_RecvMesh(recv_meshes[index_recv_mesh++],i,with_attr,comm);
if (MESH_Has_Overlaps_On_Prtn(submesh,i,MFACE))
MESH_SendMesh(send_mesh,i,with_attr,comm,
&numreq,&maxreq,&requests,&numptrs2free,&maxptrs2free,
&ptrs2free);
}
if (i > rank) {
if (MESH_Has_Overlaps_On_Prtn(submesh,i,MFACE))
MESH_SendMesh(send_mesh,i,with_attr,comm,
&numreq,&maxreq,&requests,&numptrs2free,&maxptrs2free,
&ptrs2free);
if (MESH_Has_Ghosts_From_Prtn(submesh,i,MFACE))
MESH_RecvMesh(recv_meshes[index_recv_mesh++],i,with_attr,comm);
}
}
if (MPI_Waitall(numreq,requests,MPI_STATUSES_IGNORE) != MPI_SUCCESS)
MSTK_Report("MSTK_Mesh_Distribute","Problem distributing mesh",MSTK_FATAL);
if (numreq) free(requests);
/* free all the memory allocated in sendmesh routines */
int p;
for (p = 0; p < numptrs2free; ++p) free(ptrs2free[p]);
if (numptrs2free) free(ptrs2free);
/* install the recv_meshes */
MESH_ConcatSubMesh(submesh, 2, num_recv_procs, recv_meshes);
/* delete recvmeshes */
for (i = 0; i < num_recv_procs; i++)
MESH_Delete(recv_meshes[i]);
free(recv_meshes);
MESH_Delete(send_mesh);
return 1;
}
int main(int argc, char *argv[]) {
char infname[256], outfname[256], attfname[256], setfname[256];
Mesh_ptr mesh;
int len, ok=0;
int build_classfn=1, partition=0, weave=0, use_geometry=0, parallel_check=0;
int num_ghost_layers=0, partmethod=0, attfile_given=0, setfile_given=0;
MshFmt inmeshformat, outmeshformat;
if (argc < 3) {
fprintf(stderr,"\n");
fprintf(stderr,"usage: mpirun -n NP %s <--partition=y|1|n|0> <--partition-method=0|1|2> <--parallel-check=y|1|n|0> <--attfile=attfilename> <--setfile=setfilename> infilename outfilename\n\n",progname);
fprintf(stderr,"partition-method = 0, METIS\n");
fprintf(stderr," = 1, ZOLTAN with GRAPH partioning\n");
fprintf(stderr," = 2, ZOLTAN with RCB partitioning\n");
fprintf(stderr,"Choose 2 if you want to avoid partitioning models\n");
fprintf(stderr,"with high aspect ratio along the short directions\n");
fprintf(stderr,"\n");
fprintf(stderr,"infilename = Input Exodus II File\n");
fprintf(stderr,"outfilename = Output Exodus II file name\n");
fprintf(stderr,"attfilename = Auxiliary file with real valued attributes\n");
fprintf(stderr,"setfilename = Auxiliary file with entity set specifications\n");
fprintf(stderr,"\n");
exit(-1);
}
#ifdef MSTK_HAVE_MPI
MPI_Init(&argc,&argv);
#endif
MSTK_Init();
int rank=0, numprocs=1;
#ifdef MSTK_HAVE_MPI
MSTK_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&numprocs);
#else
MSTK_Comm comm = NULL;
#endif
if (argc > 3) {
int i;
for (i = 1; i < argc-2; i++) {
if (strncmp(argv[i],"--partition=",12) == 0) {
if (strncmp(argv[i]+12,"y",1) == 0 ||
strncmp(argv[i]+12,"1",1) == 0)
partition = 1;
else if (strncmp(argv[i]+12,"n",1) == 0 ||
strncmp(argv[i]+12,"0",1) == 0)
partition = 0;
else
MSTK_Report(progname,
"--partition option should be y, n, 1 or 0",
MSTK_FATAL);
}
else if (strncmp(argv[i],"--partition-method",18) == 0 ||
strncmp(argv[i],"--partition_method",18) == 0) {
sscanf(argv[i]+19,"%d",&partmethod);
partition = 1;
}
else if (strncmp(argv[i],"--parallel-check",16) == 0 ||
strncmp(argv[i],"--parallel_check",16) == 0) {
if (strncmp(argv[i]+17,"y",1) == 0 ||
strncmp(argv[i]+17,"1",1) == 1)
parallel_check = 1;
else if (strncmp(argv[i]+17,"n",13) == 0 ||
strncmp(argv[i]+17,"0",13) == 0)
parallel_check = 0;
}
else if (strncmp(argv[i],"--attfile",9) == 0) {
sscanf(argv[i]+10,"%s",attfname);
attfile_given=1;
}
else if (strncmp(argv[i],"--setfile",9) == 0) {
sscanf(argv[i]+10,"%s",setfname);
setfile_given=1;
}
else
fprintf(stderr,"Unrecognized option...Ignoring\n");
}
/* If running on multiple processors, but partition is 0, assume
that you are partitioning a serial mesh */
if (numprocs > 1 && partition == 0) {
MSTK_Report(progname,
"Running on multiple processors but partition option not specified",
MSTK_WARN);
MSTK_Report(progname,
"Output mesh will be partitioned into as many parts as processors",
MSTK_WARN);
partition = 1;
}
}
strcpy(infname,argv[argc-2]);
strcpy(outfname,argv[argc-1]);
len = strlen(infname);
if (len > 4 && strncmp(&(infname[len-4]),".exo",4) == 0)
inmeshformat = EXODUSII;
else
MSTK_Report(progname,"Input file must have .exo extension", MSTK_FATAL);
len = strlen(outfname);
if (len > 4 && strncmp(&(outfname[len-4]),".exo",4) == 0)
outmeshformat = EXODUSII;
else if (len > 4 && strncmp(&(outfname[len-4]),".par",4) == 0)
outmeshformat = EXODUSII;
else
MSTK_Report(progname,"Output file must have .exo or .par extension", MSTK_FATAL);
/* Import the Exodus II mesh */
mesh = MESH_New(F1);
#ifdef MSTK_HAVE_MPI
if (rank == 0) {
#endif
int opts[5]={0,0,0,0,0};
ok = 0;
fprintf(stderr,"Importing mesh from ExodusII file...");
opts[0] = 0; /* don't partition while importing - do it later */
opts[1] = 0;
opts[2] = 0; /* no ghost layers */
opts[3] = 0;
ok = MESH_ImportFromFile(mesh,infname,"exodusii",opts,comm);
if (ok)
fprintf(stderr,"done\n\n");
else {
MSTK_Report(progname,"Failed\n",MSTK_FATAL);
}
/* Read in the attributes only on rank 0 and attach it to the mesh */
if (attfile_given)
import_attributes(mesh, attfname);
/* Import element set definitions and create in the mesh */
if (setfile_given)
import_sets(mesh, setfname);
#ifdef MSTK_HAVE_MPI
}
#endif
#ifdef MSTK_HAVE_MPI
if (partition) {
/* Need to make sure that we have a mesh only on processor 0 */
/* For now we cannot repartition */
int prepartitioned = 0, mesh_present = 0;
int dim = 3;
if (rank > 0) {
if (mesh && MESH_Num_Vertices(mesh) != 0)
mesh_present = 1;
}
MPI_Reduce(&mesh_present,&prepartitioned,1,MPI_INT,MPI_MAX,0,comm);
MPI_Bcast(&prepartitioned,1,MPI_INT,0,comm);
if (!prepartitioned) {
Mesh_ptr mesh0=NULL;
if (rank == 0) {
fprintf(stderr,"Partitioning mesh into %d parts...",numprocs);
if (MESH_Num_Regions(mesh))
dim = 3;
else if (MESH_Num_Faces(mesh))
dim = 2;
else {
fprintf(stderr,"Partitioning possible only for 2D or 3D meshes\n");
exit(-1);
}
mesh0 = mesh;
mesh = NULL;
}
int ring = 0; /* No ghost ring of elements */
int with_attr = 1; /* Do allow exchange of attributes */
int del_inmesh = 1; /* Delete input mesh after partitioning */
ok = MSTK_Mesh_Distribute(mesh0, &mesh, &dim, ring, with_attr,
partmethod, del_inmesh, comm);
if (rank == 0) {
if (ok)
fprintf(stderr,"done\n");
else {
fprintf(stderr,"failed\n");
exit(-1);
}
}
/* if (rank == 0)
MESH_Delete(mesh0);
*/
}
}
#endif /* MSTK_HAVE_MPI */
#ifdef MSTK_HAVE_MPI
if (numprocs > 1 && parallel_check == 1) {
/* Do a parallel consistency check too */
ok = ok && MESH_Parallel_Check(mesh,comm);
}
#endif
fprintf(stderr,"\nExporting mesh to ExodusII format...");
ok = MESH_ExportToFile(mesh,outfname,"exodusii",-1,NULL,NULL,comm);
if (ok)
fprintf(stderr,"Done\n");
else {
fprintf(stderr,"Failed\n");
exit(-1);
}
MESH_Delete(mesh);
// while (1);
#ifdef MSTK_HAVE_MPI
MPI_Finalize();
#endif
return 1;
}