PETSC_EXTERN PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
{
MPI_Comm comm;
const PetscInt dim = 3;
const char *labelName = "marker";
PLC *in, *out;
DMLabel label;
PetscInt verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = MPIU_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMGetLabel(dm, labelName, &label);CHKERRQ(ierr);
ierr = PLCCreate(&in);CHKERRQ(ierr);
ierr = PLCCreate(&out);CHKERRQ(ierr);
in->numberofpoints = vEnd - vStart;
if (in->numberofpoints > 0) {
PetscSection coordSection;
Vec coordinates;
PetscScalar *array;
ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
ierr = PetscMalloc1(in->numberofpoints, &in->pointmarkerlist);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
const PetscInt idx = v - vStart;
PetscInt off, d, m = -1;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) {
in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
}
if (label) {ierr = DMLabelGetValue(label, v, &m);CHKERRQ(ierr);}
in->pointmarkerlist[idx] = (int) m;
}
ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
}
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
in->numberofcorners = 4;
in->numberoftetrahedra = cEnd - cStart;
in->tetrahedronvolumelist = maxVolumes;
if (in->numberoftetrahedra > 0) {
ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {
const PetscInt idx = c - cStart;
PetscInt *closure = NULL;
PetscInt closureSize;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
for (v = 0; v < 4; ++v) {
in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
}
if (!rank) {
TetGenOpts t;
ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
t.in = dm; /* Should go away */
t.refine = 1;
t.varvolume = 1;
t.quality = 1;
t.edgesout = 1;
t.zeroindex = 1;
t.quiet = 1;
t.verbose = verbose; /* Change this */
ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
}
in->tetrahedronvolumelist = NULL;
{
DMLabel rlabel = NULL;
const PetscInt numCorners = 4;
const PetscInt numCells = out->numberoftetrahedra;
const PetscInt numVertices = out->numberofpoints;
double *meshCoords;
int *cells = out->tetrahedronlist;
PetscBool interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
if (sizeof (PetscReal) == sizeof (double)) {
meshCoords = (double *) out->pointlist;
}
else {
PetscInt i;
ierr = PetscMalloc1(3 * numVertices,&meshCoords);CHKERRQ(ierr);
for (i = 0; i < 3 * numVertices; i++) {
meshCoords[i] = (PetscReal) out->pointlist[i];
}
}
ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
if (sizeof (PetscReal) != sizeof (double)) {
ierr = PetscFree(meshCoords);CHKERRQ(ierr);
}
if (label) {ierr = DMCreateLabel(*dmRefined, labelName); ierr = DMGetLabel(*dmRefined, labelName, &rlabel);}
/* Set labels */
for (v = 0; v < numVertices; ++v) {
if (out->pointmarkerlist[v]) {
if (rlabel) {ierr = DMLabelSetValue(rlabel, v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);}
}
}
if (interpolate) {
PetscInt e, f;
for (e = 0; e < out->numberofedges; e++) {
if (out->edgemarkerlist[e]) {
const PetscInt vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
const PetscInt *edges;
PetscInt numEdges;
ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
if (rlabel) {ierr = DMLabelSetValue(rlabel, edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
}
}
for (f = 0; f < out->numberoftrifaces; f++) {
if (out->trifacemarkerlist[f]) {
const PetscInt vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
const PetscInt *faces;
PetscInt numFaces;
ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
if (rlabel) {ierr = DMLabelSetValue(rlabel, faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
}
}
}
ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
}
ierr = PLCDestroy(&in);CHKERRQ(ierr);
ierr = PLCDestroy(&out);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
{
MPI_Comm comm;
const PetscInt dim = 3;
const char *labelName = "marker";
PLC *in, *out;
DMLabel label;
PetscInt verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMGetLabel(boundary, labelName, &label);CHKERRQ(ierr);
ierr = PLCCreate(&in);CHKERRQ(ierr);
ierr = PLCCreate(&out);CHKERRQ(ierr);
in->numberofpoints = vEnd - vStart;
if (in->numberofpoints > 0) {
PetscSection coordSection;
Vec coordinates;
PetscScalar *array;
ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
ierr = PetscMalloc1(in->numberofpoints, &in->pointmarkerlist);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
const PetscInt idx = v - vStart;
PetscInt off, d, m = -1;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) {
in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
}
if (label) {ierr = DMLabelGetValue(label, v, &m);CHKERRQ(ierr);}
in->pointmarkerlist[idx] = (int) m;
}
ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
}
ierr = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
in->numberoffacets = fEnd - fStart;
if (in->numberoffacets > 0) {
ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
ierr = PetscMalloc1(in->numberoffacets, &in->facetmarkerlist);CHKERRQ(ierr);
for (f = fStart; f < fEnd; ++f) {
const PetscInt idx = f - fStart;
PetscInt *points = NULL, numPoints, p, numVertices = 0, v, m = -1;
polygon *poly;
in->facetlist[idx].numberofpolygons = 1;
ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
in->facetlist[idx].numberofholes = 0;
in->facetlist[idx].holelist = NULL;
ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
for (p = 0; p < numPoints*2; p += 2) {
const PetscInt point = points[p];
if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
}
poly = in->facetlist[idx].polygonlist;
poly->numberofvertices = numVertices;
ierr = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
const PetscInt vIdx = points[v] - vStart;
poly->vertexlist[v] = vIdx;
}
if (label) {ierr = DMLabelGetValue(label, f, &m);CHKERRQ(ierr);}
in->facetmarkerlist[idx] = (int) m;
ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
}
}
if (!rank) {
TetGenOpts t;
ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
t.in = boundary; /* Should go away */
t.plc = 1;
t.quality = 1;
t.edgesout = 1;
t.zeroindex = 1;
t.quiet = 1;
t.verbose = verbose;
ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
}
{
DMLabel glabel = NULL;
const PetscInt numCorners = 4;
const PetscInt numCells = out->numberoftetrahedra;
const PetscInt numVertices = out->numberofpoints;
double *meshCoords;
int *cells = out->tetrahedronlist;
if (sizeof (PetscReal) == sizeof (double)) {
meshCoords = (double *) out->pointlist;
}
else {
PetscInt i;
ierr = PetscMalloc1(3 * numVertices,&meshCoords);CHKERRQ(ierr);
for (i = 0; i < 3 * numVertices; i++) {
meshCoords[i] = (PetscReal) out->pointlist[i];
}
}
ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
if (sizeof (PetscReal) != sizeof (double)) {
ierr = PetscFree(meshCoords);CHKERRQ(ierr);
}
if (label) {ierr = DMCreateLabel(*dm, labelName); ierr = DMGetLabel(*dm, labelName, &glabel);}
/* Set labels */
for (v = 0; v < numVertices; ++v) {
if (out->pointmarkerlist[v]) {
if (glabel) {ierr = DMLabelSetValue(glabel, v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);}
}
}
if (interpolate) {
PetscInt e;
for (e = 0; e < out->numberofedges; e++) {
if (out->edgemarkerlist[e]) {
const PetscInt vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
const PetscInt *edges;
PetscInt numEdges;
ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
if (glabel) {ierr = DMLabelSetValue(glabel, edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
}
}
for (f = 0; f < out->numberoftrifaces; f++) {
if (out->trifacemarkerlist[f]) {
const PetscInt vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
const PetscInt *faces;
PetscInt numFaces;
ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
if (glabel) {ierr = DMLabelSetValue(glabel, faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
}
}
}
ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
}
ierr = PLCDestroy(&in);CHKERRQ(ierr);
ierr = PLCDestroy(&out);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexCreateExodus - Create a DMPlex mesh from an ExodusII file ID.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. exoid - The ExodusII id associated with a exodus file and obtained using ex_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Level: beginner
.keywords: mesh,ExodusII
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateExodus(MPI_Comm comm, PetscInt exoid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_EXODUSII)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from ex_get_init() */
char title[PETSC_MAX_PATH_LEN+1];
int dim = 0, numVertices = 0, numCells = 0;
int num_cs = 0, num_vs = 0, num_fs = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_EXODUSII)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open EXODUS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
ierr = PetscMemzero(title,(PETSC_MAX_PATH_LEN+1)*sizeof(char));CHKERRQ(ierr);
ierr = ex_get_init(exoid, title, &dim, &numVertices, &numCells, &num_cs, &num_vs, &num_fs);
if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"ExodusII ex_get_init() failed with error code %D\n",ierr);
if (!num_cs) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Exodus file does not contain any cell set\n");
}
ierr = MPI_Bcast(title, PETSC_MAX_PATH_LEN+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, title);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read cell sets information */
if (!rank) {
PetscInt *cone;
int c, cs, c_loc, v, v_loc;
/* Read from ex_get_elem_blk_ids() */
int *cs_id;
/* Read from ex_get_elem_block() */
char buffer[PETSC_MAX_PATH_LEN+1];
int num_cell_in_set, num_vertex_per_cell, num_attr;
/* Read from ex_get_elem_conn() */
int *cs_connect;
/* Get cell sets IDs */
ierr = PetscMalloc1(num_cs, &cs_id);CHKERRQ(ierr);
ierr = ex_get_elem_blk_ids(exoid, cs_id);CHKERRQ(ierr);
/* Read the cell set connectivity table and build mesh topology
EXO standard requires that cells in cell sets be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
for (c_loc = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, num_vertex_per_cell);CHKERRQ(ierr);
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc2(num_vertex_per_cell*num_cell_in_set,&cs_connect,num_vertex_per_cell,&cone);CHKERRQ(ierr);
ierr = ex_get_elem_conn(exoid, cs_id[cs], cs_connect);CHKERRQ(ierr);
/* EXO uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
for (v_loc = 0; v_loc < num_vertex_per_cell; ++v_loc, ++v) {
cone[v_loc] = cs_connect[v]+numCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (num_vertex_per_cell == 4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (num_vertex_per_cell == 8) {
PetscInt tmp = cone[1];
cone[1] = cone[3];
cone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMSetLabelValue(*dm, "Cell Sets", c, cs_id[cs]);CHKERRQ(ierr);
}
ierr = PetscFree2(cs_connect,cone);CHKERRQ(ierr);
}
ierr = PetscFree(cs_id);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
/* Maintain Cell Sets label */
{
DMLabel label;
ierr = DMRemoveLabel(*dm, "Cell Sets", &label);CHKERRQ(ierr);
if (label) {ierr = DMAddLabel(idm, label);CHKERRQ(ierr);}
}
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Create vertex set label */
if (!rank && (num_vs > 0)) {
int vs, v;
/* Read from ex_get_node_set_ids() */
int *vs_id;
/* Read from ex_get_node_set_param() */
int num_vertex_in_set, num_attr;
/* Read from ex_get_node_set() */
int *vs_vertex_list;
/* Get vertex set ids */
ierr = PetscMalloc1(num_vs, &vs_id);CHKERRQ(ierr);
ierr = ex_get_node_set_ids(exoid, vs_id);CHKERRQ(ierr);
for (vs = 0; vs < num_vs; ++vs) {
ierr = ex_get_node_set_param(exoid, vs_id[vs], &num_vertex_in_set, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc1(num_vertex_in_set, &vs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_node_set(exoid, vs_id[vs], vs_vertex_list);CHKERRQ(ierr);
for (v = 0; v < num_vertex_in_set; ++v) {
ierr = DMSetLabelValue(*dm, "Vertex Sets", vs_vertex_list[v]+numCells-1, vs_id[vs]);CHKERRQ(ierr);
}
ierr = PetscFree(vs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(vs_id);CHKERRQ(ierr);
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
float *x, *y, *z;
ierr = PetscMalloc3(numVertices,&x,numVertices,&y,numVertices,&z);CHKERRQ(ierr);
ierr = ex_get_coord(exoid, x, y, z);CHKERRQ(ierr);
if (dim > 0) {
for (v = 0; v < numVertices; ++v) coords[v*dim+0] = x[v];
}
if (dim > 1) {
for (v = 0; v < numVertices; ++v) coords[v*dim+1] = y[v];
}
if (dim > 2) {
for (v = 0; v < numVertices; ++v) coords[v*dim+2] = z[v];
}
ierr = PetscFree3(x,y,z);CHKERRQ(ierr);
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Create side set label */
if (!rank && interpolate && (num_fs > 0)) {
int fs, f, voff;
/* Read from ex_get_side_set_ids() */
int *fs_id;
/* Read from ex_get_side_set_param() */
int num_side_in_set, num_dist_fact_in_set;
/* Read from ex_get_side_set_node_list() */
int *fs_vertex_count_list, *fs_vertex_list;
/* Get side set ids */
ierr = PetscMalloc1(num_fs, &fs_id);CHKERRQ(ierr);
ierr = ex_get_side_set_ids(exoid, fs_id);CHKERRQ(ierr);
for (fs = 0; fs < num_fs; ++fs) {
ierr = ex_get_side_set_param(exoid, fs_id[fs], &num_side_in_set, &num_dist_fact_in_set);CHKERRQ(ierr);
ierr = PetscMalloc2(num_side_in_set,&fs_vertex_count_list,num_side_in_set*4,&fs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_side_set_node_list(exoid, fs_id[fs], fs_vertex_count_list, fs_vertex_list);CHKERRQ(ierr);
for (f = 0, voff = 0; f < num_side_in_set; ++f) {
const PetscInt *faces = NULL;
PetscInt faceSize = fs_vertex_count_list[f], numFaces;
PetscInt faceVertices[4], v;
if (faceSize > 4) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "ExodusII side cannot have %d > 4 vertices", faceSize);
for (v = 0; v < faceSize; ++v, ++voff) {
faceVertices[v] = fs_vertex_list[voff]+numCells-1;
}
ierr = DMPlexGetFullJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Invalid ExodusII side %d in set %d maps to %d faces", f, fs, numFaces);
ierr = DMSetLabelValue(*dm, "Face Sets", faces[0], fs_id[fs]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
}
ierr = PetscFree2(fs_vertex_count_list,fs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(fs_id);CHKERRQ(ierr);
}
#else
SETERRQ(comm, PETSC_ERR_SUP, "This method requires ExodusII support. Reconfigure using --download-exodusii");
#endif
PetscFunctionReturn(0);
}
/*@C
DMPlexCreateFluent - Create a DMPlex mesh from a Fluent mesh file.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Fluent mesh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://aerojet.engr.ucdavis.edu/fluenthelp/html/ug/node1490.htm
Level: beginner
.keywords: mesh, fluent, case
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateFluent(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
PetscMPIInt rank;
PetscInt c, f, v, dim = PETSC_DETERMINE, numCells = 0, numVertices = 0, numCellVertices = PETSC_DETERMINE;
PetscInt numFaces = PETSC_DETERMINE, numFaceEntries = PETSC_DETERMINE, numFaceVertices = PETSC_DETERMINE;
PetscInt *faces = NULL, *cellVertices, *faceZoneIDs = NULL;
PetscInt d, coordSize;
PetscScalar *coords, *coordsIn = NULL;
PetscSection coordSection;
Vec coordinates;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
if (!rank) {
FluentSection s;
do {
ierr = DMPlexCreateFluent_ReadSection(viewer, &s);CHKERRQ(ierr);
if (s.index == 2) { /* Dimension */
dim = s.nd;
} else if (s.index == 10 || s.index == 2010) { /* Vertices */
if (s.zoneID == 0) numVertices = s.last;
else {
if (coordsIn) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Currently no support for multiple coordinate sets in Fluent files");
coordsIn = (PetscScalar *) s.data;
}
} else if (s.index == 12 || s.index == 2012) { /* Cells */
if (s.zoneID == 0) numCells = s.last;
else {
switch (s.nd) {
case 0: numCellVertices = PETSC_DETERMINE; break;
case 1: numCellVertices = 3; break; /* triangular */
case 2: numCellVertices = 4; break; /* tetrahedral */
case 3: numCellVertices = 4; break; /* quadrilateral */
case 4: numCellVertices = 8; break; /* hexahedral */
case 5: numCellVertices = 5; break; /* pyramid */
case 6: numCellVertices = 6; break; /* wedge */
default: numCellVertices = PETSC_DETERMINE;
}
}
} else if (s.index == 13 || s.index == 2013) { /* Facets */
if (s.zoneID == 0) { /* Header section */
numFaces = s.last - s.first + 1;
if (s.nd == 0 || s.nd == 5) numFaceVertices = PETSC_DETERMINE;
else numFaceVertices = s.nd;
} else { /* Data section */
if (numFaceVertices != PETSC_DETERMINE && s.nd != numFaceVertices) {
SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mixed facets in Fluent files are not supported");
}
if (numFaces < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No header section for facets in Fluent file");
if (numFaceVertices == PETSC_DETERMINE) numFaceVertices = s.nd;
numFaceEntries = numFaceVertices + 2;
if (!faces) {ierr = PetscMalloc1(numFaces*numFaceEntries, &faces);CHKERRQ(ierr);}
if (!faceZoneIDs) {ierr = PetscMalloc1(numFaces, &faceZoneIDs);CHKERRQ(ierr);}
ierr = PetscMemcpy(&(faces[(s.first-1)*numFaceEntries]), s.data, (s.last-s.first+1)*numFaceEntries*sizeof(PetscInt));CHKERRQ(ierr);
/* Record the zoneID for each face set */
for (f = s.first -1; f < s.last; f++) faceZoneIDs[f] = s.zoneID;
ierr = PetscFree(s.data);CHKERRQ(ierr);
}
}
} while (s.index >= 0);
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
if (dim < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Fluent file does not include dimension");
/* Allocate cell-vertex mesh */
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells + numVertices);CHKERRQ(ierr);
if (!rank) {
if (numCells < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown number of cells in Fluent file");
/* If no cell type was given we assume simplices */
if (numCellVertices == PETSC_DETERMINE) numCellVertices = numFaceVertices + 1;
for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(*dm, c, numCellVertices);CHKERRQ(ierr);}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
if (!rank) {
/* Derive cell-vertex list from face-vertex and face-cell maps */
ierr = PetscMalloc1(numCells*numCellVertices, &cellVertices);CHKERRQ(ierr);
for (c = 0; c < numCells*numCellVertices; c++) cellVertices[c] = -1;
for (f = 0; f < numFaces; f++) {
PetscInt *cell;
const PetscInt cl = faces[f*numFaceEntries + numFaceVertices];
const PetscInt cr = faces[f*numFaceEntries + numFaceVertices + 1];
const PetscInt *face = &(faces[f*numFaceEntries]);
if (cl > 0) {
cell = &(cellVertices[(cl-1) * numCellVertices]);
for (v = 0; v < numFaceVertices; v++) {
PetscBool found = PETSC_FALSE;
for (c = 0; c < numCellVertices; c++) {
if (cell[c] < 0) break;
if (cell[c] == face[v]-1 + numCells) {found = PETSC_TRUE; break;}
}
if (!found) cell[c] = face[v]-1 + numCells;
}
}
if (cr > 0) {
cell = &(cellVertices[(cr-1) * numCellVertices]);
for (v = 0; v < numFaceVertices; v++) {
PetscBool found = PETSC_FALSE;
for (c = 0; c < numCellVertices; c++) {
if (cell[c] < 0) break;
if (cell[c] == face[v]-1 + numCells) {found = PETSC_TRUE; break;}
}
if (!found) cell[c] = face[v]-1 + numCells;
}
}
}
for (c = 0; c < numCells; c++) {
ierr = DMPlexSetCone(*dm, c, &(cellVertices[c*numCellVertices]));CHKERRQ(ierr);
}
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
if (!rank) {
PetscInt fi, joinSize, meetSize, *fverts, cells[2];
const PetscInt *join, *meet;
ierr = PetscMalloc1(numFaceVertices, &fverts);CHKERRQ(ierr);
/* Mark facets by finding the full join of all adjacent vertices */
for (f = 0; f < numFaces; f++) {
const PetscInt cl = faces[f*numFaceEntries + numFaceVertices] - 1;
const PetscInt cr = faces[f*numFaceEntries + numFaceVertices + 1] - 1;
if (cl > 0 && cr > 0) {
/* If we know both adjoining cells we can use a single-level meet */
cells[0] = cl; cells[1] = cr;
ierr = DMPlexGetMeet(*dm, 2, cells, &meetSize, &meet);CHKERRQ(ierr);
if (meetSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for Fluent face %d", f);
ierr = DMPlexSetLabelValue(*dm, "Face Sets", meet[0], faceZoneIDs[f]);CHKERRQ(ierr);
ierr = DMPlexRestoreMeet(*dm, numFaceVertices, fverts, &meetSize, &meet);CHKERRQ(ierr);
} else {
for (fi = 0; fi < numFaceVertices; fi++) fverts[fi] = faces[f*numFaceEntries + fi] + numCells - 1;
ierr = DMPlexGetFullJoin(*dm, numFaceVertices, fverts, &joinSize, &join);CHKERRQ(ierr);
if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for Fluent face %d", f);
ierr = DMPlexSetLabelValue(*dm, "Face Sets", join[0], faceZoneIDs[f]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, numFaceVertices, fverts, &joinSize, &join);CHKERRQ(ierr);
}
}
ierr = PetscFree(fverts);CHKERRQ(ierr);
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(comm, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*dim+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
if (!rank) {
ierr = PetscFree(cellVertices);CHKERRQ(ierr);
ierr = PetscFree(faces);CHKERRQ(ierr);
ierr = PetscFree(faceZoneIDs);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@
DMPlexCreateGmsh - Create a DMPlex mesh from a Gmsh file viewer
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Gmsh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.geuz.org/gmsh/doc/texinfo/#MSH-ASCII-file-format
and http://www.geuz.org/gmsh/doc/texinfo/#MSH-binary-file-format
Level: beginner
.keywords: mesh,Gmsh
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateGmsh(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
PetscViewerType vtype;
GmshElement *gmsh_elem;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords, *coordsIn = NULL;
PetscInt dim = 0, coordSize, c, v, d, r, cell;
int i, numVertices = 0, numCells = 0, trueNumCells = 0, numRegions = 0, snum;
PetscMPIInt num_proc, rank;
char line[PETSC_MAX_PATH_LEN];
PetscBool match, binary, bswap = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
ierr = PetscViewerGetType(viewer, &vtype);CHKERRQ(ierr);
ierr = PetscStrcmp(vtype, PETSCVIEWERBINARY, &binary);CHKERRQ(ierr);
if (!rank || binary) {
PetscBool match;
int fileType, dataSize;
float version;
/* Read header */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$MeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 3, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%f %d %d", &version, &fileType, &dataSize);
if (snum != 3) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Gmsh file header: %s", line);
if (version < 2.0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Gmsh file must be at least version 2.0");
if (dataSize != sizeof(double)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Data size %d is not valid for a Gmsh file", dataSize);
if (binary) {
int checkInt;
ierr = PetscViewerRead(viewer, &checkInt, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
if (checkInt != 1) {
ierr = PetscByteSwap(&checkInt, PETSC_ENUM, 1);CHKERRQ(ierr);
if (checkInt == 1) bswap = PETSC_TRUE;
else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh binary file", fileType);
}
} else if (fileType) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh ASCII file", fileType);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndMeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* OPTIONAL Read physical names */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$PhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (match) {
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numRegions);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
for (r = 0; r < numRegions; ++r) {
PetscInt rdim, tag;
ierr = PetscViewerRead(viewer, &rdim, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &tag, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndPhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Initial read for vertex section */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
/* Read vertices */
ierr = PetscStrncmp(line, "$Nodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numVertices);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscMalloc1(numVertices*3, &coordsIn);CHKERRQ(ierr);
if (binary) {
size_t doubleSize, intSize;
PetscInt elementSize;
char *buffer;
PetscScalar *baseptr;
ierr = PetscDataTypeGetSize(PETSC_ENUM, &intSize);CHKERRQ(ierr);
ierr = PetscDataTypeGetSize(PETSC_DOUBLE, &doubleSize);CHKERRQ(ierr);
elementSize = (intSize + 3*doubleSize);
ierr = PetscMalloc1(elementSize*numVertices, &buffer);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, buffer, elementSize*numVertices, NULL, PETSC_CHAR);CHKERRQ(ierr);
if (bswap) ierr = PetscByteSwap(buffer, PETSC_CHAR, elementSize*numVertices);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
baseptr = ((PetscScalar*)(buffer+v*elementSize+intSize));
coordsIn[v*3+0] = baseptr[0];
coordsIn[v*3+1] = baseptr[1];
coordsIn[v*3+2] = baseptr[2];
}
ierr = PetscFree(buffer);CHKERRQ(ierr);
} else {
for (v = 0; v < numVertices; ++v) {
ierr = PetscViewerRead(viewer, &i, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &(coordsIn[v*3]), 3, NULL, PETSC_DOUBLE);CHKERRQ(ierr);
if (i != (int)v+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid node number %d should be %d", i, v+1);
}
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndNodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read cells */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$Elements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
snum = sscanf(line, "%d", &numCells);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
if (!rank || binary) {
/* Gmsh elements can be of any dimension/co-dimension, so we need to traverse the
file contents multiple times to figure out the true number of cells and facets
in the given mesh. To make this more efficient we read the file contents only
once and store them in memory, while determining the true number of cells. */
ierr = DMPlexCreateGmsh_ReadElement(viewer, numCells, binary, bswap, &gmsh_elem);CHKERRQ(ierr);
for (trueNumCells=0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim > dim) {dim = gmsh_elem[c].dim; trueNumCells = 0;}
if (gmsh_elem[c].dim == dim) trueNumCells++;
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndElements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
/* For binary we read on all ranks, but only build the plex on rank 0 */
if (binary && rank) {trueNumCells = 0; numVertices = 0;};
/* Allocate the cell-vertex mesh */
ierr = DMPlexSetChart(*dm, 0, trueNumCells+numVertices);CHKERRQ(ierr);
if (!rank) {
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
ierr = DMPlexSetConeSize(*dm, cell, gmsh_elem[c].numNodes);CHKERRQ(ierr);
cell++;
}
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
/* Add cell-vertex connections */
if (!rank) {
PetscInt pcone[8], corner;
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + trueNumCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (gmsh_elem[c].numNodes == 4) {
PetscInt tmp = pcone[0];
pcone[0] = pcone[1];
pcone[1] = tmp;
}
/* Hexahedra are inverted */
if (gmsh_elem[c].numNodes == 8) {
PetscInt tmp = pcone[1];
pcone[1] = pcone[3];
pcone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, cell, pcone);CHKERRQ(ierr);
cell++;
}
}
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
if (!rank) {
/* Apply boundary IDs by finding the relevant facets with vertex joins */
PetscInt pcone[8], corner, vStart, vEnd;
ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim-1) {
PetscInt joinSize;
const PetscInt *join;
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + vStart - 1;
}
ierr = DMPlexGetFullJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for element %d", gmsh_elem[c].id);
ierr = DMSetLabelValue(*dm, "Face Sets", join[0], gmsh_elem[c].tags[0]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
}
}
/* Create cell sets */
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
if (gmsh_elem[c].numTags > 0) {
ierr = DMSetLabelValue(*dm, "Cell Sets", cell, gmsh_elem[c].tags[0]);CHKERRQ(ierr);
cell++;
}
}
}
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, trueNumCells, trueNumCells + numVertices);CHKERRQ(ierr);
for (v = trueNumCells; v < trueNumCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*3+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Clean up intermediate storage */
if (!rank || binary) ierr = PetscFree(gmsh_elem);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* This interpolates the PointSF in parallel following local interpolation */
static PetscErrorCode DMPlexInterpolatePointSF(DM dm, PetscSF pointSF, PetscInt depth)
{
PetscMPIInt numProcs, rank;
PetscInt p, c, d, dof, offset;
PetscInt numLeaves, numRoots, candidatesSize, candidatesRemoteSize;
const PetscInt *localPoints;
const PetscSFNode *remotePoints;
PetscSFNode *candidates, *candidatesRemote, *claims;
PetscSection candidateSection, candidateSectionRemote, claimSection;
PetscHashI leafhash;
PetscHashIJ roothash;
PetscHashIJKey key;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
if (numProcs < 2 || numRoots < 0) PetscFunctionReturn(0);
/* Build hashes of points in the SF for efficient lookup */
PetscHashICreate(leafhash);
PetscHashIJCreate(&roothash);
ierr = PetscHashIJSetMultivalued(roothash, PETSC_FALSE);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
PetscHashIAdd(leafhash, localPoints[p], p);
key.i = remotePoints[p].index; key.j = remotePoints[p].rank;
PetscHashIJAdd(roothash, key, p);
}
/* Build a section / SFNode array of candidate points in the single-level adjacency of leaves,
where each candidate is defined by the root entry for the other vertex that defines the edge. */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &candidateSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(candidateSection, 0, numRoots);CHKERRQ(ierr);
{
PetscInt leaf, root, idx, a, *adj = NULL;
for (p = 0; p < numLeaves; ++p) {
PetscInt adjSize = PETSC_DETERMINE;
ierr = DMPlexGetAdjacency_Internal(dm, localPoints[p], PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
for (a = 0; a < adjSize; ++a) {
PetscHashIMap(leafhash, adj[a], leaf);
if (leaf >= 0) {ierr = PetscSectionAddDof(candidateSection, localPoints[p], 1);CHKERRQ(ierr);}
}
}
ierr = PetscSectionSetUp(candidateSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(candidateSection, &candidatesSize);CHKERRQ(ierr);
ierr = PetscMalloc1(candidatesSize, &candidates);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
PetscInt adjSize = PETSC_DETERMINE;
ierr = PetscSectionGetOffset(candidateSection, localPoints[p], &offset);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency_Internal(dm, localPoints[p], PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
for (idx = 0, a = 0; a < adjSize; ++a) {
PetscHashIMap(leafhash, adj[a], root);
if (root >= 0) candidates[offset+idx++] = remotePoints[root];
}
}
ierr = PetscFree(adj);CHKERRQ(ierr);
}
/* Gather candidate section / array pair into the root partition via inverse(multi(pointSF)). */
{
PetscSF sfMulti, sfInverse, sfCandidates;
PetscInt *remoteOffsets;
ierr = PetscSFGetMultiSF(pointSF, &sfMulti);CHKERRQ(ierr);
ierr = PetscSFCreateInverseSF(sfMulti, &sfInverse);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sfInverse, candidateSection, &remoteOffsets, candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfInverse, candidateSection, remoteOffsets, candidateSectionRemote, &sfCandidates);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(candidateSectionRemote, &candidatesRemoteSize);CHKERRQ(ierr);
ierr = PetscMalloc1(candidatesRemoteSize, &candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfCandidates, MPIU_2INT, candidates, candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfCandidates, MPIU_2INT, candidates, candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfInverse);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfCandidates);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
}
/* Walk local roots and check for each remote candidate whether we know all required points,
either from owning it or having a root entry in the point SF. If we do we place a claim
by replacing the vertex number with our edge ID. */
{
PetscInt idx, root, joinSize, vertices[2];
const PetscInt *rootdegree, *join = NULL;
ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
/* Loop remote edge connections and put in a claim if both vertices are known */
for (idx = 0, p = 0; p < numRoots; ++p) {
for (d = 0; d < rootdegree[p]; ++d) {
ierr = PetscSectionGetDof(candidateSectionRemote, idx, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(candidateSectionRemote, idx, &offset);CHKERRQ(ierr);
for (c = 0; c < dof; ++c) {
/* We own both vertices, so we claim the edge by replacing vertex with edge */
if (candidatesRemote[offset+c].rank == rank) {
vertices[0] = p; vertices[1] = candidatesRemote[offset+c].index;
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) candidatesRemote[offset+c].index = join[0];
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
continue;
}
/* If we own one vertex and share a root with the other, we claim it */
key.i = candidatesRemote[offset+c].index; key.j = candidatesRemote[offset+c].rank;
PetscHashIJGet(roothash, key, &root);
if (root >= 0) {
vertices[0] = p; vertices[1] = localPoints[root];
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) {
candidatesRemote[offset+c].index = join[0];
candidatesRemote[offset+c].rank = rank;
}
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
}
}
idx++;
}
}
}
/* Push claims back to receiver via the MultiSF and derive new pointSF mapping on receiver */
{
PetscSF sfMulti, sfClaims, sfPointNew;
PetscHashI claimshash;
PetscInt size, pStart, pEnd, root, joinSize, numLocalNew;
PetscInt *remoteOffsets, *localPointsNew, vertices[2];
const PetscInt *join = NULL;
PetscSFNode *remotePointsNew;
ierr = PetscSFGetMultiSF(pointSF, &sfMulti);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &claimSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sfMulti, candidateSectionRemote, &remoteOffsets, claimSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfMulti, candidateSectionRemote, remoteOffsets, claimSection, &sfClaims);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(claimSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &claims);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfClaims, MPIU_2INT, candidatesRemote, claims);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfClaims, MPIU_2INT, candidatesRemote, claims);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfClaims);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
/* Walk the original section of local supports and add an SF entry for each updated item */
PetscHashICreate(claimshash);
for (p = 0; p < numRoots; ++p) {
ierr = PetscSectionGetDof(candidateSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(candidateSection, p, &offset);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
if (candidates[offset+d].index != claims[offset+d].index) {
key.i = candidates[offset+d].index; key.j = candidates[offset+d].rank;
PetscHashIJGet(roothash, key, &root);
if (root >= 0) {
vertices[0] = p; vertices[1] = localPoints[root];
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) PetscHashIAdd(claimshash, join[0], offset+d);
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
}
}
}
}
/* Create new pointSF from hashed claims */
PetscHashISize(claimshash, numLocalNew);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeaves + numLocalNew, &localPointsNew);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeaves + numLocalNew, &remotePointsNew);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
localPointsNew[p] = localPoints[p];
remotePointsNew[p].index = remotePoints[p].index;
remotePointsNew[p].rank = remotePoints[p].rank;
}
p = numLeaves; ierr = PetscHashIGetKeys(claimshash, &p, localPointsNew);CHKERRQ(ierr);
for (p = numLeaves; p < numLeaves + numLocalNew; ++p) {
PetscHashIMap(claimshash, localPointsNew[p], offset);
remotePointsNew[p] = claims[offset];
}
ierr = PetscSFCreate(PetscObjectComm((PetscObject) dm), &sfPointNew);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sfPointNew, pEnd-pStart, numLeaves+numLocalNew, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = DMSetPointSF(dm, sfPointNew);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfPointNew);CHKERRQ(ierr);
PetscHashIDestroy(claimshash);
}
PetscHashIDestroy(leafhash);
ierr = PetscHashIJDestroy(&roothash);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&candidateSection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&claimSection);CHKERRQ(ierr);
ierr = PetscFree(candidates);CHKERRQ(ierr);
ierr = PetscFree(candidatesRemote);CHKERRQ(ierr);
ierr = PetscFree(claims);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
