/*@
DMPlexInterpolate - Take in a cell-vertex mesh and return one with all intermediate faces, edges, etc.
Collective on DM
Input Parameter:
. dm - The DMPlex object with only cells and vertices
Output Parameter:
. dmInt - The complete DMPlex object
Level: intermediate
.keywords: mesh
.seealso: DMPlexUninterpolate(), DMPlexCreateFromCellList()
@*/
PetscErrorCode DMPlexInterpolate(DM dm, DM *dmInt)
{
DM idm, odm = dm;
PetscInt depth, dim, d;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogEventBegin(DMPLEX_Interpolate,dm,0,0,0);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
if (dim <= 1) {
ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr);
idm = dm;
}
for (d = 1; d < dim; ++d) {
/* Create interpolated mesh */
ierr = DMCreate(PetscObjectComm((PetscObject)dm), &idm);CHKERRQ(ierr);
ierr = DMSetType(idm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(idm, dim);CHKERRQ(ierr);
if (depth > 0) {ierr = DMPlexInterpolateFaces_Internal(odm, 1, idm);CHKERRQ(ierr);}
if (odm != dm) {ierr = DMDestroy(&odm);CHKERRQ(ierr);}
odm = idm;
}
*dmInt = idm;
ierr = PetscLogEventEnd(DMPLEX_Interpolate,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexCreateHexBoxMesh(MPI_Comm comm, PetscInt dim, const PetscInt cells[], DM *dm)
{
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidPointer(dm, 4);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
PetscValidLogicalCollectiveInt(*dm,dim,2);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dm, dim);CHKERRQ(ierr);
switch (dim) {
case 2:
{
PetscReal lower[2] = {0.0, 0.0};
PetscReal upper[2] = {1.0, 1.0};
ierr = DMPlexCreateSquareMesh(*dm, lower, upper, cells);CHKERRQ(ierr);
break;
}
#if 0
case 3:
{
PetscReal lower[3] = {0.0, 0.0, 0.0};
PetscReal upper[3] = {1.0, 1.0, 1.0};
ierr = DMPlexCreateCubeMesh(boundary, lower, upper, cells);CHKERRQ(ierr);
break;
}
#endif
default:
SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
}
PetscFunctionReturn(0);
}
PetscErrorCode CreateHex_3D(MPI_Comm comm, DM *newdm)
{
DM dm;
PetscInt numPoints[2] = {12, 2};
PetscInt coneSize[14] = {8, 8, 0,0,0,0,0,0,0,0,0,0,0,0};
PetscInt cones[16] = {2,5,4,3,6,7,8,9, 3,4,11,10,7,12,13,8};
PetscInt coneOrientations[16] = {0,0,0,0,0,0,0,0, 0,0, 0, 0,0, 0, 0,0};
PetscScalar vertexCoords[36] = {-0.5,0.0,0.0, 0.0,0.0,0.0, 0.0,1.0,0.0, -0.5,1.0,0.0,
-0.5,0.0,1.0, 0.0,0.0,1.0, 0.0,1.0,1.0, -0.5,1.0,1.0,
0.5,0.0,0.0, 0.5,1.0,0.0, 0.5,0.0,1.0, 0.5,1.0,1.0};
PetscInt markerPoints[24] = {2,1,3,1,4,1,5,1,6,1,7,1,8,1,9,1,10,1,11,1,12,1,13,1};
PetscInt dim = 3, depth = 1, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) dm, "hexahedral");CHKERRQ(ierr);
ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
ierr = DMPlexCreateFromDAG(dm, depth, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
for(p = 0; p < 12; ++p) {
ierr = DMPlexSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);
}
*newdm = dm;
PetscFunctionReturn(0);
}
PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscBool refinementUniform = user->refinementUniform;
PetscBool cellSimplex = user->cellSimplex;
const char *partitioner = "chaco";
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dm, dim);CHKERRQ(ierr);
switch(dim) {
case 2:
if (cellSimplex) {
ierr = CreateSimplexHybrid_2D(comm, *dm);CHKERRQ(ierr);
} else {
SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make hybrid meshes for quadrilaterals");
}
break;
default:
SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make hybrid meshes for dimension %d", dim);
}
{
DM refinedMesh = PETSC_NULL;
DM distributedMesh = PETSC_NULL;
/* Distribute mesh over processes */
ierr = DMPlexDistribute(*dm, partitioner, 0, &distributedMesh);CHKERRQ(ierr);
if (distributedMesh) {
PetscInt cMax = PETSC_DETERMINE, fMax = PETSC_DETERMINE;
/* Do not know how to preserve this after distribution */
if (rank) {
cMax = 1;
fMax = 11;
}
ierr = DMPlexSetHybridBounds(distributedMesh, cMax, PETSC_DETERMINE, fMax, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = distributedMesh;
}
if (refinementUniform) {
ierr = DMPlexSetRefinementUniform(*dm, refinementUniform);CHKERRQ(ierr);
ierr = DMRefine(*dm, comm, &refinedMesh);CHKERRQ(ierr);
if (refinedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = refinedMesh;
}
}
}
ierr = PetscObjectSetName((PetscObject) *dm, "Hybrid Mesh");CHKERRQ(ierr);
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
user->dm = *dm;
PetscFunctionReturn(0);
}
/*@
DMPlexUninterpolate - Take in a mesh with all intermediate faces, edges, etc. and return a cell-vertex mesh
Collective on DM
Input Parameter:
. dm - The complete DMPlex object
Output Parameter:
. dmUnint - The DMPlex object with only cells and vertices
Level: intermediate
.keywords: mesh
.seealso: DMPlexInterpolate(), DMPlexCreateFromCellList()
@*/
PetscErrorCode DMPlexUninterpolate(DM dm, DM *dmUnint)
{
DM udm;
PetscInt dim, vStart, vEnd, cStart, cEnd, c, maxConeSize = 0, *cone;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
if (dim <= 1) {
ierr = PetscObjectReference((PetscObject) dm);CHKERRQ(ierr);
*dmUnint = dm;
PetscFunctionReturn(0);
}
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMCreate(PetscObjectComm((PetscObject) dm), &udm);CHKERRQ(ierr);
ierr = DMSetType(udm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(udm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(udm, cStart, vEnd);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {
PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
const PetscInt p = closure[cl];
if ((p >= vStart) && (p < vEnd)) ++coneSize;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
ierr = DMPlexSetConeSize(udm, c, coneSize);CHKERRQ(ierr);
maxConeSize = PetscMax(maxConeSize, coneSize);
}
ierr = DMSetUp(udm);CHKERRQ(ierr);
ierr = PetscMalloc1(maxConeSize, &cone);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {
PetscInt *closure = NULL, closureSize, cl, coneSize = 0;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
const PetscInt p = closure[cl];
if ((p >= vStart) && (p < vEnd)) cone[coneSize++] = p;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
ierr = DMPlexSetCone(udm, c, cone);CHKERRQ(ierr);
}
ierr = PetscFree(cone);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(udm);CHKERRQ(ierr);
ierr = DMPlexStratify(udm);CHKERRQ(ierr);
*dmUnint = udm;
PetscFunctionReturn(0);
}
/*@C
DMPlexCreateFromCellList - This takes as input common mesh generator output, a list of the vertices for each cell, and produces a DM
Input Parameters:
+ comm - The communicator
. dim - The topological dimension of the mesh
. numCells - The number of cells
. numVertices - The number of vertices
. numCorners - The number of vertices for each cell
. interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
. cells - An array of numCells*numCorners numbers, the vertices for each cell
. spaceDim - The spatial dimension used for coordinates
- vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
Output Parameter:
. dm - The DM
Note: Two triangles sharing a face
$
$ 2
$ / | \
$ / | \
$ / | \
$ 0 0 | 1 3
$ \ | /
$ \ | /
$ \ | /
$ 1
would have input
$ numCells = 2, numVertices = 4
$ cells = [0 1 2 1 3 2]
$
which would result in the DMPlex
$
$ 4
$ / | \
$ / | \
$ / | \
$ 2 0 | 1 5
$ \ | /
$ \ | /
$ \ | /
$ 3
Level: beginner
.seealso: DMPlexCreate()
@*/
PetscErrorCode DMPlexCreateFromCellList(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const int cells[], PetscInt spaceDim, const double vertexCoords[], DM *dm)
{
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexBuildFromCellList_Private(*dm, numCells, numVertices, numCorners, cells);CHKERRQ(ierr);
if (interpolate) {
DM idm;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
ierr = DMPlexBuildCoordinates_Private(*dm, spaceDim, numCells, numVertices, vertexCoords);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexInterpolate(DM dm, DM *dmInt)
{
DM idm, odm = dm;
PetscInt dim, d;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
for (d = 1; d < dim; ++d) {
/* Create interpolated mesh */
ierr = DMCreate(PetscObjectComm((PetscObject)dm), &idm);CHKERRQ(ierr);
ierr = DMSetType(idm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(idm, dim);CHKERRQ(ierr);
ierr = DMPlexInterpolateFaces_Internal(odm, 1, idm);CHKERRQ(ierr);
if (odm != dm) {ierr = DMDestroy(&odm);CHKERRQ(ierr);}
odm = idm;
}
*dmInt = idm;
PetscFunctionReturn(0);
}
/*@
DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box).
Collective on MPI_Comm
Input Parameters:
+ comm - The communicator for the DM object
. dim - The spatial dimension
- interpolate - Flag to create intermediate mesh pieces (edges, faces)
Output Parameter:
. dm - The DM object
Level: beginner
.keywords: DM, create
.seealso: DMSetType(), DMCreate()
@*/
PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool interpolate, DM *dm)
{
DM boundary;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidPointer(dm, 4);
ierr = DMCreate(comm, &boundary);CHKERRQ(ierr);
PetscValidLogicalCollectiveInt(boundary,dim,2);
ierr = DMSetType(boundary, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(boundary, dim-1);CHKERRQ(ierr);
switch (dim) {
case 2:
{
PetscReal lower[2] = {0.0, 0.0};
PetscReal upper[2] = {1.0, 1.0};
PetscInt edges[2] = {2, 2};
ierr = DMPlexCreateSquareBoundary(boundary, lower, upper, edges);CHKERRQ(ierr);
break;
}
case 3:
{
PetscReal lower[3] = {0.0, 0.0, 0.0};
PetscReal upper[3] = {1.0, 1.0, 1.0};
PetscInt faces[3] = {1, 1, 1};
ierr = DMPlexCreateCubeBoundary(boundary, lower, upper, faces);CHKERRQ(ierr);
break;
}
default:
SETERRQ1(comm, PETSC_ERR_SUP, "Dimension not supported: %d", dim);
}
ierr = DMPlexGenerate(boundary, NULL, interpolate, dm);CHKERRQ(ierr);
ierr = DMDestroy(&boundary);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode CreateSimplex_2D(MPI_Comm comm, DM *newdm)
{
DM dm;
PetscInt numPoints[2] = {4, 2};
PetscInt coneSize[6] = {3, 3, 0, 0, 0, 0};
PetscInt cones[6] = {2, 3, 4, 5, 4, 3};
PetscInt coneOrientations[6] = {0, 0, 0, 0, 0, 0};
PetscScalar vertexCoords[8] = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
PetscInt markerPoints[8] = {2, 1, 3, 1, 4, 1, 5, 1};
PetscInt dim = 2, depth = 1, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) dm, "triangular");CHKERRQ(ierr);
ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
ierr = DMPlexCreateFromDAG(dm, depth, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
for (p = 0; p < 4; ++p) {
ierr = DMPlexSetLabelValue(dm, "marker", markerPoints[p*2], markerPoints[p*2+1]);CHKERRQ(ierr);
}
*newdm = dm;
PetscFunctionReturn(0);
}
/*@
DMPlexCreateGmsh - Create a DMPlex mesh from a Gmsh file.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Gmsh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.geuz.org/gmsh/doc/texinfo/#MSH-ASCII-file-format
Level: beginner
.keywords: mesh,Gmsh
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateGmsh(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
FILE *fd;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords, *coordsIn = NULL;
PetscInt dim = 0, coordSize, c, v, d;
int numVertices = 0, numCells = 0, snum;
long fpos = 0;
PetscMPIInt num_proc, rank;
char line[PETSC_MAX_PATH_LEN];
PetscBool match;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
if (!rank) {
PetscBool match;
int fileType, dataSize;
ierr = PetscViewerASCIIGetPointer(viewer, &fd);CHKERRQ(ierr);
/* Read header */
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$MeshFormat\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
snum = fscanf(fd, "2.2 %d %d\n", &fileType, &dataSize);CHKERRQ(snum != 2);
if (fileType) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh ASCII file", fileType);
if (dataSize != sizeof(double)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Data size %d is not valid for a Gmsh file", dataSize);
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$EndMeshFormat\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read vertices */
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$Nodes\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
snum = fscanf(fd, "%d\n", &numVertices);CHKERRQ(snum != 1);
ierr = PetscMalloc(numVertices*3 * sizeof(PetscScalar), &coordsIn);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
double x, y, z;
int i;
snum = fscanf(fd, "%d %lg %lg %lg\n", &i, &x, &y, &z);CHKERRQ(snum != 4);
coordsIn[v*3+0] = x; coordsIn[v*3+1] = y; coordsIn[v*3+2] = z;
if (i != v+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid node number %d should be %d", i, v+1);
}
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$EndNodes\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read cells */
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$Elements\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
snum = fscanf(fd, "%d\n", &numCells);CHKERRQ(snum != 1);
}
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
if (!rank) {
fpos = ftell(fd);
for (c = 0; c < numCells; ++c) {
PetscInt numCorners, t;
int cone[8], i, cellType, numTags, tag;
snum = fscanf(fd, "%d %d %d", &i, &cellType, &numTags);CHKERRQ(snum != 3);
if (numTags) for (t = 0; t < numTags; ++t) {snum = fscanf(fd, "%d", &tag);CHKERRQ(snum != 1);}
switch (cellType) {
case 1: /* 2-node line */
dim = 1;
numCorners = 2;
snum = fscanf(fd, "%d %d\n", &cone[0], &cone[1]);CHKERRQ(snum != numCorners);
break;
case 2: /* 3-node triangle */
dim = 2;
numCorners = 3;
snum = fscanf(fd, "%d %d %d\n", &cone[0], &cone[1], &cone[2]);CHKERRQ(snum != numCorners);
break;
case 3: /* 4-node quadrangle */
dim = 2;
numCorners = 4;
snum = fscanf(fd, "%d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3]);CHKERRQ(snum != numCorners);
break;
case 4: /* 4-node tetrahedron */
dim = 3;
numCorners = 4;
snum = fscanf(fd, "%d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3]);CHKERRQ(snum != numCorners);
break;
case 5: /* 8-node hexahedron */
dim = 3;
numCorners = 8;
snum = fscanf(fd, "%d %d %d %d %d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3], &cone[4], &cone[5], &cone[6], &cone[7]);CHKERRQ(snum != numCorners);
break;
default:
SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unsupported Gmsh element type %d", cellType);
}
ierr = DMPlexSetConeSize(*dm, c, numCorners);CHKERRQ(ierr);
if (i != c+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell number %d should be %d", i, c+1);
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
if (!rank) {
ierr = fseek(fd, fpos, SEEK_SET);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
PetscInt pcone[8], numCorners, corner, t;
int cone[8], i, cellType, numTags, tag;
snum = fscanf(fd, "%d %d %d", &i, &cellType, &numTags);CHKERRQ(snum != 3);
if (numTags) for (t = 0; t < numTags; ++t) {snum = fscanf(fd, "%d", &tag);CHKERRQ(snum != 1);}
switch (cellType) {
case 1: /* 2-node line */
dim = 1;
numCorners = 2;
snum = fscanf(fd, "%d %d\n", &cone[0], &cone[1]);CHKERRQ(snum != numCorners);
break;
case 2: /* 3-node triangle */
dim = 2;
numCorners = 3;
snum = fscanf(fd, "%d %d %d\n", &cone[0], &cone[1], &cone[2]);CHKERRQ(snum != numCorners);
break;
case 3: /* 4-node quadrangle */
dim = 2;
numCorners = 4;
snum = fscanf(fd, "%d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3]);CHKERRQ(snum != numCorners);
break;
case 4: /* 4-node tetrahedron */
dim = 3;
numCorners = 4;
snum = fscanf(fd, "%d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3]);CHKERRQ(snum != numCorners);
ierr = DMPlexInvertCell(dim, numCorners, cone);CHKERRQ(ierr);
break;
case 5: /* 8-node hexahedron */
dim = 3;
numCorners = 8;
snum = fscanf(fd, "%d %d %d %d %d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3], &cone[4], &cone[5], &cone[6], &cone[7]);CHKERRQ(snum != numCorners);
ierr = DMPlexInvertCell(dim, numCorners, cone);CHKERRQ(ierr);
break;
default:
SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unsupported Gmsh element type %d", cellType);
}
for (corner = 0; corner < numCorners; ++corner) pcone[corner] = cone[corner] + numCells-1;
ierr = DMPlexSetCone(*dm, c, pcone);CHKERRQ(ierr);
if (i != c+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell number %d should be %d", i, c+1);
}
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$EndElements\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(comm, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*3+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode TestTriangle(MPI_Comm comm, PetscBool interpolate, PetscBool transform)
{
DM dm;
PetscRandom r, ang, ang2;
PetscInt dim, t;
PetscErrorCode ierr;
PetscFunctionBegin;
/* Create reference triangle */
dim = 2;
ierr = DMCreate(comm, &dm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) dm, "triangle");CHKERRQ(ierr);
ierr = DMSetType(dm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(dm, dim);CHKERRQ(ierr);
{
PetscInt numPoints[2] = {3, 1};
PetscInt coneSize[4] = {3, 0, 0, 0};
PetscInt cones[3] = {1, 2, 3};
PetscInt coneOrientations[3] = {0, 0, 0};
PetscScalar vertexCoords[6] = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
ierr = DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords);CHKERRQ(ierr);
if (interpolate) {
DM idm;
ierr = DMPlexInterpolate(dm, &idm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) idm, "triangle");CHKERRQ(ierr);
ierr = DMPlexCopyCoordinates(dm, idm);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
dm = idm;
}
ierr = DMSetFromOptions(dm);CHKERRQ(ierr);
}
/* Check reference geometry: determinant is scaled by reference volume (2.0) */
{
PetscReal v0Ex[2] = {-1.0, -1.0};
PetscReal JEx[4] = {1.0, 0.0, 0.0, 1.0};
PetscReal invJEx[4] = {1.0, 0.0, 0.0, 1.0};
PetscReal detJEx = 1.0;
PetscReal centroidEx[2] = {-0.333333333333, -0.333333333333};
PetscReal normalEx[2] = {0.0, 0.0};
PetscReal volEx = 2.0;
ierr = CheckFEMGeometry(dm, 0, dim, v0Ex, JEx, invJEx, detJEx);CHKERRQ(ierr);
if (interpolate) {ierr = CheckFVMGeometry(dm, 0, dim, centroidEx, normalEx, volEx);CHKERRQ(ierr);}
}
/* Check random triangles: rotate, scale, then translate */
if (transform) {
ierr = PetscRandomCreate(PETSC_COMM_SELF, &r);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(r);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(r, 0.0, 10.0);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_SELF, &ang);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(ang);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(ang, 0.0, 2*PETSC_PI);CHKERRQ(ierr);
for (t = 0; t < 100; ++t) {
PetscScalar vertexCoords[6] = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0}, trans[2];
PetscReal v0Ex[2] = {-1.0, -1.0};
PetscReal JEx[4] = {1.0, 0.0, 0.0, 1.0}, R[4], rot[2], rotM[4];
PetscReal invJEx[4] = {1.0, 0.0, 0.0, 1.0};
PetscReal detJEx = 1.0, scale, phi;
PetscReal centroidEx[2] = {-0.333333333333, -0.333333333333};
PetscReal normalEx[2] = {0.0, 0.0};
PetscReal volEx = 2.0;
PetscInt d, e, f, p;
ierr = PetscRandomGetValueReal(r, &scale);CHKERRQ(ierr);
ierr = PetscRandomGetValueReal(ang, &phi);CHKERRQ(ierr);
R[0] = cos(phi); R[1] = -sin(phi);
R[2] = sin(phi); R[3] = cos(phi);
for (p = 0; p < 3; ++p) {
for (d = 0; d < dim; ++d) {
for (e = 0, rot[d] = 0.0; e < dim; ++e) {
rot[d] += R[d*dim+e] * vertexCoords[p*dim+e];
}
}
for (d = 0; d < dim; ++d) vertexCoords[p*dim+d] = rot[d];
}
for (d = 0; d < dim; ++d) {
for (e = 0, rot[d] = 0.0; e < dim; ++e) {
rot[d] += R[d*dim+e] * centroidEx[e];
}
}
for (d = 0; d < dim; ++d) centroidEx[d] = rot[d];
for (d = 0; d < dim; ++d) {
for (e = 0; e < dim; ++e) {
for (f = 0, rotM[d*dim+e] = 0.0; f < dim; ++f) {
rotM[d*dim+e] += R[d*dim+f] * JEx[f*dim+e];
}
}
}
for (d = 0; d < dim; ++d) {
for (e = 0; e < dim; ++e) {
JEx[d*dim+e] = rotM[d*dim+e];
}
}
for (d = 0; d < dim; ++d) {
for (e = 0; e < dim; ++e) {
for (f = 0, rotM[d*dim+e] = 0.0; f < dim; ++f) {
rotM[d*dim+e] += invJEx[d*dim+f] * R[e*dim+f];
}
}
}
for (d = 0; d < dim; ++d) {
for (e = 0; e < dim; ++e) {
invJEx[d*dim+e] = rotM[d*dim+e];
}
}
for (d = 0; d < dim; ++d) {
ierr = PetscRandomGetValueReal(r, &trans[d]);CHKERRQ(ierr);
for (p = 0; p < 3; ++p) {
vertexCoords[p*dim+d] *= scale;
vertexCoords[p*dim+d] += trans[d];
}
v0Ex[d] = vertexCoords[d];
for (e = 0; e < dim; ++e) {
JEx[d*dim+e] *= scale;
invJEx[d*dim+e] /= scale;
}
detJEx *= scale;
centroidEx[d] *= scale;
centroidEx[d] += trans[d];
volEx *= scale;
}
ierr = ChangeCoordinates(dm, dim, vertexCoords);CHKERRQ(ierr);
ierr = CheckFEMGeometry(dm, 0, dim, v0Ex, JEx, invJEx, detJEx);CHKERRQ(ierr);
if (interpolate) {ierr = CheckFVMGeometry(dm, 0, dim, centroidEx, normalEx, volEx);CHKERRQ(ierr);}
}
ierr = PetscRandomDestroy(&r);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&ang);CHKERRQ(ierr);
}
/* Move to 3D: Check reference geometry: determinant is scaled by reference volume (2.0) */
dim = 3;
{
PetscScalar vertexCoords[9] = {-1.0, -1.0, 0.0, 1.0, -1.0, 0.0, -1.0, 1.0, 0.0};
PetscReal v0Ex[3] = {-1.0, -1.0, 0.0};
PetscReal JEx[9] = {1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0};
PetscReal invJEx[9] = {1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0};
PetscReal detJEx = 1.0;
PetscReal centroidEx[3] = {-0.333333333333, -0.333333333333, 0.0};
PetscReal normalEx[3] = {0.0, 0.0, 1.0};
PetscReal volEx = 2.0;
ierr = ChangeCoordinates(dm, dim, vertexCoords);CHKERRQ(ierr);
ierr = CheckFEMGeometry(dm, 0, dim, v0Ex, JEx, invJEx, detJEx);CHKERRQ(ierr);
if (interpolate) {ierr = CheckFVMGeometry(dm, 0, dim, centroidEx, normalEx, volEx);CHKERRQ(ierr);}
}
/* Rotated reference element */
{
PetscScalar vertexCoords[9] = {0.0, -1.0, -1.0, 0.0, 1.0, -1.0, 0.0, -1.0, 1.0};
PetscReal v0Ex[3] = {0.0, -1.0, -1.0};
PetscReal JEx[9] = {0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0};
PetscReal invJEx[9] = {0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0};
PetscReal detJEx = 1.0;
PetscReal centroidEx[3] = {0.0, -0.333333333333, -0.333333333333};
PetscReal normalEx[3] = {1.0, 0.0, 0.0};
PetscReal volEx = 2.0;
ierr = ChangeCoordinates(dm, dim, vertexCoords);CHKERRQ(ierr);
ierr = CheckFEMGeometry(dm, 0, dim, v0Ex, JEx, invJEx, detJEx);CHKERRQ(ierr);
if (interpolate) {ierr = CheckFVMGeometry(dm, 0, dim, centroidEx, normalEx, volEx);CHKERRQ(ierr);}
}
/* Check random triangles: scale, translate, then rotate */
if (transform) {
ierr = PetscRandomCreate(PETSC_COMM_SELF, &r);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(r);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(r, 0.0, 10.0);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_SELF, &ang);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(ang);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(ang, 0.0, 2*PETSC_PI);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_SELF, &ang2);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(ang2);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(ang2, 0.0, PETSC_PI);CHKERRQ(ierr);
for (t = 0; t < 100; ++t) {
PetscScalar vertexCoords[9] = {-1.0, -1.0, 0.0, 1.0, -1.0, 0.0, -1.0, 1.0, 0.0}, trans[3];
PetscReal v0Ex[3] = {-1.0, -1.0, 0.0};
PetscReal JEx[9] = {1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0}, R[9], rot[3], rotM[9];
PetscReal invJEx[9] = {1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0};
PetscReal detJEx = 1.0, scale, phi, theta, psi = 0.0;
PetscReal centroidEx[3] = {-0.333333333333, -0.333333333333, 0.0};
PetscReal normalEx[3] = {0.0, 0.0, 1.0};
PetscReal volEx = 2.0;
PetscInt d, e, f, p;
ierr = PetscRandomGetValueReal(r, &scale);CHKERRQ(ierr);
ierr = PetscRandomGetValueReal(ang, &phi);CHKERRQ(ierr);
ierr = PetscRandomGetValueReal(ang2, &theta);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) {
ierr = PetscRandomGetValueReal(r, &trans[d]);CHKERRQ(ierr);
for (p = 0; p < 3; ++p) {
vertexCoords[p*dim+d] *= scale;
vertexCoords[p*dim+d] += trans[d];
}
centroidEx[d] *= scale;
centroidEx[d] += trans[d];
for (e = 0; e < dim-1; ++e) {
JEx[d*dim+e] *= scale;
invJEx[d*dim+e] /= scale;
}
if (d < dim-1) {
detJEx *= scale;
volEx *= scale;
}
}
R[0] = cos(theta)*cos(psi); R[1] = sin(phi)*sin(theta)*cos(psi) - cos(phi)*sin(psi); R[2] = sin(phi)*sin(psi) + cos(phi)*sin(theta)*cos(psi);
R[3] = cos(theta)*sin(psi); R[4] = cos(phi)*cos(psi) + sin(phi)*sin(theta)*sin(psi); R[5] = cos(phi)*sin(theta)*sin(psi) - sin(phi)*cos(psi);
R[6] = -sin(theta); R[7] = sin(phi)*cos(theta); R[8] = cos(phi)*cos(theta);
for (p = 0; p < 3; ++p) {
for (d = 0; d < dim; ++d) {
for (e = 0, rot[d] = 0.0; e < dim; ++e) {
rot[d] += R[d*dim+e] * vertexCoords[p*dim+e];
}
}
for (d = 0; d < dim; ++d) vertexCoords[p*dim+d] = rot[d];
}
for (d = 0; d < dim; ++d) {
for (e = 0, rot[d] = 0.0; e < dim; ++e) {
rot[d] += R[d*dim+e] * centroidEx[e];
}
}
for (d = 0; d < dim; ++d) centroidEx[d] = rot[d];
for (d = 0; d < dim; ++d) {
for (e = 0, rot[d] = 0.0; e < dim; ++e) {
rot[d] += R[d*dim+e] * normalEx[e];
}
}
for (d = 0; d < dim; ++d) normalEx[d] = rot[d];
for (d = 0; d < dim; ++d) {
v0Ex[d] = vertexCoords[d];
for (e = 0; e < dim; ++e) {
for (f = 0, rotM[d*dim+e] = 0.0; f < dim; ++f) {
rotM[d*dim+e] += R[d*dim+f] * JEx[f*dim+e];
}
}
}
for (d = 0; d < dim; ++d) {
for (e = 0; e < dim; ++e) {
JEx[d*dim+e] = rotM[d*dim+e];
}
}
for (d = 0; d < dim; ++d) {
for (e = 0; e < dim; ++e) {
for (f = 0, rotM[d*dim+e] = 0.0; f < dim; ++f) {
rotM[d*dim+e] += invJEx[d*dim+f] * R[e*dim+f];
}
}
}
for (d = 0; d < dim; ++d) {
for (e = 0; e < dim; ++e) {
invJEx[d*dim+e] = rotM[d*dim+e];
}
}
ierr = ChangeCoordinates(dm, dim, vertexCoords);CHKERRQ(ierr);
ierr = CheckFEMGeometry(dm, 0, dim, v0Ex, JEx, invJEx, detJEx);CHKERRQ(ierr);
if (interpolate) {ierr = CheckFVMGeometry(dm, 0, dim, centroidEx, normalEx, volEx);CHKERRQ(ierr);}
}
ierr = PetscRandomDestroy(&r);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&ang);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&ang2);CHKERRQ(ierr);
}
/* Cleanup */
ierr = DMDestroy(&dm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETSC_STDCALL dmplexsetdimension_(DM dm,PetscInt *dim, int *__ierr ){
*__ierr = DMPlexSetDimension(
(DM)PetscToPointer((dm) ),*dim);
}
PetscErrorCode CreateMesh(MPI_Comm comm, PetscInt testNum, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscBool cellSimplex = user->cellSimplex;
PetscBool useGenerator = user->useGenerator;
const char *filename = user->filename;
const char *partitioner = "chaco";
size_t len;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = PetscStrlen(filename, &len);CHKERRQ(ierr);
if (len) {
#if defined(PETSC_HAVE_EXODUSII)
int CPU_word_size = 0, IO_word_size = 0, exoid = -1;
float version;
if (!rank) {
exoid = ex_open(filename, EX_READ, &CPU_word_size, &IO_word_size, &version);
if (exoid <= 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_LIB, "ex_open(\"%s\",...) did not return a valid file ID", filename);
}
ierr = DMPlexCreateExodus(comm, exoid, PETSC_FALSE, dm);CHKERRQ(ierr);
if (!rank) {ierr = ex_close(exoid);CHKERRQ(ierr);}
ierr = DMPlexGetDimension(*dm, &dim);CHKERRQ(ierr);
#else
SETERRQ(comm, PETSC_ERR_SUP, "Loading meshes requires ExodusII support. Reconfigure using --download-exodusii");
#endif
} else if (useGenerator) {
if (cellSimplex) {
ierr = DMPlexCreateBoxMesh(comm, dim, PETSC_FALSE, dm);CHKERRQ(ierr);
} else {
ierr = DMPlexCreateHexBoxMesh(comm, dim, PETSC_FALSE, dm);CHKERRQ(ierr);
}
} else {
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dm, dim);CHKERRQ(ierr);
switch (dim) {
case 2:
if (cellSimplex) {
ierr = CreateSimplex_2D(comm, *dm);CHKERRQ(ierr);
} else {
ierr = CreateQuad_2D(comm, testNum, *dm);CHKERRQ(ierr);
}
break;
case 3:
if (cellSimplex) {
ierr = CreateSimplex_3D(comm, *dm);CHKERRQ(ierr);
} else {
ierr = CreateHex_3D(comm, *dm);CHKERRQ(ierr);
}
break;
default:
SETERRQ1(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %d", dim);
}
}
{
DM interpolatedMesh = NULL;
ierr = CheckMesh(*dm);CHKERRQ(ierr);
ierr = DMPlexInterpolate(*dm, &interpolatedMesh);CHKERRQ(ierr);
ierr = DMPlexCopyCoordinates(*dm, interpolatedMesh);CHKERRQ(ierr);
ierr = CompareCones(*dm, interpolatedMesh);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = interpolatedMesh;
}
{
DM distributedMesh = NULL;
/* Distribute mesh over processes */
ierr = DMPlexDistribute(*dm, partitioner, 0, &distributedMesh);CHKERRQ(ierr);
if (distributedMesh) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = distributedMesh;
}
}
ierr = PetscObjectSetName((PetscObject) *dm, "Interpolated Mesh");CHKERRQ(ierr);
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
user->dm = *dm;
PetscFunctionReturn(0);
}
/*@
DMPlexCreateExodus - Create a DMPlex mesh from an ExodusII file.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. exoid - The ExodusII id associated with a exodus file and obtained using ex_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Level: beginner
.keywords: mesh,ExodusII
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateExodus(MPI_Comm comm, PetscInt exoid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_EXODUSII)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from ex_get_init() */
char title[PETSC_MAX_PATH_LEN+1];
int dim = 0, numVertices = 0, numCells = 0;
int num_cs = 0, num_vs = 0, num_fs = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_EXODUSII)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open EXODUS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
ierr = ex_get_init(exoid, title, &dim, &numVertices, &numCells, &num_cs, &num_vs, &num_fs);
if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"ExodusII ex_get_init() failed with error code %D\n",ierr);
if (!num_cs) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Exodus file does not contain any cell set\n");
}
ierr = MPI_Bcast(title, PETSC_MAX_PATH_LEN+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, title);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read cell sets information */
if (!rank) {
PetscInt *cone;
int c, cs, c_loc, v, v_loc;
/* Read from ex_get_elem_blk_ids() */
int *cs_id;
/* Read from ex_get_elem_block() */
char buffer[PETSC_MAX_PATH_LEN+1];
int num_cell_in_set, num_vertex_per_cell, num_attr;
/* Read from ex_get_elem_conn() */
int *cs_connect;
/* Get cell sets IDs */
ierr = PetscMalloc(num_cs * sizeof(int), &cs_id);CHKERRQ(ierr);
ierr = ex_get_elem_blk_ids(exoid, cs_id);CHKERRQ(ierr);
/* Read the cell set connectivity table and build mesh topology
EXO standard requires that cells in cell sets be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
for (c_loc = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, num_vertex_per_cell);CHKERRQ(ierr);
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc2(num_vertex_per_cell*num_cell_in_set,int,&cs_connect,num_vertex_per_cell,PetscInt,&cone);CHKERRQ(ierr);
ierr = ex_get_elem_conn(exoid, cs_id[cs], cs_connect);CHKERRQ(ierr);
/* EXO uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
for (v_loc = 0; v_loc < num_vertex_per_cell; ++v_loc, ++v) {
cone[v_loc] = cs_connect[v]+numCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (num_vertex_per_cell == 4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (num_vertex_per_cell == 8) {
PetscInt tmp = cone[1];
cone[1] = cone[3];
cone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(*dm, "Cell Sets", c, cs_id[cs]);CHKERRQ(ierr);
}
ierr = PetscFree2(cs_connect,cone);CHKERRQ(ierr);
}
ierr = PetscFree(cs_id);CHKERRQ(ierr);
}
