/*@
DMPlexCopyCoordinates - Copy coordinates from one mesh to another with the same vertices
Collective on DM
Input Parameter:
. dmA - The DMPlex object with initial coordinates
Output Parameter:
. dmB - The DMPlex object with copied coordinates
Level: intermediate
Note: This is typically used when adding pieces other than vertices to a mesh
.keywords: mesh
.seealso: DMCopyLabels(), DMGetCoordinates(), DMGetCoordinatesLocal(), DMGetCoordinateDM(), DMGetCoordinateSection()
@*/
PetscErrorCode DMPlexCopyCoordinates(DM dmA, DM dmB)
{
Vec coordinatesA, coordinatesB;
PetscSection coordSectionA, coordSectionB;
PetscScalar *coordsA, *coordsB;
PetscInt spaceDim, vStartA, vStartB, vEndA, vEndB, coordSizeB, v, d;
PetscErrorCode ierr;
PetscFunctionBegin;
if (dmA == dmB) PetscFunctionReturn(0);
ierr = DMPlexGetDepthStratum(dmA, 0, &vStartA, &vEndA);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dmB, 0, &vStartB, &vEndB);CHKERRQ(ierr);
if ((vEndA-vStartA) != (vEndB-vStartB)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of vertices in first DM %d != %d in the second DM", vEndA-vStartA, vEndB-vStartB);
ierr = DMGetCoordinateSection(dmA, &coordSectionA);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dmB, &coordSectionB);CHKERRQ(ierr);
if (!coordSectionB) {
PetscInt dim;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) coordSectionA), &coordSectionB);CHKERRQ(ierr);
ierr = DMGetCoordinateDim(dmA, &dim);CHKERRQ(ierr);
ierr = DMSetCoordinateSection(dmB, dim, coordSectionB);CHKERRQ(ierr);
ierr = PetscObjectDereference((PetscObject) coordSectionB);CHKERRQ(ierr);
}
ierr = PetscSectionSetNumFields(coordSectionB, 1);CHKERRQ(ierr);
ierr = PetscSectionGetFieldComponents(coordSectionA, 0, &spaceDim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSectionB, 0, spaceDim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSectionB, vStartB, vEndB);CHKERRQ(ierr);
for (v = vStartB; v < vEndB; ++v) {
ierr = PetscSectionSetDof(coordSectionB, v, spaceDim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSectionB, v, 0, spaceDim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSectionB);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSectionB, &coordSizeB);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dmA, &coordinatesA);CHKERRQ(ierr);
ierr = VecCreate(PetscObjectComm((PetscObject) dmB), &coordinatesB);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinatesB, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinatesB, coordSizeB, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinatesB,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinatesA, &coordsA);CHKERRQ(ierr);
ierr = VecGetArray(coordinatesB, &coordsB);CHKERRQ(ierr);
for (v = 0; v < vEndB-vStartB; ++v) {
for (d = 0; d < spaceDim; ++d) {
coordsB[v*spaceDim+d] = coordsA[v*spaceDim+d];
}
}
ierr = VecRestoreArray(coordinatesA, &coordsA);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinatesB, &coordsB);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dmB, coordinatesB);CHKERRQ(ierr);
ierr = VecDestroy(&coordinatesB);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode CreateSpectralPlanes(DM dm, PetscInt numPlanes, const PetscInt planeDir[], const PetscReal planeCoord[], AppCtx *user)
{
PetscSection coordSection;
Vec coordinates;
const PetscScalar *coords;
PetscInt dim, p, vStart, vEnd, v;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (p = 0; p < numPlanes; ++p) {
DMLabel label;
char name[PETSC_MAX_PATH_LEN];
ierr = PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "spectral_plane_%D", p);CHKERRQ(ierr);
ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
ierr = DMLabelAddStratum(label, 1);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
PetscInt off;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
if (PetscAbsReal(planeCoord[p] - PetscRealPart(coords[off+planeDir[p]])) < PETSC_SMALL) {
ierr = DMLabelSetValue(label, v, 1);CHKERRQ(ierr);
}
}
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode CreatePoints_Centroid(DM dm, PetscInt *Np, PetscReal **pcoords, PetscBool *pointsAllProcs, AppCtx *ctx)
{
PetscSection coordSection;
Vec coordsLocal;
PetscInt spaceDim, p;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateDim(dm, &spaceDim);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, NULL, Np);CHKERRQ(ierr);
ierr = PetscCalloc1(*Np * spaceDim, pcoords);CHKERRQ(ierr);
for (p = 0; p < *Np; ++p) {
PetscScalar *coords = NULL;
PetscInt size, num, n, d;
ierr = DMPlexVecGetClosure(dm, coordSection, coordsLocal, p, &size, &coords);CHKERRQ(ierr);
num = size/spaceDim;
for (n = 0; n < num; ++n) {
for (d = 0; d < spaceDim; ++d) (*pcoords)[p*spaceDim+d] += PetscRealPart(coords[n*spaceDim+d]) / num;
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d]Point %D (", rank, p);CHKERRQ(ierr);
for (d = 0; d < spaceDim; ++d) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%g", (double)(*pcoords)[p*spaceDim+d]);CHKERRQ(ierr);
if (d < spaceDim-1) {ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, ", ");CHKERRQ(ierr);}
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, ")\n");CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, p, &num, &coords);CHKERRQ(ierr);
}
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD, NULL);CHKERRQ(ierr);
*pointsAllProcs = PETSC_FALSE;
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data;
FILE *fp;
PetscViewerVTKObjectLink link;
PetscSection coordSection, globalCoordSection;
PetscLayout vLayout;
Vec coordinates;
PetscReal lengthScale;
PetscInt vMax, totVertices, totCells;
PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr);
/* Vertices */
ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
if (vMax >= 0) {
PetscInt pStart, pEnd, p, localSize = 0;
ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr);
pEnd = PetscMin(pEnd, vMax);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr);
if (dof > 0) ++localSize;
}
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
} else {
ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr);
}
ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr);
ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr);
/* Cells */
ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr);
/* Vertex fields */
for (link = vtk->link; link; link = link->next) {
if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE;
if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE;
}
if (hasPoint) {
ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection, newSection = NULL;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
DMLabel subpointMap, subpointMapX;
PetscInt dim, dimX, pStart, pEnd, qStart, qEnd;
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
/* Here is where we check whether dmX is a submesh of dm */
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr);
if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) {
const PetscInt *ind = NULL;
IS subpointIS;
PetscInt n = 0, q;
ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr);
if (subpointIS) {
ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr);
ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr);
}
ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr);
for (q = qStart; q < qEnd; ++q) {
PetscInt dof, off, p;
ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr);
if (dof) {
ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr);
if (p >= pStart) {
ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr);
ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr);
}
}
}
if (subpointIS) {
ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr);
ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
}
/* No need to setup section */
section = newSection;
}
} else {
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr);
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);}
}
}
/* Cell Fields */
ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr);
if (hasCell || writePartition) {
ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
} else {
PetscContainer c;
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr);
if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
}
if (writePartition) {
ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr);
}
}
/* Cleanup */
ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
ierr = PetscFClose(comm, fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexCreateGmsh - Create a DMPlex mesh from a Gmsh file.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Gmsh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.geuz.org/gmsh/doc/texinfo/#MSH-ASCII-file-format
Level: beginner
.keywords: mesh,Gmsh
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateGmsh(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
FILE *fd;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords, *coordsIn = NULL;
PetscInt dim = 0, coordSize, c, v, d;
int numVertices = 0, numCells = 0, snum;
long fpos = 0;
PetscMPIInt num_proc, rank;
char line[PETSC_MAX_PATH_LEN];
PetscBool match;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
if (!rank) {
PetscBool match;
int fileType, dataSize;
ierr = PetscViewerASCIIGetPointer(viewer, &fd);CHKERRQ(ierr);
/* Read header */
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$MeshFormat\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
snum = fscanf(fd, "2.2 %d %d\n", &fileType, &dataSize);CHKERRQ(snum != 2);
if (fileType) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh ASCII file", fileType);
if (dataSize != sizeof(double)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Data size %d is not valid for a Gmsh file", dataSize);
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$EndMeshFormat\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read vertices */
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$Nodes\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
snum = fscanf(fd, "%d\n", &numVertices);CHKERRQ(snum != 1);
ierr = PetscMalloc(numVertices*3 * sizeof(PetscScalar), &coordsIn);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
double x, y, z;
int i;
snum = fscanf(fd, "%d %lg %lg %lg\n", &i, &x, &y, &z);CHKERRQ(snum != 4);
coordsIn[v*3+0] = x; coordsIn[v*3+1] = y; coordsIn[v*3+2] = z;
if (i != v+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid node number %d should be %d", i, v+1);
}
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$EndNodes\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read cells */
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$Elements\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
snum = fscanf(fd, "%d\n", &numCells);CHKERRQ(snum != 1);
}
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
if (!rank) {
fpos = ftell(fd);
for (c = 0; c < numCells; ++c) {
PetscInt numCorners, t;
int cone[8], i, cellType, numTags, tag;
snum = fscanf(fd, "%d %d %d", &i, &cellType, &numTags);CHKERRQ(snum != 3);
if (numTags) for (t = 0; t < numTags; ++t) {snum = fscanf(fd, "%d", &tag);CHKERRQ(snum != 1);}
switch (cellType) {
case 1: /* 2-node line */
dim = 1;
numCorners = 2;
snum = fscanf(fd, "%d %d\n", &cone[0], &cone[1]);CHKERRQ(snum != numCorners);
break;
case 2: /* 3-node triangle */
dim = 2;
numCorners = 3;
snum = fscanf(fd, "%d %d %d\n", &cone[0], &cone[1], &cone[2]);CHKERRQ(snum != numCorners);
break;
case 3: /* 4-node quadrangle */
dim = 2;
numCorners = 4;
snum = fscanf(fd, "%d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3]);CHKERRQ(snum != numCorners);
break;
case 4: /* 4-node tetrahedron */
dim = 3;
numCorners = 4;
snum = fscanf(fd, "%d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3]);CHKERRQ(snum != numCorners);
break;
case 5: /* 8-node hexahedron */
dim = 3;
numCorners = 8;
snum = fscanf(fd, "%d %d %d %d %d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3], &cone[4], &cone[5], &cone[6], &cone[7]);CHKERRQ(snum != numCorners);
break;
default:
SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unsupported Gmsh element type %d", cellType);
}
ierr = DMPlexSetConeSize(*dm, c, numCorners);CHKERRQ(ierr);
if (i != c+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell number %d should be %d", i, c+1);
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
if (!rank) {
ierr = fseek(fd, fpos, SEEK_SET);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
PetscInt pcone[8], numCorners, corner, t;
int cone[8], i, cellType, numTags, tag;
snum = fscanf(fd, "%d %d %d", &i, &cellType, &numTags);CHKERRQ(snum != 3);
if (numTags) for (t = 0; t < numTags; ++t) {snum = fscanf(fd, "%d", &tag);CHKERRQ(snum != 1);}
switch (cellType) {
case 1: /* 2-node line */
dim = 1;
numCorners = 2;
snum = fscanf(fd, "%d %d\n", &cone[0], &cone[1]);CHKERRQ(snum != numCorners);
break;
case 2: /* 3-node triangle */
dim = 2;
numCorners = 3;
snum = fscanf(fd, "%d %d %d\n", &cone[0], &cone[1], &cone[2]);CHKERRQ(snum != numCorners);
break;
case 3: /* 4-node quadrangle */
dim = 2;
numCorners = 4;
snum = fscanf(fd, "%d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3]);CHKERRQ(snum != numCorners);
break;
case 4: /* 4-node tetrahedron */
dim = 3;
numCorners = 4;
snum = fscanf(fd, "%d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3]);CHKERRQ(snum != numCorners);
ierr = DMPlexInvertCell(dim, numCorners, cone);CHKERRQ(ierr);
break;
case 5: /* 8-node hexahedron */
dim = 3;
numCorners = 8;
snum = fscanf(fd, "%d %d %d %d %d %d %d %d\n", &cone[0], &cone[1], &cone[2], &cone[3], &cone[4], &cone[5], &cone[6], &cone[7]);CHKERRQ(snum != numCorners);
ierr = DMPlexInvertCell(dim, numCorners, cone);CHKERRQ(ierr);
break;
default:
SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unsupported Gmsh element type %d", cellType);
}
for (corner = 0; corner < numCorners; ++corner) pcone[corner] = cone[corner] + numCells-1;
ierr = DMPlexSetCone(*dm, c, pcone);CHKERRQ(ierr);
if (i != c+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell number %d should be %d", i, c+1);
}
fgets(line, PETSC_MAX_PATH_LEN, fd);
ierr = PetscStrncmp(line, "$EndElements\n", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(comm, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*3+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexTSSetupGeometry(DM dm, PetscFV fvm, DMTS_Plex *dmplexts)
{
DM dmFace, dmCell;
DMLabel ghostLabel;
PetscSection sectionFace, sectionCell;
PetscSection coordSection;
Vec coordinates;
PetscReal minradius;
PetscScalar *fgeom, *cgeom;
PetscInt dim, cStart, cEnd, cEndInterior, c, fStart, fEnd, f;
PetscErrorCode ierr;
PetscFunctionBegin;
if (dmplexts->setupGeom) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
/* Make cell centroids and volumes */
ierr = DMClone(dm, &dmCell);CHKERRQ(ierr);
ierr = DMSetCoordinateSection(dmCell, coordSection);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dmCell, coordinates);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionCell);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionCell, cStart, cEnd);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {ierr = PetscSectionSetDof(sectionCell, c, sizeof(CellGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionCell);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmCell, sectionCell);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionCell);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmCell, &dmplexts->cellgeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
for (c = cStart; c < cEndInterior; ++c) {
CellGeom *cg;
ierr = DMPlexPointLocalRef(dmCell, c, cgeom, &cg);CHKERRQ(ierr);
ierr = PetscMemzero(cg, sizeof(*cg));CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dmCell, c, &cg->volume, cg->centroid, NULL);CHKERRQ(ierr);
}
/* Compute face normals and minimum cell radius */
ierr = DMClone(dm, &dmFace);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), §ionFace);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionFace, fStart, fEnd);CHKERRQ(ierr);
for (f = fStart; f < fEnd; ++f) {ierr = PetscSectionSetDof(sectionFace, f, sizeof(FaceGeom)/sizeof(PetscScalar));CHKERRQ(ierr);}
ierr = PetscSectionSetUp(sectionFace);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dmFace, sectionFace);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionFace);CHKERRQ(ierr);
ierr = DMCreateLocalVector(dmFace, &dmplexts->facegeom);CHKERRQ(ierr);
ierr = VecGetArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = DMPlexGetLabel(dm, "ghost", &ghostLabel);CHKERRQ(ierr);
minradius = PETSC_MAX_REAL;
for (f = fStart; f < fEnd; ++f) {
FaceGeom *fg;
PetscReal area;
PetscInt ghost, d;
ierr = DMLabelGetValue(ghostLabel, f, &ghost);CHKERRQ(ierr);
if (ghost >= 0) continue;
ierr = DMPlexPointLocalRef(dmFace, f, fgeom, &fg);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, f, &area, fg->centroid, fg->normal);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) fg->normal[d] *= area;
/* Flip face orientation if necessary to match ordering in support, and Update minimum radius */
{
CellGeom *cL, *cR;
const PetscInt *cells;
PetscReal *lcentroid, *rcentroid;
PetscReal v[3];
ierr = DMPlexGetSupport(dm, f, &cells);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[0], cgeom, &cL);CHKERRQ(ierr);
ierr = DMPlexPointLocalRead(dmCell, cells[1], cgeom, &cR);CHKERRQ(ierr);
lcentroid = cells[0] >= cEndInterior ? fg->centroid : cL->centroid;
rcentroid = cells[1] >= cEndInterior ? fg->centroid : cR->centroid;
WaxpyD(dim, -1, lcentroid, rcentroid, v);
if (DotRealD(dim, fg->normal, v) < 0) {
for (d = 0; d < dim; ++d) fg->normal[d] = -fg->normal[d];
}
if (DotRealD(dim, fg->normal, v) <= 0) {
if (dim == 2) SETERRQ5(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g) v (%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) v[0], (double) v[1]);
if (dim == 3) SETERRQ7(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed, normal (%g,%g,%g) v (%g,%g,%g)", f, (double) fg->normal[0], (double) fg->normal[1], (double) fg->normal[2], (double) v[0], (double) v[1], (double) v[2]);
SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Direction for face %d could not be fixed", f);
}
if (cells[0] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cL->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
if (cells[1] < cEndInterior) {
WaxpyD(dim, -1, fg->centroid, cR->centroid, v);
minradius = PetscMin(minradius, NormD(dim, v));
}
}
}
ierr = MPI_Allreduce(&minradius, &dmplexts->minradius, 1, MPIU_REAL, MPI_MIN, PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
/* Compute centroids of ghost cells */
for (c = cEndInterior; c < cEnd; ++c) {
FaceGeom *fg;
const PetscInt *cone, *support;
PetscInt coneSize, supportSize, s;
ierr = DMPlexGetConeSize(dmCell, c, &coneSize);CHKERRQ(ierr);
if (coneSize != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ghost cell %d has cone size %d != 1", c, coneSize);
ierr = DMPlexGetCone(dmCell, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(dmCell, cone[0], &supportSize);CHKERRQ(ierr);
if (supportSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %d has support size %d != 1", cone[0], supportSize);
ierr = DMPlexGetSupport(dmCell, cone[0], &support);CHKERRQ(ierr);
ierr = DMPlexPointLocalRef(dmFace, cone[0], fgeom, &fg);CHKERRQ(ierr);
for (s = 0; s < 2; ++s) {
/* Reflect ghost centroid across plane of face */
if (support[s] == c) {
const CellGeom *ci;
CellGeom *cg;
PetscReal c2f[3], a;
ierr = DMPlexPointLocalRead(dmCell, support[(s+1)%2], cgeom, &ci);CHKERRQ(ierr);
WaxpyD(dim, -1, ci->centroid, fg->centroid, c2f); /* cell to face centroid */
a = DotRealD(dim, c2f, fg->normal)/DotRealD(dim, fg->normal, fg->normal);
ierr = DMPlexPointLocalRef(dmCell, support[s], cgeom, &cg);CHKERRQ(ierr);
WaxpyD(dim, 2*a, fg->normal, ci->centroid, cg->centroid);
cg->volume = ci->volume;
}
}
}
ierr = VecRestoreArray(dmplexts->facegeom, &fgeom);CHKERRQ(ierr);
ierr = VecRestoreArray(dmplexts->cellgeom, &cgeom);CHKERRQ(ierr);
ierr = DMDestroy(&dmCell);CHKERRQ(ierr);
ierr = DMDestroy(&dmFace);CHKERRQ(ierr);
dmplexts->setupGeom = PETSC_TRUE;
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode DMPlexRefine_CTetgen(DM dm, PetscReal *maxVolumes, DM *dmRefined)
{
MPI_Comm comm;
const PetscInt dim = 3;
const char *labelName = "marker";
PLC *in, *out;
DMLabel label;
PetscInt verbose = 0, vStart, vEnd, v, cStart, cEnd, c, depth, depthGlobal;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,((PetscObject) dm)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = MPIU_Allreduce(&depth, &depthGlobal, 1, MPIU_INT, MPI_MAX, comm);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMGetLabel(dm, labelName, &label);CHKERRQ(ierr);
ierr = PLCCreate(&in);CHKERRQ(ierr);
ierr = PLCCreate(&out);CHKERRQ(ierr);
in->numberofpoints = vEnd - vStart;
if (in->numberofpoints > 0) {
PetscSection coordSection;
Vec coordinates;
PetscScalar *array;
ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
ierr = PetscMalloc1(in->numberofpoints, &in->pointmarkerlist);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
const PetscInt idx = v - vStart;
PetscInt off, d, m = -1;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) {
in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
}
if (label) {ierr = DMLabelGetValue(label, v, &m);CHKERRQ(ierr);}
in->pointmarkerlist[idx] = (int) m;
}
ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
}
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
in->numberofcorners = 4;
in->numberoftetrahedra = cEnd - cStart;
in->tetrahedronvolumelist = maxVolumes;
if (in->numberoftetrahedra > 0) {
ierr = PetscMalloc1(in->numberoftetrahedra*in->numberofcorners, &in->tetrahedronlist);CHKERRQ(ierr);
for (c = cStart; c < cEnd; ++c) {
const PetscInt idx = c - cStart;
PetscInt *closure = NULL;
PetscInt closureSize;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
if ((closureSize != 5) && (closureSize != 15)) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Mesh has cell which is not a tetrahedron, %D vertices in closure", closureSize);
for (v = 0; v < 4; ++v) {
in->tetrahedronlist[idx*in->numberofcorners + v] = closure[(v+closureSize-4)*2] - vStart;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
}
if (!rank) {
TetGenOpts t;
ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
t.in = dm; /* Should go away */
t.refine = 1;
t.varvolume = 1;
t.quality = 1;
t.edgesout = 1;
t.zeroindex = 1;
t.quiet = 1;
t.verbose = verbose; /* Change this */
ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
}
in->tetrahedronvolumelist = NULL;
{
DMLabel rlabel = NULL;
const PetscInt numCorners = 4;
const PetscInt numCells = out->numberoftetrahedra;
const PetscInt numVertices = out->numberofpoints;
double *meshCoords;
int *cells = out->tetrahedronlist;
PetscBool interpolate = depthGlobal > 1 ? PETSC_TRUE : PETSC_FALSE;
if (sizeof (PetscReal) == sizeof (double)) {
meshCoords = (double *) out->pointlist;
}
else {
PetscInt i;
ierr = PetscMalloc1(3 * numVertices,&meshCoords);CHKERRQ(ierr);
for (i = 0; i < 3 * numVertices; i++) {
meshCoords[i] = (PetscReal) out->pointlist[i];
}
}
ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dmRefined);CHKERRQ(ierr);
if (sizeof (PetscReal) != sizeof (double)) {
ierr = PetscFree(meshCoords);CHKERRQ(ierr);
}
if (label) {ierr = DMCreateLabel(*dmRefined, labelName); ierr = DMGetLabel(*dmRefined, labelName, &rlabel);}
/* Set labels */
for (v = 0; v < numVertices; ++v) {
if (out->pointmarkerlist[v]) {
if (rlabel) {ierr = DMLabelSetValue(rlabel, v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);}
}
}
if (interpolate) {
PetscInt e, f;
for (e = 0; e < out->numberofedges; e++) {
if (out->edgemarkerlist[e]) {
const PetscInt vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
const PetscInt *edges;
PetscInt numEdges;
ierr = DMPlexGetJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
if (rlabel) {ierr = DMLabelSetValue(rlabel, edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dmRefined, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
}
}
for (f = 0; f < out->numberoftrifaces; f++) {
if (out->trifacemarkerlist[f]) {
const PetscInt vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
const PetscInt *faces;
PetscInt numFaces;
ierr = DMPlexGetFullJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
if (rlabel) {ierr = DMLabelSetValue(rlabel, faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dmRefined, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
}
}
}
ierr = DMPlexSetRefinementUniform(*dmRefined, PETSC_FALSE);CHKERRQ(ierr);
}
ierr = PLCDestroy(&in);CHKERRQ(ierr);
ierr = PLCDestroy(&out);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode DMPlexGenerate_CTetgen(DM boundary, PetscBool interpolate, DM *dm)
{
MPI_Comm comm;
const PetscInt dim = 3;
const char *labelName = "marker";
PLC *in, *out;
DMLabel label;
PetscInt verbose = 0, vStart, vEnd, v, fStart, fEnd, f;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)boundary,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,((PetscObject) boundary)->prefix, "-ctetgen_verbose", &verbose, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(boundary, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMGetLabel(boundary, labelName, &label);CHKERRQ(ierr);
ierr = PLCCreate(&in);CHKERRQ(ierr);
ierr = PLCCreate(&out);CHKERRQ(ierr);
in->numberofpoints = vEnd - vStart;
if (in->numberofpoints > 0) {
PetscSection coordSection;
Vec coordinates;
PetscScalar *array;
ierr = PetscMalloc1(in->numberofpoints*dim, &in->pointlist);CHKERRQ(ierr);
ierr = PetscMalloc1(in->numberofpoints, &in->pointmarkerlist);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(boundary, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(boundary, &coordSection);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &array);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
const PetscInt idx = v - vStart;
PetscInt off, d, m = -1;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
for (d = 0; d < dim; ++d) {
in->pointlist[idx*dim + d] = PetscRealPart(array[off+d]);
}
if (label) {ierr = DMLabelGetValue(label, v, &m);CHKERRQ(ierr);}
in->pointmarkerlist[idx] = (int) m;
}
ierr = VecRestoreArray(coordinates, &array);CHKERRQ(ierr);
}
ierr = DMPlexGetHeightStratum(boundary, 0, &fStart, &fEnd);CHKERRQ(ierr);
in->numberoffacets = fEnd - fStart;
if (in->numberoffacets > 0) {
ierr = PetscMalloc1(in->numberoffacets, &in->facetlist);CHKERRQ(ierr);
ierr = PetscMalloc1(in->numberoffacets, &in->facetmarkerlist);CHKERRQ(ierr);
for (f = fStart; f < fEnd; ++f) {
const PetscInt idx = f - fStart;
PetscInt *points = NULL, numPoints, p, numVertices = 0, v, m = -1;
polygon *poly;
in->facetlist[idx].numberofpolygons = 1;
ierr = PetscMalloc1(in->facetlist[idx].numberofpolygons, &in->facetlist[idx].polygonlist);CHKERRQ(ierr);
in->facetlist[idx].numberofholes = 0;
in->facetlist[idx].holelist = NULL;
ierr = DMPlexGetTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
for (p = 0; p < numPoints*2; p += 2) {
const PetscInt point = points[p];
if ((point >= vStart) && (point < vEnd)) points[numVertices++] = point;
}
poly = in->facetlist[idx].polygonlist;
poly->numberofvertices = numVertices;
ierr = PetscMalloc1(poly->numberofvertices, &poly->vertexlist);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
const PetscInt vIdx = points[v] - vStart;
poly->vertexlist[v] = vIdx;
}
if (label) {ierr = DMLabelGetValue(label, f, &m);CHKERRQ(ierr);}
in->facetmarkerlist[idx] = (int) m;
ierr = DMPlexRestoreTransitiveClosure(boundary, f, PETSC_TRUE, &numPoints, &points);CHKERRQ(ierr);
}
}
if (!rank) {
TetGenOpts t;
ierr = TetGenOptsInitialize(&t);CHKERRQ(ierr);
t.in = boundary; /* Should go away */
t.plc = 1;
t.quality = 1;
t.edgesout = 1;
t.zeroindex = 1;
t.quiet = 1;
t.verbose = verbose;
ierr = TetGenCheckOpts(&t);CHKERRQ(ierr);
ierr = TetGenTetrahedralize(&t, in, out);CHKERRQ(ierr);
}
{
DMLabel glabel = NULL;
const PetscInt numCorners = 4;
const PetscInt numCells = out->numberoftetrahedra;
const PetscInt numVertices = out->numberofpoints;
double *meshCoords;
int *cells = out->tetrahedronlist;
if (sizeof (PetscReal) == sizeof (double)) {
meshCoords = (double *) out->pointlist;
}
else {
PetscInt i;
ierr = PetscMalloc1(3 * numVertices,&meshCoords);CHKERRQ(ierr);
for (i = 0; i < 3 * numVertices; i++) {
meshCoords[i] = (PetscReal) out->pointlist[i];
}
}
ierr = DMPlexInvertCells_Internal(dim, numCells, numCorners, cells);CHKERRQ(ierr);
ierr = DMPlexCreateFromCellList(comm, dim, numCells, numVertices, numCorners, interpolate, cells, dim, meshCoords, dm);CHKERRQ(ierr);
if (sizeof (PetscReal) != sizeof (double)) {
ierr = PetscFree(meshCoords);CHKERRQ(ierr);
}
if (label) {ierr = DMCreateLabel(*dm, labelName); ierr = DMGetLabel(*dm, labelName, &glabel);}
/* Set labels */
for (v = 0; v < numVertices; ++v) {
if (out->pointmarkerlist[v]) {
if (glabel) {ierr = DMLabelSetValue(glabel, v+numCells, out->pointmarkerlist[v]);CHKERRQ(ierr);}
}
}
if (interpolate) {
PetscInt e;
for (e = 0; e < out->numberofedges; e++) {
if (out->edgemarkerlist[e]) {
const PetscInt vertices[2] = {out->edgelist[e*2+0]+numCells, out->edgelist[e*2+1]+numCells};
const PetscInt *edges;
PetscInt numEdges;
ierr = DMPlexGetJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
if (numEdges != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Two vertices must cover only one edge, not %D", numEdges);
if (glabel) {ierr = DMLabelSetValue(glabel, edges[0], out->edgemarkerlist[e]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dm, 2, vertices, &numEdges, &edges);CHKERRQ(ierr);
}
}
for (f = 0; f < out->numberoftrifaces; f++) {
if (out->trifacemarkerlist[f]) {
const PetscInt vertices[3] = {out->trifacelist[f*3+0]+numCells, out->trifacelist[f*3+1]+numCells, out->trifacelist[f*3+2]+numCells};
const PetscInt *faces;
PetscInt numFaces;
ierr = DMPlexGetFullJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Three vertices must cover only one face, not %D", numFaces);
if (glabel) {ierr = DMLabelSetValue(glabel, faces[0], out->trifacemarkerlist[f]);CHKERRQ(ierr);}
ierr = DMPlexRestoreJoin(*dm, 3, vertices, &numFaces, &faces);CHKERRQ(ierr);
}
}
}
ierr = DMPlexSetRefinementUniform(*dm, PETSC_FALSE);CHKERRQ(ierr);
}
ierr = PLCDestroy(&in);CHKERRQ(ierr);
ierr = PLCDestroy(&out);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexCreateExodus - Create a DMPlex mesh from an ExodusII file ID.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. exoid - The ExodusII id associated with a exodus file and obtained using ex_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Level: beginner
.keywords: mesh,ExodusII
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateExodus(MPI_Comm comm, PetscInt exoid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_EXODUSII)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from ex_get_init() */
char title[PETSC_MAX_PATH_LEN+1];
int dim = 0, numVertices = 0, numCells = 0;
int num_cs = 0, num_vs = 0, num_fs = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_EXODUSII)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open EXODUS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
ierr = PetscMemzero(title,(PETSC_MAX_PATH_LEN+1)*sizeof(char));CHKERRQ(ierr);
ierr = ex_get_init(exoid, title, &dim, &numVertices, &numCells, &num_cs, &num_vs, &num_fs);
if (ierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"ExodusII ex_get_init() failed with error code %D\n",ierr);
if (!num_cs) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Exodus file does not contain any cell set\n");
}
ierr = MPI_Bcast(title, PETSC_MAX_PATH_LEN+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, title);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read cell sets information */
if (!rank) {
PetscInt *cone;
int c, cs, c_loc, v, v_loc;
/* Read from ex_get_elem_blk_ids() */
int *cs_id;
/* Read from ex_get_elem_block() */
char buffer[PETSC_MAX_PATH_LEN+1];
int num_cell_in_set, num_vertex_per_cell, num_attr;
/* Read from ex_get_elem_conn() */
int *cs_connect;
/* Get cell sets IDs */
ierr = PetscMalloc1(num_cs, &cs_id);CHKERRQ(ierr);
ierr = ex_get_elem_blk_ids(exoid, cs_id);CHKERRQ(ierr);
/* Read the cell set connectivity table and build mesh topology
EXO standard requires that cells in cell sets be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
for (c_loc = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, num_vertex_per_cell);CHKERRQ(ierr);
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (cs = 0, c = 0; cs < num_cs; cs++) {
ierr = ex_get_elem_block(exoid, cs_id[cs], buffer, &num_cell_in_set, &num_vertex_per_cell, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc2(num_vertex_per_cell*num_cell_in_set,&cs_connect,num_vertex_per_cell,&cone);CHKERRQ(ierr);
ierr = ex_get_elem_conn(exoid, cs_id[cs], cs_connect);CHKERRQ(ierr);
/* EXO uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc < num_cell_in_set; ++c_loc, ++c) {
for (v_loc = 0; v_loc < num_vertex_per_cell; ++v_loc, ++v) {
cone[v_loc] = cs_connect[v]+numCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (num_vertex_per_cell == 4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (num_vertex_per_cell == 8) {
PetscInt tmp = cone[1];
cone[1] = cone[3];
cone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMSetLabelValue(*dm, "Cell Sets", c, cs_id[cs]);CHKERRQ(ierr);
}
ierr = PetscFree2(cs_connect,cone);CHKERRQ(ierr);
}
ierr = PetscFree(cs_id);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
/* Maintain Cell Sets label */
{
DMLabel label;
ierr = DMRemoveLabel(*dm, "Cell Sets", &label);CHKERRQ(ierr);
if (label) {ierr = DMAddLabel(idm, label);CHKERRQ(ierr);}
}
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Create vertex set label */
if (!rank && (num_vs > 0)) {
int vs, v;
/* Read from ex_get_node_set_ids() */
int *vs_id;
/* Read from ex_get_node_set_param() */
int num_vertex_in_set, num_attr;
/* Read from ex_get_node_set() */
int *vs_vertex_list;
/* Get vertex set ids */
ierr = PetscMalloc1(num_vs, &vs_id);CHKERRQ(ierr);
ierr = ex_get_node_set_ids(exoid, vs_id);CHKERRQ(ierr);
for (vs = 0; vs < num_vs; ++vs) {
ierr = ex_get_node_set_param(exoid, vs_id[vs], &num_vertex_in_set, &num_attr);CHKERRQ(ierr);
ierr = PetscMalloc1(num_vertex_in_set, &vs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_node_set(exoid, vs_id[vs], vs_vertex_list);CHKERRQ(ierr);
for (v = 0; v < num_vertex_in_set; ++v) {
ierr = DMSetLabelValue(*dm, "Vertex Sets", vs_vertex_list[v]+numCells-1, vs_id[vs]);CHKERRQ(ierr);
}
ierr = PetscFree(vs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(vs_id);CHKERRQ(ierr);
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
float *x, *y, *z;
ierr = PetscMalloc3(numVertices,&x,numVertices,&y,numVertices,&z);CHKERRQ(ierr);
ierr = ex_get_coord(exoid, x, y, z);CHKERRQ(ierr);
if (dim > 0) {
for (v = 0; v < numVertices; ++v) coords[v*dim+0] = x[v];
}
if (dim > 1) {
for (v = 0; v < numVertices; ++v) coords[v*dim+1] = y[v];
}
if (dim > 2) {
for (v = 0; v < numVertices; ++v) coords[v*dim+2] = z[v];
}
ierr = PetscFree3(x,y,z);CHKERRQ(ierr);
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Create side set label */
if (!rank && interpolate && (num_fs > 0)) {
int fs, f, voff;
/* Read from ex_get_side_set_ids() */
int *fs_id;
/* Read from ex_get_side_set_param() */
int num_side_in_set, num_dist_fact_in_set;
/* Read from ex_get_side_set_node_list() */
int *fs_vertex_count_list, *fs_vertex_list;
/* Get side set ids */
ierr = PetscMalloc1(num_fs, &fs_id);CHKERRQ(ierr);
ierr = ex_get_side_set_ids(exoid, fs_id);CHKERRQ(ierr);
for (fs = 0; fs < num_fs; ++fs) {
ierr = ex_get_side_set_param(exoid, fs_id[fs], &num_side_in_set, &num_dist_fact_in_set);CHKERRQ(ierr);
ierr = PetscMalloc2(num_side_in_set,&fs_vertex_count_list,num_side_in_set*4,&fs_vertex_list);CHKERRQ(ierr);
ierr = ex_get_side_set_node_list(exoid, fs_id[fs], fs_vertex_count_list, fs_vertex_list);CHKERRQ(ierr);
for (f = 0, voff = 0; f < num_side_in_set; ++f) {
const PetscInt *faces = NULL;
PetscInt faceSize = fs_vertex_count_list[f], numFaces;
PetscInt faceVertices[4], v;
if (faceSize > 4) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "ExodusII side cannot have %d > 4 vertices", faceSize);
for (v = 0; v < faceSize; ++v, ++voff) {
faceVertices[v] = fs_vertex_list[voff]+numCells-1;
}
ierr = DMPlexGetFullJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
if (numFaces != 1) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Invalid ExodusII side %d in set %d maps to %d faces", f, fs, numFaces);
ierr = DMSetLabelValue(*dm, "Face Sets", faces[0], fs_id[fs]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, faceSize, faceVertices, &numFaces, &faces);CHKERRQ(ierr);
}
ierr = PetscFree2(fs_vertex_count_list,fs_vertex_list);CHKERRQ(ierr);
}
ierr = PetscFree(fs_id);CHKERRQ(ierr);
}
#else
SETERRQ(comm, PETSC_ERR_SUP, "This method requires ExodusII support. Reconfigure using --download-exodusii");
#endif
PetscFunctionReturn(0);
}
/*@
DMPlexCreateCGNS - Create a DMPlex mesh from a CGNS file ID.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. cgid - The CG id associated with a file and obtained using cg_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.grc.nasa.gov/WWW/cgns/CGNS_docs_current/index.html
Level: beginner
.keywords: mesh,CGNS
.seealso: DMPlexCreate(), DMPlexCreateExodus()
@*/
PetscErrorCode DMPlexCreateCGNS(MPI_Comm comm, PetscInt cgid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_CGNS)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt *cellStart, *vertStart;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from file */
char basename[CGIO_MAX_NAME_LENGTH+1];
char buffer[CGIO_MAX_NAME_LENGTH+1];
int dim = 0, physDim = 0, numVertices = 0, numCells = 0;
int nzones = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_CGNS)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open CGNS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
int nbases, z;
ierr = cg_nbases(cgid, &nbases);CHKERRQ(ierr);
if (nbases > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single base, not %d\n",nbases);
ierr = cg_base_read(cgid, 1, basename, &dim, &physDim);CHKERRQ(ierr);
ierr = cg_nzones(cgid, 1, &nzones);CHKERRQ(ierr);
ierr = PetscCalloc2(nzones+1, &cellStart, nzones+1, &vertStart);CHKERRQ(ierr);
for (z = 1; z <= nzones; ++z) {
cgsize_t sizes[3]; /* Number of vertices, number of cells, number of boundary vertices */
ierr = cg_zone_read(cgid, 1, z, buffer, sizes);CHKERRQ(ierr);
numVertices += sizes[0];
numCells += sizes[1];
cellStart[z] += sizes[1] + cellStart[z-1];
vertStart[z] += sizes[0] + vertStart[z-1];
}
for (z = 1; z <= nzones; ++z) {
vertStart[z] += numCells;
}
}
ierr = MPI_Bcast(basename, CGIO_MAX_NAME_LENGTH+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&nzones, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, basename);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read zone information */
if (!rank) {
int z, c, c_loc, v, v_loc;
/* Read the cell set connectivity table and build mesh topology
CGNS standard requires that cells in a zone be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (z = 1, c = 0; z <= nzones; ++z) {
ZoneType_t zonetype;
int nsections;
ElementType_t cellType;
cgsize_t start, end;
int nbndry, parentFlag;
PetscInt numCorners;
ierr = cg_zone_type(cgid, 1, z, &zonetype);CHKERRQ(ierr);
if (zonetype == Structured) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Can only handle Unstructured zones for CGNS");
ierr = cg_nsections(cgid, 1, z, &nsections);CHKERRQ(ierr);
if (nsections > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single section, not %d\n",nsections);
ierr = cg_section_read(cgid, 1, z, 1, buffer, &cellType, &start, &end, &nbndry, &parentFlag);CHKERRQ(ierr);
/* This alone is reason enough to bludgeon every single CGNDS developer, this must be what they describe as the "idiocy of crowds" */
if (cellType == MIXED) {
cgsize_t elementDataSize, *elements;
PetscInt off;
ierr = cg_ElementDataSize(cgid, 1, z, 1, &elementDataSize);CHKERRQ(ierr);
ierr = PetscMalloc1(elementDataSize, &elements);CHKERRQ(ierr);
ierr = cg_elements_read(cgid, 1, z, 1, elements, NULL);CHKERRQ(ierr);
for (c_loc = start, off = 0; c_loc <= end; ++c_loc, ++c) {
switch (elements[off]) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) elements[off]);
}
ierr = DMPlexSetConeSize(*dm, c, numCorners);CHKERRQ(ierr);
off += numCorners+1;
}
ierr = PetscFree(elements);CHKERRQ(ierr);
} else {
switch (cellType) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) cellType);
}
for (c_loc = start; c_loc <= end; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, numCorners);CHKERRQ(ierr);
}
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (z = 1, c = 0; z <= nzones; ++z) {
ElementType_t cellType;
cgsize_t *elements, elementDataSize, start, end;
int nbndry, parentFlag;
PetscInt *cone, numc, numCorners, maxCorners = 27;
ierr = cg_section_read(cgid, 1, z, 1, buffer, &cellType, &start, &end, &nbndry, &parentFlag);CHKERRQ(ierr);
numc = end - start;
/* This alone is reason enough to bludgeon every single CGNDS developer, this must be what they describe as the "idiocy of crowds" */
ierr = cg_ElementDataSize(cgid, 1, z, 1, &elementDataSize);CHKERRQ(ierr);
ierr = PetscMalloc2(elementDataSize,&elements,maxCorners,&cone);CHKERRQ(ierr);
ierr = cg_elements_read(cgid, 1, z, 1, elements, NULL);CHKERRQ(ierr);
if (cellType == MIXED) {
/* CGNS uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc <= numc; ++c_loc, ++c) {
switch (elements[v]) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) elements[v]);
}
++v;
for (v_loc = 0; v_loc < numCorners; ++v_loc, ++v) {
cone[v_loc] = elements[v]+numCells-1;
}
/* Tetrahedra are inverted */
if (elements[v] == TETRA_4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (elements[v] == HEXA_8) {
PetscInt tmp = cone[5];
cone[5] = cone[7];
cone[7] = tmp;
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(*dm, "zone", c, z);CHKERRQ(ierr);
}
} else {
switch (cellType) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) cellType);
}
/* CGNS uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc <= numc; ++c_loc, ++c) {
for (v_loc = 0; v_loc < numCorners; ++v_loc, ++v) {
cone[v_loc] = elements[v]+numCells-1;
}
/* Tetrahedra are inverted */
if (cellType == TETRA_4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted, and they give the top first */
if (cellType == HEXA_8) {
PetscInt tmp = cone[5];
cone[5] = cone[7];
cone[7] = tmp;
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(*dm, "zone", c, z);CHKERRQ(ierr);
}
}
ierr = PetscFree2(elements,cone);CHKERRQ(ierr);
}
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
/* Maintain zone label */
{
DMLabel label;
ierr = DMPlexRemoveLabel(*dm, "zone", &label);CHKERRQ(ierr);
if (label) {ierr = DMPlexAddLabel(idm, label);CHKERRQ(ierr);}
}
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(comm, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
PetscInt off = 0;
float *x[3];
int z, d;
ierr = PetscMalloc3(numVertices,&x[0],numVertices,&x[1],numVertices,&x[2]);CHKERRQ(ierr);
for (z = 1; z <= nzones; ++z) {
DataType_t datatype;
cgsize_t sizes[3]; /* Number of vertices, number of cells, number of boundary vertices */
cgsize_t range_min[3] = {1, 1, 1};
cgsize_t range_max[3] = {1, 1, 1};
int ngrids, ncoords;
ierr = cg_zone_read(cgid, 1, z, buffer, sizes);CHKERRQ(ierr);
range_max[0] = sizes[0];
ierr = cg_ngrids(cgid, 1, z, &ngrids);CHKERRQ(ierr);
if (ngrids > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single grid, not %d\n",ngrids);
ierr = cg_ncoords(cgid, 1, z, &ncoords);CHKERRQ(ierr);
if (ncoords != dim) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a coordinate array for each dimension, not %d\n",ncoords);
for (d = 0; d < dim; ++d) {
ierr = cg_coord_info(cgid, 1, z, 1+d, &datatype, buffer);CHKERRQ(ierr);
ierr = cg_coord_read(cgid, 1, z, buffer, RealSingle, range_min, range_max, x[d]);CHKERRQ(ierr);
}
if (dim > 0) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+0] = x[0][v];
}
if (dim > 1) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+1] = x[1][v];
}
if (dim > 2) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+2] = x[2][v];
}
off += sizes[0];
}
ierr = PetscFree3(x[0],x[1],x[2]);CHKERRQ(ierr);
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Read boundary conditions */
if (!rank) {
DMLabel label;
BCType_t bctype;
DataType_t datatype;
PointSetType_t pointtype;
cgsize_t *points;
PetscReal *normals;
int normal[3];
char *bcname = buffer;
cgsize_t npoints, nnormals;
int z, nbc, bc, c, ndatasets;
for (z = 1; z <= nzones; ++z) {
ierr = cg_nbocos(cgid, 1, z, &nbc);CHKERRQ(ierr);
for (bc = 1; bc <= nbc; ++bc) {
ierr = cg_boco_info(cgid, 1, z, bc, bcname, &bctype, &pointtype, &npoints, normal, &nnormals, &datatype, &ndatasets);CHKERRQ(ierr);
ierr = DMPlexCreateLabel(*dm, bcname);CHKERRQ(ierr);
ierr = DMPlexGetLabel(*dm, bcname, &label);CHKERRQ(ierr);
ierr = PetscMalloc2(npoints, &points, nnormals, &normals);CHKERRQ(ierr);
ierr = cg_boco_read(cgid, 1, z, bc, points, (void *) normals);CHKERRQ(ierr);
if (pointtype == ElementRange) {
/* Range of cells: assuming half-open interval since the documentation sucks */
for (c = points[0]; c < points[1]; ++c) {
ierr = DMLabelSetValue(label, c - cellStart[z-1], 1);CHKERRQ(ierr);
}
} else if (pointtype == ElementList) {
/* List of cells */
for (c = 0; c < npoints; ++c) {
ierr = DMLabelSetValue(label, points[c] - cellStart[z-1], 1);CHKERRQ(ierr);
}
} else if (pointtype == PointRange) {
GridLocation_t gridloc;
/* List of points: Oh please, someone get the CGNS developers away from a computer. This is unconscionable. */
ierr = cg_goto(cgid, 1, "Zone_t", z, "BC_t", bc, "end");CHKERRQ(ierr);
ierr = cg_gridlocation_read(&gridloc);CHKERRQ(ierr);
/* Range of points: assuming half-open interval since the documentation sucks */
for (c = points[0]; c < points[1]; ++c) {
if (gridloc == Vertex) {ierr = DMLabelSetValue(label, c - vertStart[z-1], 1);CHKERRQ(ierr);}
else {ierr = DMLabelSetValue(label, c - cellStart[z-1], 1);CHKERRQ(ierr);}
}
} else if (pointtype == PointList) {
GridLocation_t gridloc;
/* List of points: Oh please, someone get the CGNS developers away from a computer. This is unconscionable. */
ierr = cg_goto(cgid, 1, "Zone_t", z, "BC_t", bc, "end");
ierr = cg_gridlocation_read(&gridloc);
for (c = 0; c < npoints; ++c) {
if (gridloc == Vertex) {ierr = DMLabelSetValue(label, points[c] - vertStart[z-1], 1);CHKERRQ(ierr);}
else {ierr = DMLabelSetValue(label, points[c] - cellStart[z-1], 1);CHKERRQ(ierr);}
}
} else SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported point set type %d", (int) pointtype);
ierr = PetscFree2(points, normals);CHKERRQ(ierr);
}
}
ierr = PetscFree2(cellStart, vertStart);CHKERRQ(ierr);
}
#else
SETERRQ(comm, PETSC_ERR_SUP, "This method requires CGNS support. Reconfigure using --with-cgns-dir");
#endif
PetscFunctionReturn(0);
}
/*@C
DMPlexCreateFluent - Create a DMPlex mesh from a Fluent mesh file.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Fluent mesh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://aerojet.engr.ucdavis.edu/fluenthelp/html/ug/node1490.htm
Level: beginner
.keywords: mesh, fluent, case
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateFluent(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
PetscMPIInt rank;
PetscInt c, f, v, dim = PETSC_DETERMINE, numCells = 0, numVertices = 0, numCellVertices = PETSC_DETERMINE;
PetscInt numFaces = PETSC_DETERMINE, numFaceEntries = PETSC_DETERMINE, numFaceVertices = PETSC_DETERMINE;
PetscInt *faces = NULL, *cellVertices, *faceZoneIDs = NULL;
PetscInt d, coordSize;
PetscScalar *coords, *coordsIn = NULL;
PetscSection coordSection;
Vec coordinates;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
if (!rank) {
FluentSection s;
do {
ierr = DMPlexCreateFluent_ReadSection(viewer, &s);CHKERRQ(ierr);
if (s.index == 2) { /* Dimension */
dim = s.nd;
} else if (s.index == 10 || s.index == 2010) { /* Vertices */
if (s.zoneID == 0) numVertices = s.last;
else {
if (coordsIn) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Currently no support for multiple coordinate sets in Fluent files");
coordsIn = (PetscScalar *) s.data;
}
} else if (s.index == 12 || s.index == 2012) { /* Cells */
if (s.zoneID == 0) numCells = s.last;
else {
switch (s.nd) {
case 0: numCellVertices = PETSC_DETERMINE; break;
case 1: numCellVertices = 3; break; /* triangular */
case 2: numCellVertices = 4; break; /* tetrahedral */
case 3: numCellVertices = 4; break; /* quadrilateral */
case 4: numCellVertices = 8; break; /* hexahedral */
case 5: numCellVertices = 5; break; /* pyramid */
case 6: numCellVertices = 6; break; /* wedge */
default: numCellVertices = PETSC_DETERMINE;
}
}
} else if (s.index == 13 || s.index == 2013) { /* Facets */
if (s.zoneID == 0) { /* Header section */
numFaces = s.last - s.first + 1;
if (s.nd == 0 || s.nd == 5) numFaceVertices = PETSC_DETERMINE;
else numFaceVertices = s.nd;
} else { /* Data section */
if (numFaceVertices != PETSC_DETERMINE && s.nd != numFaceVertices) {
SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mixed facets in Fluent files are not supported");
}
if (numFaces < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "No header section for facets in Fluent file");
if (numFaceVertices == PETSC_DETERMINE) numFaceVertices = s.nd;
numFaceEntries = numFaceVertices + 2;
if (!faces) {ierr = PetscMalloc1(numFaces*numFaceEntries, &faces);CHKERRQ(ierr);}
if (!faceZoneIDs) {ierr = PetscMalloc1(numFaces, &faceZoneIDs);CHKERRQ(ierr);}
ierr = PetscMemcpy(&(faces[(s.first-1)*numFaceEntries]), s.data, (s.last-s.first+1)*numFaceEntries*sizeof(PetscInt));CHKERRQ(ierr);
/* Record the zoneID for each face set */
for (f = s.first -1; f < s.last; f++) faceZoneIDs[f] = s.zoneID;
ierr = PetscFree(s.data);CHKERRQ(ierr);
}
}
} while (s.index >= 0);
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
if (dim < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Fluent file does not include dimension");
/* Allocate cell-vertex mesh */
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells + numVertices);CHKERRQ(ierr);
if (!rank) {
if (numCells < 0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown number of cells in Fluent file");
/* If no cell type was given we assume simplices */
if (numCellVertices == PETSC_DETERMINE) numCellVertices = numFaceVertices + 1;
for (c = 0; c < numCells; ++c) {ierr = DMPlexSetConeSize(*dm, c, numCellVertices);CHKERRQ(ierr);}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
if (!rank) {
/* Derive cell-vertex list from face-vertex and face-cell maps */
ierr = PetscMalloc1(numCells*numCellVertices, &cellVertices);CHKERRQ(ierr);
for (c = 0; c < numCells*numCellVertices; c++) cellVertices[c] = -1;
for (f = 0; f < numFaces; f++) {
PetscInt *cell;
const PetscInt cl = faces[f*numFaceEntries + numFaceVertices];
const PetscInt cr = faces[f*numFaceEntries + numFaceVertices + 1];
const PetscInt *face = &(faces[f*numFaceEntries]);
if (cl > 0) {
cell = &(cellVertices[(cl-1) * numCellVertices]);
for (v = 0; v < numFaceVertices; v++) {
PetscBool found = PETSC_FALSE;
for (c = 0; c < numCellVertices; c++) {
if (cell[c] < 0) break;
if (cell[c] == face[v]-1 + numCells) {found = PETSC_TRUE; break;}
}
if (!found) cell[c] = face[v]-1 + numCells;
}
}
if (cr > 0) {
cell = &(cellVertices[(cr-1) * numCellVertices]);
for (v = 0; v < numFaceVertices; v++) {
PetscBool found = PETSC_FALSE;
for (c = 0; c < numCellVertices; c++) {
if (cell[c] < 0) break;
if (cell[c] == face[v]-1 + numCells) {found = PETSC_TRUE; break;}
}
if (!found) cell[c] = face[v]-1 + numCells;
}
}
}
for (c = 0; c < numCells; c++) {
ierr = DMPlexSetCone(*dm, c, &(cellVertices[c*numCellVertices]));CHKERRQ(ierr);
}
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
if (!rank) {
PetscInt fi, joinSize, meetSize, *fverts, cells[2];
const PetscInt *join, *meet;
ierr = PetscMalloc1(numFaceVertices, &fverts);CHKERRQ(ierr);
/* Mark facets by finding the full join of all adjacent vertices */
for (f = 0; f < numFaces; f++) {
const PetscInt cl = faces[f*numFaceEntries + numFaceVertices] - 1;
const PetscInt cr = faces[f*numFaceEntries + numFaceVertices + 1] - 1;
if (cl > 0 && cr > 0) {
/* If we know both adjoining cells we can use a single-level meet */
cells[0] = cl; cells[1] = cr;
ierr = DMPlexGetMeet(*dm, 2, cells, &meetSize, &meet);CHKERRQ(ierr);
if (meetSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for Fluent face %d", f);
ierr = DMPlexSetLabelValue(*dm, "Face Sets", meet[0], faceZoneIDs[f]);CHKERRQ(ierr);
ierr = DMPlexRestoreMeet(*dm, numFaceVertices, fverts, &meetSize, &meet);CHKERRQ(ierr);
} else {
for (fi = 0; fi < numFaceVertices; fi++) fverts[fi] = faces[f*numFaceEntries + fi] + numCells - 1;
ierr = DMPlexGetFullJoin(*dm, numFaceVertices, fverts, &joinSize, &join);CHKERRQ(ierr);
if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for Fluent face %d", f);
ierr = DMPlexSetLabelValue(*dm, "Face Sets", join[0], faceZoneIDs[f]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, numFaceVertices, fverts, &joinSize, &join);CHKERRQ(ierr);
}
}
ierr = PetscFree(fverts);CHKERRQ(ierr);
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(comm, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*dim+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
if (!rank) {
ierr = PetscFree(cellVertices);CHKERRQ(ierr);
ierr = PetscFree(faces);CHKERRQ(ierr);
ierr = PetscFree(faceZoneIDs);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode ComputeSpectral(DM dm, Vec u, PetscInt numPlanes, const PetscInt planeDir[], const PetscReal planeCoord[], AppCtx *user)
{
MPI_Comm comm;
PetscSection coordSection, section;
Vec coordinates, uloc;
const PetscScalar *coords, *array;
PetscInt p;
PetscMPIInt size, rank;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm, u, INSERT_VALUES, uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm, u, INSERT_VALUES, uloc);CHKERRQ(ierr);
ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, uloc, 0.0, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = VecViewFromOptions(uloc, NULL, "-sol_view");CHKERRQ(ierr);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = VecGetArrayRead(uloc, &array);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (p = 0; p < numPlanes; ++p) {
DMLabel label;
char name[PETSC_MAX_PATH_LEN];
Mat F;
Vec x, y;
IS stratum;
PetscReal *ray, *gray;
PetscScalar *rvals, *svals, *gsvals;
PetscInt *perm, *nperm;
PetscInt n, N, i, j, off, offu;
const PetscInt *points;
ierr = PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "spectral_plane_%D", p);CHKERRQ(ierr);
ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(label, 1, &stratum);CHKERRQ(ierr);
ierr = ISGetLocalSize(stratum, &n);CHKERRQ(ierr);
ierr = ISGetIndices(stratum, &points);CHKERRQ(ierr);
ierr = PetscMalloc2(n, &ray, n, &svals);CHKERRQ(ierr);
for (i = 0; i < n; ++i) {
ierr = PetscSectionGetOffset(coordSection, points[i], &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, points[i], &offu);CHKERRQ(ierr);
ray[i] = PetscRealPart(coords[off+((planeDir[p]+1)%2)]);
svals[i] = array[offu];
}
/* Gather the ray data to proc 0 */
if (size > 1) {
PetscMPIInt *cnt, *displs, p;
ierr = PetscCalloc2(size, &cnt, size, &displs);CHKERRQ(ierr);
ierr = MPI_Gather(&n, 1, MPIU_INT, cnt, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 1; p < size; ++p) displs[p] = displs[p-1] + cnt[p-1];
N = displs[size-1] + cnt[size-1];
ierr = PetscMalloc2(N, &gray, N, &gsvals);CHKERRQ(ierr);
ierr = MPI_Gatherv(ray, n, MPIU_REAL, gray, cnt, displs, MPIU_REAL, 0, comm);CHKERRQ(ierr);
ierr = MPI_Gatherv(svals, n, MPIU_SCALAR, gsvals, cnt, displs, MPIU_SCALAR, 0, comm);CHKERRQ(ierr);
ierr = PetscFree2(cnt, displs);CHKERRQ(ierr);
} else {
N = n;
gray = ray;
gsvals = svals;
}
if (!rank) {
/* Sort point along ray */
ierr = PetscMalloc2(N, &perm, N, &nperm);CHKERRQ(ierr);
for (i = 0; i < N; ++i) {perm[i] = i;}
ierr = PetscSortRealWithPermutation(N, gray, perm);CHKERRQ(ierr);
/* Count duplicates and squish mapping */
nperm[0] = perm[0];
for (i = 1, j = 1; i < N; ++i) {
if (PetscAbsReal(gray[perm[i]] - gray[perm[i-1]]) > PETSC_SMALL) nperm[j++] = perm[i];
}
/* Create FFT structs */
ierr = MatCreateFFT(PETSC_COMM_SELF, 1, &j, MATFFTW, &F);CHKERRQ(ierr);
ierr = MatCreateVecs(F, &x, &y);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, name);CHKERRQ(ierr);
ierr = VecGetArray(x, &rvals);CHKERRQ(ierr);
for (i = 0, j = 0; i < N; ++i) {
if (i > 0 && PetscAbsReal(gray[perm[i]] - gray[perm[i-1]]) < PETSC_SMALL) continue;
rvals[j] = gsvals[nperm[j]];
++j;
}
ierr = PetscFree2(perm, nperm);CHKERRQ(ierr);
if (size > 1) {ierr = PetscFree2(gray, gsvals);CHKERRQ(ierr);}
ierr = VecRestoreArray(x, &rvals);CHKERRQ(ierr);
/* Do FFT along the ray */
ierr = MatMult(F, x, y);CHKERRQ(ierr);
/* Chop FFT */
ierr = VecChop(y, PETSC_SMALL);CHKERRQ(ierr);
ierr = VecViewFromOptions(x, NULL, "-real_view");CHKERRQ(ierr);
ierr = VecViewFromOptions(y, NULL, "-fft_view");CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(stratum, &points);CHKERRQ(ierr);
ierr = ISDestroy(&stratum);CHKERRQ(ierr);
ierr = PetscFree2(ray, svals);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(uloc, &array);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &uloc);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexCreateGmsh - Create a DMPlex mesh from a Gmsh file viewer
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Gmsh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.geuz.org/gmsh/doc/texinfo/#MSH-ASCII-file-format
and http://www.geuz.org/gmsh/doc/texinfo/#MSH-binary-file-format
Level: beginner
.keywords: mesh,Gmsh
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateGmsh(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
PetscViewerType vtype;
GmshElement *gmsh_elem;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords, *coordsIn = NULL;
PetscInt dim = 0, coordSize, c, v, d, r, cell;
int i, numVertices = 0, numCells = 0, trueNumCells = 0, numRegions = 0, snum;
PetscMPIInt num_proc, rank;
char line[PETSC_MAX_PATH_LEN];
PetscBool match, binary, bswap = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
ierr = PetscViewerGetType(viewer, &vtype);CHKERRQ(ierr);
ierr = PetscStrcmp(vtype, PETSCVIEWERBINARY, &binary);CHKERRQ(ierr);
if (!rank || binary) {
PetscBool match;
int fileType, dataSize;
float version;
/* Read header */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$MeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 3, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%f %d %d", &version, &fileType, &dataSize);
if (snum != 3) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Gmsh file header: %s", line);
if (version < 2.0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Gmsh file must be at least version 2.0");
if (dataSize != sizeof(double)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Data size %d is not valid for a Gmsh file", dataSize);
if (binary) {
int checkInt;
ierr = PetscViewerRead(viewer, &checkInt, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
if (checkInt != 1) {
ierr = PetscByteSwap(&checkInt, PETSC_ENUM, 1);CHKERRQ(ierr);
if (checkInt == 1) bswap = PETSC_TRUE;
else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh binary file", fileType);
}
} else if (fileType) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh ASCII file", fileType);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndMeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* OPTIONAL Read physical names */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$PhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (match) {
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numRegions);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
for (r = 0; r < numRegions; ++r) {
PetscInt rdim, tag;
ierr = PetscViewerRead(viewer, &rdim, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &tag, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndPhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Initial read for vertex section */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
/* Read vertices */
ierr = PetscStrncmp(line, "$Nodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numVertices);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscMalloc1(numVertices*3, &coordsIn);CHKERRQ(ierr);
if (binary) {
size_t doubleSize, intSize;
PetscInt elementSize;
char *buffer;
PetscScalar *baseptr;
ierr = PetscDataTypeGetSize(PETSC_ENUM, &intSize);CHKERRQ(ierr);
ierr = PetscDataTypeGetSize(PETSC_DOUBLE, &doubleSize);CHKERRQ(ierr);
elementSize = (intSize + 3*doubleSize);
ierr = PetscMalloc1(elementSize*numVertices, &buffer);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, buffer, elementSize*numVertices, NULL, PETSC_CHAR);CHKERRQ(ierr);
if (bswap) ierr = PetscByteSwap(buffer, PETSC_CHAR, elementSize*numVertices);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
baseptr = ((PetscScalar*)(buffer+v*elementSize+intSize));
coordsIn[v*3+0] = baseptr[0];
coordsIn[v*3+1] = baseptr[1];
coordsIn[v*3+2] = baseptr[2];
}
ierr = PetscFree(buffer);CHKERRQ(ierr);
} else {
for (v = 0; v < numVertices; ++v) {
ierr = PetscViewerRead(viewer, &i, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &(coordsIn[v*3]), 3, NULL, PETSC_DOUBLE);CHKERRQ(ierr);
if (i != (int)v+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid node number %d should be %d", i, v+1);
}
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndNodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read cells */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$Elements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
snum = sscanf(line, "%d", &numCells);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
if (!rank || binary) {
/* Gmsh elements can be of any dimension/co-dimension, so we need to traverse the
file contents multiple times to figure out the true number of cells and facets
in the given mesh. To make this more efficient we read the file contents only
once and store them in memory, while determining the true number of cells. */
ierr = DMPlexCreateGmsh_ReadElement(viewer, numCells, binary, bswap, &gmsh_elem);CHKERRQ(ierr);
for (trueNumCells=0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim > dim) {dim = gmsh_elem[c].dim; trueNumCells = 0;}
if (gmsh_elem[c].dim == dim) trueNumCells++;
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndElements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
/* For binary we read on all ranks, but only build the plex on rank 0 */
if (binary && rank) {trueNumCells = 0; numVertices = 0;};
/* Allocate the cell-vertex mesh */
ierr = DMPlexSetChart(*dm, 0, trueNumCells+numVertices);CHKERRQ(ierr);
if (!rank) {
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
ierr = DMPlexSetConeSize(*dm, cell, gmsh_elem[c].numNodes);CHKERRQ(ierr);
cell++;
}
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
/* Add cell-vertex connections */
if (!rank) {
PetscInt pcone[8], corner;
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + trueNumCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (gmsh_elem[c].numNodes == 4) {
PetscInt tmp = pcone[0];
pcone[0] = pcone[1];
pcone[1] = tmp;
}
/* Hexahedra are inverted */
if (gmsh_elem[c].numNodes == 8) {
PetscInt tmp = pcone[1];
pcone[1] = pcone[3];
pcone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, cell, pcone);CHKERRQ(ierr);
cell++;
}
}
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
if (!rank) {
/* Apply boundary IDs by finding the relevant facets with vertex joins */
PetscInt pcone[8], corner, vStart, vEnd;
ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim-1) {
PetscInt joinSize;
const PetscInt *join;
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + vStart - 1;
}
ierr = DMPlexGetFullJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for element %d", gmsh_elem[c].id);
ierr = DMSetLabelValue(*dm, "Face Sets", join[0], gmsh_elem[c].tags[0]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
}
}
/* Create cell sets */
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
if (gmsh_elem[c].numTags > 0) {
ierr = DMSetLabelValue(*dm, "Cell Sets", cell, gmsh_elem[c].tags[0]);CHKERRQ(ierr);
cell++;
}
}
}
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, trueNumCells, trueNumCells + numVertices);CHKERRQ(ierr);
for (v = trueNumCells; v < trueNumCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*3+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Clean up intermediate storage */
if (!rank || binary) ierr = PetscFree(gmsh_elem);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
