/*
Simple square boundary:
18--5-17--4--16
| | |
6 10 3
| | |
19-11-20--9--15
| | |
7 8 2
| | |
12--0-13--1--14
*/
PetscErrorCode DMMeshCreateSquareBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
{
DM_Mesh *mesh = (DM_Mesh *) dm->data;
PetscInt numVertices = (edges[0]+1)*(edges[1]+1);
PetscInt numEdges = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
PetscScalar *coords;
PetscInt coordSize;
PetscMPIInt rank;
PetscInt v, vx, vy;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(((PetscObject) dm)->comm, &rank);CHKERRQ(ierr);
if (!rank) {
PetscInt e, ex, ey;
ierr = DMMeshSetChart(dm, 0, numEdges+numVertices);CHKERRQ(ierr);
for(e = 0; e < numEdges; ++e) {
ierr = DMMeshSetConeSize(dm, e, 2);CHKERRQ(ierr);
}
ierr = DMMeshSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
for(vy = 0; vy <= edges[1]; vy++) {
for(ex = 0; ex < edges[0]; ex++) {
PetscInt edge = vy*edges[0] + ex;
PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
PetscInt cone[2] = {vertex, vertex+1};
ierr = DMMeshSetCone(dm, edge, cone);CHKERRQ(ierr);
if ((vy == 0) || (vy == edges[1])) {
ierr = DMMeshSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr);
ierr = DMMeshSetLabelValue(dm, "marker", cone[0], 1);CHKERRQ(ierr);
if (ex == edges[0]-1) {
ierr = DMMeshSetLabelValue(dm, "marker", cone[1], 1);CHKERRQ(ierr);
}
}
}
}
for(vx = 0; vx <= edges[0]; vx++) {
for(ey = 0; ey < edges[1]; ey++) {
PetscInt edge = vx*edges[1] + ey + edges[0]*(edges[1]+1);
PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
PetscInt cone[2] = {vertex, vertex+edges[0]+1};
ierr = DMMeshSetCone(dm, edge, cone);CHKERRQ(ierr);
if ((vx == 0) || (vx == edges[0])) {
ierr = DMMeshSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr);
ierr = DMMeshSetLabelValue(dm, "marker", cone[0], 1);CHKERRQ(ierr);
if (ey == edges[1]-1) {
ierr = DMMeshSetLabelValue(dm, "marker", cone[1], 1);CHKERRQ(ierr);
}
}
}
}
}
ierr = DMMeshSymmetrize(dm);CHKERRQ(ierr);
ierr = DMMeshStratify(dm);CHKERRQ(ierr);
/* Build coordinates */
ierr = PetscSectionSetChart(mesh->coordSection, numEdges, numEdges + numVertices);CHKERRQ(ierr);
for(v = numEdges; v < numEdges+numVertices; ++v) {
ierr = PetscSectionSetDof(mesh->coordSection, v, 2);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(mesh->coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(mesh->coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecSetSizes(mesh->coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetFromOptions(mesh->coordinates);CHKERRQ(ierr);
ierr = VecGetArray(mesh->coordinates, &coords);CHKERRQ(ierr);
for(vy = 0; vy <= edges[1]; ++vy) {
for(vx = 0; vx <= edges[0]; ++vx) {
coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
}
}
ierr = VecRestoreArray(mesh->coordinates, &coords);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Distribute cones
- Partitioning: input partition point map and naive sf, output sf with inverse of map, distribute points
- Distribute section: input current sf, communicate sizes and offsets, output local section and offsets (only use for new sf)
- Create SF for values: input current sf and offsets, output new sf
- Distribute values: input new sf, communicate values
*/
PetscErrorCode DistributeMesh(DM dm, AppCtx *user, PetscSF *pointSF, DM *parallelDM)
{
MPI_Comm comm = ((PetscObject) dm)->comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS cellPart, part;
PetscSection cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF;
ISLocalToGlobalMapping renumbering;
PetscSF coneSF;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets, newConesSize;
PetscInt *cones, *newCones;
PetscMPIInt numProcs, rank, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMMeshGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = DMMeshCreatePartition(dm, &cellPartSection, &cellPart, height);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) {
numRemoteRanks = numProcs;
} else {
numRemoteRanks = 0;
}
ierr = PetscMalloc(numRemoteRanks * sizeof(PetscSFNode), &remoteRanks);CHKERRQ(ierr);
for(p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, PETSC_NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFView(partSF, PETSC_NULL);CHKERRQ(ierr);
/* Close the partition over the mesh */
ierr = DMMeshCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMMeshCreate(comm, parallelDM);CHKERRQ(ierr);
ierr = DMMeshSetDimension(*parallelDM, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *parallelDM, "Parallel Mesh");CHKERRQ(ierr);
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, pointSF);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFView(*pointSF, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, PETSC_NULL);CHKERRQ(ierr);
/* Cleanup */
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMMeshGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMMeshGetConeSection(*parallelDM, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(*pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMMeshSetUp(*parallelDM);CHKERRQ(ierr);
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(*pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMMeshGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMMeshGetCones(*parallelDM, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newConesSize, newCones, PETSC_NULL, newCones);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
/* Create supports and stratify sieve */
ierr = DMMeshSymmetrize(*parallelDM);CHKERRQ(ierr);
ierr = DMMeshStratify(*parallelDM);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Simple cubic boundary:
2-------3
/| /|
6-------7 |
| | | |
| 0-----|-1
|/ |/
4-------5
*/
PetscErrorCode DMMeshCreateCubeBoundary(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
{
DM_Mesh *mesh = (DM_Mesh *) dm->data;
PetscInt numVertices = (faces[0]+1)*(faces[1]+1)*(faces[2]+1);
PetscInt numFaces = 6;
PetscScalar *coords;
PetscInt coordSize;
PetscMPIInt rank;
PetscInt v, vx, vy, vz;
PetscErrorCode ierr;
PetscFunctionBegin;
if ((faces[0] < 1) || (faces[1] < 1) || (faces[2] < 1)) {SETERRQ(((PetscObject) dm)->comm, PETSC_ERR_SUP, "Must have at least 1 face per side");}
if ((faces[0] > 1) || (faces[1] > 1) || (faces[2] > 1)) {SETERRQ(((PetscObject) dm)->comm, PETSC_ERR_SUP, "Currently can't handle more than 1 face per side");}
ierr = PetscMalloc(numVertices*2 * sizeof(PetscReal), &coords);CHKERRQ(ierr);
ierr = MPI_Comm_rank(((PetscObject) dm)->comm, &rank);CHKERRQ(ierr);
if (!rank) {
PetscInt f;
ierr = DMMeshSetChart(dm, 0, numFaces+numVertices);CHKERRQ(ierr);
for(f = 0; f < numFaces; ++f) {
ierr = DMMeshSetConeSize(dm, f, 4);CHKERRQ(ierr);
}
ierr = DMMeshSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
for(v = 0; v < numFaces+numVertices; ++v) {
ierr = DMMeshSetLabelValue(dm, "marker", v, 1);CHKERRQ(ierr);
}
{ // Side 0 (Front)
PetscInt cone[4] = {numFaces+4, numFaces+5, numFaces+7, numFaces+6};
ierr = DMMeshSetCone(dm, 0, cone);CHKERRQ(ierr);
}
{ // Side 1 (Back)
PetscInt cone[4] = {numFaces+1, numFaces+0, numFaces+2, numFaces+3};
ierr = DMMeshSetCone(dm, 0, cone);CHKERRQ(ierr);
}
{ // Side 0 (Bottom)
PetscInt cone[4] = {numFaces+0, numFaces+1, numFaces+5, numFaces+4};
ierr = DMMeshSetCone(dm, 0, cone);CHKERRQ(ierr);
}
{ // Side 0 (Top)
PetscInt cone[4] = {numFaces+6, numFaces+7, numFaces+3, numFaces+2};
ierr = DMMeshSetCone(dm, 0, cone);CHKERRQ(ierr);
}
{ // Side 0 (Left)
PetscInt cone[4] = {numFaces+0, numFaces+4, numFaces+6, numFaces+2};
ierr = DMMeshSetCone(dm, 0, cone);CHKERRQ(ierr);
}
{ // Side 0 (Right)
PetscInt cone[4] = {numFaces+5, numFaces+1, numFaces+3, numFaces+7};
ierr = DMMeshSetCone(dm, 0, cone);CHKERRQ(ierr);
}
}
ierr = DMMeshSymmetrize(dm);CHKERRQ(ierr);
ierr = DMMeshStratify(dm);CHKERRQ(ierr);
/* Build coordinates */
ierr = PetscSectionSetChart(mesh->coordSection, numFaces, numFaces + numVertices);CHKERRQ(ierr);
for(v = numFaces; v < numFaces+numVertices; ++v) {
ierr = PetscSectionSetDof(mesh->coordSection, v, 3);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(mesh->coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(mesh->coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecSetSizes(mesh->coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecGetArray(mesh->coordinates, &coords);CHKERRQ(ierr);
for(vz = 0; vz <= faces[2]; ++vz) {
for(vy = 0; vy <= faces[1]; ++vy) {
for(vx = 0; vx <= faces[0]; ++vx) {
coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
}
}
}
ierr = VecRestoreArray(mesh->coordinates, &coords);CHKERRQ(ierr);
PetscFunctionReturn(0);
}