static PetscErrorCode PCBDDCScalingSetUp_Deluxe(PC pc)
{
PC_IS *pcis=(PC_IS*)pc->data;
PC_BDDC *pcbddc=(PC_BDDC*)pc->data;
PCBDDCDeluxeScaling deluxe_ctx=pcbddc->deluxe_ctx;
PCBDDCSubSchurs sub_schurs=pcbddc->sub_schurs;
PetscErrorCode ierr;
PetscFunctionBegin;
/* reset data structures if the topology has changed */
if (pcbddc->recompute_topography) {
ierr = PCBDDCScalingReset_Deluxe_Solvers(deluxe_ctx);CHKERRQ(ierr);
}
/* Compute data structures to solve sequential problems */
ierr = PCBDDCScalingSetUp_Deluxe_Private(pc);CHKERRQ(ierr);
/* diagonal scaling on interface dofs not contained in cc */
if (sub_schurs->is_vertices || sub_schurs->is_dir) {
PetscInt n_com,n_dir;
n_com = 0;
if (sub_schurs->is_vertices) {
ierr = ISGetLocalSize(sub_schurs->is_vertices,&n_com);CHKERRQ(ierr);
}
n_dir = 0;
if (sub_schurs->is_dir) {
ierr = ISGetLocalSize(sub_schurs->is_dir,&n_dir);CHKERRQ(ierr);
}
if (!deluxe_ctx->n_simple) {
deluxe_ctx->n_simple = n_dir + n_com;
ierr = PetscMalloc1(deluxe_ctx->n_simple,&deluxe_ctx->idx_simple_B);CHKERRQ(ierr);
if (sub_schurs->is_vertices) {
PetscInt nmap;
const PetscInt *idxs;
ierr = ISGetIndices(sub_schurs->is_vertices,&idxs);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApply(pcis->BtoNmap,IS_GTOLM_DROP,n_com,idxs,&nmap,deluxe_ctx->idx_simple_B);CHKERRQ(ierr);
if (nmap != n_com) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Error when mapping simply scaled dofs (is_vertices)! %D != %D",nmap,n_com);
ierr = ISRestoreIndices(sub_schurs->is_vertices,&idxs);CHKERRQ(ierr);
}
if (sub_schurs->is_dir) {
PetscInt nmap;
const PetscInt *idxs;
ierr = ISGetIndices(sub_schurs->is_dir,&idxs);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApply(pcis->BtoNmap,IS_GTOLM_DROP,n_dir,idxs,&nmap,deluxe_ctx->idx_simple_B+n_com);CHKERRQ(ierr);
if (nmap != n_dir) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Error when mapping simply scaled dofs (sub_schurs->is_dir)! %D != %D",nmap,n_dir);
ierr = ISRestoreIndices(sub_schurs->is_dir,&idxs);CHKERRQ(ierr);
}
ierr = PetscSortInt(deluxe_ctx->n_simple,deluxe_ctx->idx_simple_B);CHKERRQ(ierr);
} else {
if (deluxe_ctx->n_simple != n_dir + n_com) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Number of simply scaled dofs %D is different from the previous one computed %D",n_dir + n_com,deluxe_ctx->n_simple);
}
} else {
deluxe_ctx->n_simple = 0;
deluxe_ctx->idx_simple_B = 0;
}
PetscFunctionReturn(0);
}
/*@
ISEmbed - embed IS a into IS b by finding the locations in b that have the same indices as in a.
If c is the IS of these locations, we have a = b*c, regarded as a composition of the
corresponding ISLocalToGlobalMaps.
Not collective.
Input arguments:
+ a - IS to embed
. b - IS to embed into
- drop - flag indicating whether to drop a's indices that are not in b.
Output arguments:
. c - local embedding indices
Note:
If some of a's global indices are not among b's indices the embedding is impossible. The local indices of a
corresponding to these global indices are either mapped to -1 (if !drop) or are omitted (if drop). In the former
case the size of c is that same as that of a, in the latter case c's size may be smaller.
The resulting IS is sequential, since the index substition it encodes is purely local.
Level: advanced
.seealso ISLocalToGlobalMapping
@*/
PetscErrorCode ISEmbed(IS a, IS b, PetscBool drop, IS *c)
{
PetscErrorCode ierr;
ISLocalToGlobalMapping ltog;
ISGlobalToLocalMappingType gtoltype = IS_GTOLM_DROP;
PetscInt alen, clen, *cindices, *cindices2;
const PetscInt *aindices;
PetscFunctionBegin;
PetscValidHeaderSpecific(a, IS_CLASSID, 1);
PetscValidHeaderSpecific(b, IS_CLASSID, 2);
PetscValidPointer(c,4);
ierr = ISLocalToGlobalMappingCreateIS(b, <og);CHKERRQ(ierr);
ierr = ISGetLocalSize(a, &alen);CHKERRQ(ierr);
ierr = ISGetIndices(a, &aindices);CHKERRQ(ierr);
ierr = PetscMalloc1(alen, &cindices);CHKERRQ(ierr);
if (!drop) gtoltype = IS_GTOLM_MASK;
ierr = ISGlobalToLocalMappingApply(ltog,gtoltype,alen,aindices,&clen,cindices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(<og);CHKERRQ(ierr);
if (clen != alen) {
cindices2 = cindices;
ierr = PetscMalloc1(clen, &cindices);CHKERRQ(ierr);
ierr = PetscMemcpy(cindices,cindices2,clen*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscFree(cindices2);CHKERRQ(ierr);
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,clen,cindices,PETSC_OWN_POINTER,c);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt i,n = 4,indices[] = {0,3,9,12},m = 2,input[] = {0,2};
PetscInt output[2],inglobals[13],outlocals[13];
ISLocalToGlobalMapping mapping;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/*
Create a local to global mapping. Each processor independently
creates a mapping
*/
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,n,indices,PETSC_COPY_VALUES,&mapping);CHKERRQ(ierr);
/*
Map a set of local indices to their global values
*/
ierr = ISLocalToGlobalMappingApply(mapping,m,input,output);CHKERRQ(ierr);
ierr = PetscIntView(m,output,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Map some global indices to local, retaining the ones without a local index by -1
*/
for (i=0; i<13; i++) {
inglobals[i] = i;
}
ierr = ISGlobalToLocalMappingApply(mapping,IS_GTOLM_MASK,13,inglobals,PETSC_NULL,outlocals);CHKERRQ(ierr);
ierr = PetscIntView(13,outlocals,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Map some global indices to local, dropping the ones without a local index.
*/
ierr = ISGlobalToLocalMappingApply(mapping,IS_GTOLM_DROP,13,inglobals,&m,outlocals);CHKERRQ(ierr);
ierr = PetscIntView(m,outlocals,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Free the space used by the local to global mapping
*/
ierr = ISLocalToGlobalMappingDestroy(&mapping);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*
Rozdeleni site mezi procesory takto:
1) vrcholy jsou nakopirovany na vsechny procesory
2) bunky jsou rozdeleny rovnmerne dle petsc
3) steny jsou rozdeleny tak, ze cpu(f) = cpu(cell(f.owner))
Indexy bunek u sten jsou zmeneny na lokalni cislovani
*/
Grid DecomposeGrid(const Grid& g) {
PetscInt istart, iend;
Vec tmp;
VecCreateMPI(PETSC_COMM_WORLD, PETSC_DECIDE, g.cells().size(), &tmp);
VecGetOwnershipRange(tmp, &istart, &iend);
VecDestroy(&tmp);
Grid::CellList cells(0);
for (auto c: g.cells())
if (istart <= c.id && c.id < iend) cells.push_back(c);
Grid::FaceList faces;
for (auto f: g.internalFaces())
if (istart <= f.owner && f.owner < iend) faces.push_back(f);
for (auto bnd: g.boundaryPatches())
for (auto f: bnd.second)
if (istart <= f.owner && f.owner < iend) faces.push_back(f);
Decomposition dist(istart, iend, faces);
for (auto& f: faces) {
int glob[1], loc[1], one;
glob[0] = f.owner;
ISGlobalToLocalMappingApply(dist.locToGlobMap, IS_GTOLM_MASK, 1, glob, &one, loc);
f.owner = loc[0];
if (f.neighbour >=0) {
glob[0] = f.neighbour;
ISGlobalToLocalMappingApply(dist.locToGlobMap, IS_GTOLM_MASK, 1, glob, &one, loc);
f.neighbour = loc[0];
}
}
Grid dg(g.points(), faces, cells);
dg.userData() = dist;
return dg;
}
PetscErrorCode MatZeroRows_IS(Mat A,PetscInt n,const PetscInt rows[],PetscScalar diag,Vec x,Vec b)
{
PetscInt n_l = 0, *rows_l = NULL;
PetscErrorCode ierr;
PetscFunctionBegin;
if (x && b) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_SUP,"No support");
if (n) {
ierr = PetscMalloc1(n,&rows_l);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApply(A->rmap->mapping,IS_GTOLM_DROP,n,rows,&n_l,rows_l);CHKERRQ(ierr);
}
ierr = MatZeroRowsLocal(A,n_l,rows_l,diag,x,b);CHKERRQ(ierr);
ierr = PetscFree(rows_l);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Distribute cones
- Partitioning: input partition point map and naive sf, output sf with inverse of map, distribute points
- Distribute section: input current sf, communicate sizes and offsets, output local section and offsets (only use for new sf)
- Create SF for values: input current sf and offsets, output new sf
- Distribute values: input new sf, communicate values
*/
PetscErrorCode DistributeMesh(DM dm, AppCtx *user, PetscSF *pointSF, DM *parallelDM)
{
MPI_Comm comm = ((PetscObject) dm)->comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS cellPart, part;
PetscSection cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF;
ISLocalToGlobalMapping renumbering;
PetscSF coneSF;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets, newConesSize;
PetscInt *cones, *newCones;
PetscMPIInt numProcs, rank, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMMeshGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = DMMeshCreatePartition(dm, &cellPartSection, &cellPart, height);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) {
numRemoteRanks = numProcs;
} else {
numRemoteRanks = 0;
}
ierr = PetscMalloc(numRemoteRanks * sizeof(PetscSFNode), &remoteRanks);CHKERRQ(ierr);
for(p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, PETSC_NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFView(partSF, PETSC_NULL);CHKERRQ(ierr);
/* Close the partition over the mesh */
ierr = DMMeshCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMMeshCreate(comm, parallelDM);CHKERRQ(ierr);
ierr = DMMeshSetDimension(*parallelDM, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *parallelDM, "Parallel Mesh");CHKERRQ(ierr);
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, pointSF);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFView(*pointSF, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, PETSC_NULL);CHKERRQ(ierr);
/* Cleanup */
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMMeshGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMMeshGetConeSection(*parallelDM, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(*pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMMeshSetUp(*parallelDM);CHKERRQ(ierr);
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(*pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMMeshGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMMeshGetCones(*parallelDM, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newConesSize, newCones, PETSC_NULL, newCones);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
/* Create supports and stratify sieve */
ierr = DMMeshSymmetrize(*parallelDM);CHKERRQ(ierr);
ierr = DMMeshStratify(*parallelDM);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
