PetscErrorCode DistributeCoordinates(DM dm, PetscSF pointSF, DM parallelDM)
{
PetscSF coordSF;
PetscSection originalCoordSection, newCoordSection;
Vec coordinates, newCoordinates;
PetscScalar *coords, *newCoords;
PetscInt *remoteOffsets, coordSize;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMMeshGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMMeshGetCoordinateSection(parallelDM, &newCoordSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalCoordSection, &remoteOffsets, newCoordSection);CHKERRQ(ierr);
ierr = DMMeshGetCoordinateVec(dm, &coordinates);CHKERRQ(ierr);
ierr = DMMeshGetCoordinateVec(parallelDM, &newCoordinates);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetFromOptions(newCoordinates);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(pointSF, originalCoordSection, remoteOffsets, newCoordSection, &coordSF);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coordSF, MPIU_SCALAR, coords, newCoords);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coordSF, MPIU_SCALAR, coords, newCoords);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coordSF);CHKERRQ(ierr);
ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexDistributeField - Distribute field data to match a given PetscSF, usually the SF from mesh distribution
Collective on DM
Input Parameters:
+ dm - The DMPlex object
. pointSF - The PetscSF describing the communication pattern
. originalSection - The PetscSection for existing data layout
- originalVec - The existing data
Output Parameters:
+ newSection - The PetscSF describing the new data layout
- newVec - The new data
Level: developer
.seealso: DMPlexDistribute(), DMPlexDistributeData()
@*/
PetscErrorCode DMPlexDistributeField(DM dm, PetscSF pointSF, PetscSection originalSection, Vec originalVec, PetscSection newSection, Vec newVec)
{
PetscSF fieldSF;
PetscInt *remoteOffsets, fieldSize;
PetscScalar *originalValues, *newValues;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogEventBegin(DMPLEX_DistributeField,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalSection, &remoteOffsets, newSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newSection, &fieldSize);CHKERRQ(ierr);
ierr = VecSetSizes(newVec, fieldSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(newVec,dm->vectype);CHKERRQ(ierr);
ierr = VecGetArray(originalVec, &originalValues);CHKERRQ(ierr);
ierr = VecGetArray(newVec, &newValues);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(pointSF, originalSection, remoteOffsets, newSection, &fieldSF);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(fieldSF, MPIU_SCALAR, originalValues, newValues);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(fieldSF, MPIU_SCALAR, originalValues, newValues);CHKERRQ(ierr);
ierr = PetscSFDestroy(&fieldSF);CHKERRQ(ierr);
ierr = VecRestoreArray(newVec, &newValues);CHKERRQ(ierr);
ierr = VecRestoreArray(originalVec, &originalValues);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeField,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelDistribute_Internal(DMLabel label, PetscSF sf, PetscSection *leafSection, PetscInt **leafStrata)
{
MPI_Comm comm;
PetscInt s, l, nroots, nleaves, dof, offset, size;
PetscInt *remoteOffsets, *rootStrata, *rootIdx;
PetscSection rootSection;
PetscSF labelSF;
PetscErrorCode ierr;
PetscFunctionBegin;
if (label) {ierr = DMLabelMakeAllValid_Private(label);CHKERRQ(ierr);}
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
/* Build a section of stratum values per point, generate the according SF
and distribute point-wise stratum values to leaves. */
ierr = PetscSFGetGraph(sf, &nroots, &nleaves, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSection, 0, nroots);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
for (l = 0; l < label->stratumSizes[s]; l++) {
ierr = PetscSectionGetDof(rootSection, label->points[s][l], &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSection, label->points[s][l], dof+1);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSection);CHKERRQ(ierr);
/* Create a point-wise array of stratum values */
ierr = PetscSectionGetStorageSize(rootSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &rootStrata);CHKERRQ(ierr);
ierr = PetscCalloc1(nroots, &rootIdx);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
for (l = 0; l < label->stratumSizes[s]; l++) {
const PetscInt p = label->points[s][l];
ierr = PetscSectionGetOffset(rootSection, p, &offset);CHKERRQ(ierr);
rootStrata[offset+rootIdx[p]++] = label->stratumValues[s];
}
}
}
/* Build SF that maps label points to remote processes */
ierr = PetscSectionCreate(comm, leafSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sf, rootSection, &remoteOffsets, *leafSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, rootSection, remoteOffsets, *leafSection, &labelSF);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
/* Send the strata for each point over the derived SF */
ierr = PetscSectionGetStorageSize(*leafSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(labelSF, MPIU_INT, rootStrata, *leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(labelSF, MPIU_INT, rootStrata, *leafStrata);CHKERRQ(ierr);
/* Clean up */
ierr = PetscFree(rootStrata);CHKERRQ(ierr);
ierr = PetscFree(rootIdx);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSection);CHKERRQ(ierr);
ierr = PetscSFDestroy(&labelSF);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PetscSFScatterEnd - ends pointwise scatter operation that was started with PetscSFScatterBegin()
Collective
Input Arguments:
+ sf - star forest
. unit - data type
- multirootdata - root buffer to send to each leaf, one unit of data per leaf
Output Argument:
. leafdata - leaf data to be update with personal data from each respective root
Level: intermediate
.seealso: PetscSFComputeDegreeEnd(), PetscSFScatterEnd()
@*/
PetscErrorCode PetscSFScatterEnd(PetscSF sf,MPI_Datatype unit,const void *multirootdata,void *leafdata)
{
PetscErrorCode ierr;
PetscSF multi;
PetscFunctionBegin;
PetscValidHeaderSpecific(sf,PETSCSF_CLASSID,1);
PetscSFCheckGraphSet(sf,1);
ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
ierr = PetscSFGetMultiSF(sf,&multi);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(multi,unit,multirootdata,leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscSF sf;
PetscInt bs,i,nroots,*rootdata,nleaves,*leafdata,nlocalcols;
const PetscInt *gidxs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) {
SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
}
ierr = MatGetLocalSize(B,&nroots,NULL);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&nleaves,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscCalloc2(nroots,&rootdata,nleaves,&leafdata);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)B),&sf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,B->rmap,nleaves,NULL,PETSC_COPY_VALUES,gidxs);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
if (!d_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += d_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += o_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves;i++) {
leafdata[i] = PetscMin(leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree2(rootdata,leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexPreallocateOperator_2(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix)
{
PetscInt *tmpClosure,*tmpAdj,*visits;
PetscInt c,cStart,cEnd,pStart,pEnd;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
maxClosureSize = 2*PetscMax(PetscPowInt(mesh->maxConeSize,depth+1),PetscPowInt(mesh->maxSupportSize,depth+1));
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
npoints = pEnd - pStart;
ierr = PetscMalloc3(maxClosureSize,&tmpClosure,npoints,&lvisits,npoints,&visits);CHKERRQ(ierr);
ierr = PetscMemzero(lvisits,(pEnd-pStart)*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemzero(visits,(pEnd-pStart)*sizeof(PetscInt));CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
for (c=cStart; c<cEnd; c++) {
PetscInt *support = tmpClosure;
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_FALSE, &supportSize, (PetscInt**)&support);CHKERRQ(ierr);
for (p=0; p<supportSize; p++) lvisits[support[p]]++;
}
ierr = PetscSFReduceBegin(sf,MPIU_INT,lvisits,visits,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd (sf,MPIU_INT,lvisits,visits,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,visits,lvisits);CHKERRQ(ierr);
ierr = PetscSFBcastEnd (sf,MPIU_INT,visits,lvisits);CHKERRQ(ierr);
ierr = PetscSFGetRanks();CHKERRQ(ierr);
ierr = PetscMalloc2(maxClosureSize*maxClosureSize,&cellmat,npoints,&owner);CHKERRQ(ierr);
for (c=cStart; c<cEnd; c++) {
ierr = PetscMemzero(cellmat,maxClosureSize*maxClosureSize*sizeof(PetscInt));CHKERRQ(ierr);
/*
Depth-first walk of transitive closure.
At each leaf frame f of transitive closure that we see, add 1/visits[f] to each pair (p,q) not marked as done in cellmat.
This contribution is added to dnz if owning ranks of p and q match, to onz otherwise.
*/
}
ierr = PetscSFReduceBegin(sf,MPIU_INT,ldnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd (sf,MPIU_INT,lonz,onz,MPI_SUM);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
PetscSFCompose - Compose a new PetscSF equivalent to action to PetscSFs
Input Parameters:
+ sfA - The first PetscSF
- sfB - The second PetscSF
Output Parameters:
. sfBA - equvalent PetscSF for applying A then B
Level: developer
.seealso: PetscSF, PetscSFGetGraph(), PetscSFSetGraph()
@*/
PetscErrorCode PetscSFCompose(PetscSF sfA, PetscSF sfB, PetscSF *sfBA)
{
MPI_Comm comm;
const PetscSFNode *remotePointsA, *remotePointsB;
PetscSFNode *remotePointsBA;
const PetscInt *localPointsA, *localPointsB;
PetscInt numRootsA, numLeavesA, numRootsB, numLeavesB;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(sfA, PETSCSF_CLASSID, 1);
PetscValidHeaderSpecific(sfB, PETSCSF_CLASSID, 1);
ierr = PetscObjectGetComm((PetscObject) sfA, &comm);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sfA, &numRootsA, &numLeavesA, &localPointsA, &remotePointsA);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sfB, &numRootsB, &numLeavesB, &localPointsB, &remotePointsB);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeavesB, &remotePointsBA);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfB, MPIU_2INT, remotePointsA, remotePointsBA);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfB, MPIU_2INT, remotePointsA, remotePointsBA);CHKERRQ(ierr);
ierr = PetscSFCreate(comm, sfBA);CHKERRQ(ierr);
ierr = PetscSFSetGraph(*sfBA, numRootsA, numLeavesB, localPointsB, PETSC_COPY_VALUES, remotePointsBA, PETSC_OWN_POINTER);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscInt bs,i,nlocalcols;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
ierr = MatISComputeSF_Private(B);CHKERRQ(ierr);
}
if (!d_nnz) {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nz;
} else {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nz;
} else {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,NULL,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves;i++) {
matis->sf_leafdata[i] = PetscMin(matis->sf_leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves/bs;i++) {
matis->sf_leafdata[i] = matis->sf_leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves/bs;i++) {
matis->sf_leafdata[i] = matis->sf_leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMPlexDistributeData - Distribute field data to match a given PetscSF, usually the SF from mesh distribution
Collective on DM
Input Parameters:
+ dm - The DMPlex object
. pointSF - The PetscSF describing the communication pattern
. originalSection - The PetscSection for existing data layout
. datatype - The type of data
- originalData - The existing data
Output Parameters:
+ newSection - The PetscSF describing the new data layout
- newData - The new data
Level: developer
.seealso: DMPlexDistribute(), DMPlexDistributeField()
@*/
PetscErrorCode DMPlexDistributeData(DM dm, PetscSF pointSF, PetscSection originalSection, MPI_Datatype datatype, void *originalData, PetscSection newSection, void **newData)
{
PetscSF fieldSF;
PetscInt *remoteOffsets, fieldSize;
PetscMPIInt dataSize;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLogEventBegin(DMPLEX_DistributeData,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalSection, &remoteOffsets, newSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newSection, &fieldSize);CHKERRQ(ierr);
ierr = MPI_Type_size(datatype, &dataSize);CHKERRQ(ierr);
ierr = PetscMalloc(fieldSize * dataSize, newData);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(pointSF, originalSection, remoteOffsets, newSection, &fieldSF);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(fieldSF, datatype, originalData, *newData);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(fieldSF, datatype, originalData, *newData);CHKERRQ(ierr);
ierr = PetscSFDestroy(&fieldSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeData,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PetscSFCreateEmbeddedSF - removes edges from all but the selected roots, does not remap indices
Collective
Input Arguments:
+ sf - original star forest
. nroots - number of roots to select on this process
- selected - selected roots on this process
Output Arguments:
. newsf - new star forest
Level: advanced
Note:
To use the new PetscSF, it may be necessary to know the indices of the leaves that are still participating. This can
be done by calling PetscSFGetGraph().
.seealso: PetscSFSetGraph(), PetscSFGetGraph()
@*/
PetscErrorCode PetscSFCreateEmbeddedSF(PetscSF sf,PetscInt nroots,const PetscInt *selected,PetscSF *newsf)
{
PetscInt *rootdata, *leafdata, *ilocal;
PetscSFNode *iremote;
PetscInt leafsize = 0, nleaves = 0, n, i;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(sf,PETSCSF_CLASSID,1);
if (nroots) PetscValidPointer(selected,3);
PetscValidPointer(newsf,4);
if (sf->mine) for (i = 0; i < sf->nleaves; ++i) {leafsize = PetscMax(leafsize, sf->mine[i]+1);}
else leafsize = sf->nleaves;
ierr = PetscCalloc2(sf->nroots,&rootdata,leafsize,&leafdata);CHKERRQ(ierr);
for (i=0; i<nroots; ++i) rootdata[selected[i]] = 1;
ierr = PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
for (i = 0; i < leafsize; ++i) nleaves += leafdata[i];
ierr = PetscMalloc1(nleaves,&ilocal);CHKERRQ(ierr);
ierr = PetscMalloc1(nleaves,&iremote);CHKERRQ(ierr);
for (i = 0, n = 0; i < sf->nleaves; ++i) {
const PetscInt lidx = sf->mine ? sf->mine[i] : i;
if (leafdata[lidx]) {
ilocal[n] = lidx;
iremote[n].rank = sf->remote[i].rank;
iremote[n].index = sf->remote[i].index;
++n;
}
}
if (n != nleaves) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "There is a size mismatch in the SF embedding, %d != %d", n, nleaves);
ierr = PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,newsf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(*newsf,sf->nroots,nleaves,ilocal,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscFree2(rootdata,leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode maxIndSetAgg(IS perm,Mat Gmat,PetscBool strict_aggs,PetscCoarsenData **a_locals_llist)
{
PetscErrorCode ierr;
Mat_SeqAIJ *matA,*matB=NULL;
Mat_MPIAIJ *mpimat=NULL;
MPI_Comm comm;
PetscInt num_fine_ghosts,kk,n,ix,j,*idx,*ii,iter,Iend,my0,nremoved,gid,lid,cpid,lidj,sgid,t1,t2,slid,nDone,nselected=0,state,statej;
PetscInt *cpcol_gid,*cpcol_state,*lid_cprowID,*lid_gid,*cpcol_sel_gid,*icpcol_gid,*lid_state,*lid_parent_gid=NULL;
PetscBool *lid_removed;
PetscBool isMPI,isAIJ,isOK;
const PetscInt *perm_ix;
const PetscInt nloc = Gmat->rmap->n;
PetscCoarsenData *agg_lists;
PetscLayout layout;
PetscSF sf;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Gmat,&comm);
CHKERRQ(ierr);
/* get submatrices */
ierr = PetscObjectTypeCompare((PetscObject)Gmat,MATMPIAIJ,&isMPI);
CHKERRQ(ierr);
if (isMPI) {
mpimat = (Mat_MPIAIJ*)Gmat->data;
matA = (Mat_SeqAIJ*)mpimat->A->data;
matB = (Mat_SeqAIJ*)mpimat->B->data;
/* force compressed storage of B */
ierr = MatCheckCompressedRow(mpimat->B,matB->nonzerorowcnt,&matB->compressedrow,matB->i,Gmat->rmap->n,-1.0);
CHKERRQ(ierr);
} else {
ierr = PetscObjectTypeCompare((PetscObject)Gmat,MATSEQAIJ,&isAIJ);
CHKERRQ(ierr);
matA = (Mat_SeqAIJ*)Gmat->data;
}
ierr = MatGetOwnershipRange(Gmat,&my0,&Iend);
CHKERRQ(ierr);
ierr = PetscMalloc1(nloc,&lid_gid);
CHKERRQ(ierr); /* explicit array needed */
if (mpimat) {
for (kk=0,gid=my0; kk<nloc; kk++,gid++) {
lid_gid[kk] = gid;
}
ierr = VecGetLocalSize(mpimat->lvec, &num_fine_ghosts);
CHKERRQ(ierr);
ierr = PetscMalloc1(num_fine_ghosts,&cpcol_gid);
CHKERRQ(ierr);
ierr = PetscMalloc1(num_fine_ghosts,&cpcol_state);
CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)Gmat),&sf);
CHKERRQ(ierr);
ierr = MatGetLayouts(Gmat,&layout,NULL);
CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,num_fine_ghosts,NULL,PETSC_COPY_VALUES,mpimat->garray);
CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,lid_gid,cpcol_gid);
CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,lid_gid,cpcol_gid);
CHKERRQ(ierr);
for (kk=0; kk<num_fine_ghosts; kk++) {
cpcol_state[kk]=MIS_NOT_DONE;
}
} else num_fine_ghosts = 0;
ierr = PetscMalloc1(nloc, &lid_cprowID);
CHKERRQ(ierr);
ierr = PetscMalloc1(nloc, &lid_removed);
CHKERRQ(ierr); /* explicit array needed */
if (strict_aggs) {
ierr = PetscMalloc1(nloc,&lid_parent_gid);
CHKERRQ(ierr);
}
ierr = PetscMalloc1(nloc,&lid_state);
CHKERRQ(ierr);
/* has ghost nodes for !strict and uses local indexing (yuck) */
ierr = PetscCDCreate(strict_aggs ? nloc : num_fine_ghosts+nloc, &agg_lists);
CHKERRQ(ierr);
if (a_locals_llist) *a_locals_llist = agg_lists;
/* need an inverse map - locals */
for (kk=0; kk<nloc; kk++) {
lid_cprowID[kk] = -1;
lid_removed[kk] = PETSC_FALSE;
if (strict_aggs) {
lid_parent_gid[kk] = -1.0;
}
lid_state[kk] = MIS_NOT_DONE;
}
/* set index into cmpressed row 'lid_cprowID' */
if (matB) {
for (ix=0; ix<matB->compressedrow.nrows; ix++) {
lid = matB->compressedrow.rindex[ix];
lid_cprowID[lid] = ix;
}
}
/* MIS */
iter = nremoved = nDone = 0;
ierr = ISGetIndices(perm, &perm_ix);
CHKERRQ(ierr);
while (nDone < nloc || PETSC_TRUE) { /* asyncronous not implemented */
iter++;
/* check all vertices */
for (kk=0; kk<nloc; kk++) {
lid = perm_ix[kk];
state = lid_state[lid];
if (lid_removed[lid]) continue;
if (state == MIS_NOT_DONE) {
/* parallel test, delete if selected ghost */
isOK = PETSC_TRUE;
if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i;
n = ii[ix+1] - ii[ix];
idx = matB->j + ii[ix];
for (j=0; j<n; j++) {
cpid = idx[j]; /* compressed row ID in B mat */
gid = cpcol_gid[cpid];
statej = cpcol_state[cpid];
if (statej == MIS_NOT_DONE && gid >= Iend) { /* should be (pe>rank), use gid as pe proxy */
isOK = PETSC_FALSE; /* can not delete */
break;
}
}
} /* parallel test */
if (isOK) { /* select or remove this vertex */
nDone++;
/* check for singleton */
ii = matA->i;
n = ii[lid+1] - ii[lid];
if (n < 2) {
/* if I have any ghost adj then not a sing */
ix = lid_cprowID[lid];
if (ix==-1 || (matB->compressedrow.i[ix+1]-matB->compressedrow.i[ix])==0) {
nremoved++;
lid_removed[lid] = PETSC_TRUE;
/* should select this because it is technically in the MIS but lets not */
continue; /* one local adj (me) and no ghost - singleton */
}
}
/* SELECTED state encoded with global index */
lid_state[lid] = lid+my0; /* needed???? */
nselected++;
if (strict_aggs) {
ierr = PetscCDAppendID(agg_lists, lid, lid+my0);
CHKERRQ(ierr);
} else {
ierr = PetscCDAppendID(agg_lists, lid, lid);
CHKERRQ(ierr);
}
/* delete local adj */
idx = matA->j + ii[lid];
for (j=0; j<n; j++) {
lidj = idx[j];
statej = lid_state[lidj];
if (statej == MIS_NOT_DONE) {
nDone++;
if (strict_aggs) {
ierr = PetscCDAppendID(agg_lists, lid, lidj+my0);
CHKERRQ(ierr);
} else {
ierr = PetscCDAppendID(agg_lists, lid, lidj);
CHKERRQ(ierr);
}
lid_state[lidj] = MIS_DELETED; /* delete this */
}
}
/* delete ghost adj of lid - deleted ghost done later for strict_aggs */
if (!strict_aggs) {
if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i;
n = ii[ix+1] - ii[ix];
idx = matB->j + ii[ix];
for (j=0; j<n; j++) {
cpid = idx[j]; /* compressed row ID in B mat */
statej = cpcol_state[cpid];
if (statej == MIS_NOT_DONE) {
ierr = PetscCDAppendID(agg_lists, lid, nloc+cpid);
CHKERRQ(ierr);
}
}
}
}
} /* selected */
} /* not done vertex */
} /* vertex loop */
/* update ghost states and count todos */
if (mpimat) {
/* scatter states, check for done */
ierr = PetscSFBcastBegin(sf,MPIU_INT,lid_state,cpcol_state);
CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,lid_state,cpcol_state);
CHKERRQ(ierr);
ii = matB->compressedrow.i;
for (ix=0; ix<matB->compressedrow.nrows; ix++) {
lid = matB->compressedrow.rindex[ix]; /* local boundary node */
state = lid_state[lid];
if (state == MIS_NOT_DONE) {
/* look at ghosts */
n = ii[ix+1] - ii[ix];
idx = matB->j + ii[ix];
for (j=0; j<n; j++) {
cpid = idx[j]; /* compressed row ID in B mat */
statej = cpcol_state[cpid];
if (MIS_IS_SELECTED(statej)) { /* lid is now deleted, do it */
nDone++;
lid_state[lid] = MIS_DELETED; /* delete this */
if (!strict_aggs) {
lidj = nloc + cpid;
ierr = PetscCDAppendID(agg_lists, lidj, lid);
CHKERRQ(ierr);
} else {
sgid = cpcol_gid[cpid];
lid_parent_gid[lid] = sgid; /* keep track of proc that I belong to */
}
break;
}
}
}
}
/* all done? */
t1 = nloc - nDone;
ierr = MPI_Allreduce(&t1, &t2, 1, MPIU_INT, MPI_SUM, comm);
CHKERRQ(ierr); /* synchronous version */
if (t2 == 0) break;
} else break; /* all done */
} /* outer parallel MIS loop */
ierr = ISRestoreIndices(perm,&perm_ix);
CHKERRQ(ierr);
ierr = PetscInfo3(Gmat,"\t removed %D of %D vertices. %D selected.\n",nremoved,nloc,nselected);
CHKERRQ(ierr);
/* tell adj who my lid_parent_gid vertices belong to - fill in agg_lists selected ghost lists */
if (strict_aggs && matB) {
/* need to copy this to free buffer -- should do this globaly */
ierr = PetscMalloc1(num_fine_ghosts, &cpcol_sel_gid);
CHKERRQ(ierr);
ierr = PetscMalloc1(num_fine_ghosts, &icpcol_gid);
CHKERRQ(ierr);
for (cpid=0; cpid<num_fine_ghosts; cpid++) icpcol_gid[cpid] = cpcol_gid[cpid];
/* get proc of deleted ghost */
ierr = PetscSFBcastBegin(sf,MPIU_INT,lid_parent_gid,cpcol_sel_gid);
CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,lid_parent_gid,cpcol_sel_gid);
CHKERRQ(ierr);
for (cpid=0; cpid<num_fine_ghosts; cpid++) {
sgid = cpcol_sel_gid[cpid];
gid = icpcol_gid[cpid];
if (sgid >= my0 && sgid < Iend) { /* I own this deleted */
slid = sgid - my0;
ierr = PetscCDAppendID(agg_lists, slid, gid);
CHKERRQ(ierr);
}
}
ierr = PetscFree(icpcol_gid);
CHKERRQ(ierr);
ierr = PetscFree(cpcol_sel_gid);
CHKERRQ(ierr);
}
if (mpimat) {
ierr = PetscSFDestroy(&sf);
CHKERRQ(ierr);
ierr = PetscFree(cpcol_gid);
CHKERRQ(ierr);
ierr = PetscFree(cpcol_state);
CHKERRQ(ierr);
}
ierr = PetscFree(lid_cprowID);
CHKERRQ(ierr);
ierr = PetscFree(lid_gid);
CHKERRQ(ierr);
ierr = PetscFree(lid_removed);
CHKERRQ(ierr);
if (strict_aggs) {
ierr = PetscFree(lid_parent_gid);
CHKERRQ(ierr);
}
ierr = PetscFree(lid_state);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
PetscInt ierr;
PetscSF sf;
Vec A,Aout;
PetscScalar *bufA;
PetscScalar *bufAout;
PetscMPIInt rank, size;
PetscInt nroots, nleaves;
PetscInt i;
PetscInt *ilocal;
PetscSFNode *iremote;
PetscBool test_dupped_type;
MPI_Datatype contig;
ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Only coded for one MPI process");
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,"","PetscSF type freeing options","none");CHKERRQ(ierr);
test_dupped_type = PETSC_FALSE;
ierr = PetscOptionsBool("-test_dupped_type", "Test dupped input type","",test_dupped_type,&test_dupped_type,NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
nleaves = 1;
nroots = 1;
ierr = PetscMalloc1(nleaves,&ilocal);CHKERRQ(ierr);
for (i = 0; i<nleaves; i++) {
ilocal[i] = i;
}
ierr = PetscMalloc1(nleaves,&iremote);CHKERRQ(ierr);
iremote[0].rank = 0;
iremote[0].index = 0;
ierr = PetscSFSetGraph(sf,nroots,nleaves,ilocal,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
ierr = PetscSFView(sf,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = VecSetSizes(A,4,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetFromOptions(A);CHKERRQ(ierr);
ierr = VecSetUp(A);CHKERRQ(ierr);
ierr = VecDuplicate(A,&Aout);CHKERRQ(ierr);
ierr = VecGetArray(A,&bufA);CHKERRQ(ierr);
for (i=0; i<4; i++) {
bufA[i] = (PetscScalar)i;
}
ierr = VecRestoreArray(A,&bufA);CHKERRQ(ierr);
ierr = VecGetArrayRead(A,(const PetscScalar**)&bufA);CHKERRQ(ierr);
ierr = VecGetArray(Aout,&bufAout);CHKERRQ(ierr);
ierr = MPI_Type_contiguous(4, MPIU_SCALAR, &contig);CHKERRQ(ierr);
ierr = MPI_Type_commit(&contig);CHKERRQ(ierr);
if (test_dupped_type) {
MPI_Datatype tmp;
ierr = MPI_Type_dup(contig, &tmp);CHKERRQ(ierr);
ierr = MPI_Type_free(&contig);CHKERRQ(ierr);
contig = tmp;
}
for (i=0;i<10000;i++) {
ierr = PetscSFBcastBegin(sf,contig,bufA,bufAout);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,contig,bufA,bufAout);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(A,(const PetscScalar**)&bufA);CHKERRQ(ierr);
ierr = VecRestoreArray(Aout,&bufAout);CHKERRQ(ierr);
ierr = VecView(Aout,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&A);CHKERRQ(ierr);
ierr = VecDestroy(&Aout);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = MPI_Type_free(&contig);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@
ISBuildTwoSided - Takes an IS that describes where we will go. Generates an IS that contains new numbers from remote or local
on the IS.
Collective on IS
Input Parameters
. to - an IS describes where we will go. Negative target rank will be ignored
. toindx - an IS describes what indices should send. NULL means sending natural numbering
Output Parameter:
. rows - contains new numbers from remote or local
Level: advanced
.seealso: MatPartitioningCreate(), ISPartitioningToNumbering(), ISPartitioningCount()
@*/
PetscErrorCode ISBuildTwoSided(IS ito,IS toindx, IS *rows)
{
const PetscInt *ito_indices,*toindx_indices;
PetscInt *send_indices,rstart,*recv_indices,nrecvs,nsends;
PetscInt *tosizes,*fromsizes,i,j,*tosizes_tmp,*tooffsets_tmp,ito_ln;
PetscMPIInt *toranks,*fromranks,size,target_rank,*fromperm_newtoold,nto,nfrom;
PetscLayout isrmap;
MPI_Comm comm;
PetscSF sf;
PetscSFNode *iremote;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)ito,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = ISGetLocalSize(ito,&ito_ln);CHKERRQ(ierr);
/* why we do not have ISGetLayout? */
isrmap = ito->map;
ierr = PetscLayoutGetRange(isrmap,&rstart,NULL);CHKERRQ(ierr);
ierr = ISGetIndices(ito,&ito_indices);CHKERRQ(ierr);
ierr = PetscCalloc2(size,&tosizes_tmp,size+1,&tooffsets_tmp);CHKERRQ(ierr);
for(i=0; i<ito_ln; i++){
if(ito_indices[i]<0) continue;
#if defined(PETSC_USE_DEBUG)
if(ito_indices[i]>=size) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"target rank %d is larger than communicator size %d ",ito_indices[i],size);
#endif
tosizes_tmp[ito_indices[i]]++;
}
nto = 0;
for(i=0; i<size; i++){
tooffsets_tmp[i+1] = tooffsets_tmp[i]+tosizes_tmp[i];
if(tosizes_tmp[i]>0) nto++;
}
ierr = PetscCalloc2(nto,&toranks,2*nto,&tosizes);CHKERRQ(ierr);
nto = 0;
for(i=0; i<size; i++){
if(tosizes_tmp[i]>0){
toranks[nto] = i;
tosizes[2*nto] = tosizes_tmp[i];/* size */
tosizes[2*nto+1] = tooffsets_tmp[i];/* offset */
nto++;
}
}
nsends = tooffsets_tmp[size];
ierr = PetscCalloc1(nsends,&send_indices);CHKERRQ(ierr);
if(toindx){
ierr = ISGetIndices(toindx,&toindx_indices);CHKERRQ(ierr);
}
for(i=0; i<ito_ln; i++){
if(ito_indices[i]<0) continue;
target_rank = ito_indices[i];
send_indices[tooffsets_tmp[target_rank]] = toindx? toindx_indices[i]:(i+rstart);
tooffsets_tmp[target_rank]++;
}
if(toindx){
ierr = ISRestoreIndices(toindx,&toindx_indices);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(ito,&ito_indices);CHKERRQ(ierr);
ierr = PetscFree2(tosizes_tmp,tooffsets_tmp);CHKERRQ(ierr);
ierr = PetscCommBuildTwoSided(comm,2,MPIU_INT,nto,toranks,tosizes,&nfrom,&fromranks,&fromsizes);CHKERRQ(ierr);
ierr = PetscFree2(toranks,tosizes);CHKERRQ(ierr);
ierr = PetscCalloc1(nfrom,&fromperm_newtoold);CHKERRQ(ierr);
for(i=0; i<nfrom; i++){
fromperm_newtoold[i] = i;
}
ierr = PetscSortMPIIntWithArray(nfrom,fromranks,fromperm_newtoold);CHKERRQ(ierr);
nrecvs = 0;
for(i=0; i<nfrom; i++){
nrecvs += fromsizes[i*2];
}
ierr = PetscCalloc1(nrecvs,&recv_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(nrecvs,&iremote);CHKERRQ(ierr);
nrecvs = 0;
for(i=0; i<nfrom; i++){
for(j=0; j<fromsizes[2*fromperm_newtoold[i]]; j++){
iremote[nrecvs].rank = fromranks[i];
iremote[nrecvs++].index = fromsizes[2*fromperm_newtoold[i]+1]+j;
}
}
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nsends,nrecvs,NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
/* how to put a prefix ? */
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,send_indices,recv_indices);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,send_indices,recv_indices);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree(fromranks);CHKERRQ(ierr);
ierr = PetscFree(fromsizes);CHKERRQ(ierr);
ierr = PetscFree(fromperm_newtoold);CHKERRQ(ierr);
ierr = PetscFree(send_indices);CHKERRQ(ierr);
if(rows){
ierr = PetscSortInt(nrecvs,recv_indices);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, nrecvs,recv_indices,PETSC_OWN_POINTER,rows);CHKERRQ(ierr);
}else{
ierr = PetscFree(recv_indices);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode MCJPGreatestWeight_Private(MatColoring mc,const PetscReal *weights,PetscReal *maxweights)
{
MC_JP *jp = (MC_JP*)mc->data;
PetscErrorCode ierr;
Mat G=mc->mat,dG,oG;
PetscBool isSeq,isMPI;
Mat_MPIAIJ *aij;
Mat_SeqAIJ *daij,*oaij;
PetscInt *di,*oi,*dj,*oj;
PetscSF sf=jp->sf;
PetscLayout layout;
PetscInt dn,on;
PetscInt i,j,l;
PetscReal *dwts=jp->dwts,*owts=jp->owts;
PetscInt ncols;
const PetscInt *cols;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)G,MATSEQAIJ,&isSeq);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)G,MATMPIAIJ,&isMPI);CHKERRQ(ierr);
if (!isSeq && !isMPI) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_ARG_WRONGSTATE,"MatColoringDegrees requires an MPI/SEQAIJ Matrix");
/* get the inner matrix structure */
oG = NULL;
oi = NULL;
oj = NULL;
if (isMPI) {
aij = (Mat_MPIAIJ*)G->data;
dG = aij->A;
oG = aij->B;
daij = (Mat_SeqAIJ*)dG->data;
oaij = (Mat_SeqAIJ*)oG->data;
di = daij->i;
dj = daij->j;
oi = oaij->i;
oj = oaij->j;
ierr = MatGetSize(oG,&dn,&on);CHKERRQ(ierr);
if (!sf) {
ierr = PetscSFCreate(PetscObjectComm((PetscObject)mc),&sf);CHKERRQ(ierr);
ierr = MatGetLayouts(G,&layout,NULL);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,on,NULL,PETSC_COPY_VALUES,aij->garray);CHKERRQ(ierr);
jp->sf = sf;
}
} else {
dG = G;
ierr = MatGetSize(dG,NULL,&dn);CHKERRQ(ierr);
daij = (Mat_SeqAIJ*)dG->data;
di = daij->i;
dj = daij->j;
}
/* set up the distance-zero weights */
if (!dwts) {
ierr = PetscMalloc1(dn,&dwts);CHKERRQ(ierr);
jp->dwts = dwts;
if (oG) {
ierr = PetscMalloc1(on,&owts);CHKERRQ(ierr);
jp->owts = owts;
}
}
for (i=0;i<dn;i++) {
maxweights[i] = weights[i];
dwts[i] = maxweights[i];
}
/* get the off-diagonal weights */
if (oG) {
ierr = PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_REAL,dwts,owts);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_REAL,dwts,owts);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
}
/* check for the maximum out to the distance of the coloring */
for (l=0;l<mc->dist;l++) {
/* check for on-diagonal greater weights */
for (i=0;i<dn;i++) {
ncols = di[i+1]-di[i];
cols = &(dj[di[i]]);
for (j=0;j<ncols;j++) {
if (dwts[cols[j]] > maxweights[i]) maxweights[i] = dwts[cols[j]];
}
/* check for off-diagonal greater weights */
if (oG) {
ncols = oi[i+1]-oi[i];
cols = &(oj[oi[i]]);
for (j=0;j<ncols;j++) {
if (owts[cols[j]] > maxweights[i]) maxweights[i] = owts[cols[j]];
}
}
}
if (l < mc->dist-1) {
for (i=0;i<dn;i++) {
dwts[i] = maxweights[i];
}
if (oG) {
ierr = PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_REAL,dwts,owts);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_REAL,dwts,owts);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
}
}
}
PetscFunctionReturn(0);
}
PetscErrorCode MCJPMinColor_Private(MatColoring mc,ISColoringValue maxcolor,const ISColoringValue *colors,ISColoringValue *mincolors)
{
MC_JP *jp = (MC_JP*)mc->data;
PetscErrorCode ierr;
Mat G=mc->mat,dG,oG;
PetscBool isSeq,isMPI;
Mat_MPIAIJ *aij;
Mat_SeqAIJ *daij,*oaij;
PetscInt *di,*oi,*dj,*oj;
PetscSF sf=jp->sf;
PetscLayout layout;
PetscInt maskrounds,maskbase,maskradix;
PetscInt dn,on;
PetscInt i,j,l,k;
PetscInt *dmask=jp->dmask,*omask=jp->omask,*cmask=jp->cmask,curmask;
PetscInt ncols;
const PetscInt *cols;
PetscFunctionBegin;
maskradix = sizeof(PetscInt)*8;
maskrounds = 1 + maxcolor / (maskradix);
maskbase = 0;
ierr = PetscObjectTypeCompare((PetscObject)G,MATSEQAIJ,&isSeq);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)G,MATMPIAIJ,&isMPI);CHKERRQ(ierr);
if (!isSeq && !isMPI) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_ARG_WRONGSTATE,"MatColoringDegrees requires an MPI/SEQAIJ Matrix");
/* get the inner matrix structure */
oG = NULL;
oi = NULL;
oj = NULL;
if (isMPI) {
aij = (Mat_MPIAIJ*)G->data;
dG = aij->A;
oG = aij->B;
daij = (Mat_SeqAIJ*)dG->data;
oaij = (Mat_SeqAIJ*)oG->data;
di = daij->i;
dj = daij->j;
oi = oaij->i;
oj = oaij->j;
ierr = MatGetSize(oG,&dn,&on);CHKERRQ(ierr);
if (!sf) {
ierr = PetscSFCreate(PetscObjectComm((PetscObject)mc),&sf);CHKERRQ(ierr);
ierr = MatGetLayouts(G,&layout,NULL);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,on,NULL,PETSC_COPY_VALUES,aij->garray);CHKERRQ(ierr);
jp->sf = sf;
}
} else {
dG = G;
ierr = MatGetSize(dG,NULL,&dn);CHKERRQ(ierr);
daij = (Mat_SeqAIJ*)dG->data;
di = daij->i;
dj = daij->j;
}
for (i=0;i<dn;i++) {
mincolors[i] = IS_COLORING_MAX;
}
/* set up the distance-zero mask */
if (!dmask) {
ierr = PetscMalloc1(dn,&dmask);CHKERRQ(ierr);
ierr = PetscMalloc1(dn,&cmask);CHKERRQ(ierr);
jp->cmask = cmask;
jp->dmask = dmask;
if (oG) {
ierr = PetscMalloc1(on,&omask);CHKERRQ(ierr);
jp->omask = omask;
}
}
/* the number of colors may be more than the number of bits in a PetscInt; take multiple rounds */
for (k=0;k<maskrounds;k++) {
for (i=0;i<dn;i++) {
cmask[i] = 0;
if (colors[i] < maskbase+maskradix && colors[i] >= maskbase)
cmask[i] = 1 << (colors[i]-maskbase);
dmask[i] = cmask[i];
}
if (oG) {
ierr = PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,dmask,omask);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,dmask,omask);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
}
/* fill in the mask out to the distance of the coloring */
for (l=0;l<mc->dist;l++) {
/* fill in the on-and-off diagonal mask */
for (i=0;i<dn;i++) {
ncols = di[i+1]-di[i];
cols = &(dj[di[i]]);
for (j=0;j<ncols;j++) {
cmask[i] = cmask[i] | dmask[cols[j]];
}
if (oG) {
ncols = oi[i+1]-oi[i];
cols = &(oj[oi[i]]);
for (j=0;j<ncols;j++) {
cmask[i] = cmask[i] | omask[cols[j]];
}
}
}
for (i=0;i<dn;i++) {
dmask[i]=cmask[i];
}
if (l < mc->dist-1) {
if (oG) {
ierr = PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,dmask,omask);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,dmask,omask);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
}
}
}
/* read through the mask to see if we've discovered an acceptable color for any vertices in this round */
for (i=0;i<dn;i++) {
if (mincolors[i] == IS_COLORING_MAX) {
curmask = dmask[i];
for (j=0;j<maskradix;j++) {
if (curmask % 2 == 0) {
mincolors[i] = j+maskbase;
break;
}
curmask = curmask >> 1;
}
}
}
/* do the next maskradix colors */
maskbase += maskradix;
}
/*
* The interface should be easy to use for both MatGetSubMatrix (parallel sub-matrix) and MatGetSubMatrices (sequential sub-matrices)
* */
static PetscErrorCode MatGetSubMatrix_MPIAdj_data(Mat adj,IS irows, IS icols, PetscInt **sadj_xadj,PetscInt **sadj_adjncy,PetscInt **sadj_values)
{
PetscInt nlrows_is,icols_n,i,j,nroots,nleaves,owner,rlocalindex,*ncols_send,*ncols_recv;
PetscInt nlrows_mat,*adjncy_recv,Ncols_recv,Ncols_send,*xadj_recv,*values_recv;
PetscInt *ncols_recv_offsets,loc,rnclos,*sadjncy,*sxadj,*svalues,isvalue;
const PetscInt *irows_indices,*icols_indices,*xadj, *adjncy;
Mat_MPIAdj *a = (Mat_MPIAdj*)adj->data;
PetscLayout rmap;
MPI_Comm comm;
PetscSF sf;
PetscSFNode *iremote;
PetscBool done;
PetscErrorCode ierr;
PetscFunctionBegin;
/* communicator */
ierr = PetscObjectGetComm((PetscObject)adj,&comm);CHKERRQ(ierr);
/* Layouts */
ierr = MatGetLayouts(adj,&rmap,PETSC_NULL);CHKERRQ(ierr);
/* get rows information */
ierr = ISGetLocalSize(irows,&nlrows_is);CHKERRQ(ierr);
ierr = ISGetIndices(irows,&irows_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(nlrows_is,&iremote);CHKERRQ(ierr);
/* construct sf graph*/
nleaves = nlrows_is;
for(i=0; i<nlrows_is; i++){
owner = -1;
rlocalindex = -1;
ierr = PetscLayoutFindOwnerIndex(rmap,irows_indices[i],&owner,&rlocalindex);CHKERRQ(ierr);
iremote[i].rank = owner;
iremote[i].index = rlocalindex;
}
ierr = MatGetRowIJ(adj,0,PETSC_FALSE,PETSC_FALSE,&nlrows_mat,&xadj,&adjncy,&done);CHKERRQ(ierr);
ierr = PetscCalloc4(nlrows_mat,&ncols_send,nlrows_is,&xadj_recv,nlrows_is+1,&ncols_recv_offsets,nlrows_is,&ncols_recv);CHKERRQ(ierr);
nroots = nlrows_mat;
for(i=0; i<nlrows_mat; i++){
ncols_send[i] = xadj[i+1]-xadj[i];
}
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,ncols_send,ncols_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,ncols_send,ncols_recv);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,xadj,xadj_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,xadj,xadj_recv);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
Ncols_recv =0;
for(i=0; i<nlrows_is; i++){
Ncols_recv += ncols_recv[i];
ncols_recv_offsets[i+1] = ncols_recv[i]+ncols_recv_offsets[i];
}
Ncols_send = 0;
for(i=0; i<nlrows_mat; i++){
Ncols_send += ncols_send[i];
}
ierr = PetscCalloc1(Ncols_recv,&iremote);CHKERRQ(ierr);
ierr = PetscCalloc1(Ncols_recv,&adjncy_recv);CHKERRQ(ierr);
nleaves = Ncols_recv;
Ncols_recv = 0;
for(i=0; i<nlrows_is; i++){
ierr = PetscLayoutFindOwner(rmap,irows_indices[i],&owner);CHKERRQ(ierr);
for(j=0; j<ncols_recv[i]; j++){
iremote[Ncols_recv].rank = owner;
iremote[Ncols_recv++].index = xadj_recv[i]+j;
}
}
ierr = ISRestoreIndices(irows,&irows_indices);CHKERRQ(ierr);
/*if we need to deal with edge weights ???*/
if(a->values){isvalue=1;}else{isvalue=0;}
/*involve a global communication */
/*ierr = MPI_Allreduce(&isvalue,&isvalue,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr);*/
if(isvalue){ierr = PetscCalloc1(Ncols_recv,&values_recv);CHKERRQ(ierr);}
nroots = Ncols_send;
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,adjncy,adjncy_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,adjncy,adjncy_recv);CHKERRQ(ierr);
if(isvalue){
ierr = PetscSFBcastBegin(sf,MPIU_INT,a->values,values_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,a->values,values_recv);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = MatRestoreRowIJ(adj,0,PETSC_FALSE,PETSC_FALSE,&nlrows_mat,&xadj,&adjncy,&done);CHKERRQ(ierr);
ierr = ISGetLocalSize(icols,&icols_n);CHKERRQ(ierr);
ierr = ISGetIndices(icols,&icols_indices);CHKERRQ(ierr);
rnclos = 0;
for(i=0; i<nlrows_is; i++){
for(j=ncols_recv_offsets[i]; j<ncols_recv_offsets[i+1]; j++){
ierr = PetscFindInt(adjncy_recv[j], icols_n, icols_indices, &loc);CHKERRQ(ierr);
if(loc<0){
adjncy_recv[j] = -1;
if(isvalue) values_recv[j] = -1;
ncols_recv[i]--;
}else{
rnclos++;
}
}
}
ierr = ISRestoreIndices(icols,&icols_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(rnclos,&sadjncy);CHKERRQ(ierr);
if(isvalue) {ierr = PetscCalloc1(rnclos,&svalues);CHKERRQ(ierr);}
ierr = PetscCalloc1(nlrows_is+1,&sxadj);CHKERRQ(ierr);
rnclos = 0;
for(i=0; i<nlrows_is; i++){
for(j=ncols_recv_offsets[i]; j<ncols_recv_offsets[i+1]; j++){
if(adjncy_recv[j]<0) continue;
sadjncy[rnclos] = adjncy_recv[j];
if(isvalue) svalues[rnclos] = values_recv[j];
rnclos++;
}
}
for(i=0; i<nlrows_is; i++){
sxadj[i+1] = sxadj[i]+ncols_recv[i];
}
if(sadj_xadj) { *sadj_xadj = sxadj;}else { ierr = PetscFree(sxadj);CHKERRQ(ierr);}
if(sadj_adjncy){ *sadj_adjncy = sadjncy;}else{ ierr = PetscFree(sadjncy);CHKERRQ(ierr);}
if(sadj_values){
if(isvalue) *sadj_values = svalues; else *sadj_values=0;
}else{
if(isvalue) {ierr = PetscFree(svalues);CHKERRQ(ierr);}
}
ierr = PetscFree4(ncols_send,xadj_recv,ncols_recv_offsets,ncols_recv);CHKERRQ(ierr);
ierr = PetscFree(adjncy_recv);CHKERRQ(ierr);
if(isvalue) {ierr = PetscFree(values_recv);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PetscSectionCreateGlobalSectionLabel - Create a section describing the global field layout using
the local section and an SF describing the section point overlap.
Input Parameters:
+ s - The PetscSection for the local field layout
. sf - The SF describing parallel layout of the section points
. includeConstraints - By default this is PETSC_FALSE, meaning that the global field vector will not possess constrained dofs
. label - The label specifying the points
- labelValue - The label stratum specifying the points
Output Parameter:
. gsection - The PetscSection for the global field layout
Note: This gives negative sizes and offsets to points not owned by this process
Level: developer
.seealso: PetscSectionCreate()
@*/
PetscErrorCode PetscSectionCreateGlobalSectionLabel(PetscSection s, PetscSF sf, PetscBool includeConstraints, DMLabel label, PetscInt labelValue, PetscSection *gsection)
{
PetscInt *neg = NULL, *tmpOff = NULL;
PetscInt pStart, pEnd, p, dof, cdof, off, globalOff = 0, nroots;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) s), gsection);CHKERRQ(ierr);
ierr = PetscSectionGetChart(s, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(*gsection, pStart, pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
if (nroots >= 0) {
if (nroots < pEnd-pStart) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "PetscSF nroots %d < %d section size", nroots, pEnd-pStart);
ierr = PetscCalloc1(nroots, &neg);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
ierr = PetscCalloc1(nroots, &tmpOff);CHKERRQ(ierr);
} else {
tmpOff = &(*gsection)->atlasDof[-pStart];
}
}
/* Mark ghost points with negative dof */
for (p = pStart; p < pEnd; ++p) {
PetscInt value;
ierr = DMLabelGetValue(label, p, &value);CHKERRQ(ierr);
if (value != labelValue) continue;
ierr = PetscSectionGetDof(s, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(*gsection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(s, p, &cdof);CHKERRQ(ierr);
if (!includeConstraints && cdof > 0) {ierr = PetscSectionSetConstraintDof(*gsection, p, cdof);CHKERRQ(ierr);}
if (neg) neg[p] = -(dof+1);
}
ierr = PetscSectionSetUpBC(*gsection);CHKERRQ(ierr);
if (nroots >= 0) {
ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasDof[p-pStart] = tmpOff[p];}
}
}
/* Calculate new sizes, get proccess offset, and calculate point offsets */
for (p = 0, off = 0; p < pEnd-pStart; ++p) {
cdof = (!includeConstraints && s->bc) ? s->bc->atlasDof[p] : 0;
(*gsection)->atlasOff[p] = off;
off += (*gsection)->atlasDof[p] > 0 ? (*gsection)->atlasDof[p]-cdof : 0;
}
ierr = MPI_Scan(&off, &globalOff, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject) s));CHKERRQ(ierr);
globalOff -= off;
for (p = 0, off = 0; p < pEnd-pStart; ++p) {
(*gsection)->atlasOff[p] += globalOff;
if (neg) neg[p] = -((*gsection)->atlasOff[p]+1);
}
/* Put in negative offsets for ghost points */
if (nroots >= 0) {
ierr = PetscSFBcastBegin(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_INT, neg, tmpOff);CHKERRQ(ierr);
if (nroots > pEnd-pStart) {
for (p = pStart; p < pEnd; ++p) {if (tmpOff[p] < 0) (*gsection)->atlasOff[p-pStart] = tmpOff[p];}
}
}
if (nroots >= 0 && nroots > pEnd-pStart) {ierr = PetscFree(tmpOff);CHKERRQ(ierr);}
ierr = PetscFree(neg);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PetscSFCreateSectionSF(PetscSF sf, PetscSection section, PetscSF *sectionSF)
{
PetscInt numRanks;
const PetscInt *ranks, *rankOffsets;
const PetscMPIInt *localPoints, *remotePoints;
PetscInt numPoints, numIndices = 0;
PetscInt *remoteOffsets;
PetscInt *localIndices;
PetscSFNode *remoteIndices;
PetscInt i, r, ind;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscSFGetRanks(sf, &numRanks, &ranks, &rankOffsets, &localPoints, &remotePoints);
CHKERRQ(ierr);
numPoints = rankOffsets[numRanks];
for (i = 0; i < numPoints; ++i) {
PetscInt dof;
ierr = PetscSectionGetDof(section, localPoints[i], &dof);
CHKERRQ(ierr);
numIndices += dof;
}
/* Communicate offsets for ghosted points */
#if 0
PetscInt *localOffsets;
ierr = PetscMalloc2(numPoints,PetscInt,&localOffsets,numPoints,PetscInt,&remoteOffsets);
CHKERRQ(ierr);
for (i = 0; i < numPoints; ++i) {
ierr = PetscSectionGetOffset(section, localPoints[i], &localOffsets[i]);
CHKERRQ(ierr);
}
ierr = PetscSFBcastBegin(sf, MPIU_INT, localOffsets, remoteOffsets);
CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_INT, localOffsets, remoteOffsets);
CHKERRQ(ierr);
for (i = 0; i < numPoints; ++i) {
ierr = PetscSynchronizedPrintf(((PetscObject) sf)->comm, "remoteOffsets[%d]: %d\n", i, remoteOffsets[i]);
CHKERRQ(ierr);
}
#else
ierr = PetscMalloc((section->atlasLayout.pEnd - section->atlasLayout.pStart) * sizeof(PetscInt), &remoteOffsets);
CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf, MPIU_INT, §ion->atlasOff[-section->atlasLayout.pStart], &remoteOffsets[-section->atlasLayout.pStart]);
CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_INT, §ion->atlasOff[-section->atlasLayout.pStart], &remoteOffsets[-section->atlasLayout.pStart]);
CHKERRQ(ierr);
for (i = section->atlasLayout.pStart; i < section->atlasLayout.pEnd; ++i) {
ierr = PetscSynchronizedPrintf(((PetscObject) sf)->comm, "remoteOffsets[%d]: %d\n", i, remoteOffsets[i-section->atlasLayout.pStart]);
CHKERRQ(ierr);
}
#endif
ierr = PetscSynchronizedFlush(((PetscObject) sf)->comm);
CHKERRQ(ierr);
ierr = PetscMalloc(numIndices * sizeof(PetscInt), &localIndices);
CHKERRQ(ierr);
ierr = PetscMalloc(numIndices * sizeof(PetscSFNode), &remoteIndices);
CHKERRQ(ierr);
/* Create new index graph */
for (r = 0, ind = 0; r < numRanks; ++r) {
PetscInt rank = ranks[r];
for (i = rankOffsets[r]; i < rankOffsets[r+1]; ++i) {
PetscInt localPoint = localPoints[i];
PetscInt remoteOffset = remoteOffsets[localPoint-section->atlasLayout.pStart];
PetscInt localOffset, dof, d;
ierr = PetscSectionGetOffset(section, localPoint, &localOffset);
CHKERRQ(ierr);
ierr = PetscSectionGetDof(section, localPoint, &dof);
CHKERRQ(ierr);
for (d = 0; d < dof; ++d, ++ind) {
localIndices[ind] = localOffset+d;
remoteIndices[ind].rank = rank;
remoteIndices[ind].index = remoteOffset+d;
}
}
}
ierr = PetscFree(remoteOffsets);
CHKERRQ(ierr);
if (numIndices != ind) SETERRQ2(((PetscObject) sf)->comm, PETSC_ERR_PLIB, "Inconsistency in indices, %d should be %d", ind, numIndices);
ierr = PetscSFCreate(((PetscObject) sf)->comm, sectionSF);
CHKERRQ(ierr);
ierr = PetscSFSetGraph(*sectionSF, numIndices, numIndices, localIndices, PETSC_OWN_POINTER, remoteIndices, PETSC_OWN_POINTER);
CHKERRQ(ierr);
ierr = PetscSFView(*sectionSF, NULL);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PetscSFGetMultiSF - gets the inner SF implemeting gathers and scatters
Collective
Input Argument:
. sf - star forest that may contain roots with 0 or with more than 1 vertex
Output Arguments:
. multi - star forest with split roots, such that each root has degree exactly 1
Level: developer
Notes:
In most cases, users should use PetscSFGatherBegin() and PetscSFScatterBegin() instead of manipulating multi
directly. Since multi satisfies the stronger condition that each entry in the global space has exactly one incoming
edge, it is a candidate for future optimization that might involve its removal.
.seealso: PetscSFSetGraph(), PetscSFGatherBegin(), PetscSFScatterBegin()
@*/
PetscErrorCode PetscSFGetMultiSF(PetscSF sf,PetscSF *multi)
{
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(sf,PETSCSF_CLASSID,1);
PetscValidPointer(multi,2);
if (sf->nroots < 0) { /* Graph has not been set yet; why do we need this? */
ierr = PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,&sf->multi);CHKERRQ(ierr);
*multi = sf->multi;
PetscFunctionReturn(0);
}
if (!sf->multi) {
const PetscInt *indegree;
PetscInt i,*inoffset,*outones,*outoffset,maxlocal;
PetscSFNode *remote;
ierr = PetscSFComputeDegreeBegin(sf,&indegree);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(sf,&indegree);CHKERRQ(ierr);
for (i=0,maxlocal=0; i<sf->nleaves; i++) maxlocal = PetscMax(maxlocal,(sf->mine ? sf->mine[i] : i)+1);
ierr = PetscMalloc3(sf->nroots+1,&inoffset,maxlocal,&outones,maxlocal,&outoffset);CHKERRQ(ierr);
inoffset[0] = 0;
for (i=0; i<sf->nroots; i++) inoffset[i+1] = inoffset[i] + indegree[i];
for (i=0; i<maxlocal; i++) outones[i] = 1;
ierr = PetscSFFetchAndOpBegin(sf,MPIU_INT,inoffset,outones,outoffset,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFFetchAndOpEnd(sf,MPIU_INT,inoffset,outones,outoffset,MPI_SUM);CHKERRQ(ierr);
for (i=0; i<sf->nroots; i++) inoffset[i] -= indegree[i]; /* Undo the increment */
#if 0
#if defined(PETSC_USE_DEBUG) /* Check that the expected number of increments occurred */
for (i=0; i<sf->nroots; i++) {
if (inoffset[i] + indegree[i] != inoffset[i+1]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Incorrect result after PetscSFFetchAndOp");
}
#endif
#endif
ierr = PetscMalloc1(sf->nleaves,&remote);CHKERRQ(ierr);
for (i=0; i<sf->nleaves; i++) {
remote[i].rank = sf->remote[i].rank;
remote[i].index = outoffset[sf->mine ? sf->mine[i] : i];
}
ierr = PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,&sf->multi);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf->multi,inoffset[sf->nroots],sf->nleaves,sf->mine,PETSC_COPY_VALUES,remote,PETSC_OWN_POINTER);CHKERRQ(ierr);
if (sf->rankorder) { /* Sort the ranks */
PetscMPIInt rank;
PetscInt *inranks,*newoffset,*outranks,*newoutoffset,*tmpoffset,maxdegree;
PetscSFNode *newremote;
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);CHKERRQ(ierr);
for (i=0,maxdegree=0; i<sf->nroots; i++) maxdegree = PetscMax(maxdegree,indegree[i]);
ierr = PetscMalloc5(sf->multi->nroots,&inranks,sf->multi->nroots,&newoffset,maxlocal,&outranks,maxlocal,&newoutoffset,maxdegree,&tmpoffset);CHKERRQ(ierr);
for (i=0; i<maxlocal; i++) outranks[i] = rank;
ierr = PetscSFReduceBegin(sf->multi,MPIU_INT,outranks,inranks,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sf->multi,MPIU_INT,outranks,inranks,MPIU_REPLACE);CHKERRQ(ierr);
/* Sort the incoming ranks at each vertex, build the inverse map */
for (i=0; i<sf->nroots; i++) {
PetscInt j;
for (j=0; j<indegree[i]; j++) tmpoffset[j] = j;
ierr = PetscSortIntWithArray(indegree[i],inranks+inoffset[i],tmpoffset);CHKERRQ(ierr);
for (j=0; j<indegree[i]; j++) newoffset[inoffset[i] + tmpoffset[j]] = inoffset[i] + j;
}
ierr = PetscSFBcastBegin(sf->multi,MPIU_INT,newoffset,newoutoffset);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf->multi,MPIU_INT,newoffset,newoutoffset);CHKERRQ(ierr);
ierr = PetscMalloc1(sf->nleaves,&newremote);CHKERRQ(ierr);
for (i=0; i<sf->nleaves; i++) {
newremote[i].rank = sf->remote[i].rank;
newremote[i].index = newoutoffset[sf->mine ? sf->mine[i] : i];
}
ierr = PetscSFSetGraph(sf->multi,inoffset[sf->nroots],sf->nleaves,sf->mine,PETSC_COPY_VALUES,newremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscFree5(inranks,newoffset,outranks,newoutoffset,tmpoffset);CHKERRQ(ierr);
}
ierr = PetscFree3(inoffset,outones,outoffset);CHKERRQ(ierr);
}
*multi = sf->multi;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscSF sf,sfDup,sfInv,sfEmbed,sfA,sfB,sfBA;
const PetscInt *degree;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr;
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFReset(sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFReset(sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = CheckGraphEmpty(sf);CHKERRQ(ierr);
ierr = PetscSFReset(sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = CheckGraphEmpty(sf);CHKERRQ(ierr);
ierr = PetscSFReset(sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test setup */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = CheckRanksNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = CheckRanksNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
ierr = CheckRanksEmpty(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test setup then reset */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
ierr = PetscSFReset(sf);CHKERRQ(ierr);
ierr = CheckRanksNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test view (no graph set, no type set) */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFView(sf,NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test set graph then view (no type set) */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscSFView(sf,NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test set type then view (no graph set) */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFView(sf,NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test set type then graph then view */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscSFView(sf,NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test set graph then type */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = CheckGraphEmpty(sf);CHKERRQ(ierr);
ierr = PetscSFReset(sf);CHKERRQ(ierr);
ierr = CheckGraphNotSet(sf);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test Bcast */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPI_INT,NULL,NULL);CHKERRQ(ierr);
ierr = PetscSFBcastEnd (sf,MPI_INT,NULL,NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test Reduce */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sf,MPI_INT,NULL,NULL,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFReduceEnd (sf,MPI_INT,NULL,NULL,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sf,MPI_INT,NULL,NULL,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd (sf,MPI_INT,NULL,NULL,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test FetchAndOp */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscSFFetchAndOpBegin(sf,MPI_INT,NULL,NULL,NULL,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFFetchAndOpEnd (sf,MPI_INT,NULL,NULL,NULL,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test ComputeDegree */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_COPY_VALUES,NULL,PETSC_COPY_VALUES);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeBegin(sf,°ree);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(sf,°ree);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test PetscSFDuplicate() */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_USE_POINTER,NULL,PETSC_USE_POINTER);CHKERRQ(ierr);
ierr = PetscSFDuplicate(sf,PETSCSF_DUPLICATE_GRAPH,&sfDup);CHKERRQ(ierr);
ierr = CheckGraphEmpty(sfDup);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfDup);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test PetscSFCreateInverseSF() */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_USE_POINTER,NULL,PETSC_USE_POINTER);CHKERRQ(ierr);
ierr = PetscSFCreateInverseSF(sf,&sfInv);CHKERRQ(ierr);
ierr = CheckGraphEmpty(sfInv);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfInv);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test PetscSFCreateEmbeddedSF() */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_USE_POINTER,NULL,PETSC_USE_POINTER);CHKERRQ(ierr);
ierr = PetscSFCreateEmbeddedSF(sf,0,NULL,&sfEmbed);CHKERRQ(ierr);
ierr = CheckGraphEmpty(sfEmbed);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfEmbed);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test PetscSFCreateEmbeddedLeafSF() */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,0,0,NULL,PETSC_USE_POINTER,NULL,PETSC_USE_POINTER);CHKERRQ(ierr);
ierr = PetscSFCreateEmbeddedLeafSF(sf,0,NULL,&sfEmbed);CHKERRQ(ierr);
ierr = CheckGraphEmpty(sfEmbed);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfEmbed);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* Test PetscSFCompose() */
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sfA);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sfA,0,0,NULL,PETSC_USE_POINTER,NULL,PETSC_USE_POINTER);CHKERRQ(ierr);
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sfB);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sfB,0,0,NULL,PETSC_USE_POINTER,NULL,PETSC_USE_POINTER);CHKERRQ(ierr);
ierr = PetscSFCompose(sfA,sfB,&sfBA);CHKERRQ(ierr);
ierr = CheckGraphEmpty(sfBA);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfBA);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfA);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfB);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@
DMPlexOrient - Give a consistent orientation to the input mesh
Input Parameters:
. dm - The DM
Note: The orientation data for the DM are change in-place.
$ This routine will fail for non-orientable surfaces, such as the Moebius strip.
Level: advanced
.seealso: DMCreate(), DMPLEX
@*/
PetscErrorCode DMPlexOrient(DM dm)
{
MPI_Comm comm;
PetscSF sf;
const PetscInt *lpoints;
const PetscSFNode *rpoints;
PetscSFNode *rorntComp = NULL, *lorntComp = NULL;
PetscInt *numNeighbors, **neighbors;
PetscSFNode *nrankComp;
PetscBool *match, *flipped;
PetscBT seenCells, flippedCells, seenFaces;
PetscInt *faceFIFO, fTop, fBottom, *cellComp, *faceComp;
PetscInt numLeaves, numRoots, dim, h, cStart, cEnd, c, cell, fStart, fEnd, face, off, totNeighbors = 0;
PetscMPIInt rank, size, numComponents, comp = 0;
PetscBool flg, flg2;
PetscViewer viewer = NULL, selfviewer = NULL;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view", &flg);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view_synchronized", &flg2);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
/* Truth Table
mismatch flips do action mismatch flipA ^ flipB action
F 0 flips no F F F
F 1 flip yes F T T
F 2 flips no T F T
T 0 flips yes T T F
T 1 flip no
T 2 flips yes
*/
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscCalloc3(fEnd - fStart, &faceFIFO, cEnd-cStart, &cellComp, fEnd-fStart, &faceComp);CHKERRQ(ierr);
/*
OLD STYLE
- Add an integer array over cells and faces (component) for connected component number
Foreach component
- Mark the initial cell as seen
- Process component as usual
- Set component for all seenCells
- Wipe seenCells and seenFaces (flippedCells can stay)
- Generate parallel adjacency for component using SF and seenFaces
- Collect numComponents adj data from each proc to 0
- Build same serial graph
- Use same solver
- Use Scatterv to to send back flipped flags for each component
- Negate flippedCells by component
NEW STYLE
- Create the adj on each process
- Bootstrap to complete graph on proc 0
*/
/* Loop over components */
for (cell = cStart; cell < cEnd; ++cell) cellComp[cell-cStart] = -1;
do {
/* Look for first unmarked cell */
for (cell = cStart; cell < cEnd; ++cell) if (cellComp[cell-cStart] < 0) break;
if (cell >= cEnd) break;
/* Initialize FIFO with first cell in component */
{
const PetscInt *cone;
PetscInt coneSize;
fTop = fBottom = 0;
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
faceFIFO[fBottom++] = cone[c];
ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
}
ierr = PetscBTSet(seenCells, cell-cStart);CHKERRQ(ierr);
}
/* Consider each face in FIFO */
while (fTop < fBottom) {
ierr = DMPlexCheckFace_Internal(dm, faceFIFO, &fTop, &fBottom, cStart, fStart, fEnd, seenCells, flippedCells, seenFaces);CHKERRQ(ierr);
}
/* Set component for cells and faces */
for (cell = 0; cell < cEnd-cStart; ++cell) {
if (PetscBTLookup(seenCells, cell)) cellComp[cell] = comp;
}
for (face = 0; face < fEnd-fStart; ++face) {
if (PetscBTLookup(seenFaces, face)) faceComp[face] = comp;
}
/* Wipe seenCells and seenFaces for next component */
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
++comp;
} while (1);
numComponents = comp;
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for serial flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Now all subdomains are oriented, but we need a consistent parallel orientation */
if (numLeaves >= 0) {
/* Store orientations of boundary faces*/
ierr = PetscCalloc2(numRoots,&rorntComp,numRoots,&lorntComp);CHKERRQ(ierr);
for (face = fStart; face < fEnd; ++face) {
const PetscInt *cone, *support, *ornt;
PetscInt coneSize, supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) continue;
ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
if (dim == 1) {
/* Use cone position instead, shifted to -1 or 1 */
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = 1-c*2;
else rorntComp[face].rank = c*2-1;
} else {
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = ornt[c] < 0 ? -1 : 1;
else rorntComp[face].rank = ornt[c] < 0 ? 1 : -1;
}
rorntComp[face].index = faceComp[face-fStart];
}
/* Communicate boundary edge orientations */
ierr = PetscSFBcastBegin(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
}
/* Get process adjacency */
ierr = PetscMalloc2(numComponents, &numNeighbors, numComponents, &neighbors);CHKERRQ(ierr);
viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)dm));
if (flg2) {ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
for (comp = 0; comp < numComponents; ++comp) {
PetscInt l, n;
numNeighbors[comp] = 0;
ierr = PetscMalloc1(PetscMax(numLeaves, 0), &neighbors[comp]);CHKERRQ(ierr);
/* I know this is p^2 time in general, but for bounded degree its alright */
for (l = 0; l < numLeaves; ++l) {
const PetscInt face = lpoints[l];
/* Find a representative face (edge) separating pairs of procs */
if ((face >= fStart) && (face < fEnd) && (faceComp[face-fStart] == comp)) {
const PetscInt rrank = rpoints[l].rank;
const PetscInt rcomp = lorntComp[face].index;
for (n = 0; n < numNeighbors[comp]; ++n) if ((rrank == rpoints[neighbors[comp][n]].rank) && (rcomp == lorntComp[lpoints[neighbors[comp][n]]].index)) break;
if (n >= numNeighbors[comp]) {
PetscInt supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
if (flg) {ierr = PetscViewerASCIIPrintf(selfviewer, "[%d]: component %d, Found representative leaf %d (face %d) connecting to face %d on (%d, %d) with orientation %d\n", rank, comp, l, face, rpoints[l].index, rrank, rcomp, lorntComp[face].rank);CHKERRQ(ierr);}
neighbors[comp][numNeighbors[comp]++] = l;
}
}
}
totNeighbors += numNeighbors[comp];
}
ierr = PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (flg2) {ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscMalloc2(totNeighbors, &nrankComp, totNeighbors, &match);CHKERRQ(ierr);
for (comp = 0, off = 0; comp < numComponents; ++comp) {
PetscInt n;
for (n = 0; n < numNeighbors[comp]; ++n, ++off) {
const PetscInt face = lpoints[neighbors[comp][n]];
const PetscInt o = rorntComp[face].rank*lorntComp[face].rank;
if (o < 0) match[off] = PETSC_TRUE;
else if (o > 0) match[off] = PETSC_FALSE;
else SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid face %d (%d, %d) neighbor: %d comp: %d", face, rorntComp[face], lorntComp[face], neighbors[comp][n], comp);
nrankComp[off].rank = rpoints[neighbors[comp][n]].rank;
nrankComp[off].index = lorntComp[lpoints[neighbors[comp][n]]].index;
}
ierr = PetscFree(neighbors[comp]);CHKERRQ(ierr);
}
/* Collect the graph on 0 */
if (numLeaves >= 0) {
Mat G;
PetscBT seenProcs, flippedProcs;
PetscInt *procFIFO, pTop, pBottom;
PetscInt *N = NULL, *Noff;
PetscSFNode *adj = NULL;
PetscBool *val = NULL;
PetscMPIInt *recvcounts = NULL, *displs = NULL, *Nc, p, o;
PetscMPIInt size = 0;
ierr = PetscCalloc1(numComponents, &flipped);CHKERRQ(ierr);
if (!rank) {ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);}
ierr = PetscCalloc4(size, &recvcounts, size+1, &displs, size, &Nc, size+1, &Noff);CHKERRQ(ierr);
ierr = MPI_Gather(&numComponents, 1, MPI_INT, Nc, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
if (!rank) {ierr = PetscMalloc1(displs[size],&N);CHKERRQ(ierr);}
ierr = MPI_Gatherv(numNeighbors, numComponents, MPIU_INT, N, Nc, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0, o = 0; p < size; ++p) {
recvcounts[p] = 0;
for (c = 0; c < Nc[p]; ++c, ++o) recvcounts[p] += N[o];
displs[p+1] = displs[p] + recvcounts[p];
}
if (!rank) {ierr = PetscMalloc2(displs[size], &adj, displs[size], &val);CHKERRQ(ierr);}
ierr = MPI_Gatherv(nrankComp, totNeighbors, MPIU_2INT, adj, recvcounts, displs, MPIU_2INT, 0, comm);CHKERRQ(ierr);
ierr = MPI_Gatherv(match, totNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
if (!rank) {
for (p = 1; p <= size; ++p) {Noff[p] = Noff[p-1] + Nc[p-1];}
if (flg) {
PetscInt n;
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
ierr = PetscPrintf(PETSC_COMM_SELF, "Proc %d Comp %d:\n", p, c);CHKERRQ(ierr);
for (n = 0; n < N[Noff[p]+c]; ++n, ++off) {
ierr = PetscPrintf(PETSC_COMM_SELF, " edge (%d, %d) (%d):\n", adj[off].rank, adj[off].index, val[off]);CHKERRQ(ierr);
}
}
}
}
/* Symmetrize the graph */
ierr = MatCreate(PETSC_COMM_SELF, &G);CHKERRQ(ierr);
ierr = MatSetSizes(G, Noff[size], Noff[size], Noff[size], Noff[size]);CHKERRQ(ierr);
ierr = MatSetUp(G);CHKERRQ(ierr);
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
const PetscInt r = Noff[p]+c;
PetscInt n;
for (n = 0; n < N[r]; ++n, ++off) {
const PetscInt q = Noff[adj[off].rank] + adj[off].index;
const PetscScalar o = val[off] ? 1.0 : 0.0;
ierr = MatSetValues(G, 1, &r, 1, &q, &o, INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(G, 1, &q, 1, &r, &o, INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &seenProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], seenProcs);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &flippedProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], flippedProcs);CHKERRQ(ierr);
ierr = PetscMalloc1(Noff[size], &procFIFO);CHKERRQ(ierr);
pTop = pBottom = 0;
for (p = 0; p < Noff[size]; ++p) {
if (PetscBTLookup(seenProcs, p)) continue;
/* Initialize FIFO with next proc */
procFIFO[pBottom++] = p;
ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
/* Consider each proc in FIFO */
while (pTop < pBottom) {
const PetscScalar *ornt;
const PetscInt *neighbors;
PetscInt proc, nproc, seen, flippedA, flippedB, mismatch, numNeighbors, n;
proc = procFIFO[pTop++];
flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
ierr = MatGetRow(G, proc, &numNeighbors, &neighbors, &ornt);CHKERRQ(ierr);
/* Loop over neighboring procs */
for (n = 0; n < numNeighbors; ++n) {
nproc = neighbors[n];
mismatch = PetscRealPart(ornt[n]) > 0.5 ? 0 : 1;
seen = PetscBTLookup(seenProcs, nproc);
flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
if (mismatch ^ (flippedA ^ flippedB)) {
if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
if (!flippedB) {
ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
} else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
if (!seen) {
procFIFO[pBottom++] = nproc;
ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
}
}
}
}
ierr = PetscFree(procFIFO);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
ierr = PetscFree2(adj, val);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
}
/* Scatter flip flags */
{
PetscBool *flips = NULL;
if (!rank) {
ierr = PetscMalloc1(Noff[size], &flips);CHKERRQ(ierr);
for (p = 0; p < Noff[size]; ++p) {
flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
if (flg && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc+Comp %d:\n", p);CHKERRQ(ierr);}
}
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
}
ierr = MPI_Scatterv(flips, Nc, displs, MPIU_BOOL, flipped, numComponents, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree(flips);CHKERRQ(ierr);
}
if (!rank) {ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);}
ierr = PetscFree(N);CHKERRQ(ierr);
ierr = PetscFree4(recvcounts, displs, Nc, Noff);CHKERRQ(ierr);
ierr = PetscFree2(nrankComp, match);CHKERRQ(ierr);
/* Decide whether to flip cells in each component */
for (c = 0; c < cEnd-cStart; ++c) {if (flipped[cellComp[c]]) {ierr = PetscBTNegate(flippedCells, c);CHKERRQ(ierr);}}
ierr = PetscFree(flipped);CHKERRQ(ierr);
}
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for parallel flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Reverse flipped cells in the mesh */
for (c = cStart; c < cEnd; ++c) {
if (PetscBTLookup(flippedCells, c-cStart)) {
ierr = DMPlexReverseCell(dm, c);CHKERRQ(ierr);
}
}
ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
ierr = PetscFree2(rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscFree3(faceFIFO, cellComp, faceComp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Distribute cones
- Partitioning: input partition point map and naive sf, output sf with inverse of map, distribute points
- Distribute section: input current sf, communicate sizes and offsets, output local section and offsets (only use for new sf)
- Create SF for values: input current sf and offsets, output new sf
- Distribute values: input new sf, communicate values
*/
PetscErrorCode DistributeMesh(DM dm, AppCtx *user, PetscSF *pointSF, DM *parallelDM)
{
MPI_Comm comm = ((PetscObject) dm)->comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS cellPart, part;
PetscSection cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF;
ISLocalToGlobalMapping renumbering;
PetscSF coneSF;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets, newConesSize;
PetscInt *cones, *newCones;
PetscMPIInt numProcs, rank, p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMMeshGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = DMMeshCreatePartition(dm, &cellPartSection, &cellPart, height);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) {
numRemoteRanks = numProcs;
} else {
numRemoteRanks = 0;
}
ierr = PetscMalloc(numRemoteRanks * sizeof(PetscSFNode), &remoteRanks);CHKERRQ(ierr);
for(p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, PETSC_NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFView(partSF, PETSC_NULL);CHKERRQ(ierr);
/* Close the partition over the mesh */
ierr = DMMeshCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMMeshCreate(comm, parallelDM);CHKERRQ(ierr);
ierr = DMMeshSetDimension(*parallelDM, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *parallelDM, "Parallel Mesh");CHKERRQ(ierr);
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, pointSF);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFView(*pointSF, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, PETSC_NULL);CHKERRQ(ierr);
/* Cleanup */
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMMeshGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMMeshGetConeSection(*parallelDM, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(*pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMMeshSetUp(*parallelDM);CHKERRQ(ierr);
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(*pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMMeshGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMMeshGetCones(*parallelDM, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApply(renumbering, IS_GTOLM_MASK, newConesSize, newCones, PETSC_NULL, newCones);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
/* Debugging */
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, PETSC_NULL);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
/* Create supports and stratify sieve */
ierr = DMMeshSymmetrize(*parallelDM);CHKERRQ(ierr);
ierr = DMMeshStratify(*parallelDM);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPatchSolve(DM dm)
{
MPI_Comm comm;
MPI_Comm commz;
DM dmc;
PetscSF sfz, sfzr;
Vec XC;
MatStencil patchSize, commSize, gridRank, lower, upper;
PetscInt M, N, P, i, j, k, l, m, n, p = 0;
PetscMPIInt rank, size;
PetscInt debug = 0;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMPatchGetCoarse(dm, &dmc);CHKERRQ(ierr);
ierr = DMPatchGetPatchSize(dm, &patchSize);CHKERRQ(ierr);
ierr = DMPatchGetCommSize(dm, &commSize);CHKERRQ(ierr);
ierr = DMPatchGetCommSize(dm, &commSize);CHKERRQ(ierr);
ierr = DMGetGlobalVector(dmc, &XC);CHKERRQ(ierr);
ierr = DMDAGetInfo(dmc, 0, &M, &N, &P, &l, &m, &n, 0,0,0,0,0,0);CHKERRQ(ierr);
M = PetscMax(M, 1); l = PetscMax(l, 1);
N = PetscMax(N, 1); m = PetscMax(m, 1);
P = PetscMax(P, 1); n = PetscMax(n, 1);
gridRank.i = rank % l;
gridRank.j = rank/l % m;
gridRank.k = rank/(l*m) % n;
if (commSize.i*commSize.j*commSize.k == size || commSize.i*commSize.j*commSize.k == 0) {
commSize.i = l; commSize.j = m; commSize.k = n;
commz = comm;
} else if (commSize.i*commSize.j*commSize.k == 1) {
commz = PETSC_COMM_SELF;
} else {
const PetscMPIInt newComm = ((gridRank.k/commSize.k)*(m/commSize.j) + gridRank.j/commSize.j)*(l/commSize.i) + (gridRank.i/commSize.i);
const PetscMPIInt newRank = ((gridRank.k%commSize.k)*commSize.j + gridRank.j%commSize.j)*commSize.i + (gridRank.i%commSize.i);
ierr = MPI_Comm_split(comm, newComm, newRank, &commz);CHKERRQ(ierr);
if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "Rank %d color %d key %d commz %d\n", rank, newComm, newRank, *((PetscMPIInt*) &commz));CHKERRQ(ierr);}
}
/*
Assumptions:
- patchSize divides gridSize
- commSize divides gridSize
- commSize divides l,m,n
Ignore multiple patches per rank for now
Multiple ranks per patch:
- l,m,n divides patchSize
- commSize divides patchSize
*/
for (k = 0; k < P; k += PetscMax(patchSize.k, 1)) {
for (j = 0; j < N; j += PetscMax(patchSize.j, 1)) {
for (i = 0; i < M; i += PetscMax(patchSize.i, 1), ++p) {
MPI_Comm commp = MPI_COMM_NULL;
DM dmz = NULL;
#if 0
DM dmf = NULL;
Mat interpz = NULL;
#endif
Vec XZ = NULL;
PetscScalar *xcarray = NULL;
PetscScalar *xzarray = NULL;
if ((gridRank.k/commSize.k == p/(l/commSize.i * m/commSize.j) % n/commSize.k) &&
(gridRank.j/commSize.j == p/(l/commSize.i) % m/commSize.j) &&
(gridRank.i/commSize.i == p % l/commSize.i)) {
if (debug) {ierr = PetscPrintf(PETSC_COMM_SELF, "Rank %d is accepting Patch %d\n", rank, p);CHKERRQ(ierr);}
commp = commz;
}
/* Zoom to coarse patch */
lower.i = i; lower.j = j; lower.k = k;
upper.i = i + patchSize.i; upper.j = j + patchSize.j; upper.k = k + patchSize.k;
ierr = DMPatchZoom(dmc, XC, lower, upper, commp, &dmz, &sfz, &sfzr);CHKERRQ(ierr);
lower.c = 0; /* initialize member, otherwise compiler issues warnings */
upper.c = 0; /* initialize member, otherwise compiler issues warnings */
/* Debug */
ierr = PetscPrintf(comm, "Patch %d: (%d, %d, %d)--(%d, %d, %d)\n", p, lower.i, lower.j, lower.k, upper.i, upper.j, upper.k);CHKERRQ(ierr);
if (dmz) {ierr = DMView(dmz, PETSC_VIEWER_STDOUT_(commz));CHKERRQ(ierr);}
ierr = PetscSFView(sfz, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);
ierr = PetscSFView(sfzr, PETSC_VIEWER_STDOUT_(comm));CHKERRQ(ierr);
/* Scatter Xcoarse -> Xzoom */
if (dmz) {ierr = DMGetGlobalVector(dmz, &XZ);CHKERRQ(ierr);}
if (XZ) {ierr = VecGetArray(XZ, &xzarray);CHKERRQ(ierr);}
ierr = VecGetArray(XC, &xcarray);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfz, MPIU_SCALAR, xcarray, xzarray);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfz, MPIU_SCALAR, xcarray, xzarray);CHKERRQ(ierr);
ierr = VecRestoreArray(XC, &xcarray);CHKERRQ(ierr);
if (XZ) {ierr = VecRestoreArray(XZ, &xzarray);CHKERRQ(ierr);}
#if 0
/* Interpolate Xzoom -> Xfine, note that this may be on subcomms */
ierr = DMRefine(dmz, MPI_COMM_NULL, &dmf);CHKERRQ(ierr);
ierr = DMCreateInterpolation(dmz, dmf, &interpz, NULL);CHKERRQ(ierr);
ierr = DMInterpolate(dmz, interpz, dmf);CHKERRQ(ierr);
/* Smooth Xfine using two-step smoother, normal smoother plus Kaczmarz---moves back and forth from dmzoom to dmfine */
/* Compute residual Rfine */
/* Restrict Rfine to Rzoom_restricted */
#endif
/* Scatter Rzoom_restricted -> Rcoarse_restricted */
if (XZ) {ierr = VecGetArray(XZ, &xzarray);CHKERRQ(ierr);}
ierr = VecGetArray(XC, &xcarray);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sfzr, MPIU_SCALAR, xzarray, xcarray, MPIU_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfzr, MPIU_SCALAR, xzarray, xcarray, MPIU_SUM);CHKERRQ(ierr);
ierr = VecRestoreArray(XC, &xcarray);CHKERRQ(ierr);
if (XZ) {ierr = VecRestoreArray(XZ, &xzarray);CHKERRQ(ierr);}
if (dmz) {ierr = DMRestoreGlobalVector(dmz, &XZ);CHKERRQ(ierr);}
/* Compute global residual Rcoarse */
/* TauCoarse = Rcoarse - Rcoarse_restricted */
ierr = PetscSFDestroy(&sfz);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfzr);CHKERRQ(ierr);
ierr = DMDestroy(&dmz);CHKERRQ(ierr);
}
}
}
ierr = DMRestoreGlobalVector(dmc, &XC);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMLabelDistribute(DMLabel label, PetscSF sf, DMLabel *labelNew)
{
MPI_Comm comm;
PetscSection rootSection, leafSection;
PetscSF labelSF;
PetscInt p, pStart, pEnd, l, lStart, lEnd, s, nroots, nleaves, size, dof, offset, stratum;
PetscInt *remoteOffsets, *rootStrata, *rootIdx, *leafStrata, *strataIdx;
char *name;
PetscInt nameSize;
size_t len = 0;
PetscMPIInt rank, numProcs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (label) {ierr = DMLabelMakeAllValid_Private(label);CHKERRQ(ierr);}
ierr = PetscObjectGetComm((PetscObject)sf, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
/* Bcast name */
if (!rank) {ierr = PetscStrlen(label->name, &len);CHKERRQ(ierr);}
nameSize = len;
ierr = MPI_Bcast(&nameSize, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1(nameSize+1, &name);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy(name, label->name, nameSize+1);CHKERRQ(ierr);}
ierr = MPI_Bcast(name, nameSize+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = DMLabelCreate(name, labelNew);CHKERRQ(ierr);
ierr = PetscFree(name);CHKERRQ(ierr);
/* Bcast numStrata */
if (!rank) (*labelNew)->numStrata = label->numStrata;
ierr = MPI_Bcast(&(*labelNew)->numStrata, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
/* Bcast stratumValues */
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->stratumValues);CHKERRQ(ierr);
if (!rank) {ierr = PetscMemcpy((*labelNew)->stratumValues, label->stratumValues, label->numStrata * sizeof(PetscInt));CHKERRQ(ierr);}
ierr = MPI_Bcast((*labelNew)->stratumValues, (*labelNew)->numStrata, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscMalloc1((*labelNew)->numStrata, &(*labelNew)->arrayValid);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) (*labelNew)->arrayValid[s] = PETSC_TRUE;
/* Build a section detailing strata-per-point, distribute and build SF
from that and then send our points. */
ierr = PetscSFGetGraph(sf, &nroots, &nleaves, NULL, NULL);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSection, 0, nroots);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
lStart = 0;
lEnd = label->stratumSizes[s];
for (l=lStart; l<lEnd; l++) {
ierr = PetscSectionGetDof(rootSection, label->points[s][l], &dof);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSection, label->points[s][l], dof+1);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSection);CHKERRQ(ierr);
/* Create a point-wise array of point strata */
ierr = PetscSectionGetStorageSize(rootSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &rootStrata);CHKERRQ(ierr);
ierr = PetscCalloc1(nroots, &rootIdx);CHKERRQ(ierr);
if (label) {
for (s = 0; s < label->numStrata; ++s) {
lStart = 0;
lEnd = label->stratumSizes[s];
for (l=lStart; l<lEnd; l++) {
p = label->points[s][l];
ierr = PetscSectionGetOffset(rootSection, p, &offset);CHKERRQ(ierr);
rootStrata[offset+rootIdx[p]++] = s;
}
}
}
/* Build SF that maps label points to remote processes */
ierr = PetscSectionCreate(comm, &leafSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sf, rootSection, &remoteOffsets, leafSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, rootSection, remoteOffsets, leafSection, &labelSF);CHKERRQ(ierr);
/* Send the strata for each point over the derived SF */
ierr = PetscSectionGetStorageSize(leafSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(labelSF, MPIU_INT, rootStrata, leafStrata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(labelSF, MPIU_INT, rootStrata, leafStrata);CHKERRQ(ierr);
/* Rebuild the point strata on the receiver */
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->stratumSizes);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
(*labelNew)->stratumSizes[leafStrata[offset+s]]++;
}
}
ierr = PetscCalloc1((*labelNew)->numStrata,&(*labelNew)->ht);CHKERRQ(ierr);
ierr = PetscMalloc1((*labelNew)->numStrata,&(*labelNew)->points);CHKERRQ(ierr);
for (s = 0; s < (*labelNew)->numStrata; ++s) {
PetscHashICreate((*labelNew)->ht[s]);
ierr = PetscMalloc1((*labelNew)->stratumSizes[s], &(*labelNew)->points[s]);CHKERRQ(ierr);
}
/* Insert points into new strata */
ierr = PetscCalloc1((*labelNew)->numStrata, &strataIdx);CHKERRQ(ierr);
ierr = PetscSectionGetChart(leafSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p=pStart; p<pEnd; p++) {
ierr = PetscSectionGetDof(leafSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(leafSection, p, &offset);CHKERRQ(ierr);
for (s=0; s<dof; s++) {
stratum = leafStrata[offset+s];
(*labelNew)->points[stratum][strataIdx[stratum]++] = p;
}
}
ierr = PetscFree(rootStrata);CHKERRQ(ierr);
ierr = PetscFree(leafStrata);CHKERRQ(ierr);
ierr = PetscFree(rootIdx);CHKERRQ(ierr);
ierr = PetscFree(strataIdx);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSection);CHKERRQ(ierr);
ierr = PetscSFDestroy(&labelSF);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
PetscInt ierr;
PetscSF sf;
Vec A,Aout;
Vec B,Bout;
PetscScalar *bufA;
PetscScalar *bufAout;
PetscScalar *bufB;
PetscScalar *bufBout;
PetscMPIInt rank, size;
PetscInt nroots, nleaves;
PetscInt i;
PetscInt *ilocal;
PetscSFNode *iremote;
ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 2) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_USER, "Only coded for two MPI processes\n");
ierr = PetscSFCreate(PETSC_COMM_WORLD,&sf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
nleaves = 2;
nroots = 1;
ierr = PetscMalloc1(nleaves,&ilocal);CHKERRQ(ierr);
for (i = 0; i<nleaves; i++) {
ilocal[i] = i;
}
ierr = PetscMalloc1(nleaves,&iremote);CHKERRQ(ierr);
if (rank == 0) {
iremote[0].rank = 0;
iremote[0].index = 0;
iremote[1].rank = 1;
iremote[1].index = 0;
} else {
iremote[0].rank = 1;
iremote[0].index = 0;
iremote[1].rank = 0;
iremote[1].index = 0;
}
ierr = PetscSFSetGraph(sf,nroots,nleaves,ilocal,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetUp(sf);CHKERRQ(ierr);
ierr = PetscSFView(sf,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = VecSetSizes(A,2,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetFromOptions(A);CHKERRQ(ierr);
ierr = VecSetUp(A);CHKERRQ(ierr);
ierr = VecDuplicate(A,&B);CHKERRQ(ierr);
ierr = VecDuplicate(A,&Aout);CHKERRQ(ierr);
ierr = VecDuplicate(A,&Bout);CHKERRQ(ierr);
ierr = VecGetArray(A,&bufA);CHKERRQ(ierr);
ierr = VecGetArray(B,&bufB);CHKERRQ(ierr);
for (i=0; i<2; i++) {
bufA[i] = (PetscScalar)rank;
bufB[i] = (PetscScalar)(rank) + 10.0;
}
ierr = VecRestoreArray(A,&bufA);CHKERRQ(ierr);
ierr = VecRestoreArray(B,&bufB);CHKERRQ(ierr);
ierr = VecGetArrayRead(A,(const PetscScalar**)&bufA);CHKERRQ(ierr);
ierr = VecGetArrayRead(B,(const PetscScalar**)&bufB);CHKERRQ(ierr);
ierr = VecGetArray(Aout,&bufAout);CHKERRQ(ierr);
ierr = VecGetArray(Bout,&bufBout);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_SCALAR,(const void*)bufA,(void *)bufAout);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_SCALAR,(const void*)bufB,(void *)bufBout);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_SCALAR,(const void*)bufA,(void *)bufAout);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_SCALAR,(const void*)bufB,(void *)bufBout);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(A,(const PetscScalar**)&bufA);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(B,(const PetscScalar**)&bufB);CHKERRQ(ierr);
ierr = VecRestoreArray(Aout,&bufAout);CHKERRQ(ierr);
ierr = VecRestoreArray(Bout,&bufBout);CHKERRQ(ierr);
ierr = VecView(Aout,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecView(Bout,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&A);CHKERRQ(ierr);
ierr = VecDestroy(&B);CHKERRQ(ierr);
ierr = VecDestroy(&Aout);CHKERRQ(ierr);
ierr = VecDestroy(&Bout);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
static PetscErrorCode GreedyColoringLocalDistanceOne_Private(MatColoring mc,PetscReal *wts,PetscInt *lperm,ISColoringValue *colors)
{
PetscInt i,j,k,s,e,n,no,nd,nd_global,n_global,idx,ncols,maxcolors,masksize,ccol,*mask;
PetscErrorCode ierr;
Mat m=mc->mat;
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)m->data;
Mat md=NULL,mo=NULL;
const PetscInt *md_i,*mo_i,*md_j,*mo_j;
PetscBool isMPIAIJ,isSEQAIJ;
ISColoringValue pcol;
const PetscInt *cidx;
PetscInt *lcolors,*ocolors;
PetscReal *owts=NULL;
PetscSF sf;
PetscLayout layout;
PetscFunctionBegin;
ierr = MatGetSize(m,&n_global,NULL);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(m,&s,&e);CHKERRQ(ierr);
n=e-s;
masksize=20;
nd_global = 0;
/* get the matrix communication structures */
ierr = PetscObjectTypeCompare((PetscObject)m, MATMPIAIJ, &isMPIAIJ); CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)m, MATSEQAIJ, &isSEQAIJ); CHKERRQ(ierr);
if (isMPIAIJ) {
/* get the CSR data for on and off diagonal portions of m */
Mat_SeqAIJ *dseq;
Mat_SeqAIJ *oseq;
md=aij->A;
dseq = (Mat_SeqAIJ*)md->data;
mo=aij->B;
oseq = (Mat_SeqAIJ*)mo->data;
md_i = dseq->i;
md_j = dseq->j;
mo_i = oseq->i;
mo_j = oseq->j;
} else if (isSEQAIJ) {
/* get the CSR data for m */
Mat_SeqAIJ *dseq;
/* no off-processor nodes */
md=m;
dseq = (Mat_SeqAIJ*)md->data;
mo=NULL;
no=0;
md_i = dseq->i;
md_j = dseq->j;
mo_i = NULL;
mo_j = NULL;
} else SETERRQ(PetscObjectComm((PetscObject)mc),PETSC_ERR_ARG_WRONG,"Matrix must be AIJ for greedy coloring");
ierr = MatColoringGetMaxColors(mc,&maxcolors);CHKERRQ(ierr);
if (mo) {
ierr = VecGetSize(aij->lvec,&no);CHKERRQ(ierr);
ierr = PetscMalloc2(no,&ocolors,no,&owts);CHKERRQ(ierr);
for(i=0;i<no;i++) {
ocolors[i]=maxcolors;
}
}
ierr = PetscMalloc1(masksize,&mask);CHKERRQ(ierr);
ierr = PetscMalloc1(n,&lcolors);CHKERRQ(ierr);
for(i=0;i<n;i++) {
lcolors[i]=maxcolors;
}
for (i=0;i<masksize;i++) {
mask[i]=-1;
}
if (mo) {
/* transfer neighbor weights */
ierr = PetscSFCreate(PetscObjectComm((PetscObject)m),&sf);CHKERRQ(ierr);
ierr = MatGetLayouts(m,&layout,NULL);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,no,NULL,PETSC_COPY_VALUES,aij->garray);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_REAL,wts,owts);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_REAL,wts,owts);CHKERRQ(ierr);
}
while (nd_global < n_global) {
nd=n;
/* assign lowest possible color to each local vertex */
ierr = PetscLogEventBegin(MATCOLORING_Local,mc,0,0,0);CHKERRQ(ierr);
for (i=0;i<n;i++) {
idx=lperm[i];
if (lcolors[idx] == maxcolors) {
ncols = md_i[idx+1]-md_i[idx];
cidx = &(md_j[md_i[idx]]);
for (j=0;j<ncols;j++) {
if (lcolors[cidx[j]] != maxcolors) {
ccol=lcolors[cidx[j]];
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
if (mo) {
ncols = mo_i[idx+1]-mo_i[idx];
cidx = &(mo_j[mo_i[idx]]);
for (j=0;j<ncols;j++) {
if (ocolors[cidx[j]] != maxcolors) {
ccol=ocolors[cidx[j]];
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
}
for (j=0;j<masksize;j++) {
if (mask[j]!=idx) {
break;
}
}
pcol=j;
if (pcol>maxcolors)pcol=maxcolors;
lcolors[idx]=pcol;
}
}
ierr = PetscLogEventEnd(MATCOLORING_Local,mc,0,0,0);CHKERRQ(ierr);
if (mo) {
/* transfer neighbor colors */
ierr = PetscLogEventBegin(MATCOLORING_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,lcolors,ocolors);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,lcolors,ocolors);CHKERRQ(ierr);
/* check for conflicts -- this is merely checking if any adjacent off-processor rows have the same color and marking the ones that are lower weight locally for changing */
for (i=0;i<n;i++) {
ncols = mo_i[i+1]-mo_i[i];
cidx = &(mo_j[mo_i[i]]);
for (j=0;j<ncols;j++) {
/* in the case of conflicts, the highest weight one stays and the others go */
if ((ocolors[cidx[j]] == lcolors[i]) && (owts[cidx[j]] > wts[i]) && lcolors[i] < maxcolors) {
lcolors[i]=maxcolors;
nd--;
}
}
}
nd_global=0;
}
ierr = MPIU_Allreduce(&nd,&nd_global,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)mc));CHKERRQ(ierr);
}
for (i=0;i<n;i++) {
colors[i] = (ISColoringValue)lcolors[i];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
ierr = PetscFree(lcolors);CHKERRQ(ierr);
if (mo) {
ierr = PetscFree2(ocolors,owts);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode GreedyColoringLocalDistanceTwo_Private(MatColoring mc,PetscReal *wts,PetscInt *lperm,ISColoringValue *colors)
{
MC_Greedy *gr = (MC_Greedy *) mc->data;
PetscInt i,j,k,l,s,e,n,nd,nd_global,n_global,idx,ncols,maxcolors,mcol,mcol_global,nd1cols,*mask,masksize,*d1cols,*bad,*badnext,nbad,badsize,ccol,no,cbad;
Mat m = mc->mat, mt;
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)m->data;
Mat md=NULL,mo=NULL;
const PetscInt *md_i,*mo_i,*md_j,*mo_j;
const PetscInt *rmd_i,*rmo_i,*rmd_j,*rmo_j;
PetscBool isMPIAIJ,isSEQAIJ;
PetscInt pcol,*dcolors,*ocolors;
ISColoringValue *badidx;
const PetscInt *cidx;
PetscReal *owts,*colorweights;
PetscInt *oconf,*conf;
PetscSF sf;
PetscLayout layout;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MatGetSize(m,&n_global,NULL);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(m,&s,&e);CHKERRQ(ierr);
n=e-s;
nd_global = 0;
/* get the matrix communication structures */
ierr = PetscObjectTypeCompare((PetscObject)m, MATMPIAIJ, &isMPIAIJ); CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)m, MATSEQAIJ, &isSEQAIJ); CHKERRQ(ierr);
if (isMPIAIJ) {
Mat_SeqAIJ *dseq;
Mat_SeqAIJ *oseq;
md=aij->A;
dseq = (Mat_SeqAIJ*)md->data;
mo=aij->B;
oseq = (Mat_SeqAIJ*)mo->data;
md_i = dseq->i;
md_j = dseq->j;
mo_i = oseq->i;
mo_j = oseq->j;
rmd_i = dseq->i;
rmd_j = dseq->j;
rmo_i = oseq->i;
rmo_j = oseq->j;
} else if (isSEQAIJ) {
Mat_SeqAIJ *dseq;
/* no off-processor nodes */
md=m;
dseq = (Mat_SeqAIJ*)md->data;
md_i = dseq->i;
md_j = dseq->j;
mo_i = NULL;
mo_j = NULL;
rmd_i = dseq->i;
rmd_j = dseq->j;
rmo_i = NULL;
rmo_j = NULL;
} else SETERRQ(PetscObjectComm((PetscObject)mc),PETSC_ERR_ARG_WRONG,"Matrix must be AIJ for greedy coloring");
if (!gr->symmetric) {
ierr = MatTranspose(m, MAT_INITIAL_MATRIX, &mt);CHKERRQ(ierr);
if (isSEQAIJ) {
Mat_SeqAIJ *dseq = (Mat_SeqAIJ*) mt->data;
rmd_i = dseq->i;
rmd_j = dseq->j;
rmo_i = NULL;
rmo_j = NULL;
} else SETERRQ(PetscObjectComm((PetscObject) mc), PETSC_ERR_SUP, "Nonsymmetric greedy coloring only works in serial");
}
/* create the vectors and communication structures if necessary */
no=0;
if (mo) {
ierr = VecGetLocalSize(aij->lvec,&no);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)m),&sf);CHKERRQ(ierr);
ierr = MatGetLayouts(m,&layout,NULL);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,no,NULL,PETSC_COPY_VALUES,aij->garray);CHKERRQ(ierr);
}
ierr = MatColoringGetMaxColors(mc,&maxcolors);CHKERRQ(ierr);
masksize=n;
nbad=0;
badsize=n;
ierr = PetscMalloc1(masksize,&mask);CHKERRQ(ierr);
ierr = PetscMalloc4(n,&d1cols,n,&dcolors,n,&conf,n,&bad);CHKERRQ(ierr);
ierr = PetscMalloc2(badsize,&badidx,badsize,&badnext);CHKERRQ(ierr);
for(i=0;i<masksize;i++) {
mask[i]=-1;
}
for (i=0;i<n;i++) {
dcolors[i]=maxcolors;
bad[i]=-1;
}
for (i=0;i<badsize;i++) {
badnext[i]=-1;
}
if (mo) {
ierr = PetscMalloc3(no,&owts,no,&oconf,no,&ocolors);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_REAL,wts,owts);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_REAL,wts,owts);CHKERRQ(ierr);
for (i=0;i<no;i++) {
ocolors[i]=maxcolors;
}
} else { /* Appease overzealous -Wmaybe-initialized */
owts = NULL;
oconf = NULL;
ocolors = NULL;
}
mcol=0;
while (nd_global < n_global) {
nd=n;
/* assign lowest possible color to each local vertex */
mcol_global=0;
ierr = PetscLogEventBegin(MATCOLORING_Local,mc,0,0,0);CHKERRQ(ierr);
for (i=0;i<n;i++) {
idx=lperm[i];
if (dcolors[idx] == maxcolors) {
/* entries in bad */
cbad=bad[idx];
while (cbad>=0) {
ccol=badidx[cbad];
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
cbad=badnext[cbad];
}
/* diagonal distance-one rows */
nd1cols=0;
ncols = rmd_i[idx+1]-rmd_i[idx];
cidx = &(rmd_j[rmd_i[idx]]);
for (j=0;j<ncols;j++) {
d1cols[nd1cols] = cidx[j];
nd1cols++;
ccol=dcolors[cidx[j]];
if (ccol != maxcolors) {
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
/* off-diagonal distance-one rows */
if (mo) {
ncols = rmo_i[idx+1]-rmo_i[idx];
cidx = &(rmo_j[rmo_i[idx]]);
for (j=0;j<ncols;j++) {
ccol=ocolors[cidx[j]];
if (ccol != maxcolors) {
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
}
/* diagonal distance-two rows */
for (j=0;j<nd1cols;j++) {
ncols = md_i[d1cols[j]+1]-md_i[d1cols[j]];
cidx = &(md_j[md_i[d1cols[j]]]);
for (l=0;l<ncols;l++) {
ccol=dcolors[cidx[l]];
if (ccol != maxcolors) {
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
}
/* off-diagonal distance-two rows */
if (mo) {
for (j=0;j<nd1cols;j++) {
ncols = mo_i[d1cols[j]+1]-mo_i[d1cols[j]];
cidx = &(mo_j[mo_i[d1cols[j]]]);
for (l=0;l<ncols;l++) {
ccol=ocolors[cidx[l]];
if (ccol != maxcolors) {
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
}
}
/* assign this one the lowest color possible by seeing if there's a gap in the sequence of sorted neighbor colors */
for (j=0;j<masksize;j++) {
if (mask[j]!=idx) {
break;
}
}
pcol=j;
if (pcol>maxcolors) pcol=maxcolors;
dcolors[idx]=pcol;
if (pcol>mcol) mcol=pcol;
}
}
ierr = PetscLogEventEnd(MATCOLORING_Local,mc,0,0,0);CHKERRQ(ierr);
if (mo) {
/* transfer neighbor colors */
ierr = PetscSFBcastBegin(sf,MPIU_INT,dcolors,ocolors);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,dcolors,ocolors);CHKERRQ(ierr);
/* find the maximum color assigned locally and allocate a mask */
ierr = MPIU_Allreduce(&mcol,&mcol_global,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)mc));CHKERRQ(ierr);
ierr = PetscMalloc1(mcol_global+1,&colorweights);CHKERRQ(ierr);
/* check for conflicts */
for (i=0;i<n;i++) {
conf[i]=PETSC_FALSE;
}
for (i=0;i<no;i++) {
oconf[i]=PETSC_FALSE;
}
for (i=0;i<n;i++) {
ncols = mo_i[i+1]-mo_i[i];
cidx = &(mo_j[mo_i[i]]);
if (ncols > 0) {
/* fill in the mask */
for (j=0;j<mcol_global+1;j++) {
colorweights[j]=0;
}
colorweights[dcolors[i]]=wts[i];
/* fill in the off-diagonal part of the mask */
for (j=0;j<ncols;j++) {
ccol=ocolors[cidx[j]];
if (ccol < maxcolors) {
if (colorweights[ccol] < owts[cidx[j]]) {
colorweights[ccol] = owts[cidx[j]];
}
}
}
/* fill in the on-diagonal part of the mask */
ncols = md_i[i+1]-md_i[i];
cidx = &(md_j[md_i[i]]);
for (j=0;j<ncols;j++) {
ccol=dcolors[cidx[j]];
if (ccol < maxcolors) {
if (colorweights[ccol] < wts[cidx[j]]) {
colorweights[ccol] = wts[cidx[j]];
}
}
}
/* go back through and set up on and off-diagonal conflict vectors */
ncols = md_i[i+1]-md_i[i];
cidx = &(md_j[md_i[i]]);
for (j=0;j<ncols;j++) {
ccol=dcolors[cidx[j]];
if (ccol < maxcolors) {
if (colorweights[ccol] > wts[cidx[j]]) {
conf[cidx[j]]=PETSC_TRUE;
}
}
}
ncols = mo_i[i+1]-mo_i[i];
cidx = &(mo_j[mo_i[i]]);
for (j=0;j<ncols;j++) {
ccol=ocolors[cidx[j]];
if (ccol < maxcolors) {
if (colorweights[ccol] > owts[cidx[j]]) {
oconf[cidx[j]]=PETSC_TRUE;
}
}
}
}
}
nd_global=0;
ierr = PetscFree(colorweights);CHKERRQ(ierr);
ierr = PetscLogEventBegin(MATCOLORING_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sf,MPIU_INT,oconf,conf,MPIU_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sf,MPIU_INT,oconf,conf,MPIU_SUM);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MATCOLORING_Comm,mc,0,0,0);CHKERRQ(ierr);
/* go through and unset local colors that have conflicts */
for (i=0;i<n;i++) {
if (conf[i]>0) {
/* push this color onto the bad stack */
badidx[nbad]=dcolors[i];
badnext[nbad]=bad[i];
bad[i]=nbad;
nbad++;
if (nbad>=badsize) {
PetscInt *newbadnext;
ISColoringValue *newbadidx;
ierr = PetscMalloc2(badsize*2,&newbadidx,badsize*2,&newbadnext);CHKERRQ(ierr);
for(k=0;k<2*badsize;k++) {
newbadnext[k]=-1;
}
for(k=0;k<badsize;k++) {
newbadidx[k]=badidx[k];
newbadnext[k]=badnext[k];
}
ierr = PetscFree2(badidx,badnext);CHKERRQ(ierr);
badidx=newbadidx;
badnext=newbadnext;
badsize*=2;
}
dcolors[i] = maxcolors;
nd--;
}
}
}
ierr = MPIU_Allreduce(&nd,&nd_global,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)mc));CHKERRQ(ierr);
}
if (mo) {
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree3(owts,oconf,ocolors);CHKERRQ(ierr);
}
for (i=0;i<n;i++) {
colors[i]=dcolors[i];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
ierr = PetscFree4(d1cols,dcolors,conf,bad);CHKERRQ(ierr);
ierr = PetscFree2(badidx,badnext);CHKERRQ(ierr);
if (!gr->symmetric) {ierr = MatDestroy(&mt);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
/* This interpolates the PointSF in parallel following local interpolation */
static PetscErrorCode DMPlexInterpolatePointSF(DM dm, PetscSF pointSF, PetscInt depth)
{
PetscMPIInt numProcs, rank;
PetscInt p, c, d, dof, offset;
PetscInt numLeaves, numRoots, candidatesSize, candidatesRemoteSize;
const PetscInt *localPoints;
const PetscSFNode *remotePoints;
PetscSFNode *candidates, *candidatesRemote, *claims;
PetscSection candidateSection, candidateSectionRemote, claimSection;
PetscHashI leafhash;
PetscHashIJ roothash;
PetscHashIJKey key;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject) dm), &numProcs);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
if (numProcs < 2 || numRoots < 0) PetscFunctionReturn(0);
/* Build hashes of points in the SF for efficient lookup */
PetscHashICreate(leafhash);
PetscHashIJCreate(&roothash);
ierr = PetscHashIJSetMultivalued(roothash, PETSC_FALSE);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
PetscHashIAdd(leafhash, localPoints[p], p);
key.i = remotePoints[p].index; key.j = remotePoints[p].rank;
PetscHashIJAdd(roothash, key, p);
}
/* Build a section / SFNode array of candidate points in the single-level adjacency of leaves,
where each candidate is defined by the root entry for the other vertex that defines the edge. */
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &candidateSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(candidateSection, 0, numRoots);CHKERRQ(ierr);
{
PetscInt leaf, root, idx, a, *adj = NULL;
for (p = 0; p < numLeaves; ++p) {
PetscInt adjSize = PETSC_DETERMINE;
ierr = DMPlexGetAdjacency_Internal(dm, localPoints[p], PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
for (a = 0; a < adjSize; ++a) {
PetscHashIMap(leafhash, adj[a], leaf);
if (leaf >= 0) {ierr = PetscSectionAddDof(candidateSection, localPoints[p], 1);CHKERRQ(ierr);}
}
}
ierr = PetscSectionSetUp(candidateSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(candidateSection, &candidatesSize);CHKERRQ(ierr);
ierr = PetscMalloc1(candidatesSize, &candidates);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
PetscInt adjSize = PETSC_DETERMINE;
ierr = PetscSectionGetOffset(candidateSection, localPoints[p], &offset);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency_Internal(dm, localPoints[p], PETSC_FALSE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj);CHKERRQ(ierr);
for (idx = 0, a = 0; a < adjSize; ++a) {
PetscHashIMap(leafhash, adj[a], root);
if (root >= 0) candidates[offset+idx++] = remotePoints[root];
}
}
ierr = PetscFree(adj);CHKERRQ(ierr);
}
/* Gather candidate section / array pair into the root partition via inverse(multi(pointSF)). */
{
PetscSF sfMulti, sfInverse, sfCandidates;
PetscInt *remoteOffsets;
ierr = PetscSFGetMultiSF(pointSF, &sfMulti);CHKERRQ(ierr);
ierr = PetscSFCreateInverseSF(sfMulti, &sfInverse);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sfInverse, candidateSection, &remoteOffsets, candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfInverse, candidateSection, remoteOffsets, candidateSectionRemote, &sfCandidates);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(candidateSectionRemote, &candidatesRemoteSize);CHKERRQ(ierr);
ierr = PetscMalloc1(candidatesRemoteSize, &candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfCandidates, MPIU_2INT, candidates, candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfCandidates, MPIU_2INT, candidates, candidatesRemote);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfInverse);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfCandidates);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
}
/* Walk local roots and check for each remote candidate whether we know all required points,
either from owning it or having a root entry in the point SF. If we do we place a claim
by replacing the vertex number with our edge ID. */
{
PetscInt idx, root, joinSize, vertices[2];
const PetscInt *rootdegree, *join = NULL;
ierr = PetscSFComputeDegreeBegin(pointSF, &rootdegree);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(pointSF, &rootdegree);CHKERRQ(ierr);
/* Loop remote edge connections and put in a claim if both vertices are known */
for (idx = 0, p = 0; p < numRoots; ++p) {
for (d = 0; d < rootdegree[p]; ++d) {
ierr = PetscSectionGetDof(candidateSectionRemote, idx, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(candidateSectionRemote, idx, &offset);CHKERRQ(ierr);
for (c = 0; c < dof; ++c) {
/* We own both vertices, so we claim the edge by replacing vertex with edge */
if (candidatesRemote[offset+c].rank == rank) {
vertices[0] = p; vertices[1] = candidatesRemote[offset+c].index;
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) candidatesRemote[offset+c].index = join[0];
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
continue;
}
/* If we own one vertex and share a root with the other, we claim it */
key.i = candidatesRemote[offset+c].index; key.j = candidatesRemote[offset+c].rank;
PetscHashIJGet(roothash, key, &root);
if (root >= 0) {
vertices[0] = p; vertices[1] = localPoints[root];
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) {
candidatesRemote[offset+c].index = join[0];
candidatesRemote[offset+c].rank = rank;
}
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
}
}
idx++;
}
}
}
/* Push claims back to receiver via the MultiSF and derive new pointSF mapping on receiver */
{
PetscSF sfMulti, sfClaims, sfPointNew;
PetscHashI claimshash;
PetscInt size, pStart, pEnd, root, joinSize, numLocalNew;
PetscInt *remoteOffsets, *localPointsNew, vertices[2];
const PetscInt *join = NULL;
PetscSFNode *remotePointsNew;
ierr = PetscSFGetMultiSF(pointSF, &sfMulti);CHKERRQ(ierr);
ierr = PetscSectionCreate(PetscObjectComm((PetscObject) dm), &claimSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(sfMulti, candidateSectionRemote, &remoteOffsets, claimSection);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfMulti, candidateSectionRemote, remoteOffsets, claimSection, &sfClaims);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(claimSection, &size);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &claims);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sfClaims, MPIU_2INT, candidatesRemote, claims);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sfClaims, MPIU_2INT, candidatesRemote, claims);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfClaims);CHKERRQ(ierr);
ierr = PetscFree(remoteOffsets);CHKERRQ(ierr);
/* Walk the original section of local supports and add an SF entry for each updated item */
PetscHashICreate(claimshash);
for (p = 0; p < numRoots; ++p) {
ierr = PetscSectionGetDof(candidateSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(candidateSection, p, &offset);CHKERRQ(ierr);
for (d = 0; d < dof; ++d) {
if (candidates[offset+d].index != claims[offset+d].index) {
key.i = candidates[offset+d].index; key.j = candidates[offset+d].rank;
PetscHashIJGet(roothash, key, &root);
if (root >= 0) {
vertices[0] = p; vertices[1] = localPoints[root];
ierr = DMPlexGetJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
if (joinSize == 1) PetscHashIAdd(claimshash, join[0], offset+d);
ierr = DMPlexRestoreJoin(dm, 2, vertices, &joinSize, &join);CHKERRQ(ierr);
}
}
}
}
/* Create new pointSF from hashed claims */
PetscHashISize(claimshash, numLocalNew);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeaves + numLocalNew, &localPointsNew);CHKERRQ(ierr);
ierr = PetscMalloc1(numLeaves + numLocalNew, &remotePointsNew);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
localPointsNew[p] = localPoints[p];
remotePointsNew[p].index = remotePoints[p].index;
remotePointsNew[p].rank = remotePoints[p].rank;
}
p = numLeaves; ierr = PetscHashIGetKeys(claimshash, &p, localPointsNew);CHKERRQ(ierr);
for (p = numLeaves; p < numLeaves + numLocalNew; ++p) {
PetscHashIMap(claimshash, localPointsNew[p], offset);
remotePointsNew[p] = claims[offset];
}
ierr = PetscSFCreate(PetscObjectComm((PetscObject) dm), &sfPointNew);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sfPointNew, pEnd-pStart, numLeaves+numLocalNew, localPointsNew, PETSC_OWN_POINTER, remotePointsNew, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = DMSetPointSF(dm, sfPointNew);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sfPointNew);CHKERRQ(ierr);
PetscHashIDestroy(claimshash);
}
PetscHashIDestroy(leafhash);
ierr = PetscHashIJDestroy(&roothash);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&candidateSection);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&candidateSectionRemote);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&claimSection);CHKERRQ(ierr);
ierr = PetscFree(candidates);CHKERRQ(ierr);
ierr = PetscFree(candidatesRemote);CHKERRQ(ierr);
ierr = PetscFree(claims);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
