bool Foam::sampledTriSurfaceMesh::update()
{
if (!needsUpdate_)
{
return false;
}
// Find the cells the triangles of the surface are in.
// Does approximation by looking at the face centres only
const pointField& fc = surface_.faceCentres();
meshSearch meshSearcher(mesh(), false);
const indexedOctree<treeDataPoint>& cellCentreTree =
meshSearcher.cellCentreTree();
// Global numbering for cells - only used to uniquely identify local cells.
globalIndex globalCells(mesh().nCells());
List<nearInfo> nearest(fc.size());
forAll(nearest, i)
{
nearest[i].first() = GREAT;
nearest[i].second() = labelMax;
}
Foam::labelList Foam::refinementParameters::findCells(const polyMesh& mesh)
const
{
// Force calculation of tet-diag decomposition (for use in findCell)
(void)mesh.tetBasePtIs();
// Global calculation engine
globalIndex globalCells(mesh.nCells());
// Cell label per point
labelList cellLabels(keepPoints_.size());
forAll(keepPoints_, i)
{
const point& keepPoint = keepPoints_[i];
label localCellI = mesh.findCell(keepPoint);
label globalCellI = -1;
if (localCellI != -1)
{
globalCellI = globalCells.toGlobal(localCellI);
}
reduce(globalCellI, maxOp<label>());
if (globalCellI == -1)
{
FatalErrorInFunction
<< "Point " << keepPoint
<< " is not inside the mesh or on a face or edge." << nl
<< "Bounding box of the mesh:" << mesh.bounds()
<< exit(FatalError);
}
label procI = globalCells.whichProcID(globalCellI);
label procCellI = globalCells.toLocal(procI, globalCellI);
Info<< "Found point " << keepPoint << " in cell " << procCellI
<< " on processor " << procI << endl;
if (globalCells.isLocal(globalCellI))
{
cellLabels[i] = localCellI;
}
else
{
cellLabels[i] = -1;
}
}
return cellLabels;
}
Foam::labelList Foam::refinementParameters::findCells(const polyMesh& mesh)
const
{
// Global calculation engine
globalIndex globalCells(mesh.nCells());
// Cell label per point
labelList cellLabels(keepPoints_.size());
forAll(keepPoints_, i)
{
const point& keepPoint = keepPoints_[i];
label localCellI = mesh.findCell(keepPoint);
label globalCellI = -1;
if (localCellI != -1)
{
Pout<< "Found point " << keepPoint << " in cell " << localCellI
<< " on processor " << Pstream::myProcNo() << endl;
globalCellI = globalCells.toGlobal(localCellI);
}
reduce(globalCellI, maxOp<label>());
if (globalCellI == -1)
{
FatalErrorIn
(
"refinementParameters::findCells(const polyMesh&) const"
) << "Point " << keepPoint
<< " is not inside the mesh or on a face or edge." << nl
<< "Bounding box of the mesh:" << mesh.bounds()
<< exit(FatalError);
}
if (globalCells.isLocal(globalCellI))
{
cellLabels[i] = localCellI;
}
else
{
cellLabels[i] = -1;
}
}
return cellLabels;
}
// Call scotch with options from dictionary.
Foam::label Foam::ptscotchDecomp::decompose
(
const fileName& meshPath,
const label adjncySize,
const label adjncy[],
const label xadjSize,
const label xadj[],
const scalarField& cWeights,
List<label>& finalDecomp
) const
{
if (debug)
{
Pout<< "ptscotchDecomp : entering with xadj:" << xadjSize << endl;
}
// Dump graph
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
if (scotchCoeffs.lookupOrDefault("writeGraph", false))
{
OFstream str
(
meshPath + "_" + Foam::name(Pstream::myProcNo()) + ".dgr"
);
Pout<< "Dumping Scotch graph file to " << str.name() << endl
<< "Use this in combination with dgpart." << endl;
globalIndex globalCells(xadjSize-1);
// Distributed graph file (.grf)
label version = 2;
str << version << nl;
// Number of files (procglbnbr)
str << Pstream::nProcs();
// My file number (procloc)
str << ' ' << Pstream::myProcNo() << nl;
// Total number of vertices (vertglbnbr)
str << globalCells.size();
// Total number of connections (edgeglbnbr)
str << ' ' << returnReduce(xadj[xadjSize-1], sumOp<label>())
<< nl;
// Local number of vertices (vertlocnbr)
str << xadjSize-1;
// Local number of connections (edgelocnbr)
str << ' ' << xadj[xadjSize-1] << nl;
// Numbering starts from 0
label baseval = 0;
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs
str << baseval << ' ' << "000" << nl;
for (label celli = 0; celli < xadjSize-1; celli++)
{
label start = xadj[celli];
label end = xadj[celli+1];
str << end-start;
for (label i = start; i < end; i++)
{
str << ' ' << adjncy[i];
}
str << nl;
}
}
}
// Strategy
// ~~~~~~~~
// Default.
SCOTCH_Strat stradat;
check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
string strategy;
if (scotchCoeffs.readIfPresent("strategy", strategy))
{
if (debug)
{
Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
}
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
//fprintf(stdout, "\n");
}
}
// Graph
// ~~~~~
List<label> velotab;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
scalar maxWeights = gMax(cWeights);
if (maxWeights > minWeights)
{
if (minWeights <= 0)
{
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != xadjSize-1)
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << xadjSize-1
<< exit(FatalError);
}
}
scalar velotabSum = gSum(cWeights)/minWeights;
scalar rangeScale(1.0);
if (Pstream::master())
{
if (velotabSum > scalar(labelMax - 1))
{
// 0.9 factor of safety to avoid floating point round-off in
// rangeScale tipping the subsequent sum over the integer limit.
rangeScale = 0.9*scalar(labelMax - 1)/velotabSum;
WarningInFunction
<< "Sum of weights has overflowed integer: " << velotabSum
<< ", compressing weight scale by a factor of " << rangeScale
<< endl;
}
}
Pstream::scatter(rangeScale);
if (maxWeights > minWeights)
{
if (cWeights.size())
{
// Convert to integers.
velotab.setSize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int((cWeights[i]/minWeights - 1)*rangeScale) + 1;
}
}
Foam::label Foam::scotchDecomp::decompose
(
const fileName& meshPath,
const List<label>& adjncy,
const List<label>& xadj,
const scalarField& cWeights,
List<label>& finalDecomp
)
{
if (!Pstream::parRun())
{
decomposeOneProc
(
meshPath,
adjncy,
xadj,
cWeights,
finalDecomp
);
}
else
{
if (debug)
{
Info<< "scotchDecomp : running in parallel."
<< " Decomposing all of graph on master processor." << endl;
}
globalIndex globalCells(xadj.size()-1);
label nTotalConnections = returnReduce(adjncy.size(), sumOp<label>());
// Send all to master. Use scheduled to save some storage.
if (Pstream::master())
{
Field<label> allAdjncy(nTotalConnections);
Field<label> allXadj(globalCells.size()+1);
scalarField allWeights(globalCells.size());
// Insert my own
label nTotalCells = 0;
forAll(cWeights, cellI)
{
allXadj[nTotalCells] = xadj[cellI];
allWeights[nTotalCells++] = cWeights[cellI];
}
nTotalConnections = 0;
forAll(adjncy, i)
{
allAdjncy[nTotalConnections++] = adjncy[i];
}
for (int slave=1; slave<Pstream::nProcs(); slave++)
{
IPstream fromSlave(Pstream::scheduled, slave);
Field<label> nbrAdjncy(fromSlave);
Field<label> nbrXadj(fromSlave);
scalarField nbrWeights(fromSlave);
// Append.
//label procStart = nTotalCells;
forAll(nbrXadj, cellI)
{
allXadj[nTotalCells] = nTotalConnections+nbrXadj[cellI];
allWeights[nTotalCells++] = nbrWeights[cellI];
}
// No need to renumber xadj since already global.
forAll(nbrAdjncy, i)
{
allAdjncy[nTotalConnections++] = nbrAdjncy[i];
}
}
allXadj[nTotalCells] = nTotalConnections;
Field<label> allFinalDecomp;
decomposeOneProc
(
meshPath,
allAdjncy,
allXadj,
allWeights,
allFinalDecomp
);
// Send allFinalDecomp back
for (int slave=1; slave<Pstream::nProcs(); slave++)
{
OPstream toSlave(Pstream::scheduled, slave);
toSlave << SubField<label>
(
allFinalDecomp,
globalCells.localSize(slave),
globalCells.offset(slave)
);
}
// Get my own part (always first)
finalDecomp = SubField<label>
(
allFinalDecomp,
globalCells.localSize()
);
}
// Call scotch with options from dictionary.
Foam::label Foam::ptscotchDecomp::decompose
(
const fileName& meshPath,
const List<int>& adjncy,
const List<int>& xadj,
const scalarField& cWeights,
List<int>& finalDecomp
) const
{
if (debug)
{
Pout<< "ptscotchDecomp : entering with xadj:" << xadj.size() << endl;
}
// Dump graph
if (decompositionDict_.found("ptscotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("ptscotchCoeffs");
if (scotchCoeffs.lookupOrDefault("writeGraph", false))
{
OFstream str
(
meshPath + "_" + Foam::name(Pstream::myProcNo()) + ".dgr"
);
Pout<< "Dumping Scotch graph file to " << str.name() << endl
<< "Use this in combination with dgpart." << endl;
globalIndex globalCells(xadj.size()-1);
// Distributed graph file (.grf)
label version = 2;
str << version << nl;
// Number of files (procglbnbr)
str << Pstream::nProcs();
// My file number (procloc)
str << ' ' << Pstream::myProcNo() << nl;
// Total number of vertices (vertglbnbr)
str << globalCells.size();
// Total number of connections (edgeglbnbr)
str << ' ' << returnReduce(xadj[xadj.size()-1], sumOp<label>())
<< nl;
// Local number of vertices (vertlocnbr)
str << xadj.size()-1;
// Local number of connections (edgelocnbr)
str << ' ' << xadj[xadj.size()-1] << nl;
// Numbering starts from 0
label baseval = 0;
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs
str << baseval << ' ' << "000" << nl;
for (label cellI = 0; cellI < xadj.size()-1; cellI++)
{
label start = xadj[cellI];
label end = xadj[cellI+1];
str << end-start;
for (label i = start; i < end; i++)
{
str << ' ' << adjncy[i];
}
str << nl;
}
}
}
// Strategy
// ~~~~~~~~
// Default.
SCOTCH_Strat stradat;
check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
string strategy;
if (scotchCoeffs.readIfPresent("strategy", strategy))
{
if (debug)
{
Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
}
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
//fprintf(stdout, "\n");
}
}
// Graph
// ~~~~~
List<int> velotab;
// Check for externally provided cellweights and if so initialise weights
scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
{
if (minWeights <= 0)
{
WarningIn
(
"ptscotchDecomp::decompose(..)"
) << "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != xadj.size()-1)
{
FatalErrorIn
(
"ptscotchDecomp::decompose(..)"
) << "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << xadj.size()-1
<< exit(FatalError);
}
// Convert to integers.
velotab.setSize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int(cWeights[i]/minWeights);
}
}
if (debug)
{
Pout<< "SCOTCH_dgraphInit" << endl;
}
SCOTCH_Dgraph grafdat;
check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
if (debug)
{
Pout<< "SCOTCH_dgraphBuild with:" << nl
<< "xadj.size()-1 : " << xadj.size()-1 << nl
<< "xadj : " << long(xadj.begin()) << nl
<< "velotab : " << long(velotab.begin()) << nl
<< "adjncy.size() : " << adjncy.size() << nl
<< "adjncy : " << long(adjncy.begin()) << nl
<< endl;
}
check
(
SCOTCH_dgraphBuild
(
&grafdat, // grafdat
0, // baseval, c-style numbering
xadj.size()-1, // vertlocnbr, nCells
xadj.size()-1, // vertlocmax
const_cast<SCOTCH_Num*>(xadj.begin()),
// vertloctab, start index per cell into
// adjncy
const_cast<SCOTCH_Num*>(&xadj[1]),// vendloctab, end index ,,
const_cast<SCOTCH_Num*>(velotab.begin()),// veloloctab, vtx weights
NULL, // vlblloctab
adjncy.size(), // edgelocnbr, number of arcs
adjncy.size(), // edgelocsiz
const_cast<SCOTCH_Num*>(adjncy.begin()), // edgeloctab
NULL, // edgegsttab
NULL // edlotab, edge weights
),
"SCOTCH_dgraphBuild"
);
if (debug)
{
Pout<< "SCOTCH_dgraphCheck" << endl;
}
check(SCOTCH_dgraphCheck(&grafdat), "SCOTCH_dgraphCheck");
// Architecture
// ~~~~~~~~~~~~
// (fully connected network topology since using switch)
if (debug)
{
Pout<< "SCOTCH_archInit" << endl;
}
SCOTCH_Arch archdat;
check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
List<label> processorWeights;
if (decompositionDict_.found("scotchCoeffs"))
{
const dictionary& scotchCoeffs =
decompositionDict_.subDict("scotchCoeffs");
scotchCoeffs.readIfPresent("processorWeights", processorWeights);
}
if (processorWeights.size())
{
if (debug)
{
Info<< "ptscotchDecomp : Using procesor weights "
<< processorWeights
<< endl;
}
check
(
SCOTCH_archCmpltw(&archdat, nProcessors_, processorWeights.begin()),
"SCOTCH_archCmpltw"
);
}
else
{
if (debug)
{
Pout<< "SCOTCH_archCmplt" << endl;
}
check
(
SCOTCH_archCmplt(&archdat, nProcessors_),
"SCOTCH_archCmplt"
);
}
//SCOTCH_Mapping mapdat;
//SCOTCH_dgraphMapInit(&grafdat, &mapdat, &archdat, NULL);
//SCOTCH_dgraphMapCompute(&grafdat, &mapdat, &stradat); /*Perform mapping*/
//SCOTCHdgraphMapExit(&grafdat, &mapdat);
// Hack:switch off fpu error trapping
# ifdef LINUX_GNUC
int oldExcepts = fedisableexcept
(
FE_DIVBYZERO
| FE_INVALID
| FE_OVERFLOW
);
# endif
if (debug)
{
Pout<< "SCOTCH_dgraphMap" << endl;
}
finalDecomp.setSize(xadj.size()-1);
finalDecomp = 0;
check
(
SCOTCH_dgraphMap
(
&grafdat,
&archdat,
&stradat, // const SCOTCH_Strat *
finalDecomp.begin() // parttab
),
"SCOTCH_graphMap"
);
# ifdef LINUX_GNUC
feenableexcept(oldExcepts);
# endif
//finalDecomp.setSize(xadj.size()-1);
//check
//(
// SCOTCH_dgraphPart
// (
// &grafdat,
// nProcessors_, // partnbr
// &stradat, // const SCOTCH_Strat *
// finalDecomp.begin() // parttab
// ),
// "SCOTCH_graphPart"
//);
if (debug)
{
Pout<< "SCOTCH_dgraphExit" << endl;
}
// Release storage for graph
SCOTCH_dgraphExit(&grafdat);
// Release storage for strategy
SCOTCH_stratExit(&stradat);
// Release storage for network topology
SCOTCH_archExit(&archdat);
return 0;
}
//- Does prevention of 0 cell domains and calls ptscotch.
Foam::label Foam::ptscotchDecomp::decomposeZeroDomains
(
const fileName& meshPath,
const List<int>& initadjncy,
const List<int>& initxadj,
const scalarField& initcWeights,
List<int>& finalDecomp
) const
{
globalIndex globalCells(initxadj.size()-1);
bool hasZeroDomain = false;
for (label procI = 0; procI < Pstream::nProcs(); procI++)
{
if (globalCells.localSize(procI) == 0)
{
hasZeroDomain = true;
break;
}
}
if (!hasZeroDomain)
{
return decompose
(
meshPath,
initadjncy,
initxadj,
initcWeights,
finalDecomp
);
}
if (debug)
{
Info<< "ptscotchDecomp : have graphs with locally 0 cells."
<< " trickling down." << endl;
}
// Make sure every domain has at least one cell
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// (scotch does not like zero sized domains)
// Trickle cells from processors that have them up to those that
// don't.
// Number of cells to send to the next processor
// (is same as number of cells next processor has to receive)
List<int> nSendCells(Pstream::nProcs(), 0);
for (label procI = nSendCells.size()-1; procI >=1; procI--)
{
label nLocalCells = globalCells.localSize(procI);
if (nLocalCells-nSendCells[procI] < 1)
{
nSendCells[procI-1] = nSendCells[procI]-nLocalCells+1;
}
}
// First receive (so increasing the sizes of all arrays)
Field<int> xadj(initxadj);
Field<int> adjncy(initadjncy);
scalarField cWeights(initcWeights);
if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
{
// Receive cells from previous processor
IPstream fromPrevProc(Pstream::blocking, Pstream::myProcNo()-1);
Field<int> prevXadj(fromPrevProc);
Field<int> prevAdjncy(fromPrevProc);
scalarField prevCellWeights(fromPrevProc);
if (prevXadj.size() != nSendCells[Pstream::myProcNo()-1])
{
FatalErrorIn("ptscotchDecomp::decompose(..)")
<< "Expected from processor " << Pstream::myProcNo()-1
<< " connectivity for " << nSendCells[Pstream::myProcNo()-1]
<< " nCells but only received " << prevXadj.size()
<< abort(FatalError);
}
// Insert adjncy
prepend(prevAdjncy, adjncy);
// Adapt offsets and prepend xadj
xadj += prevAdjncy.size();
prepend(prevXadj, xadj);
// Weights
prepend(prevCellWeights, cWeights);
}
// Send to my next processor
if (nSendCells[Pstream::myProcNo()] > 0)
{
// Send cells to next processor
OPstream toNextProc(Pstream::blocking, Pstream::myProcNo()+1);
int nCells = nSendCells[Pstream::myProcNo()];
int startCell = xadj.size()-1 - nCells;
int startFace = xadj[startCell];
int nFaces = adjncy.size()-startFace;
// Send for all cell data: last nCells elements
// Send for all face data: last nFaces elements
toNextProc
<< Field<int>::subField(xadj, nCells, startCell)-startFace
<< Field<int>::subField(adjncy, nFaces, startFace)
<<
(
cWeights.size()
? static_cast<const scalarField&>
(
scalarField::subField(cWeights, nCells, startCell)
)
: scalarField(0)
);
// Remove data that has been sent
if (cWeights.size())
{
cWeights.setSize(cWeights.size()-nCells);
}
adjncy.setSize(adjncy.size()-nFaces);
xadj.setSize(xadj.size() - nCells);
}
// Do decomposition as normal. Sets finalDecomp.
label result = decompose(meshPath, adjncy, xadj, cWeights, finalDecomp);
if (debug)
{
Info<< "ptscotchDecomp : have graphs with locally 0 cells."
<< " trickling up." << endl;
}
// If we sent cells across make sure we undo it
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// Receive back from next processor if I sent something
if (nSendCells[Pstream::myProcNo()] > 0)
{
IPstream fromNextProc(Pstream::blocking, Pstream::myProcNo()+1);
List<int> nextFinalDecomp(fromNextProc);
if (nextFinalDecomp.size() != nSendCells[Pstream::myProcNo()])
{
FatalErrorIn("parMetisDecomp::decompose(..)")
<< "Expected from processor " << Pstream::myProcNo()+1
<< " decomposition for " << nSendCells[Pstream::myProcNo()]
<< " nCells but only received " << nextFinalDecomp.size()
<< abort(FatalError);
}
append(nextFinalDecomp, finalDecomp);
}
// Send back to previous processor.
if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
{
OPstream toPrevProc(Pstream::blocking, Pstream::myProcNo()-1);
int nToPrevious = nSendCells[Pstream::myProcNo()-1];
toPrevProc <<
SubList<int>
(
finalDecomp,
nToPrevious,
finalDecomp.size()-nToPrevious
);
// Remove locally what has been sent
finalDecomp.setSize(finalDecomp.size()-nToPrevious);
}
return result;
}
// Given a subset of cells determine the new global indices. The problem
// is in the cells from neighbouring processors which need to be renumbered.
void Foam::multiLevelDecomp::subsetGlobalCellCells
(
const label nDomains,
const label domainI,
const labelList& dist,
const labelListList& cellCells,
const labelList& set,
labelListList& subCellCells,
labelList& cutConnections
) const
{
// Determine new index for cells by inverting subset
labelList oldToNew(invert(cellCells.size(), set));
globalIndex globalCells(cellCells.size());
// Subset locally the elements for which I have data
subCellCells = UIndirectList<labelList>(cellCells, set);
// Get new indices for neighbouring processors
List<Map<label>> compactMap;
mapDistribute map(globalCells, subCellCells, compactMap);
map.distribute(oldToNew);
labelList allDist(dist);
map.distribute(allDist);
// Now we have:
// oldToNew : the locally-compact numbering of all our cellCells. -1 if
// cellCell is not in set.
// allDist : destination domain for all our cellCells
// subCellCells : indexes into oldToNew and allDist
// Globally compact numbering for cells in set.
globalIndex globalSubCells(set.size());
// Now subCellCells contains indices into oldToNew which are the
// new locations of the neighbouring cells.
cutConnections.setSize(nDomains);
cutConnections = 0;
forAll(subCellCells, subCelli)
{
labelList& cCells = subCellCells[subCelli];
// Keep the connections to valid mapped cells
label newI = 0;
forAll(cCells, i)
{
// Get locally-compact cell index of neighbouring cell
label nbrCelli = oldToNew[cCells[i]];
if (nbrCelli == -1)
{
cutConnections[allDist[cCells[i]]]++;
}
else
{
// Reconvert local cell index into global one
// Get original neighbour
label celli = set[subCelli];
label oldNbrCelli = cellCells[celli][i];
// Get processor from original neighbour
label proci = globalCells.whichProcID(oldNbrCelli);
// Convert into global compact numbering
cCells[newI++] = globalSubCells.toGlobal(proci, nbrCelli);
}
}