Foam::lduSolverPerformance Foam::cufflink_CG::solve
(
scalarField& x,
const scalarField& b,
const direction cmpt
) const
{
OFSolverPerformance OFSP;//container for solver performance
#include "../CFL_Headers/getGPUStorage.H"
// --- Setup class containing solver performance data
lduSolverPerformance solverPerf("cufflink_CG",fieldName());
register label nCells = x.size();
register label NFaces = matrix().lower().size();
OFSP.nCells = nCells;
OFSP.nFaces = NFaces;
OFSP.debugCusp = false;
if (lduMatrix::debug >= 2) {
OFSP.debugCusp = true;
}
#include "../CFL_Headers/initializeCusp.H"
CFL_CG(&CES, &OFSP);//call the CUDA code to solve the system
thrust::copy(CES.X.begin(),CES.X.end(),x.begin());//copy the x vector back to Openfoam
solverPerf.initialResidual() = OFSP.iRes;//return initial residual
solverPerf.finalResidual() = OFSP.fRes;//return final residual
solverPerf.nIterations() = OFSP.nIterations;//return the number of iterations
//solverPerf.checkConvergence(tolerance(), relTolerance()) ; what should be passed here and does this waste time?
return solverPerf;
}
Foam::lduSolverPerformance Foam::cufflink_DiagPCG_Parallel::solve
(
scalarField& x,
const scalarField& b,
const direction cmpt
) const
{
if (!Pstream::parRun()) {
FatalErrorIn("cufflink:\nMulti-GPU Solver cannot be run serially. Choose the serial version of the solver")<< "Fatal Error"<< abort(FatalError);
}
OFSolverPerformance OFSP;//container for solver performance
#include "../CFL_Headers/getGPUStorage.H"
// --- Setup class containing solver performance data
lduSolverPerformance solverPerf("cufflink_DiagPCG_Parallel",fieldName());
register label nCells = x.size();
register label NFaces = matrix().lower().size();
OFSP.nCells = nCells;
OFSP.nFaces = NFaces;
OFSP.debugCusp = false;
if (lduMatrix::debug >= 2) {
OFSP.debugCusp = true;
}
//interface gathering
//used when launching kernals in machines with multiple gpus
int gpusPerMachine = readInt(dict().lookup("gpusPerMachine"));
//get all the interface information
#include "../CFL_Headers/getInterfaces.H"
OFInterfaces.gpusPerMachine = gpusPerMachine;
if(!OFInterfaces.checkGPUCount(OFInterfaces.gpusPerMachine)) {
FatalErrorIn("checkGPUCount:\nThe number of GPUs per machine is not correct.\nChange the gpusPerMachine variable in fvSolution")<< "Fatal Error"<< abort(FatalError);
}
//end interface gathering
#include "../CFL_Headers/initializeCusp.H"
//set the device
cudaError_t cudareturn;
cudareturn = cudaSetDevice(Pstream::myProcNo() % (gpusPerMachine));
if (cudareturn == cudaErrorInvalidDevice){ perror("cudaSetDevice returned cudaErrorInvalidDevice"); }
CFL_DiagPCG_Parallel(&CES, &OFSP, &OFInterfaces);//call the CUDA code to solve the system
thrust::copy(CES.X.begin(),CES.X.end(),x.begin());//copy the x vector back to Openfoam
solverPerf.initialResidual() = OFSP.iRes;//return initial residual
solverPerf.finalResidual() = OFSP.fRes;//return final residual
solverPerf.nIterations() = OFSP.nIterations;//return the number of iterations
//solverPerf.checkConvergence(tolerance(), relTolerance()) ; what should be passed here and does this waste time?
return solverPerf;
}
Foam::lduSolverPerformance Foam::bicgStabSolver::solve
(
scalarField& x,
const scalarField& b,
const direction cmpt
) const
{
// Prepare solver performance
lduSolverPerformance solverPerf(typeName, fieldName());
scalarField p(x.size());
scalarField r(x.size());
// Calculate initial residual
matrix_.Amul(p, x, coupleBouCoeffs_, interfaces_, cmpt);
scalar normFactor = this->normFactor(x, b, p, r, cmpt);
if (lduMatrix::debug >= 2)
{
Info<< " Normalisation factor = " << normFactor << endl;
}
// Calculate residual
forAll (r, i)
{
r[i] = b[i] - p[i];
}
Foam::lduSolverPerformance Foam::amgSolver::solve
(
scalarField& x,
const scalarField& b,
const direction cmpt
) const
{
// Prepare solver performance
lduSolverPerformance solverPerf(typeName, fieldName());
scalar norm = this->normFactor(x, b, cmpt);
// Calculate initial residual
solverPerf.initialResidual() = gSumMag(amg_.residual(x, b, cmpt))/norm;
solverPerf.finalResidual() = solverPerf.initialResidual();
if (!stop(solverPerf))
{
do
{
amg_.cycle(x, b, cmpt);
// Re-calculate residual
solverPerf.finalResidual() =
gSumMag(amg_.residual(x, b, cmpt))/norm;
solverPerf.nIterations()++;
} while (!stop(solverPerf));
}
return solverPerf;
}
Foam::lduSolverPerformance Foam::gmresSolver::solve
(
scalarField& x,
const scalarField& b,
const direction cmpt
) const
{
// Prepare solver performance
lduSolverPerformance solverPerf(typeName, fieldName());
scalarField wA(x.size());
scalarField rA(x.size());
// Calculate initial residual
matrix_.Amul(wA, x, coupleBouCoeffs_, interfaces_, cmpt);
// Use rA as scratch space when calculating the normalisation factor
scalar normFactor = this->normFactor(x, b, wA, rA, cmpt);
if (lduMatrix::debug >= 2)
{
Info<< " Normalisation factor = " << normFactor << endl;
}
// Calculate residual
forAll (rA, i)
{
rA[i] = b[i] - wA[i];
}
Foam::lduMatrix::solverPerformance Foam::gPCG::solve
(
scalarField& psi,
const scalarField& source,
const direction cmpt
) const
{
scalar initialResidual;
scalar finalResidual;
int nIterations = 0;
bool converged = false;
label rows = matrix_.diag().size();
label nonZeroNumber= matrix_.lower().size() + matrix_.upper().size() + matrix_.diag().size();
pcgsolve(rows, nonZeroNumber,
psi.data(),
source.cdata(),
matrix_.upper().size(),
matrix_.lduAddr().upperAddr().cdata(),
matrix_.lduAddr().lowerAddr().cdata(),
matrix_.upper().cdata(),
matrix_.lower().cdata(),
matrix_.diag().cdata(),
maxIter_,
relTol_,
tolerance_,
initialResidual,
finalResidual,
nIterations,
converged
);
// --- Setup class containing solver performance data
lduMatrix::solverPerformance solverPerf
(
lduMatrix::preconditioner::getName(controlDict_) + typeName,
fieldName_,
initialResidual,
finalResidual,
nIterations,
converged
);
return solverPerf;
}
Foam::coupledSolverPerformance Foam::coupledBicgSolver::solve
(
FieldField<Field, scalar>& x,
const FieldField<Field, scalar>& b,
const direction cmpt
) const
{
// Prepare solver performance
coupledSolverPerformance solverPerf(typeName, fieldName());
FieldField<Field, scalar> wA(x.size());
FieldField<Field, scalar> rA(x.size());
forAll (x, rowI)
{
wA.set(rowI, new scalarField(x[rowI].size(), 0));
rA.set(rowI, new scalarField(x[rowI].size(), 0));
}
Foam::coupledSolverPerformance Foam::coupledSmoothSolver::solve
(
FieldField<Field, scalar>& x,
const FieldField<Field, scalar>& b,
const direction cmpt
) const
{
// Prepare solver performance
coupledSolverPerformance solverPerf(typeName, fieldName());
// Do a minimum number of sweeps
// HJ, 19/Jan/2009
if (minIter() > 0)
{
autoPtr<coupledLduSmoother> smootherPtr = coupledLduSmoother::New
(
matrix_,
bouCoeffs_,
intCoeffs_,
interfaces_,
dict()
);
smootherPtr->smooth
(
x,
b,
cmpt,
minIter()
);
solverPerf.nIterations() += minIter();
}
// Now do normal sweeps. HJ, 19/Jan/2009
FieldField<Field, scalar> Ax(x.size());
FieldField<Field, scalar> temp(x.size());
forAll (x, rowI)
{
Ax.set(rowI, new scalarField(x[rowI].size(), 0));
temp.set(rowI, new scalarField(x[rowI].size(), 0));
}
Foam::lduSolverPerformance Foam::mpeAmgSolver::solve
(
scalarField& x,
const scalarField& b,
const direction cmpt
) const
{
// Prepare solver performance
lduSolverPerformance solverPerf(typeName, fieldName());
scalar normFactor = this->normFactor(x, b, cmpt);
// Calculate initial residual
solverPerf.initialResidual() =
gSumMag(amg_.residual(x, b, cmpt))/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// Solver loop
if (!stop(solverPerf))
{
// Krylov vectors
typedef FieldField<Field, scalar> scalarFieldField;
scalarFieldField xSave(kDimension_ + 1);
forAll (xSave, i)
{
xSave.set(i, new scalarField(x.size()));
}
scalarFieldField xFirstDiffs(kDimension_ - 1);
forAll (xFirstDiffs, i)
{
xFirstDiffs.set(i, new scalarField(x.size()));
}
Foam::lduMatrix::solverPerformance Foam::paralution_PFGMRES::solve
(
scalarField& psi,
const scalarField& source,
const direction cmpt
) const
{
word precond_name = lduMatrix::preconditioner::getName(controlDict_);
double div = controlDict_.lookupOrDefault<double>("div", 1e+08);
int basis = controlDict_.lookupOrDefault<int>("BasisSize", 30);
bool accel = controlDict_.lookupOrDefault<bool>("useAccelerator", true);
word mformat = controlDict_.lookupOrDefault<word>("MatrixFormat", "CSR");
word pformat = controlDict_.lookupOrDefault<word>("PrecondFormat", "CSR");
int ILUp = controlDict_.lookupOrDefault<int>("ILUp", 0);
int ILUq = controlDict_.lookupOrDefault<int>("ILUq", 1);
int MEp = controlDict_.lookupOrDefault<int>("MEp", 1);
word LBPre = controlDict_.lookupOrDefault<word>("LastBlockPrecond", "paralution_Jacobi");
lduMatrix::solverPerformance solverPerf(typeName + '(' + precond_name + ')', fieldName_);
register label nCells = psi.size();
scalarField pA(nCells);
scalarField wA(nCells);
// --- Calculate A.psi
matrix_.Amul(wA, psi, interfaceBouCoeffs_, interfaces_, cmpt);
// --- Calculate initial residual field
scalarField rA(source - wA);
// --- Calculate normalisation factor
scalar normFactor = this->normFactor(psi, source, wA, pA);
// --- Calculate normalised residual norm
solverPerf.initialResidual() = gSumMag(rA)/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// TODO check why we cannot skip 1 iteration when initial residual < relTol_ or why initial residual actually
// does not drop below relTol_
if (!solverPerf.checkConvergence(tolerance_, relTol_)) {
paralution::_matrix_format mf = paralution::CSR;
if (mformat == "CSR") mf = paralution::CSR;
else if (mformat == "DIA") mf = paralution::DIA;
else if (mformat == "HYB") mf = paralution::HYB;
else if (mformat == "ELL") mf = paralution::ELL;
else if (mformat == "MCSR") mf = paralution::MCSR;
else if (mformat == "BCSR") mf = paralution::BCSR;
else if (mformat == "COO") mf = paralution::COO;
else if (mformat == "DENSE") mf = paralution::DENSE;
paralution::init_paralution();
paralution::LocalVector<double> x;
paralution::LocalVector<double> rhs;
paralution::LocalMatrix<double> mat;
paralution::FGMRES<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double> ls;
import_openfoam_matrix(matrix(), &mat);
import_openfoam_vector(source, &rhs);
import_openfoam_vector(psi, &x);
ls.Clear();
if (accel) {
mat.MoveToAccelerator();
rhs.MoveToAccelerator();
x.MoveToAccelerator();
}
paralution::Preconditioner<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double > *precond = NULL;
precond = GetPreconditioner<double>(precond_name, LBPre, pformat, ILUp, ILUq, MEp);
if (precond != NULL) ls.SetPreconditioner(*precond);
ls.SetOperator(mat);
ls.SetBasisSize(basis);
ls.Verbose(0);
ls.Init(tolerance_*normFactor, // abs
relTol_, // rel
div, // div
maxIter_); // max iter
ls.Build();
switch(mf) {
case paralution::DENSE:
mat.ConvertToDENSE();
break;
case paralution::CSR:
mat.ConvertToCSR();
break;
case paralution::MCSR:
mat.ConvertToMCSR();
break;
case paralution::BCSR:
mat.ConvertToBCSR();
break;
case paralution::COO:
mat.ConvertToCOO();
break;
case paralution::DIA:
mat.ConvertToDIA();
break;
case paralution::ELL:
mat.ConvertToELL();
break;
case paralution::HYB:
mat.ConvertToHYB();
break;
}
// mat.info();
ls.Solve(rhs, &x);
export_openfoam_vector(x, &psi);
solverPerf.finalResidual() = ls.GetCurrentResidual() / normFactor; // divide by normFactor, see lduMatrixSolver.C
solverPerf.nIterations() = ls.GetIterationCount();
solverPerf.checkConvergence(tolerance_, relTol_);
ls.Clear();
if (precond != NULL) {
precond->Clear();
delete precond;
}
paralution::stop_paralution();
}
return solverPerf;
}
Foam::solverPerformance Foam::smoothSolver::solve
(
scalarField& psi_s,
const scalarField& source,
const direction cmpt
) const
{
FieldWrapper<solveScalar, scalar> tpsi(psi_s);
solveScalarField& psi = tpsi.constCast();
// Setup class containing solver performance data
solverPerformance solverPerf(typeName, fieldName_);
// If the nSweeps_ is negative do a fixed number of sweeps
if (nSweeps_ < 0)
{
addProfiling(solve, "lduMatrix::smoother." + fieldName_);
autoPtr<lduMatrix::smoother> smootherPtr = lduMatrix::smoother::New
(
fieldName_,
matrix_,
interfaceBouCoeffs_,
interfaceIntCoeffs_,
interfaces_,
controlDict_
);
smootherPtr->smooth
(
psi,
source,
cmpt,
-nSweeps_
);
solverPerf.nIterations() -= nSweeps_;
}
else
{
solveScalar normFactor = 0;
solveScalarField residual;
ConstFieldWrapper<solveScalar, scalar> tsource(source);
{
solveScalarField Apsi(psi.size());
solveScalarField temp(psi.size());
// Calculate A.psi
matrix_.Amul(Apsi, psi, interfaceBouCoeffs_, interfaces_, cmpt);
// Calculate normalisation factor
normFactor = this->normFactor(psi, source, Apsi, temp);
residual = tsource() - Apsi;
matrix().setResidualField
(
ConstFieldWrapper<scalar, solveScalar>(residual)(),
fieldName_,
false
);
// Calculate residual magnitude
solverPerf.initialResidual() =
gSumMag(residual, matrix().mesh().comm())/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
}
if (lduMatrix::debug >= 2)
{
Info.masterStream(matrix().mesh().comm())
<< " Normalisation factor = " << normFactor << endl;
}
// Check convergence, solve if not converged
if
(
minIter_ > 0
|| !solverPerf.checkConvergence(tolerance_, relTol_)
)
{
addProfiling(solve, "lduMatrix::smoother." + fieldName_);
autoPtr<lduMatrix::smoother> smootherPtr = lduMatrix::smoother::New
(
fieldName_,
matrix_,
interfaceBouCoeffs_,
interfaceIntCoeffs_,
interfaces_,
controlDict_
);
// Smoothing loop
do
{
smootherPtr->smooth
(
psi,
source,
cmpt,
nSweeps_
);
residual =
matrix_.residual
(
psi,
source,
interfaceBouCoeffs_,
interfaces_,
cmpt
);
// Calculate the residual to check convergence
solverPerf.finalResidual() =
gSumMag(residual, matrix().mesh().comm())/normFactor;
} while
(
(
(solverPerf.nIterations() += nSweeps_) < maxIter_
&& !solverPerf.checkConvergence(tolerance_, relTol_)
)
|| solverPerf.nIterations() < minIter_
);
}
matrix().setResidualField
(
ConstFieldWrapper<scalar, solveScalar>(residual)(),
fieldName_,
false
);
}
return solverPerf;
}
Foam::lduSolverPerformance Foam::PBiCG::solve
(
scalarField& x,
const scalarField& b,
const direction cmpt
) const
{
// --- Setup class containing solver performance data
lduSolverPerformance solverPerf
(
lduMatrix::preconditioner::getName(dict()) + typeName,
fieldName()
);
register label nCells = x.size();
scalar* __restrict__ xPtr = x.begin();
scalarField pA(nCells);
scalar* __restrict__ pAPtr = pA.begin();
scalarField pT(nCells, 0.0);
scalar* __restrict__ pTPtr = pT.begin();
scalarField wA(nCells);
scalar* __restrict__ wAPtr = wA.begin();
scalarField wT(nCells);
scalar* __restrict__ wTPtr = wT.begin();
scalar wArT = matrix_.great_;
scalar wArTold = wArT;
// Calculate A.x and T.x
matrix_.Amul(wA, x, coupleBouCoeffs_, interfaces_, cmpt);
matrix_.Tmul(wT, x, coupleIntCoeffs_, interfaces_, cmpt);
// Calculate initial residual and transpose residual fields
scalarField rA(b - wA);
scalarField rT(b - wT);
scalar* __restrict__ rAPtr = rA.begin();
scalar* __restrict__ rTPtr = rT.begin();
// Calculate normalisation factor
scalar normFactor = this->normFactor(x, b, wA, pA, cmpt);
if (lduMatrix::debug >= 2)
{
Info<< " Normalisation factor = " << normFactor << endl;
}
// Calculate normalised residual norm
solverPerf.initialResidual() = gSumMag(rA)/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// Check convergence, solve if not converged
if (!stop(solverPerf))
{
// Select and construct the preconditioner
autoPtr<lduPreconditioner> preconPtr;
preconPtr =
lduPreconditioner::New
(
matrix_,
coupleBouCoeffs_,
coupleIntCoeffs_,
interfaces_,
dict()
);
// Solver iteration
do
{
// Store previous wArT
wArTold = wArT;
// Precondition residuals
preconPtr->precondition(wA, rA, cmpt);
preconPtr->preconditionT(wT, rT, cmpt);
// Update search directions:
wArT = gSumProd(wA, rT);
if (solverPerf.nIterations() == 0)
{
for (register label cell=0; cell<nCells; cell++)
{
pAPtr[cell] = wAPtr[cell];
pTPtr[cell] = wTPtr[cell];
}
}
else
{
scalar beta = wArT/wArTold;
for (register label cell=0; cell<nCells; cell++)
{
pAPtr[cell] = wAPtr[cell] + beta*pAPtr[cell];
pTPtr[cell] = wTPtr[cell] + beta*pTPtr[cell];
}
}
// Update preconditioned residuals
matrix_.Amul(wA, pA, coupleBouCoeffs_, interfaces_, cmpt);
matrix_.Tmul(wT, pT, coupleIntCoeffs_, interfaces_, cmpt);
scalar wApT = gSumProd(wA, pT);
// Test for singularity
if (solverPerf.checkSingularity(mag(wApT)/normFactor)) break;
// Update solution and residual:
scalar alpha = wArT/wApT;
for (register label cell=0; cell<nCells; cell++)
{
xPtr[cell] += alpha*pAPtr[cell];
rAPtr[cell] -= alpha*wAPtr[cell];
rTPtr[cell] -= alpha*wTPtr[cell];
}
solverPerf.finalResidual() = gSumMag(rA)/normFactor;
solverPerf.nIterations()++;
} while (!stop(solverPerf));
}
return solverPerf;
}
Foam::solverPerformance Foam::GAMGSolver::solve
(
scalargpuField& psi,
const scalargpuField& source,
const direction cmpt
) const
{
// Setup class containing solver performance data
solverPerformance solverPerf(typeName, fieldName_);
// Calculate A.psi used to calculate the initial residual
scalargpuField Apsi(psi.size());
matrix_.Amul(Apsi, psi, interfaceBouCoeffs_, interfaces_, cmpt);
// Create the storage for the finestCorrection which may be used as a
// temporary in normFactor
scalargpuField finestCorrection(psi.size());
// Calculate normalisation factor
scalar normFactor = this->normFactor(psi, source, Apsi, finestCorrection);
if (debug >= 2)
{
Pout<< " Normalisation factor = " << normFactor << endl;
}
// Calculate initial finest-grid residual field
scalargpuField finestResidual(source - Apsi);
// Calculate normalised residual for convergence test
solverPerf.initialResidual() = gSumMag
(
finestResidual,
matrix().mesh().comm()
)/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// Check convergence, solve if not converged
if
(
minIter_ > 0
|| !solverPerf.checkConvergence(tolerance_, relTol_)
)
{
// Create coarse grid correction fields
PtrList<scalargpuField> coarseCorrFields;
// Create coarse grid sources
PtrList<scalargpuField> coarseSources;
// Create the smoothers for all levels
PtrList<lduMatrix::smoother> smoothers;
// Scratch fields if processor-agglomerated coarse level meshes
// are bigger than original. Usually not needed
scalargpuField scratch1;
scalargpuField scratch2;
// Initialise the above data structures
initVcycle
(
coarseCorrFields,
coarseSources,
smoothers,
scratch1,
scratch2
);
do
{
Vcycle
(
smoothers,
psi,
source,
Apsi,
finestCorrection,
finestResidual,
(scratch1.size() ? scratch1 : Apsi),
(scratch2.size() ? scratch2 : finestCorrection),
coarseCorrFields,
coarseSources,
cmpt
);
// Calculate finest level residual field
matrix_.Amul(Apsi, psi, interfaceBouCoeffs_, interfaces_, cmpt);
finestResidual = source;
finestResidual -= Apsi;
solverPerf.finalResidual() = gSumMag
(
finestResidual,
matrix().mesh().comm()
)/normFactor;
if (debug >= 2)
{
solverPerf.print(Info.masterStream(matrix().mesh().comm()));
}
} while
(
(
++solverPerf.nIterations() < maxIter_
&& !solverPerf.checkConvergence(tolerance_, relTol_)
)
|| solverPerf.nIterations() < minIter_
);
}
return solverPerf;
}
Foam::solverPerformance Foam::PBiCG::solve
(
scalarField& psi,
const scalarField& source,
const direction cmpt
) const
{
// --- Setup class containing solver performance data
solverPerformance solverPerf
(
lduMatrix::preconditioner::getName(controlDict_) + typeName,
fieldName_
);
label nCells = psi.size();
scalar* __restrict__ psiPtr = psi.begin();
scalarField pA(nCells);
scalar* __restrict__ pAPtr = pA.begin();
scalarField pT(nCells, 0.0);
scalar* __restrict__ pTPtr = pT.begin();
scalarField wA(nCells);
scalar* __restrict__ wAPtr = wA.begin();
scalarField wT(nCells);
scalar* __restrict__ wTPtr = wT.begin();
scalar wArT = solverPerf.great_;
scalar wArTold = wArT;
// --- Calculate A.psi and T.psi
matrix_.Amul(wA, psi, interfaceBouCoeffs_, interfaces_, cmpt);
matrix_.Tmul(wT, psi, interfaceIntCoeffs_, interfaces_, cmpt);
// --- Calculate initial residual and transpose residual fields
scalarField rA(source - wA);
scalarField rT(source - wT);
scalar* __restrict__ rAPtr = rA.begin();
scalar* __restrict__ rTPtr = rT.begin();
// --- Calculate normalisation factor
scalar normFactor = this->normFactor(psi, source, wA, pA);
if (lduMatrix::debug >= 2)
{
Info<< " Normalisation factor = " << normFactor << endl;
}
// --- Calculate normalised residual norm
solverPerf.initialResidual() =
gSumMag(rA, matrix().mesh().comm())
/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// --- Check convergence, solve if not converged
if
(
minIter_ > 0
|| !solverPerf.checkConvergence(tolerance_, relTol_)
)
{
// --- Select and construct the preconditioner
autoPtr<lduMatrix::preconditioner> preconPtr =
lduMatrix::preconditioner::New
(
*this,
controlDict_
);
// --- Solver iteration
do
{
// --- Store previous wArT
wArTold = wArT;
// --- Precondition residuals
preconPtr->precondition(wA, rA, cmpt);
preconPtr->preconditionT(wT, rT, cmpt);
// --- Update search directions:
wArT = gSumProd(wA, rT, matrix().mesh().comm());
if (solverPerf.nIterations() == 0)
{
for (label cell=0; cell<nCells; cell++)
{
pAPtr[cell] = wAPtr[cell];
pTPtr[cell] = wTPtr[cell];
}
}
else
{
scalar beta = wArT/wArTold;
for (label cell=0; cell<nCells; cell++)
{
pAPtr[cell] = wAPtr[cell] + beta*pAPtr[cell];
pTPtr[cell] = wTPtr[cell] + beta*pTPtr[cell];
}
}
// --- Update preconditioned residuals
matrix_.Amul(wA, pA, interfaceBouCoeffs_, interfaces_, cmpt);
matrix_.Tmul(wT, pT, interfaceIntCoeffs_, interfaces_, cmpt);
scalar wApT = gSumProd(wA, pT, matrix().mesh().comm());
// --- Test for singularity
if (solverPerf.checkSingularity(mag(wApT)/normFactor))
{
break;
}
// --- Update solution and residual:
scalar alpha = wArT/wApT;
for (label cell=0; cell<nCells; cell++)
{
psiPtr[cell] += alpha*pAPtr[cell];
rAPtr[cell] -= alpha*wAPtr[cell];
rTPtr[cell] -= alpha*wTPtr[cell];
}
solverPerf.finalResidual() =
gSumMag(rA, matrix().mesh().comm())
/normFactor;
} while
(
(
solverPerf.nIterations()++ < maxIter_
&& !solverPerf.checkConvergence(tolerance_, relTol_)
)
|| solverPerf.nIterations() < minIter_
);
}
return solverPerf;
}
Foam::solverPerformance Foam::paralution_AMG::solve
(
scalarField& psi,
const scalarField& source,
const direction cmpt
) const
{
word precond_name = lduMatrix::preconditioner::getName(controlDict_);
double div = controlDict_.lookupOrDefault<double>("div", 1e+08);
bool accel = controlDict_.lookupOrDefault<bool>("useAccelerator", true);
word mformat = controlDict_.lookupOrDefault<word>("MatrixFormat", "CSR");
word pformat = controlDict_.lookupOrDefault<word>("PrecondFormat", "CSR");
word sformat = controlDict_.lookupOrDefault<word>("SmootherFormat", "CSR");
word solver_name = controlDict_.lookupOrDefault<word>("CoarseGridSolver", "CG");
word smoother_name = controlDict_.lookupOrDefault<word>("smoother", "paralution_MultiColoredGS");
int MEp = controlDict_.lookupOrDefault<int>("MEp", 1);
word LBPre = controlDict_.lookupOrDefault<word>("LastBlockPrecond", "paralution_Jacobi");
int iterPreSmooth = controlDict_.lookupOrDefault<int>("nPreSweeps", 1);
int iterPostSmooth = controlDict_.lookupOrDefault<int>("nPostSweeps", 2);
double epsCoupling = controlDict_.lookupOrDefault<double>("couplingStrength", 0.01);
int coarsestCells = controlDict_.lookupOrDefault<int>("nCellsInCoarsestLevel", 300);
int ILUp = controlDict_.lookupOrDefault<int>("ILUp", 0);
int ILUq = controlDict_.lookupOrDefault<int>("ILUq", 1);
double relax = controlDict_.lookupOrDefault<double>("Relaxation", 1.0);
double aggrrelax = controlDict_.lookupOrDefault<double>("AggrRelax", 2./3.);
bool scaling = controlDict_.lookupOrDefault<bool>("scaleCorrection", true);
word interp_name = controlDict_.lookupOrDefault<word>("InterpolationType", "SmoothedAggregation");
solverPerformance solverPerf(typeName + '(' + precond_name + ')', fieldName_);
register label nCells = psi.size();
scalarField pA(nCells);
scalarField wA(nCells);
// --- Calculate A.psi
matrix_.Amul(wA, psi, interfaceBouCoeffs_, interfaces_, cmpt);
// --- Calculate initial residual field
scalarField rA(source - wA);
// --- Calculate normalisation factor
scalar normFactor = this->normFactor(psi, source, wA, pA);
// --- Calculate normalised residual norm
solverPerf.initialResidual() = gSumMag(rA)/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
if ( !solverPerf.checkConvergence(tolerance_, relTol_) ) {
paralution::_matrix_format mf = paralution::CSR;
if (mformat == "CSR") mf = paralution::CSR;
else if (mformat == "DIA") mf = paralution::DIA;
else if (mformat == "HYB") mf = paralution::HYB;
else if (mformat == "ELL") mf = paralution::ELL;
else if (mformat == "MCSR") mf = paralution::MCSR;
else if (mformat == "BCSR") mf = paralution::BCSR;
else if (mformat == "COO") mf = paralution::COO;
else if (mformat == "DENSE") mf = paralution::DENSE;
paralution::_interp ip = paralution::SmoothedAggregation;
if (interp_name == "SmoothedAggregation") ip = paralution::SmoothedAggregation;
else if (interp_name == "Aggregation") ip = paralution::Aggregation;
paralution::LocalVector<double> x;
paralution::LocalVector<double> rhs;
paralution::LocalMatrix<double> mat;
paralution::AMG<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double> ls;
paralution::import_openfoam_matrix(matrix(), &mat);
paralution::import_openfoam_vector(source, &rhs);
paralution::import_openfoam_vector(psi, &x);
ls.SetOperator(mat);
// coupling strength
ls.SetCouplingStrength(epsCoupling);
// number of unknowns on coarsest level
ls.SetCoarsestLevel(coarsestCells);
// interpolation type for grid transfer operators
ls.SetInterpolation(ip);
// Relaxation parameter for smoothed interpolation aggregation
ls.SetInterpRelax(aggrrelax);
// Manual smoothers
ls.SetManualSmoothers(true);
// Manual course grid solver
ls.SetManualSolver(true);
// grid transfer scaling
ls.SetScaling(scaling);
// operator format
ls.SetOperatorFormat(mf);
ls.SetSmootherPreIter(iterPreSmooth);
ls.SetSmootherPostIter(iterPostSmooth);
ls.BuildHierarchy();
int levels = ls.GetNumLevels();
// Smoother via preconditioned FixedPoint iteration
paralution::IterativeLinearSolver<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double > **fp = NULL;
fp = new paralution::IterativeLinearSolver<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double >*[levels-1];
paralution::Preconditioner<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double > **sm = NULL;
sm = new paralution::Preconditioner<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double >*[levels-1];
for (int i=0; i<levels-1; ++i) {
fp[i] = paralution::GetIterativeLinearSolver<double>("paralution_FixedPoint", relax);
sm[i] = paralution::GetPreconditioner<double>(smoother_name, LBPre, sformat, ILUp, ILUq, MEp);
fp[i]->SetPreconditioner(*sm[i]);
fp[i]->Verbose(0);
}
// Coarse Grid Solver and its Preconditioner
paralution::IterativeLinearSolver<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double > *cgs = NULL;
cgs = paralution::GetIterativeLinearSolver<double>(solver_name, relax);
cgs->Verbose(0);
paralution::Preconditioner<paralution::LocalMatrix<double>,
paralution::LocalVector<double>,
double > *cgp = NULL;
cgp = paralution::GetPreconditioner<double>(precond_name, LBPre, pformat, ILUp, ILUq, MEp);
if (cgp != NULL) cgs->SetPreconditioner(*cgp);
ls.SetSmoother(fp);
ls.SetSolver(*cgs);
// Switch to L1 norm to be consistent with OpenFOAM solvers
ls.SetResidualNorm(1);
ls.Init(tolerance_*normFactor, // abs
relTol_, // rel
div, // div
maxIter_); // max iter
ls.Build();
if (accel) {
mat.MoveToAccelerator();
rhs.MoveToAccelerator();
x.MoveToAccelerator();
ls.MoveToAccelerator();
}
switch(mf) {
case paralution::DENSE:
mat.ConvertToDENSE();
break;
case paralution::CSR:
mat.ConvertToCSR();
break;
case paralution::MCSR:
mat.ConvertToMCSR();
break;
case paralution::BCSR:
mat.ConvertToBCSR();
break;
case paralution::COO:
mat.ConvertToCOO();
break;
case paralution::DIA:
mat.ConvertToDIA();
break;
case paralution::ELL:
mat.ConvertToELL();
break;
case paralution::HYB:
mat.ConvertToHYB();
break;
}
ls.Verbose(0);
// Solve linear system
ls.Solve(rhs, &x);
paralution::export_openfoam_vector(x, &psi);
solverPerf.finalResidual() = ls.GetCurrentResidual() / normFactor; // divide by normFactor, see lduMatrixSolver.C
solverPerf.nIterations() = ls.GetIterationCount();
solverPerf.checkConvergence(tolerance_, relTol_);
// Clear MultiGrid object
ls.Clear();
// Free all structures
for (int i=0; i<levels-1; ++i) {
delete fp[i];
delete sm[i];
}
cgs->Clear();
if (cgp != NULL)
delete cgp;
delete[] fp;
delete[] sm;
delete cgs;
}
return solverPerf;
}
Foam::lduMatrix::solverPerformance Foam::PCG::solve
(
scalarField& psi,
const scalarField& source,
const direction cmpt
) const
{
// --- Setup class containing solver performance data
lduMatrix::solverPerformance solverPerf
(
lduMatrix::preconditioner::getName(controlDict_) + typeName,
fieldName_
);
register label nCells = psi.size();
scalar* __restrict__ psiPtr = psi.begin();
scalarField pA(nCells);
scalar* __restrict__ pAPtr = pA.begin();
scalarField wA(nCells);
scalar* __restrict__ wAPtr = wA.begin();
scalar wArA = matrix_.great_;
scalar wArAold = wArA;
// --- Calculate A.psi
matrix_.Amul(wA, psi, interfaceBouCoeffs_, interfaces_, cmpt);
// --- Calculate initial residual field
scalarField rA(source - wA);
scalar* __restrict__ rAPtr = rA.begin();
// --- Calculate normalisation factor
scalar normFactor = this->normFactor(psi, source, wA, pA);
if (lduMatrix::debug >= 2)
{
Info<< " Normalisation factor = " << normFactor << endl;
}
// --- Calculate normalised residual norm
solverPerf.initialResidual() = gSumMag(rA)/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// --- Check convergence, solve if not converged
if (!solverPerf.checkConvergence(tolerance_, relTol_))
{
// --- Select and construct the preconditioner
autoPtr<lduMatrix::preconditioner> preconPtr =
lduMatrix::preconditioner::New
(
*this,
controlDict_
);
// --- Solver iteration
do
{
// --- Store previous wArA
wArAold = wArA;
// --- Precondition residual
preconPtr->precondition(wA, rA, cmpt);
// --- Update search directions:
wArA = gSumProd(wA, rA);
if (solverPerf.nIterations() == 0)
{
for (register label cell=0; cell<nCells; cell++)
{
pAPtr[cell] = wAPtr[cell];
}
}
else
{
scalar beta = wArA/wArAold;
for (register label cell=0; cell<nCells; cell++)
{
pAPtr[cell] = wAPtr[cell] + beta*pAPtr[cell];
}
}
// --- Update preconditioned residual
matrix_.Amul(wA, pA, interfaceBouCoeffs_, interfaces_, cmpt);
scalar wApA = gSumProd(wA, pA);
// --- Test for singularity
if (solverPerf.checkSingularity(mag(wApA)/normFactor)) break;
// --- Update solution and residual:
scalar alpha = wArA/wApA;
for (register label cell=0; cell<nCells; cell++)
{
psiPtr[cell] += alpha*pAPtr[cell];
rAPtr[cell] -= alpha*wAPtr[cell];
}
solverPerf.finalResidual() = gSumMag(rA)/normFactor;
} while
(
solverPerf.nIterations()++ < maxIter_
&& !(solverPerf.checkConvergence(tolerance_, relTol_))
);
}
return solverPerf;
}
Foam::solverPerformance Foam::PBiCGStab::solve
(
scalarField& psi,
const scalarField& source,
const direction cmpt
) const
{
// --- Setup class containing solver performance data
solverPerformance solverPerf
(
lduMatrix::preconditioner::getName(controlDict_) + typeName,
fieldName_
);
const label nCells = psi.size();
scalar* __restrict__ psiPtr = psi.begin();
scalarField pA(nCells);
scalar* __restrict__ pAPtr = pA.begin();
scalarField yA(nCells);
scalar* __restrict__ yAPtr = yA.begin();
// --- Calculate A.psi
matrix_.Amul(yA, psi, interfaceBouCoeffs_, interfaces_, cmpt);
// --- Calculate initial residual field
scalarField rA(source - yA);
scalar* __restrict__ rAPtr = rA.begin();
// --- Calculate normalisation factor
const scalar normFactor = this->normFactor(psi, source, yA, pA);
if (lduMatrix::debug >= 2)
{
Info<< " Normalisation factor = " << normFactor << endl;
}
// --- Calculate normalised residual norm
solverPerf.initialResidual() =
gSumMag(rA, matrix().mesh().comm())
/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// --- Check convergence, solve if not converged
if
(
minIter_ > 0
|| !solverPerf.checkConvergence(tolerance_, relTol_)
)
{
scalarField AyA(nCells);
scalar* __restrict__ AyAPtr = AyA.begin();
scalarField sA(nCells);
scalar* __restrict__ sAPtr = sA.begin();
scalarField zA(nCells);
scalar* __restrict__ zAPtr = zA.begin();
scalarField tA(nCells);
scalar* __restrict__ tAPtr = tA.begin();
// --- Store initial residual
const scalarField rA0(rA);
// --- Initial values not used
scalar rA0rA = 0;
scalar alpha = 0;
scalar omega = 0;
// --- Select and construct the preconditioner
autoPtr<lduMatrix::preconditioner> preconPtr =
lduMatrix::preconditioner::New
(
*this,
controlDict_
);
// --- Solver iteration
do
{
// --- Store previous rA0rA
const scalar rA0rAold = rA0rA;
rA0rA = gSumProd(rA0, rA, matrix().mesh().comm());
// --- Test for singularity
if (solverPerf.checkSingularity(mag(rA0rA)))
{
break;
}
// --- Update pA
if (solverPerf.nIterations() == 0)
{
for (label cell=0; cell<nCells; cell++)
{
pAPtr[cell] = rAPtr[cell];
}
}
else
{
// --- Test for singularity
if (solverPerf.checkSingularity(mag(omega)))
{
break;
}
const scalar beta = (rA0rA/rA0rAold)*(alpha/omega);
for (label cell=0; cell<nCells; cell++)
{
pAPtr[cell] =
rAPtr[cell] + beta*(pAPtr[cell] - omega*AyAPtr[cell]);
}
}
// --- Precondition pA
preconPtr->precondition(yA, pA, cmpt);
// --- Calculate AyA
matrix_.Amul(AyA, yA, interfaceBouCoeffs_, interfaces_, cmpt);
const scalar rA0AyA = gSumProd(rA0, AyA, matrix().mesh().comm());
alpha = rA0rA/rA0AyA;
// --- Calculate sA
for (label cell=0; cell<nCells; cell++)
{
sAPtr[cell] = rAPtr[cell] - alpha*AyAPtr[cell];
}
// --- Test sA for convergence
solverPerf.finalResidual() =
gSumMag(sA, matrix().mesh().comm())/normFactor;
if (solverPerf.checkConvergence(tolerance_, relTol_))
{
for (label cell=0; cell<nCells; cell++)
{
psiPtr[cell] += alpha*yAPtr[cell];
}
solverPerf.nIterations()++;
return solverPerf;
}
// --- Precondition sA
preconPtr->precondition(zA, sA, cmpt);
// --- Calculate tA
matrix_.Amul(tA, zA, interfaceBouCoeffs_, interfaces_, cmpt);
const scalar tAtA = gSumSqr(tA, matrix().mesh().comm());
// --- Calculate omega from tA and sA
// (cheaper than using zA with preconditioned tA)
omega = gSumProd(tA, sA, matrix().mesh().comm())/tAtA;
// --- Update solution and residual
for (label cell=0; cell<nCells; cell++)
{
psiPtr[cell] += alpha*yAPtr[cell] + omega*zAPtr[cell];
rAPtr[cell] = sAPtr[cell] - omega*tAPtr[cell];
}
solverPerf.finalResidual() =
gSumMag(rA, matrix().mesh().comm())
/normFactor;
} while
(
(
solverPerf.nIterations()++ < maxIter_
&& !solverPerf.checkConvergence(tolerance_, relTol_)
)
|| solverPerf.nIterations() < minIter_
);
}
return solverPerf;
}