void SequentialSolver::solve_local(vec& res,
const system_type& sys,
const vec& rhs,
bool correct) const {
if( d_iterateOnBilaterals.getValue() || !m_localSystem.H.nonZeros() )
return solve_impl( res, sys, rhs, correct );
const size_t localsize = m_localSystem.size();
vec localrhs(localsize), localres(localsize);
// reordering rhs
m_localSub.toLocal( localrhs, rhs );
// reordering res, for warm_start...
m_localSub.toLocal( localres, res );
// performing the solve on the reorganized system
solve_impl( localres, m_localSystem, localrhs, correct );
// reordering res
m_localSub.fromLocal( res, localres );
}
int shylu_dist_solve<Epetra_CrsMatrix,Epetra_MultiVector>(
shylu_symbolic<Epetra_CrsMatrix,Epetra_MultiVector> *ssym,
shylu_data<Epetra_CrsMatrix,Epetra_MultiVector> *data,
shylu_config<Epetra_CrsMatrix,Epetra_MultiVector> *config,
const Epetra_MultiVector& X,
Epetra_MultiVector& Y
)
{
int err;
AztecOO *solver = 0;
assert(X.Map().SameAs(Y.Map()));
//assert(X.Map().SameAs(A_->RowMap()));
const Epetra_MultiVector *newX;
newX = &X;
//rd_->redistribute(X, newX);
int nvectors = newX->NumVectors();
// May have to use importer/exporter
Epetra_Map BsMap(-1, data->Snr, data->SRowElems, 0, X.Comm());
Epetra_Map BdMap(-1, data->Dnr, data->DRowElems, 0, X.Comm());
Epetra_MultiVector Bs(BsMap, nvectors);
Epetra_Import BsImporter(BsMap, newX->Map());
assert(BsImporter.SourceMap().SameAs(newX->Map()));
assert((newX->Map()).SameAs(BsImporter.SourceMap()));
Bs.Import(*newX, BsImporter, Insert);
Epetra_MultiVector Xs(BsMap, nvectors);
Epetra_SerialComm LComm; // Use Serial Comm for the local vectors.
Epetra_Map LocalBdMap(-1, data->Dnr, data->DRowElems, 0, LComm);
Epetra_MultiVector localrhs(LocalBdMap, nvectors);
Epetra_MultiVector locallhs(LocalBdMap, nvectors);
Epetra_MultiVector Z(BdMap, nvectors);
Epetra_MultiVector Bd(BdMap, nvectors);
Epetra_Import BdImporter(BdMap, newX->Map());
assert(BdImporter.SourceMap().SameAs(newX->Map()));
assert((newX->Map()).SameAs(BdImporter.SourceMap()));
Bd.Import(*newX, BdImporter, Insert);
int lda;
double *values;
err = Bd.ExtractView(&values, &lda);
assert (err == 0);
int nrows = ssym->C->RowMap().NumMyElements();
// copy to local vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = localrhs.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
// TODO : Do we need to reset the lhs and rhs here ?
if (config->amesosForDiagonal)
{
ssym->LP->SetRHS(&localrhs);
ssym->LP->SetLHS(&locallhs);
ssym->Solver->Solve();
}
else
{
ssym->ifSolver->ApplyInverse(localrhs, locallhs);
}
err = locallhs.ExtractView(&values, &lda);
assert (err == 0);
// copy to distributed vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = Z.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
Epetra_MultiVector temp1(BsMap, nvectors);
ssym->R->Multiply(false, Z, temp1);
Bs.Update(-1.0, temp1, 1.0);
Xs.PutScalar(0.0);
Epetra_LinearProblem Problem(data->Sbar.get(), &Xs, &Bs);
if (config->schurSolver == "Amesos")
{
Amesos_BaseSolver *solver2 = data->dsolver;
data->LP2->SetLHS(&Xs);
data->LP2->SetRHS(&Bs);
//cout << "Calling solve *****************************" << endl;
solver2->Solve();
//cout << "Out of solve *****************************" << endl;
}
else
{
if (config->libName == "Belos")
{
solver = data->innersolver;
solver->SetLHS(&Xs);
solver->SetRHS(&Bs);
}
else
{
// See the comment above on why we are not able to reuse the solver
// when outer solve is AztecOO as well.
solver = new AztecOO();
//solver.SetPrecOperator(precop_);
solver->SetAztecOption(AZ_solver, AZ_gmres);
// Do not use AZ_none
solver->SetAztecOption(AZ_precond, AZ_dom_decomp);
//solver->SetAztecOption(AZ_precond, AZ_none);
//solver->SetAztecOption(AZ_precond, AZ_Jacobi);
////solver->SetAztecOption(AZ_precond, AZ_Neumann);
//solver->SetAztecOption(AZ_overlap, 3);
//solver->SetAztecOption(AZ_subdomain_solve, AZ_ilu);
//solver->SetAztecOption(AZ_output, AZ_all);
//solver->SetAztecOption(AZ_diagnostics, AZ_all);
solver->SetProblem(Problem);
}
// What should be a good inner_tolerance :-) ?
solver->Iterate(config->inner_maxiters, config->inner_tolerance);
}
Epetra_MultiVector temp(BdMap, nvectors);
ssym->C->Multiply(false, Xs, temp);
temp.Update(1.0, Bd, -1.0);
//Epetra_SerialComm LComm; // Use Serial Comm for the local vectors.
//Epetra_Map LocalBdMap(-1, data->Dnr, data->DRowElems, 0, LComm);
//Epetra_MultiVector localrhs(LocalBdMap, nvectors);
//Epetra_MultiVector locallhs(LocalBdMap, nvectors);
//int lda;
//double *values;
err = temp.ExtractView(&values, &lda);
assert (err == 0);
//int nrows = data->Cptr->RowMap().NumMyElements();
// copy to local vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = localrhs.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
if (config->amesosForDiagonal)
{
ssym->LP->SetRHS(&localrhs);
ssym->LP->SetLHS(&locallhs);
ssym->Solver->Solve();
}
else
{
ssym->ifSolver->ApplyInverse(localrhs, locallhs);
}
err = locallhs.ExtractView(&values, &lda);
assert (err == 0);
// copy to distributed vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = temp.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
// For checking faults
//if (NumApplyInverse_ == 5) temp.ReplaceMyValue(0, 0, 0.0);
Epetra_Export XdExporter(BdMap, Y.Map());
Y.Export(temp, XdExporter, Insert);
Epetra_Export XsExporter(BsMap, Y.Map());
Y.Export(Xs, XsExporter, Insert);
if (config->libName == "Belos" || config->schurSolver == "Amesos")
{
// clean up
}
else
{
delete solver;
}
return 0;
}//end shylu_dist_solve <epetra,epetra>
