int shylu_local_solve<Epetra_CrsMatrix, Epetra_MultiVector>
(
shylu_symbolic<Epetra_CrsMatrix,Epetra_MultiVector> *ssym,
shylu_data<Epetra_CrsMatrix,Epetra_MultiVector> *data,
shylu_config<Epetra_CrsMatrix,Epetra_MultiVector> *config,
const Epetra_MultiVector& X,
Epetra_MultiVector& Y
)
{
int err;
#ifndef NDEBUG
int nvectors = X.NumVectors();
assert (nvectors == data->localrhs->NumVectors());
#endif // NDEBUG
// Initialize the X vector for iterative solver
data->Xs->PutScalar(0.0);
// Get local portion of X
data->localrhs->Import(X, *(data->BdImporter), Insert);
data->localrhs->Print(std::cout);
std::cout << " " << std::endl;
data->locallhs->Print(std::cout);
// locallhs is z in paper
if (config->amesosForDiagonal) {
std::cout << "calling amesos for diagon" << endl;
ssym->OrigLP->SetRHS((data->localrhs).getRawPtr());
ssym->OrigLP->SetLHS((data->locallhs).getRawPtr());
std::cout << "set RHS and LHS " << std::endl;
ssym->ReIdx_LP->fwd();
ssym->Solver->Solve();
}
else {
ssym->ifSolver->ApplyInverse(*(data->localrhs), *(data->locallhs));
}
err = ssym->R->Multiply(false, *(data->locallhs), *(data->temp1));
assert (err == 0);
// Export temp1 to a dist vector - temp2
data->temp2->Import(*(data->temp1), *(data->DistImporter), Insert);
//Epetra_MultiVector Bs(SMap, nvectors); // b_2 - R * z in ShyLU paper
data->Bs->Import(X, *(data->BsImporter), Insert);
data->Bs->Update(-1.0, *(data->temp2), 1.0);
AztecOO *solver = 0;
Epetra_LinearProblem Problem(data->Sbar.get(),
(data->Xs).getRawPtr(), (data->Bs).getRawPtr());
if ((config->schurSolver == "G") || (config->schurSolver == "IQR"))
{
IFPACK_CHK_ERR(data->iqrSolver->Solve(*(data->schur_op),
*(data->Bs), *(data->Xs)));
}
else if (config->schurSolver == "Amesos")
{
Amesos_BaseSolver *solver2 = data->dsolver;
data->OrigLP2->SetLHS((data->Xs).getRawPtr());
data->OrigLP2->SetRHS((data->Bs).getRawPtr());
data->ReIdx_LP2->fwd();
//cout << "Calling solve *****************************" << endl;
solver2->Solve();
//cout << "Out of solve *****************************" << endl;
}
else
{
if (config->libName == "Belos")
{
solver = data->innersolver;
solver->SetLHS((data->Xs).getRawPtr());
solver->SetRHS((data->Bs).getRawPtr());
}
else
{
// See the comment above on why we are not able to reuse the solver
// when outer solve is AztecOO as well.
solver = new AztecOO();
//solver.SetPrecOperator(precop_);
solver->SetAztecOption(AZ_solver, AZ_gmres);
// Do not use AZ_none
solver->SetAztecOption(AZ_precond, AZ_dom_decomp);
//solver->SetAztecOption(AZ_precond, AZ_none);
//solver->SetAztecOption(AZ_precond, AZ_Jacobi);
////solver->SetAztecOption(AZ_precond, AZ_Neumann);
//solver->SetAztecOption(AZ_overlap, 3);
//solver->SetAztecOption(AZ_subdomain_solve, AZ_ilu);
//solver->SetAztecOption(AZ_output, AZ_all);
//solver->SetAztecOption(AZ_diagnostics, AZ_all);
solver->SetProblem(Problem);
}
// What should be a good inner_tolerance :-) ?
solver->Iterate(config->inner_maxiters, config->inner_tolerance);
}
// Import Xs locally
data->LocalXs->Import(*(data->Xs), *(data->XsImporter), Insert);
err = ssym->C->Multiply(false, *(data->LocalXs), *(data->temp3));
assert (err == 0);
data->temp3->Update(1.0, *(data->localrhs), -1.0);
if (config->amesosForDiagonal) {
ssym->OrigLP->SetRHS((data->temp3).getRawPtr());
ssym->OrigLP->SetLHS((data->locallhs).getRawPtr());
ssym->ReIdx_LP->fwd();
ssym->Solver->Solve();
}
else {
ssym->ifSolver->ApplyInverse(*(data->temp3), *(data->locallhs));
}
Y.Export(*(data->locallhs), *(data->XdExporter), Insert);
Y.Export(*(data->LocalXs), *(data->XsExporter), Insert);
if (config->libName == "Belos" || config->schurSolver == "Amesos")
{
// clean up
}
else
{
delete solver;
}
return 0;
}
int main(int argc, char *argv[]) {
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
Teuchos::ParameterList GaleriList;
int nx = 30;
GaleriList.set("nx", nx);
// GaleriList.set("ny", nx * Comm.NumProc());
GaleriList.set("ny", nx);
GaleriList.set("mx", 1);
GaleriList.set("my", Comm.NumProc());
GaleriList.set("alpha", .0);
GaleriList.set("diff", 1.0);
GaleriList.set("conv", 100.0);
Teuchos::RefCountPtr<Epetra_Map> Map = Teuchos::rcp( Galeri::CreateMap("Cartesian2D", Comm, GaleriList) );
Teuchos::RefCountPtr<Epetra_CrsMatrix> A = Teuchos::rcp( Galeri::CreateCrsMatrix("UniFlow2D", &*Map, GaleriList) );
Teuchos::RefCountPtr<Epetra_MultiVector> LHS = Teuchos::rcp( new Epetra_MultiVector(*Map, 1) );
Teuchos::RefCountPtr<Epetra_MultiVector> RHS = Teuchos::rcp( new Epetra_MultiVector(*Map, 1) );
LHS->PutScalar(0.0); RHS->Random();
Ifpack Factory;
int Niters = 100;
// ============================= //
// Construct IHSS preconditioner //
// ============================= //
Teuchos::RefCountPtr<Ifpack_Preconditioner> Prec = Teuchos::rcp( Factory.Create("IHSS", &*A,0) );
Teuchos::ParameterList List;
List.set("ihss: hermetian type","ILU");
List.set("ihss: skew hermetian type","ILU");
List.set("ihss: ratio eigenvalue",100.0);
// Could set sublist values here to better control the ILU, but this isn't needed for this example.
IFPACK_CHK_ERR(Prec->SetParameters(List));
IFPACK_CHK_ERR(Prec->Compute());
// ============================= //
// Create solver Object //
// ============================= //
AztecOO solver;
solver.SetUserMatrix(&*A);
solver.SetLHS(&*LHS);
solver.SetRHS(&*RHS);
solver.SetAztecOption(AZ_solver,AZ_gmres);
solver.SetPrecOperator(&*Prec);
solver.SetAztecOption(AZ_output, 1);
solver.Iterate(Niters, 1e-8);
// ============================= //
// Construct SORa preconditioner //
// ============================= //
Teuchos::RefCountPtr<Ifpack_Preconditioner> Prec2 = Teuchos::rcp( Factory.Create("SORa", &*A,0) );
Teuchos::ParameterList List2;
List2.set("sora: sweeps",1);
// Could set sublist values here to better control the ILU, but this isn't needed for this example.
IFPACK_CHK_ERR(Prec2->SetParameters(List2));
IFPACK_CHK_ERR(Prec2->Compute());
// ============================= //
// Create solver Object //
// ============================= //
AztecOO solver2;
LHS->PutScalar(0.0);
solver2.SetUserMatrix(&*A);
solver2.SetLHS(&*LHS);
solver2.SetRHS(&*RHS);
solver2.SetAztecOption(AZ_solver,AZ_gmres);
solver2.SetPrecOperator(&*Prec2);
solver2.SetAztecOption(AZ_output, 1);
solver2.Iterate(Niters, 1e-8);
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return(EXIT_SUCCESS);
}
int shylu_dist_solve<Epetra_CrsMatrix,Epetra_MultiVector>(
shylu_symbolic<Epetra_CrsMatrix,Epetra_MultiVector> *ssym,
shylu_data<Epetra_CrsMatrix,Epetra_MultiVector> *data,
shylu_config<Epetra_CrsMatrix,Epetra_MultiVector> *config,
const Epetra_MultiVector& X,
Epetra_MultiVector& Y
)
{
int err;
AztecOO *solver = 0;
assert(X.Map().SameAs(Y.Map()));
//assert(X.Map().SameAs(A_->RowMap()));
const Epetra_MultiVector *newX;
newX = &X;
//rd_->redistribute(X, newX);
int nvectors = newX->NumVectors();
// May have to use importer/exporter
Epetra_Map BsMap(-1, data->Snr, data->SRowElems, 0, X.Comm());
Epetra_Map BdMap(-1, data->Dnr, data->DRowElems, 0, X.Comm());
Epetra_MultiVector Bs(BsMap, nvectors);
Epetra_Import BsImporter(BsMap, newX->Map());
assert(BsImporter.SourceMap().SameAs(newX->Map()));
assert((newX->Map()).SameAs(BsImporter.SourceMap()));
Bs.Import(*newX, BsImporter, Insert);
Epetra_MultiVector Xs(BsMap, nvectors);
Epetra_SerialComm LComm; // Use Serial Comm for the local vectors.
Epetra_Map LocalBdMap(-1, data->Dnr, data->DRowElems, 0, LComm);
Epetra_MultiVector localrhs(LocalBdMap, nvectors);
Epetra_MultiVector locallhs(LocalBdMap, nvectors);
Epetra_MultiVector Z(BdMap, nvectors);
Epetra_MultiVector Bd(BdMap, nvectors);
Epetra_Import BdImporter(BdMap, newX->Map());
assert(BdImporter.SourceMap().SameAs(newX->Map()));
assert((newX->Map()).SameAs(BdImporter.SourceMap()));
Bd.Import(*newX, BdImporter, Insert);
int lda;
double *values;
err = Bd.ExtractView(&values, &lda);
assert (err == 0);
int nrows = ssym->C->RowMap().NumMyElements();
// copy to local vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = localrhs.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
// TODO : Do we need to reset the lhs and rhs here ?
if (config->amesosForDiagonal)
{
ssym->LP->SetRHS(&localrhs);
ssym->LP->SetLHS(&locallhs);
ssym->Solver->Solve();
}
else
{
ssym->ifSolver->ApplyInverse(localrhs, locallhs);
}
err = locallhs.ExtractView(&values, &lda);
assert (err == 0);
// copy to distributed vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = Z.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
Epetra_MultiVector temp1(BsMap, nvectors);
ssym->R->Multiply(false, Z, temp1);
Bs.Update(-1.0, temp1, 1.0);
Xs.PutScalar(0.0);
Epetra_LinearProblem Problem(data->Sbar.get(), &Xs, &Bs);
if (config->schurSolver == "Amesos")
{
Amesos_BaseSolver *solver2 = data->dsolver;
data->LP2->SetLHS(&Xs);
data->LP2->SetRHS(&Bs);
//cout << "Calling solve *****************************" << endl;
solver2->Solve();
//cout << "Out of solve *****************************" << endl;
}
else
{
if (config->libName == "Belos")
{
solver = data->innersolver;
solver->SetLHS(&Xs);
solver->SetRHS(&Bs);
}
else
{
// See the comment above on why we are not able to reuse the solver
// when outer solve is AztecOO as well.
solver = new AztecOO();
//solver.SetPrecOperator(precop_);
solver->SetAztecOption(AZ_solver, AZ_gmres);
// Do not use AZ_none
solver->SetAztecOption(AZ_precond, AZ_dom_decomp);
//solver->SetAztecOption(AZ_precond, AZ_none);
//solver->SetAztecOption(AZ_precond, AZ_Jacobi);
////solver->SetAztecOption(AZ_precond, AZ_Neumann);
//solver->SetAztecOption(AZ_overlap, 3);
//solver->SetAztecOption(AZ_subdomain_solve, AZ_ilu);
//solver->SetAztecOption(AZ_output, AZ_all);
//solver->SetAztecOption(AZ_diagnostics, AZ_all);
solver->SetProblem(Problem);
}
// What should be a good inner_tolerance :-) ?
solver->Iterate(config->inner_maxiters, config->inner_tolerance);
}
Epetra_MultiVector temp(BdMap, nvectors);
ssym->C->Multiply(false, Xs, temp);
temp.Update(1.0, Bd, -1.0);
//Epetra_SerialComm LComm; // Use Serial Comm for the local vectors.
//Epetra_Map LocalBdMap(-1, data->Dnr, data->DRowElems, 0, LComm);
//Epetra_MultiVector localrhs(LocalBdMap, nvectors);
//Epetra_MultiVector locallhs(LocalBdMap, nvectors);
//int lda;
//double *values;
err = temp.ExtractView(&values, &lda);
assert (err == 0);
//int nrows = data->Cptr->RowMap().NumMyElements();
// copy to local vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = localrhs.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
if (config->amesosForDiagonal)
{
ssym->LP->SetRHS(&localrhs);
ssym->LP->SetLHS(&locallhs);
ssym->Solver->Solve();
}
else
{
ssym->ifSolver->ApplyInverse(localrhs, locallhs);
}
err = locallhs.ExtractView(&values, &lda);
assert (err == 0);
// copy to distributed vector //TODO: OMP ?
assert(lda == nrows);
for (int v = 0; v < nvectors; v++)
{
for (int i = 0; i < nrows; i++)
{
err = temp.ReplaceMyValue(i, v, values[i+v*lda]);
assert (err == 0);
}
}
// For checking faults
//if (NumApplyInverse_ == 5) temp.ReplaceMyValue(0, 0, 0.0);
Epetra_Export XdExporter(BdMap, Y.Map());
Y.Export(temp, XdExporter, Insert);
Epetra_Export XsExporter(BsMap, Y.Map());
Y.Export(Xs, XsExporter, Insert);
if (config->libName == "Belos" || config->schurSolver == "Amesos")
{
// clean up
}
else
{
delete solver;
}
return 0;
}//end shylu_dist_solve <epetra,epetra>
int main(int argc, char *argv[]) {
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int MyPID = Comm.MyPID();
bool verbose = false;
if (MyPID==0) verbose = true;
Teuchos::ParameterList GaleriList;
int nx = 30;
GaleriList.set("nx", nx);
GaleriList.set("ny", nx * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", Comm.NumProc());
Teuchos::RefCountPtr<Epetra_Map> Map = Teuchos::rcp( Galeri::CreateMap("Cartesian2D", Comm, GaleriList) );
Teuchos::RefCountPtr<Epetra_CrsMatrix> A = Teuchos::rcp( Galeri::CreateCrsMatrix("Laplace2D", &*Map, GaleriList) );
Teuchos::RefCountPtr<Epetra_MultiVector> LHS = Teuchos::rcp( new Epetra_MultiVector(*Map, 1) );
Teuchos::RefCountPtr<Epetra_MultiVector> RHS = Teuchos::rcp( new Epetra_MultiVector(*Map, 1) );
LHS->PutScalar(0.0); RHS->Random();
// ============================ //
// Construct ILU preconditioner //
// ---------------------------- //
// I wanna test funky values to be sure that they have the same
// influence on the algorithms, both old and new
int LevelFill = 2;
double DropTol = 0.3333;
double Athresh = 0.0123;
double Rthresh = 0.9876;
double Relax = 0.1;
int Overlap = 2;
Teuchos::RefCountPtr<Ifpack_IlukGraph> Graph;
Teuchos::RefCountPtr<Ifpack_CrsRiluk> RILU;
Graph = Teuchos::rcp( new Ifpack_IlukGraph(A->Graph(), LevelFill, Overlap) );
int ierr;
ierr = Graph->ConstructFilledGraph();
IFPACK_CHK_ERR(ierr);
RILU = Teuchos::rcp( new Ifpack_CrsRiluk(*Graph) );
RILU->SetAbsoluteThreshold(Athresh);
RILU->SetRelativeThreshold(Rthresh);
RILU->SetRelaxValue(Relax);
int initerr = RILU->InitValues(*A);
if (initerr!=0) cout << Comm << "*ERR* InitValues = " << initerr;
RILU->Factor();
// Define label for printing out during the solve phase
string label = "Ifpack_CrsRiluk Preconditioner: LevelFill = " + toString(LevelFill) +
" Overlap = " + toString(Overlap) +
" Athresh = " + toString(Athresh) +
" Rthresh = " + toString(Rthresh);
// Here we create an AztecOO object
LHS->PutScalar(0.0);
int Niters = 1200;
AztecOO solver;
solver.SetUserMatrix(&*A);
solver.SetLHS(&*LHS);
solver.SetRHS(&*RHS);
solver.SetAztecOption(AZ_solver,AZ_gmres);
solver.SetPrecOperator(&*RILU);
solver.SetAztecOption(AZ_output, 16);
solver.Iterate(Niters, 5.0e-5);
int OldIters = solver.NumIters();
// now rebuild the same preconditioner using RILU, we expect the same
// number of iterations
Ifpack Factory;
Teuchos::RefCountPtr<Ifpack_Preconditioner> Prec = Teuchos::rcp( Factory.Create("ILU", &*A, Overlap) );
Teuchos::ParameterList List;
List.get("fact: level-of-fill", LevelFill);
List.get("fact: drop tolerance", DropTol);
List.get("fact: absolute threshold", Athresh);
List.get("fact: relative threshold", Rthresh);
List.get("fact: relax value", Relax);
IFPACK_CHK_ERR(Prec->SetParameters(List));
IFPACK_CHK_ERR(Prec->Compute());
// Here we create an AztecOO object
LHS->PutScalar(0.0);
solver.SetUserMatrix(&*A);
solver.SetLHS(&*LHS);
solver.SetRHS(&*RHS);
solver.SetAztecOption(AZ_solver,AZ_gmres);
solver.SetPrecOperator(&*Prec);
solver.SetAztecOption(AZ_output, 16);
solver.Iterate(Niters, 5.0e-5);
int NewIters = solver.NumIters();
if (OldIters != NewIters)
IFPACK_CHK_ERR(-1);
#ifdef HAVE_IFPACK_SUPERLU
// Now test w/ SuperLU's ILU, if we've got it
Teuchos::RefCountPtr<Ifpack_Preconditioner> Prec2 = Teuchos::rcp( Factory.Create("SILU", &*A,0) );
Teuchos::ParameterList SList;
SList.set("fact: drop tolerance",1e-4);
SList.set("fact: zero pivot threshold",.1);
SList.set("fact: maximum fill factor",10.0);
// NOTE: There is a bug in SuperLU 4.0 which will crash the code if the maximum fill factor is set too low.
// This bug was reported to Sherry Li on 4/8/10.
SList.set("fact: silu drop rule",9);
IFPACK_CHK_ERR(Prec2->SetParameters(SList));
IFPACK_CHK_ERR(Prec2->Compute());
LHS->PutScalar(0.0);
solver.SetPrecOperator(&*Prec2);
solver.Iterate(Niters, 5.0e-5);
Prec2->Print(cout);
#endif
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return(EXIT_SUCCESS);
}