void DistributedOverlapMatrix<Rank>::SolveOverlapRank(Wavefunction<Rank> &srcPsi, Wavefunction<Rank> &destPsi, int opRank, bool fastAlgo)
//void DistributedOverlapMatrix<Rank>::SolveOverlapRank(Wavefunction<Rank> &psi, int opRank, bool fastAlgo)
{
SetupRank(srcPsi, opRank);
if (fastAlgo)
{
//Setup source multivector
WavefunctionToMultiVector(srcPsi, InputVector(opRank), opRank);
//Solve
AMESOS_CHK_ERRV(Solvers(opRank)->Solve());
//Put solution multivector into destination wavefunction
MultiVectorToWavefunction(destPsi, OutputVector(opRank), opRank);
}
else
{
//Setup source multivector
Epetra_MultiVector_Ptr srcVec = SetupMultivector(srcPsi, opRank);
//Output multivector (copy of input)
Epetra_MultiVector_Ptr destVec = Epetra_MultiVector_Ptr(new Epetra_MultiVector(*srcVec));
//Set up Epetra LinearProblem with overlap for this rank and input/output multivectors
Epetra_LinearProblem Problem(OverlapMatrices(opRank).get(), destVec.get(), srcVec.get());
//Initialize Amesos solver and factory
Amesos_BaseSolver* Solver;
Amesos Factory;
std::string SolverType = "Amesos_Superludist";
Solver = Factory.Create(SolverType, Problem);
//Check that requested solver exists in Amesos
if (Solver == 0)
{
throw std::runtime_error("Specified Amesos solver not available");
}
//Setup the parameter list for the solver
Teuchos::ParameterList List;
//List.set("PrintTiming", true);
//List.set("PrintStatus", true);
Solver->SetParameters(List);
//Factorization and solve
Solver->SymbolicFactorization();
Solver->NumericFactorization();
AMESOS_CHK_ERRV(Solver->Solve());
//Put solution multivector into destination wavefunction
MultiVectorToWavefunction(destPsi, destVec, opRank);
//Solver->PrintTiming();
}
}
int PartialFactorizationOneStep( const char* AmesosClass,
const Epetra_Comm &Comm,
bool transpose,
bool verbose,
Teuchos::ParameterList ParamList,
Epetra_CrsMatrix *& Amat,
double Rcond,
int Steps )
{
assert( Steps >= 0 && Steps < MaxNumSteps ) ;
int iam = Comm.MyPID() ;
int errors = 0 ;
const Epetra_Map *RangeMap =
transpose?&Amat->OperatorDomainMap():&Amat->OperatorRangeMap() ;
const Epetra_Map *DomainMap =
transpose?&Amat->OperatorRangeMap():&Amat->OperatorDomainMap() ;
Epetra_Vector xexact(*DomainMap);
Epetra_Vector x(*DomainMap);
Epetra_Vector b(*RangeMap);
Epetra_Vector bcheck(*RangeMap);
Epetra_Vector difference(*DomainMap);
Epetra_LinearProblem Problem;
Amesos_BaseSolver* Abase ;
Amesos Afactory;
Abase = Afactory.Create( AmesosClass, Problem ) ;
std::string AC = AmesosClass ;
if ( AC == "Amesos_Mumps" ) {
ParamList.set( "NoDestroy", true );
Abase->SetParameters( ParamList ) ;
}
double relresidual = 0 ;
if ( Steps > 0 ) {
//
// Phase 1: Compute b = A' A' A xexact
//
Problem.SetOperator( Amat );
//
// We only set transpose if we have to - this allows valgrind to check
// that transpose is set to a default value before it is used.
//
if ( transpose ) OUR_CHK_ERR( Abase->SetUseTranspose( transpose ) );
// if (verbose) ParamList.set( "DebugLevel", 1 );
// if (verbose) ParamList.set( "OutputLevel", 1 );
if ( Steps > 1 ) {
OUR_CHK_ERR( Abase->SetParameters( ParamList ) );
if ( Steps > 2 ) {
xexact.Random();
xexact.PutScalar(1.0);
//
// Compute cAx = A' xexact
//
Amat->Multiply( transpose, xexact, b ) ; // b = A x2 = A A' A'' xexact
#if 0
std::cout << __FILE__ << "::" << __LINE__ << "b = " << std::endl ;
b.Print( std::cout ) ;
std::cout << __FILE__ << "::" << __LINE__ << "xexact = " << std::endl ;
xexact.Print( std::cout ) ;
std::cout << __FILE__ << "::" << __LINE__ << "x = " << std::endl ;
x.Print( std::cout ) ;
#endif
//
// Phase 2: Solve A' A' A x = b
//
//
// Solve A sAAx = b
//
Problem.SetLHS( &x );
Problem.SetRHS( &b );
OUR_CHK_ERR( Abase->SymbolicFactorization( ) );
if ( Steps > 2 ) {
OUR_CHK_ERR( Abase->SymbolicFactorization( ) );
if ( Steps > 3 ) {
OUR_CHK_ERR( Abase->NumericFactorization( ) );
if ( Steps > 4 ) {
OUR_CHK_ERR( Abase->NumericFactorization( ) );
if ( Steps > 5 ) {
OUR_CHK_ERR( Abase->Solve( ) );
if ( Steps > 6 ) {
OUR_CHK_ERR( Abase->Solve( ) );
Amat->Multiply( transpose, x, bcheck ) ; // temp = A" x2
double norm_diff ;
double norm_one ;
difference.Update( 1.0, x, -1.0, xexact, 0.0 ) ;
difference.Norm2( &norm_diff ) ;
x.Norm2( &norm_one ) ;
relresidual = norm_diff / norm_one ;
if (iam == 0 ) {
if ( relresidual * Rcond > 1e-16 ) {
if (verbose) std::cout << __FILE__ << "::"<< __LINE__
<< " norm( x - xexact ) / norm(x) = "
<< norm_diff /norm_one << std::endl ;
errors += 1 ;
}
}
}
}
}
}
}
}
}
}
delete Abase;
return errors;
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
bool verbose = (Comm.MyPID() == 0);
double TotalResidual = 0.0;
// Create the Map, defined as a grid, of size nx x ny x nz,
// subdivided into mx x my x mz cubes, each assigned to a
// different processor.
#if 0
ParameterList GaleriList;
GaleriList.set("nx", 4);
GaleriList.set("ny", 4);
GaleriList.set("nz", 4 * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", 1);
GaleriList.set("mz", Comm.NumProc());
Epetra_Map* Map = CreateMap("Cartesian3D", Comm, GaleriList);
// Create a matrix, in this case corresponding to a 3D Laplacian
// discretized using a classical 7-point stencil. Please refer to
// the Galeri documentation for an overview of available matrices.
//
// NOTE: matrix must be symmetric if DSCPACK is used.
Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace3D", Map, GaleriList);
#else
bool transpose = false ;
bool distribute = false ;
bool symmetric ;
Epetra_CrsMatrix *Matrix = 0 ;
Epetra_Map *Map = 0 ;
CreateCrsMatrix( "ibm.triU", Comm, Map, transpose, distribute, &symmetric, Matrix ) ;
#endif
// build vectors, in this case with 1 vector
Epetra_MultiVector LHS(*Map, 1);
Epetra_MultiVector RHS(*Map, 1);
// create a linear problem object
Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);
// use this list to set up parameters, now it is required
// to use all the available processes (if supported by the
// underlying solver). Uncomment the following two lines
// to let Amesos print out some timing and status information.
ParameterList List;
List.set("PrintTiming",true);
List.set("PrintStatus",true);
List.set("MaxProcs",Comm.NumProc());
std::vector<std::string> SolverType;
SolverType.push_back("Amesos_Paraklete");
Epetra_Time Time(Comm);
// this is the Amesos factory object that will create
// a specific Amesos solver.
Amesos Factory;
// Cycle over all solvers.
// Only installed solvers will be tested.
for (unsigned int i = 0 ; i < SolverType.size() ; ++i)
{
// Check whether the solver is available or not
if (Factory.Query(SolverType[i]))
{
// 1.- set exact solution (constant vector)
LHS.PutScalar(1.0);
// 2.- create corresponding rhs
Matrix->Multiply(false, LHS, RHS);
// 3.- randomize solution vector
LHS.Random();
// 4.- create the amesos solver object
Amesos_BaseSolver* Solver = Factory.Create(SolverType[i], Problem);
assert (Solver != 0);
Solver->SetParameters(List);
// 5.- factorize and solve
Time.ResetStartTime();
AMESOS_CHK_ERR(Solver->SymbolicFactorization());
// 7.- delete the object
delete Solver;
}
}
delete Matrix;
delete Map;
if (TotalResidual > 1e-9)
exit(EXIT_FAILURE);
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} // end of main()
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
bool verbose = (Comm.MyPID() == 0);
double TotalResidual = 0.0;
// Create the Map, defined as a grid, of size nx x ny x nz,
// subdivided into mx x my x mz cubes, each assigned to a
// different processor.
#ifndef FILENAME_SPECIFIED_ON_COMMAND_LINE
ParameterList GaleriList;
GaleriList.set("nx", 4);
GaleriList.set("ny", 4);
GaleriList.set("nz", 4 * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", 1);
GaleriList.set("mz", Comm.NumProc());
Epetra_Map* Map = CreateMap("Cartesian3D", Comm, GaleriList);
// Create a matrix, in this case corresponding to a 3D Laplacian
// discretized using a classical 7-point stencil. Please refer to
// the Galeri documentation for an overview of available matrices.
//
// NOTE: matrix must be symmetric if DSCPACK is used.
Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace3D", Map, GaleriList);
#else
bool transpose = false ;
bool distribute = false ;
bool symmetric ;
Epetra_CrsMatrix *Matrix = 0 ;
Epetra_Map *Map = 0 ;
MyCreateCrsMatrix( argv[1], Comm, Map, transpose, distribute, symmetric, Matrix ) ;
#endif
// build vectors, in this case with 1 vector
Epetra_MultiVector LHS(*Map, 1);
Epetra_MultiVector RHS(*Map, 1);
// create a linear problem object
Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);
// use this list to set up parameters, now it is required
// to use all the available processes (if supported by the
// underlying solver). Uncomment the following two lines
// to let Amesos print out some timing and status information.
ParameterList List;
List.set("PrintTiming",true);
List.set("PrintStatus",true);
List.set("MaxProcs",Comm.NumProc());
std::vector<std::string> SolverType;
SolverType.push_back("Amesos_Paraklete");
SolverType.push_back("Amesos_Klu");
Comm.Barrier() ;
#if 1
SolverType.push_back("Amesos_Lapack");
SolverType.push_back("Amesos_Umfpack");
SolverType.push_back("Amesos_Pardiso");
SolverType.push_back("Amesos_Taucs");
SolverType.push_back("Amesos_Superlu");
SolverType.push_back("Amesos_Superludist");
SolverType.push_back("Amesos_Mumps");
SolverType.push_back("Amesos_Dscpack");
SolverType.push_back("Amesos_Scalapack");
#endif
Epetra_Time Time(Comm);
// this is the Amesos factory object that will create
// a specific Amesos solver.
Amesos Factory;
// Cycle over all solvers.
// Only installed solvers will be tested.
for (unsigned int i = 0 ; i < SolverType.size() ; ++i)
{
// Check whether the solver is available or not
if (Factory.Query(SolverType[i]))
{
// 1.- set exact solution (constant vector)
LHS.PutScalar(1.0);
// 2.- create corresponding rhs
Matrix->Multiply(false, LHS, RHS);
// 3.- randomize solution vector
LHS.Random();
// 4.- create the amesos solver object
Amesos_BaseSolver* Solver = Factory.Create(SolverType[i], Problem);
assert (Solver != 0);
Solver->SetParameters(List);
Solver->SetUseTranspose( true) ;
// 5.- factorize and solve
Comm.Barrier() ;
if (verbose)
std::cout << std::endl
<< "Solver " << SolverType[i]
<< ", verbose = " << verbose << std::endl ;
Comm.Barrier() ;
Time.ResetStartTime();
AMESOS_CHK_ERR(Solver->SymbolicFactorization());
if (verbose)
std::cout << std::endl
<< "Solver " << SolverType[i]
<< ", symbolic factorization time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
AMESOS_CHK_ERR(Solver->NumericFactorization());
if (verbose)
std::cout << "Solver " << SolverType[i]
<< ", numeric factorization time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
AMESOS_CHK_ERR(Solver->Solve());
if (verbose)
std::cout << "Solver " << SolverType[i]
<< ", solve time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
// 6.- compute difference between exact solution and Amesos one
// (there are other ways of doing this in Epetra, but let's
// keep it simple)
double d = 0.0, d_tot = 0.0;
for (int j = 0 ; j< LHS.Map().NumMyElements() ; ++j)
d += (LHS[0][j] - 1.0) * (LHS[0][j] - 1.0);
Comm.SumAll(&d,&d_tot,1);
if (verbose)
std::cout << "Solver " << SolverType[i] << ", ||x - x_exact||_2 = "
<< sqrt(d_tot) << std::endl;
// 7.- delete the object
delete Solver;
TotalResidual += d_tot;
}
}
delete Matrix;
delete Map;
if (TotalResidual > 1e-9)
exit(EXIT_FAILURE);
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} // end of main()
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int NumGlobalRows = 10000; // must be a square for the
// matrix generator.
int NumVectors = 1; // number of rhs's. Amesos
// supports single or
// multiple RHS.
// Initializes an Gallery object.
// NOTE: this example uses the Trilinos package Galeri
// to define in an easy way the linear system matrix.
// The user can easily change the matrix type; consult the
// Galeri documentation for mode details.
//
// Here the problem has size nx x ny, and the 2D Cartesian
// grid is divided into mx x my subdomains.
ParameterList GaleriList;
GaleriList.set("nx", 100);
GaleriList.set("ny", 100 * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", Comm.NumProc());
Epetra_Map* Map = CreateMap("Cartesian2D", Comm, GaleriList);
Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace2D", Map, GaleriList);
// Creates vectors for right-hand side and solution, and the
// linear problem container.
Epetra_Vector LHS(*Map); LHS.PutScalar(0.0); // zero solution
Epetra_Vector RHS(*Map); RHS.Random(); // random rhs
Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);
// ===================================================== //
// B E G I N N I N G O F T H E AM E S O S P A R T //
// ===================================================== //
// Initializes the Amesos solver. This is the base class for
// Amesos. It is a pure virtual class (hence objects of this
// class cannot be allocated, and can exist only as pointers
// or references).
//
Amesos_BaseSolver* Solver;
// Initializes the Factory. Factory is a function class (a
// class that contains methods only, no data). Factory
// will be used to create Amesos_BaseSolver derived objects.
//
Amesos Factory;
Solver = Factory.Create("Klu", Problem);
// Parameters for all Amesos solvers are set through
// a call to SetParameters(List). List is a Teuchos
// parameter list (Amesos requires Teuchos to compile).
// In most cases, users can proceed without calling
// SetParameters(). Please refer to the Amesos guide
// for more details.
// NOTE: you can skip this call; then the solver will
// use default parameters.
//
// Parameters in the list are set using
// List.set("parameter-name", ParameterValue);
// In this example, we specify that we want more output.
//
Teuchos::ParameterList List;
List.set("PrintTiming", true);
List.set("PrintStatus", true);
Solver->SetParameters(List);
// Now we are ready to solve. Generally, users will
// call SymbolicFactorization(), then NumericFactorization(),
// and finally Solve(). Note that:
// - the numerical values of the linear system matrix
// are *not* required before NumericFactorization();
// - solution and rhs are *not* required before calling
// Solve().
if (Comm.MyPID() == 0)
cout << "Starting symbolic factorization..." << endl;
Solver->SymbolicFactorization();
// you can change the matrix values here
if (Comm.MyPID() == 0)
cout << "Starting numeric factorization..." << endl;
Solver->NumericFactorization();
// you can change LHS and RHS here
if (Comm.MyPID() == 0)
cout << "Starting solution phase..." << endl;
Solver->Solve();
// =========================================== //
// E N D O F T H E A M E S O S P A R T //
// =========================================== //
// delete Solver. MPI calls can occur.
delete Solver;
// delete the objects created by Galeri
delete Matrix;
delete Map;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} // end of main()
