void DistributedOverlapMatrix<Rank>::SolveOverlapRank(Wavefunction<Rank> &srcPsi, Wavefunction<Rank> &destPsi, int opRank, bool fastAlgo)
//void DistributedOverlapMatrix<Rank>::SolveOverlapRank(Wavefunction<Rank> &psi, int opRank, bool fastAlgo)
{
SetupRank(srcPsi, opRank);
if (fastAlgo)
{
//Setup source multivector
WavefunctionToMultiVector(srcPsi, InputVector(opRank), opRank);
//Solve
AMESOS_CHK_ERRV(Solvers(opRank)->Solve());
//Put solution multivector into destination wavefunction
MultiVectorToWavefunction(destPsi, OutputVector(opRank), opRank);
}
else
{
//Setup source multivector
Epetra_MultiVector_Ptr srcVec = SetupMultivector(srcPsi, opRank);
//Output multivector (copy of input)
Epetra_MultiVector_Ptr destVec = Epetra_MultiVector_Ptr(new Epetra_MultiVector(*srcVec));
//Set up Epetra LinearProblem with overlap for this rank and input/output multivectors
Epetra_LinearProblem Problem(OverlapMatrices(opRank).get(), destVec.get(), srcVec.get());
//Initialize Amesos solver and factory
Amesos_BaseSolver* Solver;
Amesos Factory;
std::string SolverType = "Amesos_Superludist";
Solver = Factory.Create(SolverType, Problem);
//Check that requested solver exists in Amesos
if (Solver == 0)
{
throw std::runtime_error("Specified Amesos solver not available");
}
//Setup the parameter list for the solver
Teuchos::ParameterList List;
//List.set("PrintTiming", true);
//List.set("PrintStatus", true);
Solver->SetParameters(List);
//Factorization and solve
Solver->SymbolicFactorization();
Solver->NumericFactorization();
AMESOS_CHK_ERRV(Solver->Solve());
//Put solution multivector into destination wavefunction
MultiVectorToWavefunction(destPsi, destVec, opRank);
//Solver->PrintTiming();
}
}
int main(int argc, char *argv[]) {
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
// initialize an Gallery object
CrsMatrixGallery Gallery("laplace_2d", Comm, false); // CJ TODO FIXME: change for Epetra64
Gallery.Set("problem_size", 100); //must be a square number
// get pointers to the linear problem, containing matrix, LHS and RHS.
// if you need to access them, you can for example uncomment the following
// code:
// Epetra_CrsMatrix* Matrix = Gallery.GetMatrix();
// Epetra_MultiVector* LHS = Gallery.GetStartingSolution();
// Epetra_MultiVector* RHS = Gallery.GetRHS();
//
// NOTE: StartingSolution and RHS are pointers to Gallery's internally stored
// vectors. Using StartingSolution and RHS, we can verify the residual
// after the solution of the linear system. However, users may define as well
// their own vectors for solution and RHS.
Epetra_LinearProblem* Problem = Gallery.GetLinearProblem();
// initialize Amesos solver:
// `Solver' is the pointer to the Amesos solver
// (note the use of the base class Amesos_BaseSolver)
Amesos_BaseSolver* Solver;
// Amesos_Factory is the function class used to create the solver.
// This class contains no data.
Amesos Amesos_Factory;
// empty parameter list
Teuchos::ParameterList List;
// may also try: "Amesos_Umfpack", "Amesos_Lapack", ...
string SolverType = "Amesos_Klu";
Solver = Amesos_Factory.Create(SolverType, *Problem);
// Amesos_Factory returns 0 is the selected solver is not
// available
assert (Solver);
// start solving
Solver->SymbolicFactorization();
Solver->NumericFactorization();
Solver->Solve();
// verify that residual is really small
double residual, diff;
Gallery.ComputeResidual(&residual);
Gallery.ComputeDiffBetweenStartingAndExactSolutions(&diff);
if( Comm.MyPID() == 0 ) {
cout << "||b-Ax||_2 = " << residual << endl;
cout << "||x_exact - x||_2 = " << diff << endl;
}
// delete Solver
delete Solver;
if (residual > 1e-5)
exit(EXIT_FAILURE);
#ifdef HAVE_MPI
MPI_Finalize();
#endif
exit(EXIT_SUCCESS);
}
int PartialFactorizationOneStep( const char* AmesosClass,
const Epetra_Comm &Comm,
bool transpose,
bool verbose,
Teuchos::ParameterList ParamList,
Epetra_CrsMatrix *& Amat,
double Rcond,
int Steps )
{
assert( Steps >= 0 && Steps < MaxNumSteps ) ;
int iam = Comm.MyPID() ;
int errors = 0 ;
const Epetra_Map *RangeMap =
transpose?&Amat->OperatorDomainMap():&Amat->OperatorRangeMap() ;
const Epetra_Map *DomainMap =
transpose?&Amat->OperatorRangeMap():&Amat->OperatorDomainMap() ;
Epetra_Vector xexact(*DomainMap);
Epetra_Vector x(*DomainMap);
Epetra_Vector b(*RangeMap);
Epetra_Vector bcheck(*RangeMap);
Epetra_Vector difference(*DomainMap);
Epetra_LinearProblem Problem;
Amesos_BaseSolver* Abase ;
Amesos Afactory;
Abase = Afactory.Create( AmesosClass, Problem ) ;
std::string AC = AmesosClass ;
if ( AC == "Amesos_Mumps" ) {
ParamList.set( "NoDestroy", true );
Abase->SetParameters( ParamList ) ;
}
double relresidual = 0 ;
if ( Steps > 0 ) {
//
// Phase 1: Compute b = A' A' A xexact
//
Problem.SetOperator( Amat );
//
// We only set transpose if we have to - this allows valgrind to check
// that transpose is set to a default value before it is used.
//
if ( transpose ) OUR_CHK_ERR( Abase->SetUseTranspose( transpose ) );
// if (verbose) ParamList.set( "DebugLevel", 1 );
// if (verbose) ParamList.set( "OutputLevel", 1 );
if ( Steps > 1 ) {
OUR_CHK_ERR( Abase->SetParameters( ParamList ) );
if ( Steps > 2 ) {
xexact.Random();
xexact.PutScalar(1.0);
//
// Compute cAx = A' xexact
//
Amat->Multiply( transpose, xexact, b ) ; // b = A x2 = A A' A'' xexact
#if 0
std::cout << __FILE__ << "::" << __LINE__ << "b = " << std::endl ;
b.Print( std::cout ) ;
std::cout << __FILE__ << "::" << __LINE__ << "xexact = " << std::endl ;
xexact.Print( std::cout ) ;
std::cout << __FILE__ << "::" << __LINE__ << "x = " << std::endl ;
x.Print( std::cout ) ;
#endif
//
// Phase 2: Solve A' A' A x = b
//
//
// Solve A sAAx = b
//
Problem.SetLHS( &x );
Problem.SetRHS( &b );
OUR_CHK_ERR( Abase->SymbolicFactorization( ) );
if ( Steps > 2 ) {
OUR_CHK_ERR( Abase->SymbolicFactorization( ) );
if ( Steps > 3 ) {
OUR_CHK_ERR( Abase->NumericFactorization( ) );
if ( Steps > 4 ) {
OUR_CHK_ERR( Abase->NumericFactorization( ) );
if ( Steps > 5 ) {
OUR_CHK_ERR( Abase->Solve( ) );
if ( Steps > 6 ) {
OUR_CHK_ERR( Abase->Solve( ) );
Amat->Multiply( transpose, x, bcheck ) ; // temp = A" x2
double norm_diff ;
double norm_one ;
difference.Update( 1.0, x, -1.0, xexact, 0.0 ) ;
difference.Norm2( &norm_diff ) ;
x.Norm2( &norm_one ) ;
relresidual = norm_diff / norm_one ;
if (iam == 0 ) {
if ( relresidual * Rcond > 1e-16 ) {
if (verbose) std::cout << __FILE__ << "::"<< __LINE__
<< " norm( x - xexact ) / norm(x) = "
<< norm_diff /norm_one << std::endl ;
errors += 1 ;
}
}
}
}
}
}
}
}
}
}
delete Abase;
return errors;
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
bool verbose = (Comm.MyPID() == 0);
double TotalResidual = 0.0;
// Create the Map, defined as a grid, of size nx x ny x nz,
// subdivided into mx x my x mz cubes, each assigned to a
// different processor.
#if 0
ParameterList GaleriList;
GaleriList.set("nx", 4);
GaleriList.set("ny", 4);
GaleriList.set("nz", 4 * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", 1);
GaleriList.set("mz", Comm.NumProc());
Epetra_Map* Map = CreateMap("Cartesian3D", Comm, GaleriList);
// Create a matrix, in this case corresponding to a 3D Laplacian
// discretized using a classical 7-point stencil. Please refer to
// the Galeri documentation for an overview of available matrices.
//
// NOTE: matrix must be symmetric if DSCPACK is used.
Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace3D", Map, GaleriList);
#else
bool transpose = false ;
bool distribute = false ;
bool symmetric ;
Epetra_CrsMatrix *Matrix = 0 ;
Epetra_Map *Map = 0 ;
CreateCrsMatrix( "ibm.triU", Comm, Map, transpose, distribute, &symmetric, Matrix ) ;
#endif
// build vectors, in this case with 1 vector
Epetra_MultiVector LHS(*Map, 1);
Epetra_MultiVector RHS(*Map, 1);
// create a linear problem object
Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);
// use this list to set up parameters, now it is required
// to use all the available processes (if supported by the
// underlying solver). Uncomment the following two lines
// to let Amesos print out some timing and status information.
ParameterList List;
List.set("PrintTiming",true);
List.set("PrintStatus",true);
List.set("MaxProcs",Comm.NumProc());
std::vector<std::string> SolverType;
SolverType.push_back("Amesos_Paraklete");
Epetra_Time Time(Comm);
// this is the Amesos factory object that will create
// a specific Amesos solver.
Amesos Factory;
// Cycle over all solvers.
// Only installed solvers will be tested.
for (unsigned int i = 0 ; i < SolverType.size() ; ++i)
{
// Check whether the solver is available or not
if (Factory.Query(SolverType[i]))
{
// 1.- set exact solution (constant vector)
LHS.PutScalar(1.0);
// 2.- create corresponding rhs
Matrix->Multiply(false, LHS, RHS);
// 3.- randomize solution vector
LHS.Random();
// 4.- create the amesos solver object
Amesos_BaseSolver* Solver = Factory.Create(SolverType[i], Problem);
assert (Solver != 0);
Solver->SetParameters(List);
// 5.- factorize and solve
Time.ResetStartTime();
AMESOS_CHK_ERR(Solver->SymbolicFactorization());
// 7.- delete the object
delete Solver;
}
}
delete Matrix;
delete Map;
if (TotalResidual > 1e-9)
exit(EXIT_FAILURE);
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} // end of main()
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
bool verbose = (Comm.MyPID() == 0);
double TotalResidual = 0.0;
// Create the Map, defined as a grid, of size nx x ny x nz,
// subdivided into mx x my x mz cubes, each assigned to a
// different processor.
#ifndef FILENAME_SPECIFIED_ON_COMMAND_LINE
ParameterList GaleriList;
GaleriList.set("nx", 4);
GaleriList.set("ny", 4);
GaleriList.set("nz", 4 * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", 1);
GaleriList.set("mz", Comm.NumProc());
Epetra_Map* Map = CreateMap("Cartesian3D", Comm, GaleriList);
// Create a matrix, in this case corresponding to a 3D Laplacian
// discretized using a classical 7-point stencil. Please refer to
// the Galeri documentation for an overview of available matrices.
//
// NOTE: matrix must be symmetric if DSCPACK is used.
Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace3D", Map, GaleriList);
#else
bool transpose = false ;
bool distribute = false ;
bool symmetric ;
Epetra_CrsMatrix *Matrix = 0 ;
Epetra_Map *Map = 0 ;
MyCreateCrsMatrix( argv[1], Comm, Map, transpose, distribute, symmetric, Matrix ) ;
#endif
// build vectors, in this case with 1 vector
Epetra_MultiVector LHS(*Map, 1);
Epetra_MultiVector RHS(*Map, 1);
// create a linear problem object
Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);
// use this list to set up parameters, now it is required
// to use all the available processes (if supported by the
// underlying solver). Uncomment the following two lines
// to let Amesos print out some timing and status information.
ParameterList List;
List.set("PrintTiming",true);
List.set("PrintStatus",true);
List.set("MaxProcs",Comm.NumProc());
std::vector<std::string> SolverType;
SolverType.push_back("Amesos_Paraklete");
SolverType.push_back("Amesos_Klu");
Comm.Barrier() ;
#if 1
SolverType.push_back("Amesos_Lapack");
SolverType.push_back("Amesos_Umfpack");
SolverType.push_back("Amesos_Pardiso");
SolverType.push_back("Amesos_Taucs");
SolverType.push_back("Amesos_Superlu");
SolverType.push_back("Amesos_Superludist");
SolverType.push_back("Amesos_Mumps");
SolverType.push_back("Amesos_Dscpack");
SolverType.push_back("Amesos_Scalapack");
#endif
Epetra_Time Time(Comm);
// this is the Amesos factory object that will create
// a specific Amesos solver.
Amesos Factory;
// Cycle over all solvers.
// Only installed solvers will be tested.
for (unsigned int i = 0 ; i < SolverType.size() ; ++i)
{
// Check whether the solver is available or not
if (Factory.Query(SolverType[i]))
{
// 1.- set exact solution (constant vector)
LHS.PutScalar(1.0);
// 2.- create corresponding rhs
Matrix->Multiply(false, LHS, RHS);
// 3.- randomize solution vector
LHS.Random();
// 4.- create the amesos solver object
Amesos_BaseSolver* Solver = Factory.Create(SolverType[i], Problem);
assert (Solver != 0);
Solver->SetParameters(List);
Solver->SetUseTranspose( true) ;
// 5.- factorize and solve
Comm.Barrier() ;
if (verbose)
std::cout << std::endl
<< "Solver " << SolverType[i]
<< ", verbose = " << verbose << std::endl ;
Comm.Barrier() ;
Time.ResetStartTime();
AMESOS_CHK_ERR(Solver->SymbolicFactorization());
if (verbose)
std::cout << std::endl
<< "Solver " << SolverType[i]
<< ", symbolic factorization time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
AMESOS_CHK_ERR(Solver->NumericFactorization());
if (verbose)
std::cout << "Solver " << SolverType[i]
<< ", numeric factorization time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
AMESOS_CHK_ERR(Solver->Solve());
if (verbose)
std::cout << "Solver " << SolverType[i]
<< ", solve time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
// 6.- compute difference between exact solution and Amesos one
// (there are other ways of doing this in Epetra, but let's
// keep it simple)
double d = 0.0, d_tot = 0.0;
for (int j = 0 ; j< LHS.Map().NumMyElements() ; ++j)
d += (LHS[0][j] - 1.0) * (LHS[0][j] - 1.0);
Comm.SumAll(&d,&d_tot,1);
if (verbose)
std::cout << "Solver " << SolverType[i] << ", ||x - x_exact||_2 = "
<< sqrt(d_tot) << std::endl;
// 7.- delete the object
delete Solver;
TotalResidual += d_tot;
}
}
delete Matrix;
delete Map;
if (TotalResidual > 1e-9)
exit(EXIT_FAILURE);
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} // end of main()
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
Teuchos::GlobalMPISession mpiSession(&argc, &argv, 0);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int nProcs, myPID ;
Teuchos::ParameterList pLUList ; // ParaLU parameters
Teuchos::ParameterList isoList ; // Isorropia parameters
string ipFileName = "ShyLU.xml"; // TODO : Accept as i/p
nProcs = mpiSession.getNProc();
myPID = Comm.MyPID();
if (myPID == 0)
{
cout <<"Parallel execution: nProcs="<< nProcs << endl;
}
// =================== Read input xml file =============================
Teuchos::updateParametersFromXmlFile(ipFileName, &pLUList);
isoList = pLUList.sublist("Isorropia Input");
// Get matrix market file name
string MMFileName = Teuchos::getParameter<string>(pLUList, "mm_file");
string prec_type = Teuchos::getParameter<string>(pLUList, "preconditioner");
if (myPID == 0)
{
cout << "Input :" << endl;
cout << "ParaLU params " << endl;
pLUList.print(std::cout, 2, true, true);
cout << "Matrix market file name: " << MMFileName << endl;
}
// ==================== Read input Matrix ==============================
Epetra_CrsMatrix *A;
int err = EpetraExt::MatrixMarketFileToCrsMatrix(MMFileName.c_str(), Comm, A);
//EpetraExt::MatlabFileToCrsMatrix(MMFileName.c_str(), Comm, A);
//assert(err != 0);
cout <<"Done reading the matrix"<< endl;
int n = A->NumGlobalRows();
cout <<"n="<< n << endl;
// Create input vectors
Epetra_Map vecMap(n, 0, Comm);
Epetra_MultiVector x(vecMap, 1);
Epetra_MultiVector b(vecMap, 1, false);
b.PutScalar(1.0); // TODO : Accept it as input
// Partition the matrix with hypergraph partitioning and redisstribute
Isorropia::Epetra::Partitioner *partitioner = new
Isorropia::Epetra::Partitioner(A, isoList, false);
partitioner->partition();
Isorropia::Epetra::Redistributor rd(partitioner);
Epetra_CrsMatrix *newA;
Epetra_MultiVector *newX, *newB;
rd.redistribute(*A, newA);
delete A;
A = newA;
rd.redistribute(x, newX);
rd.redistribute(b, newB);
Epetra_LinearProblem problem(A, newX, newB);
Amesos Factory;
char* SolverType = "Amesos_Klu";
bool IsAvailable = Factory.Query(SolverType);
Epetra_LinearProblem *LP = new Epetra_LinearProblem();
LP->SetOperator(A);
LP->SetLHS(newX);
LP->SetRHS(newB);
Amesos_BaseSolver *Solver = Factory.Create(SolverType, *LP);
Solver->SymbolicFactorization();
Teuchos::Time ftime("setup time");
ftime.start();
Solver->NumericFactorization();
cout << "Numeric Factorization" << endl;
Solver->Solve();
cout << "Solve done" << endl;
ftime.stop();
cout << "Time to setup" << ftime.totalElapsedTime() << endl;
// compute ||Ax - b||
double Norm;
Epetra_MultiVector Ax(vecMap, 1);
Epetra_MultiVector *newAx;
rd.redistribute(Ax, newAx);
A->Multiply(false, *newX, *newAx);
newAx->Update(1.0, *newB, -1.0);
newAx->Norm2(&Norm);
double ANorm = A->NormOne();
cout << "|Ax-b |/|A| = " << Norm/ANorm << endl;
delete newAx;
delete newX;
delete newB;
delete A;
delete partitioner;
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int NumGlobalRows = 10000; // must be a square for the
// matrix generator.
int NumVectors = 1; // number of rhs's. Amesos
// supports single or
// multiple RHS.
// Initializes an Gallery object.
// NOTE: this example uses the Trilinos package Galeri
// to define in an easy way the linear system matrix.
// The user can easily change the matrix type; consult the
// Galeri documentation for mode details.
//
// Here the problem has size nx x ny, and the 2D Cartesian
// grid is divided into mx x my subdomains.
ParameterList GaleriList;
GaleriList.set("nx", 100);
GaleriList.set("ny", 100 * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", Comm.NumProc());
Epetra_Map* Map = CreateMap("Cartesian2D", Comm, GaleriList);
Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace2D", Map, GaleriList);
// Creates vectors for right-hand side and solution, and the
// linear problem container.
Epetra_Vector LHS(*Map); LHS.PutScalar(0.0); // zero solution
Epetra_Vector RHS(*Map); RHS.Random(); // random rhs
Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);
// ===================================================== //
// B E G I N N I N G O F T H E AM E S O S P A R T //
// ===================================================== //
// Initializes the Amesos solver. This is the base class for
// Amesos. It is a pure virtual class (hence objects of this
// class cannot be allocated, and can exist only as pointers
// or references).
//
Amesos_BaseSolver* Solver;
// Initializes the Factory. Factory is a function class (a
// class that contains methods only, no data). Factory
// will be used to create Amesos_BaseSolver derived objects.
//
Amesos Factory;
Solver = Factory.Create("Klu", Problem);
// Parameters for all Amesos solvers are set through
// a call to SetParameters(List). List is a Teuchos
// parameter list (Amesos requires Teuchos to compile).
// In most cases, users can proceed without calling
// SetParameters(). Please refer to the Amesos guide
// for more details.
// NOTE: you can skip this call; then the solver will
// use default parameters.
//
// Parameters in the list are set using
// List.set("parameter-name", ParameterValue);
// In this example, we specify that we want more output.
//
Teuchos::ParameterList List;
List.set("PrintTiming", true);
List.set("PrintStatus", true);
Solver->SetParameters(List);
// Now we are ready to solve. Generally, users will
// call SymbolicFactorization(), then NumericFactorization(),
// and finally Solve(). Note that:
// - the numerical values of the linear system matrix
// are *not* required before NumericFactorization();
// - solution and rhs are *not* required before calling
// Solve().
if (Comm.MyPID() == 0)
cout << "Starting symbolic factorization..." << endl;
Solver->SymbolicFactorization();
// you can change the matrix values here
if (Comm.MyPID() == 0)
cout << "Starting numeric factorization..." << endl;
Solver->NumericFactorization();
// you can change LHS and RHS here
if (Comm.MyPID() == 0)
cout << "Starting solution phase..." << endl;
Solver->Solve();
// =========================================== //
// E N D O F T H E A M E S O S P A R T //
// =========================================== //
// delete Solver. MPI calls can occur.
delete Solver;
// delete the objects created by Galeri
delete Matrix;
delete Map;
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} // end of main()
