// ======================================================================
void Eig(const Operator& Op, MultiVector& ER, MultiVector& EI)
{
int ierr;
if (Op.GetDomainSpace() != Op.GetRangeSpace())
ML_THROW("Matrix is not square", -1);
ER.Reshape(Op.GetDomainSpace());
EI.Reshape(Op.GetDomainSpace());
Epetra_LinearProblem Problem;
Problem.SetOperator(const_cast<Epetra_RowMatrix*>(Op.GetRowMatrix()));
Amesos_Lapack Lapack(Problem);
Epetra_Vector ER_Epetra(Op.GetRowMatrix()->RowMatrixRowMap());
Epetra_Vector EI_Epetra(Op.GetRowMatrix()->RowMatrixRowMap());
ierr = Lapack.GEEV(ER_Epetra, EI_Epetra);
if (ierr)
ML_THROW("GEEV returned error code = " + GetString(ierr), -1);
for (int i = 0 ; i < ER.GetMyLength() ; ++i) {
ER(i) = ER_Epetra[i];
EI(i) = EI_Epetra[i];
}
}
void show_matrix(const char *txt, const Epetra_LinearProblem &problem, const Epetra_Comm &comm)
{
int me = comm.MyPID();
if (comm.NumProc() > 10){
if (me == 0){
std::cout << txt << std::endl;
std::cout << "Printed matrix format only works for 10 or fewer processes" << std::endl;
}
return;
}
Epetra_RowMatrix *matrix = problem.GetMatrix();
Epetra_MultiVector *lhs = problem.GetLHS();
Epetra_MultiVector *rhs = problem.GetRHS();
int numRows = matrix->NumGlobalRows();
int numCols = matrix->NumGlobalCols();
if ((numRows > 200) || (numCols > 500)){
if (me == 0){
std::cerr << txt << std::endl;
std::cerr << "show_matrix: problem is too large to display" << std::endl;
}
return;
}
int *myA = new int [numRows * numCols];
make_my_A(*matrix, myA, comm);
int *myX = new int [numCols];
int *myB = new int [numRows];
memset(myX, 0, sizeof(int) * numCols);
memset(myB, 0, sizeof(int) * numRows);
const Epetra_BlockMap &lhsMap = lhs->Map();
const Epetra_BlockMap &rhsMap = rhs->Map();
int base = lhsMap.IndexBase();
for (int j=0; j < lhsMap.NumMyElements(); j++){
int colGID = lhsMap.GID(j);
myX[colGID - base] = me + 1;
}
for (int i=0; i < rhsMap.NumMyElements(); i++){
int rowGID = rhsMap.GID(i);
myB[rowGID - base] = me + 1;
}
printMatrix(txt, myA, myX, myB, numRows, numCols, comm);
delete [] myA;
delete [] myX;
delete [] myB;
}
// ======================================================================
void Krylov(const Operator& A, const MultiVector& LHS,
const MultiVector& RHS, const BaseOperator& Prec,
Teuchos::ParameterList& List)
{
#ifndef HAVE_ML_AZTECOO
std::cerr << "Please configure ML with --enable-aztecoo to use" << std::endl;
std::cerr << "MLAPI Krylov solvers" << std::endl;
exit(EXIT_FAILURE);
#else
if (LHS.GetNumVectors() != 1)
ML_THROW("FIXME: only one vector is currently supported", -1);
Epetra_LinearProblem Problem;
const Epetra_RowMatrix& A_Epetra = *(A.GetRowMatrix());
Epetra_Vector LHS_Epetra(View,A_Epetra.OperatorDomainMap(),
(double*)&(LHS(0)));
Epetra_Vector RHS_Epetra(View,A_Epetra.OperatorRangeMap(),
(double*)&(RHS(0)));
// FIXME: this works only for Epetra-based operators
Problem.SetOperator((const_cast<Epetra_RowMatrix*>(&A_Epetra)));
Problem.SetLHS(&LHS_Epetra);
Problem.SetRHS(&RHS_Epetra);
AztecOO solver(Problem);
EpetraBaseOperator Prec_Epetra(A_Epetra.OperatorDomainMap(),Prec);
solver.SetPrecOperator(&Prec_Epetra);
// get options from List
int NumIters = List.get("krylov: max iterations", 1550);
double Tol = List.get("krylov: tolerance", 1e-9);
std::string type = List.get("krylov: type", "gmres");
int output = List.get("krylov: output level", GetPrintLevel());
// set options in `solver'
if (type == "cg")
solver.SetAztecOption(AZ_solver, AZ_cg);
else if (type == "cg_condnum")
solver.SetAztecOption(AZ_solver, AZ_cg_condnum);
else if (type == "gmres")
solver.SetAztecOption(AZ_solver, AZ_gmres);
else if (type == "gmres_condnum")
solver.SetAztecOption(AZ_solver, AZ_gmres_condnum);
else if (type == "fixed point")
solver.SetAztecOption(AZ_solver, AZ_fixed_pt);
else
ML_THROW("krylov: type has incorrect value (" +
type + ")", -1);
solver.SetAztecOption(AZ_output, output);
solver.Iterate(NumIters, Tol);
#endif
}
// ======================================================================
bool BasicTest(string PrecType,
CrsMatrixGallery& Gallery)
{
// The following methods of CrsMatrixGallery are used to get pointers
// to internally stored Epetra_RowMatrix and Epetra_LinearProblem.
Epetra_RowMatrix* A = Gallery.GetMatrix();
Epetra_LinearProblem* Problem = Gallery.GetLinearProblem();
Epetra_MultiVector& RHS = *(Problem->GetRHS());
Epetra_MultiVector& LHS = *(Problem->GetLHS());
LHS.PutScalar(0.0);
// Set up the list
Teuchos::ParameterList List;
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: sweeps",1550);
List.set("relaxation: type", PrecType);
Ifpack_PointRelaxation Point(A);
Point.SetParameters(List);
Point.Compute();
// use the preconditioner as solver, with 1550 iterations
Point.ApplyInverse(RHS,LHS);
// compute the real residual
double residual, diff;
Gallery.ComputeResidual(&residual);
Gallery.ComputeDiffBetweenStartingAndExactSolutions(&diff);
if (verbose && A->Comm().MyPID()==0) {
cout << "||b-Ax||_2 = " << residual << endl;
cout << "||x_exact - x||_2 = " << diff << endl;
}
// Jacobi is very slow to converge here
if (residual < 1e-2) {
if (verbose)
cout << "Test passed" << endl;
return(true);
}
else {
if (verbose)
cout << "Test failed!" << endl;
return(false);
}
}
void NOX::Epetra::Scaling::computeScaling(const Epetra_LinearProblem& problem)
{
Epetra_Vector* diagonal = 0;
for (unsigned int i = 0; i < scaleVector.size(); i ++) {
if (sourceType[i] == RowSum) {
diagonal = scaleVector[i].get();
// Make sure the Jacobian is an Epetra_RowMatrix, otherwise we can't
// perform a row sum scale!
const Epetra_RowMatrix* test = 0;
test = dynamic_cast<const Epetra_RowMatrix*>(problem.GetOperator());
if (test == 0) {
std::cout << "ERROR: NOX::Epetra::Scaling::scaleLinearSystem() - "
<< "For \"Row Sum\" scaling, the Matrix must be an "
<< "Epetra_RowMatrix derived object!" << std::endl;
throw "NOX Error";
}
test->InvRowSums(*diagonal);
diagonal->Reciprocal(*diagonal);
}
else if (sourceType[i] == ColSum) {
diagonal = scaleVector[i].get();
// Make sure the Jacobian is an Epetra_RowMatrix, otherwise we can't
// perform a row sum scale!
const Epetra_RowMatrix* test = 0;
test = dynamic_cast<const Epetra_RowMatrix*>(problem.GetOperator());
if (test == 0) {
std::cout << "ERROR: NOX::Epetra::Scaling::scaleLinearSystem() - "
<< "For \"Column Sum\" scaling, the Matrix must be an "
<< "Epetra_RowMatrix derived object!" << std::endl;
throw "NOX Error";
}
test->InvColSums(*diagonal);
diagonal->Reciprocal(*diagonal);
}
}
}
void NOX::Epetra::Scaling::unscaleLinearSystem(Epetra_LinearProblem& problem)
{
Epetra_Vector* diagonal = 0;
for (unsigned int i = 0; i < scaleVector.size(); i ++) {
diagonal = scaleVector[i].get();
if (scaleType[i] == Left) {
problem.LeftScale(*diagonal);
}
else if (scaleType[i] == Right) {
problem.RightScale(*diagonal);
}
}
}
int
AmesosGenOp::Apply (const Epetra_MultiVector& X, Epetra_MultiVector& Y) const
{
if (problem_ == NULL) {
throw std::logic_error ("AmesosGenOp::Apply: problem_ is NULL");
}
if (massMtx_.is_null ()) {
throw std::logic_error ("AmesosGenOp::Apply: massMtx_ is null");
}
if (solver_.is_null ()) {
throw std::logic_error ("AmesosGenOp::Apply: solver_ is null");
}
if (! useTranspose_) {
// Storage for M*X
Epetra_MultiVector MX (X.Map (), X.NumVectors ());
// Apply M*X
massMtx_->Apply (X, MX);
Y.PutScalar (0.0);
// Set the LHS and RHS
problem_->SetRHS (&MX);
problem_->SetLHS (&Y);
// Solve the linear system A*Y = MX
solver_->Solve ();
}
else { // apply the transposed operator
// Storage for A^{-T}*X
Epetra_MultiVector ATX (X.Map (), X.NumVectors ());
Epetra_MultiVector tmpX = const_cast<Epetra_MultiVector&> (X);
// Set the LHS and RHS
problem_->SetRHS (&tmpX);
problem_->SetLHS (&ATX);
// Solve the linear system A^T*Y = X
solver_->Solve ();
// Apply M*ATX
massMtx_->Apply (ATX, Y);
}
return 0; // the method completed correctly
}
int AmesosBucklingOp::Apply(const Epetra_MultiVector& X, Epetra_MultiVector& Y ) const
{
// Storage for A*X
Epetra_MultiVector AX(X.Map(),X.NumVectors());
// Apply A*X
stiffMtx_->Apply(X, AX);
Y.PutScalar(0.0);
// Set the LHS and RHS
problem_->SetRHS(&AX);
problem_->SetLHS(&Y);
// Solve the linear system (A-sigma*M)*Y = AX
solver_->Solve();
return 0;
}
// ======================================================================
int CompareBlockSizes(string PrecType,
CrsMatrixGallery& Gallery, int NumParts)
{
Epetra_RowMatrix* A = Gallery.GetMatrix();
Epetra_LinearProblem* Problem = Gallery.GetLinearProblem();
Epetra_MultiVector& RHS = *(Problem->GetRHS());
Epetra_MultiVector& LHS = *(Problem->GetLHS());
Teuchos::ParameterList List;
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: type", PrecType);
List.set("relaxation: sweeps",1);
List.set("partitioner: type", "greedy");
List.set("partitioner: local parts", NumParts);
RHS.PutScalar(1.0);
LHS.PutScalar(0.0);
Ifpack_AdditiveSchwarz<Ifpack_BlockRelaxation<Ifpack_DenseContainer> > Prec(A);
Prec.SetParameters(List);
Prec.Compute();
// set AztecOO solver object
AztecOO AztecOOSolver(*Problem);
AztecOOSolver.SetAztecOption(AZ_solver,Solver);
if (verbose)
AztecOOSolver.SetAztecOption(AZ_output,32);
else
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetPrecOperator(&Prec);
AztecOOSolver.Iterate(1550,1e-8);
return(AztecOOSolver.NumIters());
}
int AmesosGenOp::Apply(const Epetra_MultiVector& X, Epetra_MultiVector& Y ) const
{
if (!useTranspose_) {
// Storage for M*X
Epetra_MultiVector MX(X.Map(),X.NumVectors());
// Apply M*X
massMtx_->Apply(X, MX);
Y.PutScalar(0.0);
// Set the LHS and RHS
problem_->SetRHS(&MX);
problem_->SetLHS(&Y);
// Solve the linear system A*Y = MX
solver_->Solve();
}
else {
// Storage for A^{-T}*X
Epetra_MultiVector ATX(X.Map(),X.NumVectors());
Epetra_MultiVector tmpX = const_cast<Epetra_MultiVector&>(X);
// Set the LHS and RHS
problem_->SetRHS(&tmpX);
problem_->SetLHS(&ATX);
// Solve the linear system A^T*Y = X
solver_->Solve();
// Apply M*ATX
massMtx_->Apply(ATX, Y);
}
return 0;
}
int main(int argc, char *argv[]) {
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
cout << Comm << endl;
int MyPID = Comm.MyPID();
bool verbose = false;
bool verbose1 = true;
if (MyPID==0) verbose = true;
if(argc < 2 && verbose) {
cerr << "Usage: " << argv[0]
<< " HB_filename [level_fill [level_overlap [absolute_threshold [ relative_threshold]]]]" << endl
<< "where:" << endl
<< "HB_filename - filename and path of a Harwell-Boeing data set" << endl
<< "level_fill - The amount of fill to use for ILU(k) preconditioner (default 0)" << endl
<< "level_overlap - The amount of overlap used for overlapping Schwarz subdomains (default 0)" << endl
<< "absolute_threshold - The minimum value to place on the diagonal prior to factorization (default 0.0)" << endl
<< "relative_threshold - The relative amount to perturb the diagonal prior to factorization (default 1.0)" << endl << endl
<< "To specify a non-default value for one of these parameters, you must specify all" << endl
<< " preceding values but not any subsequent parameters. Example:" << endl
<< "ifpackHpcSerialMsr.exe mymatrix.hpc 1 - loads mymatrix.hpc, uses level fill of one, all other values are defaults" << endl
<< endl;
return(1);
}
// Uncomment the next three lines to debug in mpi mode
//int tmp;
//if (MyPID==0) cin >> tmp;
//Comm.Barrier();
Epetra_Map * readMap;
Epetra_CrsMatrix * readA;
Epetra_Vector * readx;
Epetra_Vector * readb;
Epetra_Vector * readxexact;
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra(argv[1], Comm, readMap, readA, readx, readb, readxexact);
// Create uniform distributed map
Epetra_Map map(readMap->NumGlobalElements(), 0, Comm);
// Create Exporter to distribute read-in matrix and vectors
Epetra_Export exporter(*readMap, map);
Epetra_CrsMatrix A(Copy, map, 0);
Epetra_Vector x(map);
Epetra_Vector b(map);
Epetra_Vector xexact(map);
Epetra_Time FillTimer(Comm);
x.Export(*readx, exporter, Add);
b.Export(*readb, exporter, Add);
xexact.Export(*readxexact, exporter, Add);
Comm.Barrier();
double vectorRedistributeTime = FillTimer.ElapsedTime();
A.Export(*readA, exporter, Add);
Comm.Barrier();
double matrixRedistributeTime = FillTimer.ElapsedTime() - vectorRedistributeTime;
assert(A.FillComplete()==0);
Comm.Barrier();
double fillCompleteTime = FillTimer.ElapsedTime() - matrixRedistributeTime;
if (Comm.MyPID()==0) {
cout << "\n\n****************************************************" << endl;
cout << "\n Vector redistribute time (sec) = " << vectorRedistributeTime<< endl;
cout << " Matrix redistribute time (sec) = " << matrixRedistributeTime << endl;
cout << " Transform to Local time (sec) = " << fillCompleteTime << endl<< endl;
}
Epetra_Vector tmp1(*readMap);
Epetra_Vector tmp2(map);
readA->Multiply(false, *readxexact, tmp1);
A.Multiply(false, xexact, tmp2);
double residual;
tmp1.Norm2(&residual);
if (verbose) cout << "Norm of Ax from file = " << residual << endl;
tmp2.Norm2(&residual);
if (verbose) cout << "Norm of Ax after redistribution = " << residual << endl << endl << endl;
//cout << "A from file = " << *readA << endl << endl << endl;
//cout << "A after dist = " << A << endl << endl << endl;
delete readA;
delete readx;
delete readb;
delete readxexact;
delete readMap;
Comm.Barrier();
bool smallProblem = false;
if (A.RowMap().NumGlobalElements()<100) smallProblem = true;
if (smallProblem)
cout << "Original Matrix = " << endl << A << endl;
x.PutScalar(0.0);
Epetra_LinearProblem FullProblem(&A, &x, &b);
double normb, norma;
b.NormInf(&normb);
norma = A.NormInf();
if (verbose)
cout << "Inf norm of Original Matrix = " << norma << endl
<< "Inf norm of Original RHS = " << normb << endl;
Epetra_Time ReductionTimer(Comm);
Epetra_CrsSingletonFilter SingletonFilter;
Comm.Barrier();
double reduceInitTime = ReductionTimer.ElapsedTime();
SingletonFilter.Analyze(&A);
Comm.Barrier();
double reduceAnalyzeTime = ReductionTimer.ElapsedTime() - reduceInitTime;
if (SingletonFilter.SingletonsDetected())
cout << "Singletons found" << endl;
else {
cout << "Singletons not found" << endl;
exit(1);
}
SingletonFilter.ConstructReducedProblem(&FullProblem);
Comm.Barrier();
double reduceConstructTime = ReductionTimer.ElapsedTime() - reduceInitTime;
double totalReduceTime = ReductionTimer.ElapsedTime();
if (verbose)
cout << "\n\n****************************************************" << endl
<< " Reduction init time (sec) = " << reduceInitTime<< endl
<< " Reduction Analyze time (sec) = " << reduceAnalyzeTime << endl
<< " Construct Reduced Problem time (sec) = " << reduceConstructTime << endl
<< " Reduction Total time (sec) = " << totalReduceTime << endl<< endl;
Statistics(SingletonFilter);
Epetra_LinearProblem * ReducedProblem = SingletonFilter.ReducedProblem();
Epetra_CrsMatrix * Ap = dynamic_cast<Epetra_CrsMatrix *>(ReducedProblem->GetMatrix());
Epetra_Vector * bp = (*ReducedProblem->GetRHS())(0);
Epetra_Vector * xp = (*ReducedProblem->GetLHS())(0);
if (smallProblem)
cout << " Reduced Matrix = " << endl << *Ap << endl
<< " LHS before sol = " << endl << *xp << endl
<< " RHS = " << endl << *bp << endl;
// Construct ILU preconditioner
double elapsed_time, total_flops, MFLOPs;
Epetra_Time timer(Comm);
int LevelFill = 0;
if (argc > 2) LevelFill = atoi(argv[2]);
if (verbose) cout << "Using Level Fill = " << LevelFill << endl;
int Overlap = 0;
if (argc > 3) Overlap = atoi(argv[3]);
if (verbose) cout << "Using Level Overlap = " << Overlap << endl;
double Athresh = 0.0;
if (argc > 4) Athresh = atof(argv[4]);
if (verbose) cout << "Using Absolute Threshold Value of = " << Athresh << endl;
double Rthresh = 1.0;
if (argc > 5) Rthresh = atof(argv[5]);
if (verbose) cout << "Using Relative Threshold Value of = " << Rthresh << endl;
Ifpack_IlukGraph * IlukGraph = 0;
Ifpack_CrsRiluk * ILUK = 0;
if (LevelFill>-1) {
elapsed_time = timer.ElapsedTime();
IlukGraph = new Ifpack_IlukGraph(Ap->Graph(), LevelFill, Overlap);
assert(IlukGraph->ConstructFilledGraph()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
if (verbose) cout << "Time to construct ILUK graph = " << elapsed_time << endl;
Epetra_Flops fact_counter;
elapsed_time = timer.ElapsedTime();
ILUK = new Ifpack_CrsRiluk(*IlukGraph);
ILUK->SetFlopCounter(fact_counter);
ILUK->SetAbsoluteThreshold(Athresh);
ILUK->SetRelativeThreshold(Rthresh);
//assert(ILUK->InitValues()==0);
int initerr = ILUK->InitValues(*Ap);
if (initerr!=0) {
cout << endl << Comm << endl << " InitValues error = " << initerr;
if (initerr==1) cout << " Zero diagonal found, warning error only";
cout << endl << endl;
}
assert(ILUK->Factor()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = ILUK->Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute preconditioner values = "
<< elapsed_time << endl
<< "MFLOPS for Factorization = " << MFLOPs << endl;
//cout << *ILUK << endl;
double Condest;
ILUK->Condest(false, Condest);
if (verbose) cout << "Condition number estimate for this preconditioner = " << Condest << endl;
}
int Maxiter = 100;
double Tolerance = 1.0E-8;
Epetra_Flops counter;
Ap->SetFlopCounter(counter);
xp->SetFlopCounter(*Ap);
bp->SetFlopCounter(*Ap);
if (ILUK!=0) ILUK->SetFlopCounter(*Ap);
elapsed_time = timer.ElapsedTime();
double normreducedb, normreduceda;
bp->NormInf(&normreducedb);
normreduceda = Ap->NormInf();
if (verbose)
cout << "Inf norm of Reduced Matrix = " << normreduceda << endl
<< "Inf norm of Reduced RHS = " << normreducedb << endl;
BiCGSTAB(*Ap, *xp, *bp, ILUK, Maxiter, Tolerance, &residual, verbose);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = counter.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute solution = "
<< elapsed_time << endl
<< "Number of operations in solve = " << total_flops << endl
<< "MFLOPS for Solve = " << MFLOPs<< endl << endl;
SingletonFilter.ComputeFullSolution();
if (smallProblem)
cout << " Reduced LHS after sol = " << endl << *xp << endl
<< " Full LHS after sol = " << endl << x << endl
<< " Full Exact LHS = " << endl << xexact << endl;
Epetra_Vector resid(x);
resid.Update(1.0, x, -1.0, xexact, 0.0); // resid = xcomp - xexact
resid.Norm2(&residual);
double normx, normxexact;
x.Norm2(&normx);
xexact.Norm2(&normxexact);
if (verbose)
cout << "2-norm of computed solution = " << normx << endl
<< "2-norm of exact solution = " << normxexact << endl
<< "2-norm of difference between computed and exact solution = " << residual << endl;
if (verbose1 && residual>1.0e-5) {
if (verbose)
cout << "Difference between computed and exact solution appears large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
Epetra_Vector bdiff(b);
assert(A.Multiply(false, resid, bdiff)==0);
assert(bdiff.Norm2(&residual)==0);
if (verbose)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
if (verbose)
cout << "********************************************************" << endl
<< " Solving again with 2*Ax=2*b" << endl
<< "********************************************************" << endl;
A.Scale(1.0); // A = 2*A
b.Scale(1.0); // b = 2*b
x.PutScalar(0.0);
b.NormInf(&normb);
norma = A.NormInf();
if (verbose)
cout << "Inf norm of Original Matrix = " << norma << endl
<< "Inf norm of Original RHS = " << normb << endl;
double updateReducedProblemTime = ReductionTimer.ElapsedTime();
SingletonFilter.UpdateReducedProblem(&FullProblem);
Comm.Barrier();
updateReducedProblemTime = ReductionTimer.ElapsedTime() - updateReducedProblemTime;
if (verbose)
cout << "\n\n****************************************************" << endl
<< " Update Reduced Problem time (sec) = " << updateReducedProblemTime<< endl
<< "****************************************************" << endl;
Statistics(SingletonFilter);
if (LevelFill>-1) {
Epetra_Flops fact_counter;
elapsed_time = timer.ElapsedTime();
int initerr = ILUK->InitValues(*Ap);
if (initerr!=0) {
cout << endl << Comm << endl << " InitValues error = " << initerr;
if (initerr==1) cout << " Zero diagonal found, warning error only";
cout << endl << endl;
}
assert(ILUK->Factor()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = ILUK->Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute preconditioner values = "
<< elapsed_time << endl
<< "MFLOPS for Factorization = " << MFLOPs << endl;
double Condest;
ILUK->Condest(false, Condest);
if (verbose) cout << "Condition number estimate for this preconditioner = " << Condest << endl;
}
bp->NormInf(&normreducedb);
normreduceda = Ap->NormInf();
if (verbose)
cout << "Inf norm of Reduced Matrix = " << normreduceda << endl
<< "Inf norm of Reduced RHS = " << normreducedb << endl;
BiCGSTAB(*Ap, *xp, *bp, ILUK, Maxiter, Tolerance, &residual, verbose);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = counter.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute solution = "
<< elapsed_time << endl
<< "Number of operations in solve = " << total_flops << endl
<< "MFLOPS for Solve = " << MFLOPs<< endl << endl;
SingletonFilter.ComputeFullSolution();
if (smallProblem)
cout << " Reduced LHS after sol = " << endl << *xp << endl
<< " Full LHS after sol = " << endl << x << endl
<< " Full Exact LHS = " << endl << xexact << endl;
resid.Update(1.0, x, -1.0, xexact, 0.0); // resid = xcomp - xexact
resid.Norm2(&residual);
x.Norm2(&normx);
xexact.Norm2(&normxexact);
if (verbose)
cout << "2-norm of computed solution = " << normx << endl
<< "2-norm of exact solution = " << normxexact << endl
<< "2-norm of difference between computed and exact solution = " << residual << endl;
if (verbose1 && residual>1.0e-5) {
if (verbose)
cout << "Difference between computed and exact solution appears large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
Epetra_Vector bdiff(b);
assert(A.Multiply(false, resid, bdiff)==0);
assert(bdiff.Norm2(&residual)==0);
if (verbose)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
if (ILUK!=0) delete ILUK;
if (IlukGraph!=0) delete IlukGraph;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return 0 ;
}
int main(int argc, char *argv[]) {
int returnierr=0;
using std::cout;
using std::endl;
using std::flush;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
int size; // Number of MPI processes, My process ID
MPI_Comm_size(MPI_COMM_WORLD, &size);
if (size > 1) {
cout << "This example cannot be run on more than one processor!" << endl;
MPI_Finalize();
returnierr = -1;
return returnierr;
}
#endif
bool verbose = false;
// Check if we should print results to standard out
if (argc>1) if (argv[1][0]=='-' && argv[1][1]=='v') verbose = true;
#ifdef EPETRA_MPI
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
if (!verbose) Comm.SetTracebackMode(0); // This should shut down any error traceback reporting
if (verbose) {
cout << EpetraExt::EpetraExt_Version() << endl << endl;
cout << Comm << endl << flush;
}
Comm.Barrier();
int NumMyElements = 3;
Epetra_Map Map( NumMyElements, 0, Comm );
Epetra_CrsGraph Graph( Copy, Map, 1 );
int index[2];
index[0] = 2;
Graph.InsertGlobalIndices( 0, 1, &index[0] );
index[0] = 0;
index[1] = 2;
Graph.InsertGlobalIndices( 1, 2, &index[0] );
index[0] = 1;
Graph.InsertGlobalIndices( 2, 1, &index[0] );
Graph.FillComplete();
if (verbose) {
cout << "***************** PERFORMING BTF TRANSFORM ON CRS_GRAPH *****************" <<endl<<endl;
cout << "CrsGraph *before* BTF transform: " << endl << endl;
cout << Graph << endl;
}
EpetraExt::AmesosBTF_CrsGraph BTFTrans( true, verbose );
Epetra_CrsGraph & NewBTFGraph = BTFTrans( Graph );
if (verbose) {
cout << "CrsGraph *after* BTF transform: " << endl << endl;
cout << NewBTFGraph << endl;
}
// Use BTF graph transformation to solve linear system.
// Create an Epetra::CrsMatrix
Epetra_CrsMatrix Matrix( Copy, Graph );
double value[2];
index[0] = 2; value[0] = 3.0;
Matrix.ReplaceMyValues( 0, 1, &value[0], &index[0] );
index[0] = 0; index[1] = 2;
value[0] = 2.0; value[1] = 2.5;
Matrix.ReplaceMyValues( 1, 2, &value[0], &index[0] );
index[0] = 1; value[0] = 1.0;
Matrix.ReplaceMyValues( 2, 1, &value[0], &index[0] );
Matrix.FillComplete();
// Create the solution and right-hand side vectors.
Epetra_MultiVector RHS( Map, 1 ), LHS( Map, 1 );
LHS.PutScalar( 0.0 );
RHS.ReplaceMyValue( 0, 0, 3.0 );
RHS.ReplaceMyValue( 1, 0, 4.5 );
RHS.ReplaceMyValue( 2, 0, 1.0 );
Epetra_LinearProblem problem( &Matrix, &LHS, &RHS );
if (verbose) {
cout << "*************** PERFORMING BTF TRANSFORM ON LINEAR_PROBLEM **************" <<endl<<endl;
cout << "CrsMatrix *before* BTF transform: " << endl << endl;
cout << Matrix << endl;
cout << "MultiVector RHS *before* BTF transform: " << endl << endl;
RHS.Print( cout );
}
// Create the linear problem transform.
EpetraExt::LinearProblem_GraphTrans * LPTrans =
new EpetraExt::LinearProblem_GraphTrans(
*(dynamic_cast<EpetraExt::StructuralSameTypeTransform<Epetra_CrsGraph>*>(&BTFTrans)) );
Epetra_LinearProblem* tProblem = &((*LPTrans)( problem ));
LPTrans->fwd();
if (verbose) {
cout << "CrsMatrix *after* BTF transform: " << endl << endl;
dynamic_cast<Epetra_CrsMatrix*>(tProblem->GetMatrix())->Print( cout );
cout << "MultiVector RHS *after* BTF transform: " << endl << endl;
tProblem->GetRHS()->Print( cout );
}
if (verbose) {
cout << endl << "*************** PERFORMING REINDEXING ON LINEAR_PROBLEM **************" <<endl<<endl;
}
EpetraExt::ViewTransform<Epetra_LinearProblem> * ReIdx_LPTrans =
new EpetraExt::LinearProblem_Reindex( 0 );
Epetra_LinearProblem* tProblem2 = &((*ReIdx_LPTrans)( *tProblem ));
ReIdx_LPTrans->fwd();
if (verbose) {
cout << endl << "CrsMatrix *after* BTF transform *and* reindexing: " << endl << endl;
dynamic_cast<Epetra_CrsMatrix*>(tProblem2->GetMatrix())->Print( cout );
cout << endl <<"Column Map *before* reindexing: " << endl << endl;
cout << dynamic_cast<Epetra_CrsMatrix*>(tProblem->GetMatrix())->ColMap() << endl;
cout << "Column Map *after* reindexing: " << endl << endl;
cout << dynamic_cast<Epetra_CrsMatrix*>(tProblem2->GetMatrix())->ColMap() << endl;
}
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
return returnierr;
}
// ======================================================================
bool ComparePointAndBlock(string PrecType,
CrsMatrixGallery& Gallery, int sweeps)
{
Epetra_RowMatrix* A = Gallery.GetMatrix();
Epetra_LinearProblem* Problem = Gallery.GetLinearProblem();
Epetra_MultiVector& RHS = *(Problem->GetRHS());
Epetra_MultiVector& LHS = *(Problem->GetLHS());
// Set up the list
Teuchos::ParameterList List;
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: type", PrecType);
List.set("relaxation: sweeps",sweeps);
List.set("partitioner: type", "linear");
List.set("partitioner: local parts", A->NumMyRows());
int ItersPoint, ItersBlock;
// ================================================== //
// get the number of iterations with point relaxation //
// ================================================== //
{
RHS.PutScalar(1.0);
LHS.PutScalar(0.0);
Ifpack_AdditiveSchwarz<Ifpack_PointRelaxation> Point(A);
Point.SetParameters(List);
Point.Compute();
// set AztecOO solver object
AztecOO AztecOOSolver(*Problem);
AztecOOSolver.SetAztecOption(AZ_solver,Solver);
if (verbose)
AztecOOSolver.SetAztecOption(AZ_output,32);
else
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetPrecOperator(&Point);
AztecOOSolver.Iterate(1550,1e-8);
double TrueResidual = AztecOOSolver.TrueResidual();
ItersPoint = AztecOOSolver.NumIters();
// some output
if (verbose && Problem->GetMatrix()->Comm().MyPID() == 0) {
cout << "Iterations = " << ItersPoint << endl;
cout << "Norm of the true residual = " << TrueResidual << endl;
}
}
// ================================================== //
// get the number of iterations with block relaxation //
// ================================================== //
{
RHS.PutScalar(1.0);
LHS.PutScalar(0.0);
Ifpack_AdditiveSchwarz<Ifpack_BlockRelaxation<Ifpack_DenseContainer> > Block(A);
Block.SetParameters(List);
Block.Compute();
// set AztecOO solver object
AztecOO AztecOOSolver(*Problem);
AztecOOSolver.SetAztecOption(AZ_solver,Solver);
if (verbose)
AztecOOSolver.SetAztecOption(AZ_output,32);
else
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetPrecOperator(&Block);
AztecOOSolver.Iterate(1550,1e-8);
double TrueResidual = AztecOOSolver.TrueResidual();
ItersBlock = AztecOOSolver.NumIters();
// some output
if (verbose && Problem->GetMatrix()->Comm().MyPID() == 0) {
cout << "Iterations " << ItersBlock << endl;
cout << "Norm of the true residual = " << TrueResidual << endl;
}
}
if (ItersPoint != ItersBlock) {
if (verbose)
cout << "TEST FAILED!" << endl;
return(false);
}
else {
if (verbose)
cout << "TEST PASSED" << endl;
return(true);
}
}
// ======================================================================
bool KrylovTest(string PrecType,
CrsMatrixGallery& Gallery)
{
// The following methods of CrsMatrixGallery are used to get pointers
// to internally stored Epetra_RowMatrix and Epetra_LinearProblem.
Epetra_RowMatrix* A = Gallery.GetMatrix();
Epetra_LinearProblem* Problem = Gallery.GetLinearProblem();
Epetra_MultiVector& LHS = *(Problem->GetLHS());
LHS.PutScalar(0.0);
// Set up the list
Teuchos::ParameterList List;
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: type", PrecType);
int Iters1, Iters10;
// ============================================== //
// get the number of iterations with 1 sweep only //
// ============================================== //
{
List.set("relaxation: sweeps",1);
Ifpack_AdditiveSchwarz<Ifpack_PointRelaxation> Point(A);
Point.SetParameters(List);
Point.Compute();
// set AztecOO solver object
AztecOO AztecOOSolver(*Problem);
AztecOOSolver.SetAztecOption(AZ_solver,Solver);
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetPrecOperator(&Point);
AztecOOSolver.Iterate(1550,1e-8);
double TrueResidual = AztecOOSolver.TrueResidual();
// some output
if (verbose && Problem->GetMatrix()->Comm().MyPID() == 0) {
cout << "Norm of the true residual = " << TrueResidual << endl;
}
Iters1 = AztecOOSolver.NumIters();
}
// ======================================================== //
// now re-run with 10 sweeps, solver should converge faster
// ======================================================== //
{
List.set("relaxation: sweeps",10);
Ifpack_AdditiveSchwarz<Ifpack_PointRelaxation> Point(A);
Point.SetParameters(List);
Point.Compute();
LHS.PutScalar(0.0);
// set AztecOO solver object
AztecOO AztecOOSolver(*Problem);
AztecOOSolver.SetAztecOption(AZ_solver,Solver);
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetPrecOperator(&Point);
AztecOOSolver.Iterate(1550,1e-8);
double TrueResidual = AztecOOSolver.TrueResidual();
// some output
if (verbose && Problem->GetMatrix()->Comm().MyPID() == 0) {
cout << "Norm of the true residual = " << TrueResidual << endl;
}
Iters10 = AztecOOSolver.NumIters();
}
if (verbose) {
cout << "Iters_1 = " << Iters1 << ", Iters_10 = " << Iters10 << endl;
cout << "(second number should be smaller than first one)" << endl;
}
if (Iters10 > Iters1) {
if (verbose)
cout << "TEST FAILED!" << endl;
return(false);
}
else {
if (verbose)
cout << "TEST PASSED" << endl;
return(true);
}
}
bool CompareWithAztecOO(Epetra_LinearProblem& Problem, const std::string what,
int Overlap, int ival)
{
using std::cout;
using std::endl;
AztecOO AztecOOSolver(Problem);
AztecOOSolver.SetAztecOption(AZ_solver,AZ_gmres);
AztecOOSolver.SetAztecOption(AZ_output,AZ_none);
AztecOOSolver.SetAztecOption(AZ_overlap,Overlap);
AztecOOSolver.SetAztecOption(AZ_graph_fill,ival);
AztecOOSolver.SetAztecOption(AZ_poly_ord, ival);
AztecOOSolver.SetAztecParam(AZ_drop, 0.0);
AztecOOSolver.SetAztecParam(AZ_athresh, 0.0);
AztecOOSolver.SetAztecParam(AZ_rthresh, 0.0);
Epetra_MultiVector& RHS = *(Problem.GetRHS());
Epetra_MultiVector& LHS = *(Problem.GetLHS());
Teuchos::RefCountPtr<Epetra_RowMatrix> A = Teuchos::rcp(Problem.GetMatrix(), false);
LHS.Random();
A->Multiply(false,LHS,RHS);
Teuchos::ParameterList List;
List.set("fact: level-of-fill", ival);
List.set("relaxation: sweeps", ival);
List.set("relaxation: damping factor", 1.0);
List.set("relaxation: zero starting solution", true);
//default combine mode is as for AztecOO
List.set("schwarz: combine mode", Zero);
Epetra_Time Time(A->Comm());
Teuchos::RefCountPtr<Ifpack_Preconditioner> Prec;
if (what == "Jacobi") {
Prec = Teuchos::rcp( new Ifpack_PointRelaxation(&*A) );
List.set("relaxation: type", "Jacobi");
AztecOOSolver.SetAztecOption(AZ_precond,AZ_Jacobi);
AztecOOSolver.SetAztecOption(AZ_reorder,0);
}
else if (what == "IC no reord") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_IC>(&*A,Overlap) );
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_icc);
AztecOOSolver.SetAztecOption(AZ_reorder,0);
}
else if (what == "IC reord") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_IC>(&*A,Overlap) );
List.set("schwarz: use reordering", true);
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_icc);
AztecOOSolver.SetAztecOption(AZ_reorder,1);
}
else if (what == "ILU no reord") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_ILU>(&*A,Overlap) );
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_ilu);
AztecOOSolver.SetAztecOption(AZ_reorder,0);
}
else if (what == "ILU reord") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_ILU>(&*A,Overlap) );
List.set("schwarz: use reordering", true);
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_ilu);
AztecOOSolver.SetAztecOption(AZ_reorder,1);
}
#ifdef HAVE_IFPACK_AMESOS
else if (what == "LU") {
Prec = Teuchos::rcp( new Ifpack_AdditiveSchwarz<Ifpack_Amesos>(&*A,Overlap) );
List.set("amesos: solver type", "Klu");
AztecOOSolver.SetAztecOption(AZ_precond,AZ_dom_decomp);
AztecOOSolver.SetAztecOption(AZ_subdomain_solve,AZ_lu);
}
#endif
else {
cerr << "Option not recognized" << endl;
exit(EXIT_FAILURE);
}
// ==================================== //
// Solve with AztecOO's preconditioners //
// ==================================== //
LHS.PutScalar(0.0);
Time.ResetStartTime();
AztecOOSolver.Iterate(150,1e-5);
if (verbose) {
cout << endl;
cout << "==================================================" << endl;
cout << "Testing `" << what << "', Overlap = "
<< Overlap << ", ival = " << ival << endl;
cout << endl;
cout << "[AztecOO] Total time = " << Time.ElapsedTime() << " (s)" << endl;
cout << "[AztecOO] Residual = " << AztecOOSolver.TrueResidual() << " (s)" << endl;
cout << "[AztecOO] Iterations = " << AztecOOSolver.NumIters() << endl;
cout << endl;
}
int AztecOOPrecIters = AztecOOSolver.NumIters();
// =========================================== //
// Create the IFPACK preconditioner and solver //
// =========================================== //
Epetra_Time Time2(A->Comm());
assert(Prec != Teuchos::null);
IFPACK_CHK_ERR(Prec->SetParameters(List));
Time.ResetStartTime();
IFPACK_CHK_ERR(Prec->Initialize());
if (verbose)
cout << "[IFPACK] Time for Initialize() = "
<< Time.ElapsedTime() << " (s)" << endl;
Time.ResetStartTime();
IFPACK_CHK_ERR(Prec->Compute());
if (verbose)
cout << "[IFPACK] Time for Compute() = "
<< Time.ElapsedTime() << " (s)" << endl;
AztecOOSolver.SetPrecOperator(&*Prec);
LHS.PutScalar(0.0);
Time.ResetStartTime();
AztecOOSolver.Iterate(150,1e-5);
if (verbose) {
cout << "[IFPACK] Total time = " << Time2.ElapsedTime() << " (s)" << endl;
cout << "[IFPACK] Residual = " << AztecOOSolver.TrueResidual() << " (s)" << endl;
cout << "[IFPACK] Iterations = " << AztecOOSolver.NumIters() << endl;
cout << endl;
}
int IFPACKPrecIters = AztecOOSolver.NumIters();
if (IFPACK_ABS(AztecOOPrecIters - IFPACKPrecIters) > 3) {
cerr << "TEST FAILED (" << AztecOOPrecIters << " != "
<< IFPACKPrecIters << ")" << endl;
return(false);
}
else
return(true);
}
int PartialFactorizationOneStep( const char* AmesosClass,
const Epetra_Comm &Comm,
bool transpose,
bool verbose,
Teuchos::ParameterList ParamList,
Epetra_CrsMatrix *& Amat,
double Rcond,
int Steps )
{
assert( Steps >= 0 && Steps < MaxNumSteps ) ;
int iam = Comm.MyPID() ;
int errors = 0 ;
const Epetra_Map *RangeMap =
transpose?&Amat->OperatorDomainMap():&Amat->OperatorRangeMap() ;
const Epetra_Map *DomainMap =
transpose?&Amat->OperatorRangeMap():&Amat->OperatorDomainMap() ;
Epetra_Vector xexact(*DomainMap);
Epetra_Vector x(*DomainMap);
Epetra_Vector b(*RangeMap);
Epetra_Vector bcheck(*RangeMap);
Epetra_Vector difference(*DomainMap);
Epetra_LinearProblem Problem;
Amesos_BaseSolver* Abase ;
Amesos Afactory;
Abase = Afactory.Create( AmesosClass, Problem ) ;
std::string AC = AmesosClass ;
if ( AC == "Amesos_Mumps" ) {
ParamList.set( "NoDestroy", true );
Abase->SetParameters( ParamList ) ;
}
double relresidual = 0 ;
if ( Steps > 0 ) {
//
// Phase 1: Compute b = A' A' A xexact
//
Problem.SetOperator( Amat );
//
// We only set transpose if we have to - this allows valgrind to check
// that transpose is set to a default value before it is used.
//
if ( transpose ) OUR_CHK_ERR( Abase->SetUseTranspose( transpose ) );
// if (verbose) ParamList.set( "DebugLevel", 1 );
// if (verbose) ParamList.set( "OutputLevel", 1 );
if ( Steps > 1 ) {
OUR_CHK_ERR( Abase->SetParameters( ParamList ) );
if ( Steps > 2 ) {
xexact.Random();
xexact.PutScalar(1.0);
//
// Compute cAx = A' xexact
//
Amat->Multiply( transpose, xexact, b ) ; // b = A x2 = A A' A'' xexact
#if 0
std::cout << __FILE__ << "::" << __LINE__ << "b = " << std::endl ;
b.Print( std::cout ) ;
std::cout << __FILE__ << "::" << __LINE__ << "xexact = " << std::endl ;
xexact.Print( std::cout ) ;
std::cout << __FILE__ << "::" << __LINE__ << "x = " << std::endl ;
x.Print( std::cout ) ;
#endif
//
// Phase 2: Solve A' A' A x = b
//
//
// Solve A sAAx = b
//
Problem.SetLHS( &x );
Problem.SetRHS( &b );
OUR_CHK_ERR( Abase->SymbolicFactorization( ) );
if ( Steps > 2 ) {
OUR_CHK_ERR( Abase->SymbolicFactorization( ) );
if ( Steps > 3 ) {
OUR_CHK_ERR( Abase->NumericFactorization( ) );
if ( Steps > 4 ) {
OUR_CHK_ERR( Abase->NumericFactorization( ) );
if ( Steps > 5 ) {
OUR_CHK_ERR( Abase->Solve( ) );
if ( Steps > 6 ) {
OUR_CHK_ERR( Abase->Solve( ) );
Amat->Multiply( transpose, x, bcheck ) ; // temp = A" x2
double norm_diff ;
double norm_one ;
difference.Update( 1.0, x, -1.0, xexact, 0.0 ) ;
difference.Norm2( &norm_diff ) ;
x.Norm2( &norm_one ) ;
relresidual = norm_diff / norm_one ;
if (iam == 0 ) {
if ( relresidual * Rcond > 1e-16 ) {
if (verbose) std::cout << __FILE__ << "::"<< __LINE__
<< " norm( x - xexact ) / norm(x) = "
<< norm_diff /norm_one << std::endl ;
errors += 1 ;
}
}
}
}
}
}
}
}
}
}
delete Abase;
return errors;
}
int TestMultiLevelPreconditioner(char ProblemType[],
Teuchos::ParameterList & MLList,
Epetra_LinearProblem & Problem, double & TotalErrorResidual,
double & TotalErrorExactSol)
{
Epetra_MultiVector* lhs = Problem.GetLHS();
Epetra_MultiVector* rhs = Problem.GetRHS();
Epetra_RowMatrix* A = Problem.GetMatrix();
// ======================================== //
// create a rhs corresponding to lhs or 1's //
// ======================================== //
lhs->PutScalar(1.0);
A->Multiply(false,*lhs,*rhs);
lhs->PutScalar(0.0);
Epetra_Time Time(A->Comm());
Epetra_MultiVector lhs2(*lhs);
Epetra_MultiVector rhs2(*rhs);
// =================== //
// call ML and AztecOO //
// =================== //
AztecOO solver(Problem);
MLList.set("ML output", 0);
ML_set_random_seed(24601);
ML_Epetra::MultiLevelPreconditioner * MLPrec = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
// tell AztecOO to use this preconditioner, then solve
solver.SetPrecOperator(MLPrec);
solver.SetAztecOption(AZ_solver, AZ_gmres);
solver.SetAztecOption(AZ_output, 32);
solver.SetAztecOption(AZ_kspace, 160);
solver.Iterate(1550, 1e-12);
delete MLPrec;
// ================================= //
// call ML and AztecOO a second time //
// ================================= //
Epetra_LinearProblem Problem2(A,&lhs2,&rhs2);
AztecOO solver2(Problem2);
ML_set_random_seed(24601);
ML_Epetra::MultiLevelPreconditioner * MLPrec2 = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
// tell AztecOO to use this preconditioner, then solve
solver2.SetPrecOperator(MLPrec2);
solver2.SetAztecOption(AZ_solver, AZ_gmres);
solver2.SetAztecOption(AZ_output, 32);
solver2.SetAztecOption(AZ_kspace, 160);
solver2.Iterate(1550, 1e-12);
// ==================================================== //
// compute difference between the two ML solutions //
// ==================================================== //
double d = 0.0, d_tot = 0.0;
for( int i=0 ; i<lhs->Map().NumMyElements() ; ++i )
d += ((*lhs)[0][i] - lhs2[0][i]) * ((*lhs)[0][i] - lhs2[0][i]);
A->Comm().SumAll(&d,&d_tot,1);
string msg = ProblemType;
if (A->Comm().MyPID() == 0) {
cout << msg << "......Using " << A->Comm().NumProc() << " processes" << endl;
cout << msg << "......||x_1 - x_2||_2 = " << sqrt(d_tot) << endl;
cout << msg << "......Total Time = " << Time.ElapsedTime() << endl;
}
TotalErrorExactSol += sqrt(d_tot);
return( solver.NumIters() );
}
int main(int argc, char *argv[]) {
Teuchos::GlobalMPISession mpisess(&argc,&argv,&cout);
bool run_me = (mpisess.getRank() == 0);
bool testFailed = false;
if (run_me) {
try {
// useful typedefs
typedef double ST;
typedef Teuchos::ScalarTraits<ST> STT;
typedef STT::magnitudeType MT;
typedef Teuchos::ScalarTraits<MT> MTT;
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<ST,MV> MVT;
typedef Anasazi::OperatorTraits<ST,MV,OP> OPT;
// parse here, so everyone knows about failure
bool verbose = false;
bool debug = false;
std::string k_filename = "linearized_qevp_A.hb";
std::string m_filename = "linearized_qevp_B.hb";
int blockSize = 1;
int numBlocks = 30;
int nev = 20;
int maxRestarts = 0;
int extraBlocks = 0;
int stepSize = 1;
int numPrint = 536;
MT tol = 1e-8;
Teuchos::CommandLineProcessor cmdp(true,true);
cmdp.setOption("verbose","quiet",&verbose,"Print messages and results.");
cmdp.setOption("debug","normal",&debug,"Print debugging information.");
cmdp.setOption("A-filename",&k_filename,"Filename and path of the stiffness matrix.");
cmdp.setOption("B-filename",&m_filename,"Filename and path of the mass matrix.");
cmdp.setOption("extra-blocks",&extraBlocks,"Extra blocks to keep on a restart.");
cmdp.setOption("block-size",&blockSize,"Block size.");
cmdp.setOption("num-blocks",&numBlocks,"Number of blocks in Krylov basis.");
cmdp.setOption("step-size",&stepSize,"Step size.");
cmdp.setOption("nev",&nev,"Number of eigenvalues to compute.");
cmdp.setOption("num-restarts",&maxRestarts,"Maximum number of restarts.");
cmdp.setOption("tol",&tol,"Convergence tolerance.");
cmdp.setOption("num-print",&numPrint,"Number of Ritz values to print.");
// parse() will throw an exception on error
cmdp.parse(argc,argv);
// Get the stiffness and mass matrices
Epetra_SerialComm Comm;
RCP<Epetra_Map> Map;
RCP<Epetra_CrsMatrix> A, B;
EpetraExt::readEpetraLinearSystem( k_filename, Comm, &A, &Map );
EpetraExt::readEpetraLinearSystem( m_filename, Comm, &B, &Map );
//
// *******************************************************
// Set up Amesos direct solver for inner iteration
// *******************************************************
//
// Create Epetra linear problem class to solve "Kx = b"
Epetra_LinearProblem AmesosProblem;
AmesosProblem.SetOperator(A.get());
// Create Amesos factory and solver for solving "Kx = b" using a direct factorization
Amesos amesosFactory;
RCP<Amesos_BaseSolver> AmesosSolver = rcp( amesosFactory.Create( "Klu", AmesosProblem ) );
// The AmesosGenOp class assumes that the symbolic/numeric factorizations have already
// been performed on the linear problem.
AmesosSolver->SymbolicFactorization();
AmesosSolver->NumericFactorization();
//
// ************************************
// Start the block Arnoldi iteration
// ************************************
//
// Variables used for the Block Arnoldi Method
//
int verbosity = Anasazi::Errors + Anasazi::Warnings + Anasazi::FinalSummary;
if (verbose) {
verbosity += Anasazi::IterationDetails;
}
if (debug) {
verbosity += Anasazi::Debug;
}
//
// Create parameter list to pass into solver
//
Teuchos::ParameterList MyPL;
MyPL.set( "Verbosity", verbosity );
MyPL.set( "Which", "LM" );
MyPL.set( "Block Size", blockSize );
MyPL.set( "Num Blocks", numBlocks );
MyPL.set( "Maximum Restarts", maxRestarts );
MyPL.set( "Convergence Tolerance", tol );
MyPL.set( "Step Size", stepSize );
MyPL.set( "Extra NEV Blocks", extraBlocks );
MyPL.set( "Print Number of Ritz Values", numPrint );
// Create an Epetra_MultiVector for an initial vector to start the solver.
// Note: This needs to have the same number of columns as the blocksize.
RCP<Epetra_MultiVector> ivec = rcp( new Epetra_MultiVector(A->Map(), blockSize) );
// MVT::MvRandom( *ivec ); // FINISH: put this back in
MVT::MvInit(*ivec,1.0);
// Create the Epetra_Operator for the spectral transformation using the Amesos direct solver.
RCP<AmesosGenOp> Aop = rcp( new AmesosGenOp(AmesosProblem, AmesosSolver, B) );
// standard inner product; B is not symmetric positive definite, and Op has no symmetry.
RCP<Anasazi::BasicEigenproblem<ST,MV,OP> > MyProblem =
rcp( new Anasazi::BasicEigenproblem<ST,MV,OP>(Aop, ivec) );
MyProblem->setHermitian(false);
MyProblem->setNEV( nev );
// Inform the eigenproblem that you are finished passing it information
TEUCHOS_TEST_FOR_EXCEPTION( MyProblem->setProblem() != true,
std::runtime_error, "Anasazi::BasicEigenproblem::setProblem() returned with error.");
// Initialize the Block Arnoldi solver
Anasazi::BlockKrylovSchurSolMgr<ST,MV,OP> MySolverMgr(MyProblem, MyPL);
// Solve the problem to the specified tolerances or length
Anasazi::ReturnType returnCode = MySolverMgr.solve();
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<ST,MV> sol = MyProblem->getSolution();
vector<Anasazi::Value<ST> > evals = sol.Evals;
RCP<MV> evecs = sol.Evecs;
vector<int> index = sol.index;
int numev = sol.numVecs;
if (returnCode != Anasazi::Converged) {
cout << "Anasazi::EigensolverMgr::solve() returned unconverged, computing " << numev << " of " << nev << endl;
}
else {
cout << "Anasazi::EigensolverMgr::solve() returned converged, computing " << numev << " of " << nev << endl;
}
if (numev > 0) {
// Compute residuals.
Teuchos::LAPACK<int,MT> lapack;
vector<MT> normA(numev);
// Back-transform the eigenvalues
for (int i=0; i<numev; ++i) {
MT mag2 = lapack.LAPY2(evals[i].realpart,evals[i].imagpart);
mag2 = mag2*mag2;
evals[i].realpart /= mag2;
evals[i].imagpart /= (-mag2);
}
// The problem is non-Hermitian.
vector<int> curind(1);
vector<MT> resnorm(1), tempnrm(1);
Teuchos::RCP<const MV> Av_r, Av_i, Bv_r, Bv_i;
Epetra_MultiVector Aevec(*Map,numev), Bevec(*Map,numev), res(*Map,1);
// Compute A*evecs, B*evecs
OPT::Apply( *A, *evecs, Aevec );
OPT::Apply( *B, *evecs, Bevec );
for (int i=0; i<numev;) {
if (index[i]==0) {
// Get views of the real part of A*evec,B*evec
curind[0] = i;
Av_r = MVT::CloneView( Aevec, curind );
Bv_r = MVT::CloneView( Bevec, curind );
// Compute set res = lambda*B*evec - A*evec
// eigenvalue and eigenvector are both real
MVT::MvAddMv(evals[i].realpart,*Bv_r,-1.0,*Av_r,res);
// Compute the norm of the residual and increment counter
MVT::MvNorm( res, resnorm );
// Scale the residual
normA[i] = resnorm[0];
MT mag = MTT::magnitude(evals[i].realpart);
if ( mag > MTT::one() ) {
normA[i] /= mag;
}
// done with this eigenvector
i++;
} else {
// Get a copy of A*evec, B*evec
curind[0] = i;
Av_r = MVT::CloneCopy( Aevec, curind );
Bv_r = MVT::CloneView( Bevec, curind );
// Get the imaginary part of A*evec,B*evec
curind[0] = i+1;
Av_i = MVT::CloneCopy( Aevec, curind );
Bv_i = MVT::CloneView( Bevec, curind );
// grab temp copies of the eigenvalue
MT l_r = evals[i].realpart,
l_i = evals[i].imagpart;
// Compute real part of B*evec*lambda - A*evec
// B is real
// evec = evec_r + evec_i*i
// lambda = lambda_r + lambda_i*i
//
// evec*lambda = evec_r*lambda_r - evec_i*lambda_i + evec_r*lambda_i*i + evec_i*lambda_r*i
//
// res = B*evec*lambda - A*evec
// = B*(evec_r*lambda_r - evec_i*lambda_i + evec_r*lambda_i*i + evec_i*lambda_r*i) - A*evec_r - A*evec_i*i
// = (B*evec_r*lambda_r - B*evec_i*lambda_i - A*evec_r) + (B*evec_r*lambda_i + B*evec_i*lambda_r - A*evec_i)*i
// real(res) = B*evec_r*lambda_r - B*evec_i*lambda_i - A*evec_r
// imag(res) = B*evec_r*lambda_i + B*evec_i*lambda_r - A*evec_i
// compute real part of residual and its norm
MVT::MvAddMv(l_r,*Bv_r, -l_i,*Bv_i, res);
MVT::MvAddMv(MTT::one(),res, -MTT::one(),*Av_r, res);
MVT::MvNorm(res,tempnrm);
// compute imag part of residual and its norm
MVT::MvAddMv(l_i,*Bv_r, l_r,*Bv_i, res);
MVT::MvAddMv(MTT::one(),res, -MTT::one(),*Av_i, res);
MVT::MvNorm(res,resnorm);
// Compute the norms and scale by magnitude of eigenvalue
normA[i] = lapack.LAPY2( tempnrm[0], resnorm[0] );
MT mag = lapack.LAPY2(evals[i].realpart, evals[i].imagpart);
if (mag > MTT::one()) {
normA[i] /= mag;
}
normA[i+1] = normA[i];
// done with this conjugate pair
i=i+2;
}
}
// Output computed eigenvalues and their direct residuals
cout.setf(std::ios_base::right, std::ios_base::adjustfield);
cout<<endl<< "Actual Residuals"<<endl;
cout<< std::setw(16) << "Real Part"
<< std::setw(16) << "Imag Part"
<< std::setw(20) << "Direct Residual"<< endl;
cout<<"-----------------------------------------------------------"<<endl;
for (int j=0; j<numev; j++) {
cout<< std::setw(16) << evals[j].realpart
<< std::setw(16) << evals[j].imagpart
<< std::setw(20) << normA[j] << endl;
if (normA[j] > tol) {
testFailed = true;
}
}
cout<<"-----------------------------------------------------------"<<endl;
}
}
catch (std::exception &e) {
cout << "Caught exception: " << endl << e.what() << endl;
testFailed = true;
}
}
else { // run_me == false
cout << "\nNot running on processor " << mpisess.getRank() << ". Serial problem only." << endl;
}
if (mpisess.getRank() == 0) {
if (testFailed) {
cout << "End Result: TEST FAILED" << endl;
}
else {
cout << "End Result: TEST PASSED" << endl;
}
}
return 0;
}
int TestMultiLevelPreconditioner(char ProblemType[],
Teuchos::ParameterList & MLList,
Epetra_LinearProblem & Problem, double & TotalErrorResidual,
double & TotalErrorExactSol,bool cg=false)
{
Epetra_MultiVector* lhs = Problem.GetLHS();
Epetra_MultiVector* rhs = Problem.GetRHS();
Epetra_RowMatrix* A = Problem.GetMatrix();
// ======================================== //
// create a rhs corresponding to lhs or 1's //
// ======================================== //
lhs->PutScalar(1.0);
A->Multiply(false,*lhs,*rhs);
lhs->PutScalar(0.0);
Epetra_Time Time(A->Comm());
// =================== //
// call ML and AztecOO //
// =================== //
AztecOO solver(Problem);
MLList.set("ML output", 10);
ML_Epetra::MultiLevelPreconditioner * MLPrec = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
// tell AztecOO to use this preconditioner, then solve
solver.SetPrecOperator(MLPrec);
if(cg) solver.SetAztecOption(AZ_solver, AZ_cg);
else solver.SetAztecOption(AZ_solver, AZ_gmres);
solver.SetAztecOption(AZ_output, 32);
solver.SetAztecOption(AZ_kspace, 160);
solver.Iterate(1550, 1e-12);
delete MLPrec;
// ==================================================== //
// compute difference between exact solution and ML one //
// ==================================================== //
double d = 0.0, d_tot = 0.0;
for( int i=0 ; i<lhs->Map().NumMyElements() ; ++i )
d += ((*lhs)[0][i] - 1.0) * ((*lhs)[0][i] - 1.0);
A->Comm().SumAll(&d,&d_tot,1);
// ================== //
// compute ||Ax - b|| //
// ================== //
double Norm;
Epetra_Vector Ax(rhs->Map());
A->Multiply(false, *lhs, Ax);
Ax.Update(1.0, *rhs, -1.0);
Ax.Norm2(&Norm);
string msg = ProblemType;
if (A->Comm().MyPID() == 0) {
cout << msg << "......Using " << A->Comm().NumProc() << " processes" << endl;
cout << msg << "......||A x - b||_2 = " << Norm << endl;
cout << msg << "......||x_exact - x||_2 = " << sqrt(d_tot) << endl;
cout << msg << "......Total Time = " << Time.ElapsedTime() << endl;
}
TotalErrorExactSol += sqrt(d_tot);
TotalErrorResidual += Norm;
return( solver.NumIters() );
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
Teuchos::GlobalMPISession mpiSession(&argc, &argv, 0);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int nProcs, myPID ;
Teuchos::ParameterList pLUList ; // ParaLU parameters
Teuchos::ParameterList isoList ; // Isorropia parameters
string ipFileName = "ShyLU.xml"; // TODO : Accept as i/p
nProcs = mpiSession.getNProc();
myPID = Comm.MyPID();
if (myPID == 0)
{
cout <<"Parallel execution: nProcs="<< nProcs << endl;
}
// =================== Read input xml file =============================
Teuchos::updateParametersFromXmlFile(ipFileName, &pLUList);
isoList = pLUList.sublist("Isorropia Input");
// Get matrix market file name
string MMFileName = Teuchos::getParameter<string>(pLUList, "mm_file");
string prec_type = Teuchos::getParameter<string>(pLUList, "preconditioner");
if (myPID == 0)
{
cout << "Input :" << endl;
cout << "ParaLU params " << endl;
pLUList.print(std::cout, 2, true, true);
cout << "Matrix market file name: " << MMFileName << endl;
}
// ==================== Read input Matrix ==============================
Epetra_CrsMatrix *A;
int err = EpetraExt::MatrixMarketFileToCrsMatrix(MMFileName.c_str(), Comm, A);
//EpetraExt::MatlabFileToCrsMatrix(MMFileName.c_str(), Comm, A);
//assert(err != 0);
cout <<"Done reading the matrix"<< endl;
int n = A->NumGlobalRows();
cout <<"n="<< n << endl;
// Create input vectors
Epetra_Map vecMap(n, 0, Comm);
Epetra_MultiVector x(vecMap, 1);
Epetra_MultiVector b(vecMap, 1, false);
b.PutScalar(1.0); // TODO : Accept it as input
// Partition the matrix with hypergraph partitioning and redisstribute
Isorropia::Epetra::Partitioner *partitioner = new
Isorropia::Epetra::Partitioner(A, isoList, false);
partitioner->partition();
Isorropia::Epetra::Redistributor rd(partitioner);
Epetra_CrsMatrix *newA;
Epetra_MultiVector *newX, *newB;
rd.redistribute(*A, newA);
delete A;
A = newA;
rd.redistribute(x, newX);
rd.redistribute(b, newB);
Epetra_LinearProblem problem(A, newX, newB);
Amesos Factory;
char* SolverType = "Amesos_Klu";
bool IsAvailable = Factory.Query(SolverType);
Epetra_LinearProblem *LP = new Epetra_LinearProblem();
LP->SetOperator(A);
LP->SetLHS(newX);
LP->SetRHS(newB);
Amesos_BaseSolver *Solver = Factory.Create(SolverType, *LP);
Solver->SymbolicFactorization();
Teuchos::Time ftime("setup time");
ftime.start();
Solver->NumericFactorization();
cout << "Numeric Factorization" << endl;
Solver->Solve();
cout << "Solve done" << endl;
ftime.stop();
cout << "Time to setup" << ftime.totalElapsedTime() << endl;
// compute ||Ax - b||
double Norm;
Epetra_MultiVector Ax(vecMap, 1);
Epetra_MultiVector *newAx;
rd.redistribute(Ax, newAx);
A->Multiply(false, *newX, *newAx);
newAx->Update(1.0, *newB, -1.0);
newAx->Norm2(&Norm);
double ANorm = A->NormOne();
cout << "|Ax-b |/|A| = " << Norm/ANorm << endl;
delete newAx;
delete newX;
delete newB;
delete A;
delete partitioner;
}
int
main (int argc, char *argv[])
{
using Teuchos::RCP;
using Teuchos::rcp;
using std::cerr;
using std::cout;
using std::endl;
// Anasazi solvers have the following template parameters:
//
// - Scalar: The type of dot product results.
// - MV: The type of (multi)vectors.
// - OP: The type of operators (functions from multivector to
// multivector). A matrix (like Epetra_CrsMatrix) is an example
// of an operator; an Ifpack preconditioner is another example.
//
// Here, Scalar is double, MV is Epetra_MultiVector, and OP is
// Epetra_Operator.
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<double, MV> MVT;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init (&argc, &argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif // EPETRA_MPI
const int MyPID = Comm.MyPID ();
//
// Set up the test problem
//
// Dimensionality of the spatial domain to discretize
const int space_dim = 2;
// Size of each of the dimensions of the (discrete) domain
std::vector<double> brick_dim (space_dim);
brick_dim[0] = 1.0;
brick_dim[1] = 1.0;
// Number of elements in each of the dimensions of the domain
std::vector<int> elements (space_dim);
elements[0] = 10;
elements[1] = 10;
// Create the test problem.
RCP<ModalProblem> testCase =
rcp (new ModeLaplace2DQ2 (Comm, brick_dim[0], elements[0],
brick_dim[1], elements[1]));
// Get the stiffness and mass matrices.
//
// rcp (T*, false) returns a nonowning (doesn't deallocate) RCP.
RCP<Epetra_CrsMatrix> K =
rcp (const_cast<Epetra_CrsMatrix* > (testCase->getStiffness ()), false);
RCP<Epetra_CrsMatrix> M =
rcp (const_cast<Epetra_CrsMatrix* > (testCase->getMass ()), false);
//
// Create linear solver for linear systems with K
//
// Anasazi uses shift and invert, with a "shift" of zero, to find
// the eigenvalues of least magnitude. In this example, we
// implement the "invert" part of shift and invert by using an
// Amesos direct solver.
//
// Create Epetra linear problem class for solving linear systems
// with K. This implements the inverse operator for shift and
// invert.
Epetra_LinearProblem AmesosProblem;
// Tell the linear problem about the matrix K. Epetra_LinearProblem
// doesn't know about RCP, so we have to give it a raw pointer.
AmesosProblem.SetOperator (K.get ());
// Create Amesos factory and solver for solving linear systems with
// K. The solver uses the KLU library to do a sparse LU
// factorization.
//
// Note that the AmesosProblem object "absorbs" K. Anasazi doesn't
// see K, just the operator that implements K^{-1} M.
Amesos amesosFactory;
RCP<Amesos_BaseSolver> AmesosSolver;
// Amesos can interface to many different solvers. The following
// loop picks a solver that Amesos supports. The loop order
// reflects solver preference, only in the sense that using LAPACK
// here is a suboptimal fall-back. (With the LAPACK option, Amesos
// makes a dense version of the sparse matrix and uses LAPACK to
// compute the factorization. The other options are true sparse
// direct factorizations.)
const int numSolverNames = 9;
const char* solverNames[9] = {
"Klu", "Umfpack", "Superlu", "Superludist", "Mumps",
"Paradiso", "Taucs", "CSparse", "Lapack"
};
for (int k = 0; k < numSolverNames; ++k) {
const char* const solverName = solverNames[k];
if (amesosFactory.Query (solverName)) {
AmesosSolver = rcp (amesosFactory.Create (solverName, AmesosProblem));
if (MyPID == 0) {
cout << "Amesos solver: \"" << solverName << "\"" << endl;
}
}
}
if (AmesosSolver.is_null ()) {
throw std::runtime_error ("Amesos appears not to have any solvers enabled.");
}
// The AmesosGenOp class assumes that the symbolic and numeric
// factorizations have already been performed on the linear problem.
AmesosSolver->SymbolicFactorization ();
AmesosSolver->NumericFactorization ();
//
// Set parameters for the block Krylov-Schur eigensolver
//
double tol = 1.0e-8; // convergence tolerance
int nev = 10; // number of eigenvalues for which to solve
int blockSize = 3; // block size (number of eigenvectors processed at once)
int numBlocks = 3 * nev / blockSize; // restart length
int maxRestarts = 5; // maximum number of restart cycles
// We're looking for the largest-magnitude eigenvalues of the
// _inverse_ operator, thus, the smallest-magnitude eigenvalues of
// the original operator.
std::string which = "LM";
int verbosity = Anasazi::Errors + Anasazi::Warnings + Anasazi::FinalSummary;
// Create ParameterList to pass into eigensolver
Teuchos::ParameterList MyPL;
MyPL.set ("Verbosity", verbosity);
MyPL.set ("Which", which);
MyPL.set ("Block Size", blockSize);
MyPL.set ("Num Blocks", numBlocks);
MyPL.set ("Maximum Restarts", maxRestarts);
MyPL.set ("Convergence Tolerance", tol);
// Create an initial set of vectors to start the eigensolver. Note:
// This needs to have the same number of columns as the block size.
RCP<MV> ivec = rcp (new MV (K->Map (), blockSize));
ivec->Random ();
// Create the Epetra_Operator for the spectral transformation using
// the Amesos direct solver.
//
// The AmesosGenOp object is the operator we give to Anasazi. Thus,
// Anasazi just sees an operator that computes y = K^{-1} M x. The
// matrix K got absorbed into AmesosProblem (the
// Epetra_LinearProblem object). Later, when we set up the Anasazi
// eigensolver, we will need to tell it about M, so that it can
// orthogonalize basis vectors with respect to the inner product
// defined by M (since it is symmetric positive definite).
RCP<AmesosGenOp> Aop = rcp (new AmesosGenOp (AmesosSolver, M));
// Create the eigenproblem. This object holds all the stuff about
// your problem that Anasazi will see.
//
// Anasazi only needs M so that it can orthogonalize basis vectors
// with respect to the M inner product. Wouldn't it be nice if
// Anasazi didn't require M in two different places? Alas, this is
// not currently the case.
RCP<Anasazi::BasicEigenproblem<double,MV,OP> > MyProblem =
rcp (new Anasazi::BasicEigenproblem<double,MV,OP> (Aop, M, ivec));
// Tell the eigenproblem that the matrix pencil (K,M) is symmetric.
MyProblem->setHermitian (true);
// Set the number of eigenvalues requested
MyProblem->setNEV (nev);
// Tell the eigenproblem that you are finished passing it information.
const bool boolret = MyProblem->setProblem ();
if (boolret != true) {
if (MyPID == 0) {
cerr << "Anasazi::BasicEigenproblem::setProblem() returned with error." << endl;
}
#ifdef EPETRA_MPI
MPI_Finalize ();
#endif // EPETRA_MPI
return -1;
}
// Create the Block Krylov-Schur eigensolver.
Anasazi::BlockKrylovSchurSolMgr<double, MV, OP> MySolverMgr (MyProblem, MyPL);
// Solve the eigenvalue problem.
//
// Note that creating the eigensolver is separate from solving it.
// After creating the eigensolver, you may call solve() multiple
// times with different parameters or initial vectors. This lets
// you reuse intermediate state, like allocated basis vectors.
Anasazi::ReturnType returnCode = MySolverMgr.solve ();
if (returnCode != Anasazi::Converged && MyPID == 0) {
cout << "Anasazi eigensolver did not converge." << endl;
}
// Get the eigenvalues and eigenvectors from the eigenproblem.
Anasazi::Eigensolution<double,MV> sol = MyProblem->getSolution ();
// Anasazi returns eigenvalues as Anasazi::Value, so that if
// Anasazi's Scalar type is real-valued (as it is in this case), but
// some eigenvalues are complex, you can still access the
// eigenvalues correctly. In this case, there are no complex
// eigenvalues, since the matrix pencil is symmetric.
std::vector<Anasazi::Value<double> > evals = sol.Evals;
RCP<MV> evecs = sol.Evecs;
int numev = sol.numVecs;
if (numev > 0) {
// Reconstruct the eigenvalues. The ones that Anasazi gave back
// are the inverses of the original eigenvalues. Reconstruct the
// eigenvectors too.
MV tempvec (K->Map (), MVT::GetNumberVecs (*evecs));
K->Apply (*evecs, tempvec);
Teuchos::SerialDenseMatrix<int,double> dmatr (numev, numev);
MVT::MvTransMv (1.0, tempvec, *evecs, dmatr);
if (MyPID == 0) {
double compeval = 0.0;
cout.setf (std::ios_base::right, std::ios_base::adjustfield);
cout << "Actual Eigenvalues (obtained by Rayleigh quotient) : " << endl;
cout << "------------------------------------------------------" << endl;
cout << std::setw(16) << "Real Part"
<< std::setw(16) << "Rayleigh Error" << endl;
cout << "------------------------------------------------------" << endl;
for (int i = 0; i < numev; ++i) {
compeval = dmatr(i,i);
cout << std::setw(16) << compeval
<< std::setw(16)
<< std::fabs (compeval - 1.0/evals[i].realpart)
<< endl;
}
cout << "------------------------------------------------------" << endl;
}
}
#ifdef EPETRA_MPI
MPI_Finalize ();
#endif // EPETRA_MPI
return 0;
}
// ============================================================================
void
Solve(const Epetra_LinearProblem Problem)
{
Solve(Problem.GetMatrix(), Problem.GetLHS(), Problem.GetRHS());
}
static int run_test(Teuchos::RCP<Epetra_CrsMatrix> matrix,
bool verbose, // display the graph before & after
bool contract, // set global number of partitions to 1/2 num procs
int partitioningType, // hypergraph or graph partitioning, or simple
int vertexWeightType, // use vertex weights?
int edgeWeightType, // use edge/hyperedge weights?
int objectType) // use isorropia's CrsMatrix or CrsGraph
{
int rc=0, fail = 0;
#ifdef HAVE_EPETRAEXT
int localProc = 0;
double balance1, balance2, cutn1, cutn2, cutl1, cutl2;
double balance3, cutn3, cutl3;
double cutWgt1, cutWgt2, cutWgt3;
int numCuts1, numCuts2, numCuts3, valid;
int numPartitions = 0;
int keepDenseEdges = 0;
int numProcs = 1;
#ifdef HAVE_MPI
const Epetra_MpiComm &Comm = dynamic_cast<const Epetra_MpiComm &>(matrix->Comm());
localProc = Comm.MyPID();
numProcs = Comm.NumProc();
#else
const Epetra_SerialComm &Comm = dynamic_cast<const Epetra_SerialComm &>(matrix->Comm());
#endif
int numRows = matrix->NumGlobalRows();
if (numRows < (numProcs * 100)){
// By default Zoltan throws out dense edges, defined as those
// whose number of non-zeros exceeds 25% of the number of vertices.
//
// If dense edges are thrown out of a small matrix, there may be nothing left.
keepDenseEdges = 1;
}
double myShareBefore = 1.0 / numProcs;
double myShare = myShareBefore;
if (contract){
numPartitions = numProcs / 2;
if (numPartitions > numRows)
numPartitions = numRows;
if (numPartitions > 0){
if (localProc < numPartitions){
myShare = 1.0 / numPartitions;
}
else{
myShare = 0.0;
}
}
else{
contract = 0;
}
}
// If we want Zoltan's or Isorropia's default weights, then we don't
// need to supply a CostDescriber object to createBalancedCopy,
// so we get to test the API functions that don't take a CostDescriber.
bool noCosts = ((vertexWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS) &&
(edgeWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS));
// Test the interface that has no parameters, if possible
bool noParams =
((partitioningType == HYPERGRAPH_PARTITIONING) && // default, so requires no params
(numPartitions == 0) && // >0 would require a parameter
(keepDenseEdges == 0)); // >0 would require a parameter
// Maps for original object
const Epetra_Map &sourceRowMap = matrix->RowMap();
const Epetra_Map &sourceRangeMap = matrix->RangeMap();
// const Epetra_Map &sourceColMap = matrix->ColMap();
const Epetra_Map &sourceDomainMap = matrix->DomainMap();
int numCols = matrix->NumGlobalCols();
int nMyRows = sourceRowMap.NumMyElements();
int base = sourceRowMap.IndexBase();
// Compute vertex and edge weights
Isorropia::Epetra::CostDescriber costs;
Teuchos::RCP<Epetra_Vector> vptr;
Teuchos::RCP<Epetra_CrsMatrix> eptr;
Teuchos::RCP<Epetra_Vector> hyperEdgeWeights;
if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){
if (partitioningType == GRAPH_PARTITIONING){
// Create graph edge weights.
eptr = Teuchos::rcp(new Epetra_CrsMatrix(*matrix));
if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){
eptr->PutScalar(1.0); // set all nonzeros to 1.0
}
else{
int maxRowSize = eptr->MaxNumEntries();
double *newVal = NULL;
if (maxRowSize > 0){
newVal = new double [maxRowSize];
for (int j=0; j<maxRowSize; j++){
newVal[j] = localProc + 1 + j;
}
}
int numEntries;
int *idx;
double *val;
for (int i=0; i<nMyRows; i++){
rc = eptr->ExtractMyRowView(i, numEntries, val, idx);
for (int j=0; j<numEntries; j++){
val[j] = newVal[j];
}
}
if (newVal) delete [] newVal;
}
eptr->FillComplete(sourceDomainMap, sourceRangeMap);
costs.setGraphEdgeWeights(eptr);
}
else{
// Create hyperedge weights. (Note that the list of hyperedges that a
// process provides weights for has no relation to the columns
// that it has non-zeroes for, or the rows that is has. Hypergraphs
// in general are not square. Also more than one process can provide
// a weight for the same edge. Zoltan combines the weights according
// to the value of the PHG_EDGE_WEIGHT_OPERATION parameter. The default
// for this parameter is to use the maximum edge weight provided by any
// process for a given hyperedge.)
Epetra_Map hyperEdgeMap(numCols, base, Comm);
hyperEdgeWeights = Teuchos::rcp(new Epetra_Vector(hyperEdgeMap));
int *edgeGIDs = NULL;
double *weights = NULL;
int numHEweights = hyperEdgeMap.NumMyElements();
if (numHEweights){
edgeGIDs = new int [numHEweights];
weights = new double [numHEweights];
if (edgeWeightType == SUPPLY_EQUAL_WEIGHTS){
for (int i=0; i<numHEweights; i++){
edgeGIDs[i] = hyperEdgeMap.GID(i);
weights[i] = 1.0;
}
}
else{
int hiVolumeStart = matrix->NumGlobalCols() / 3;
int hiVolumeEnd = hiVolumeStart * 2;
for (int i=0; i<numHEweights; i++){
edgeGIDs[i] = hyperEdgeMap.GID(i);
if ((edgeGIDs[i] < hiVolumeStart) || (edgeGIDs[i] >= hiVolumeEnd)){
weights[i] = 1.0;
}
else{
weights[i] = 3.0;
}
}
}
hyperEdgeWeights->ReplaceGlobalValues(numHEweights, weights, edgeGIDs);
}
if (weights){
delete [] weights;
delete [] edgeGIDs;
}
costs.setHypergraphEdgeWeights(hyperEdgeWeights);
}
}
bool need_importer = false;
if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){
need_importer = true; // to redistribute row weights
double *val = NULL;
if (nMyRows){
val = new double [nMyRows];
if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){
for (int i=0; i<nMyRows; i++){
val[i] = 1.0;
}
}
else if (vertexWeightType == SUPPLY_UNEQUAL_WEIGHTS){
for (int i=0; i<nMyRows; i++){
val[i] = 1.0 + ((localProc+1) / 2);
}
}
}
vptr = Teuchos::rcp(new Epetra_Vector(Copy, sourceRowMap, val));
if (val) delete [] val;
costs.setVertexWeights(vptr);
}
// Calculate partition quality metrics before calling Zoltan
if (partitioningType == GRAPH_PARTITIONING){
rc = ispatest::compute_graph_metrics(matrix->Graph(), costs,
myShare, balance1, numCuts1, cutWgt1, cutn1, cutl1);
if (contract){
// balance wrt target of balancing weight over *all* procs
rc = ispatest::compute_graph_metrics(matrix->Graph(), costs,
myShareBefore, balance3, numCuts3, cutWgt3, cutn3, cutl3);
}
}
else{
rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs,
myShare, balance1, cutn1, cutl1);
if (contract){
// balance wrt target of balancing weight over *all* procs
rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs,
myShareBefore, balance3, cutn3, cutl3);
}
}
if (rc){
ERROREXIT((localProc==0), "Error in computing partitioning metrics")
}
Teuchos::ParameterList params;
#ifdef HAVE_ISORROPIA_ZOLTAN
if (!noParams){
// We're using Zoltan for partitioning and supplying
// parameters, overriding defaults.
Teuchos::ParameterList &sublist = params.sublist("Zoltan");
if (partitioningType == GRAPH_PARTITIONING){
params.set("PARTITIONING METHOD", "GRAPH");
sublist.set("GRAPH_PACKAGE", "PHG");
}
else{
params.set("PARTITIONING METHOD", "HYPERGRAPH");
sublist.set("LB_APPROACH", "PARTITION");
sublist.set("PHG_CUT_OBJECTIVE", "CONNECTIVITY"); // "cutl"
}
if (keepDenseEdges){
// only throw out rows that have no zeroes, default is to
// throw out if .25 or more of the columns are non-zero
sublist.set("PHG_EDGE_SIZE_THRESHOLD", "1.0");
}
if (numPartitions > 0){
// test #Partitions < #Processes
std::ostringstream os;
os << numPartitions;
std::string s = os.str();
// sublist.set("NUM_GLOBAL_PARTS", s);
params.set("NUM PARTS", s);
}
//sublist.set("DEBUG_LEVEL", "1"); // Zoltan will print out parameters
//sublist.set("DEBUG_LEVEL", "5"); // proc 0 will trace Zoltan calls
//sublist.set("DEBUG_MEMORY", "2"); // Zoltan will trace alloc & free
}
#else
ERROREXIT((localProc==0),
"Zoltan partitioning required but Zoltan not available.")
#endif
// Function scope values
Teuchos::RCP<Epetra_Vector> newvwgts;
Teuchos::RCP<Epetra_CrsMatrix> newewgts;
// Function scope values required for LinearProblem
Epetra_LinearProblem *problem = NULL;
Epetra_Map *LHSmap = NULL;
Epetra_MultiVector *RHS = NULL;
Epetra_MultiVector *LHS = NULL;
// Reference counted pointer to balanced object
Epetra_CrsMatrix *matrixPtr=NULL;
Epetra_CrsGraph *graphPtr=NULL;
Epetra_RowMatrix *rowMatrixPtr=NULL;
Epetra_LinearProblem *problemPtr=NULL;
// Row map for balanced object
const Epetra_BlockMap *targetBlockRowMap=NULL; // for input CrsGraph
const Epetra_Map *targetRowMap=NULL; // for all other inputs
// Column map for balanced object
const Epetra_BlockMap *targetBlockColMap=NULL; // for input CrsGraph
const Epetra_Map *targetColMap=NULL; // for all other inputs
if (objectType == EPETRA_CRSMATRIX){
if (noParams && noCosts){
matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix);
}
else if (noCosts){
matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params);
}
targetRowMap = &(matrixPtr->RowMap());
targetColMap = &(matrixPtr->ColMap());
}
else if (objectType == EPETRA_CRSGRAPH){
const Epetra_CrsGraph graph = matrix->Graph();
if (noParams && noCosts){
graphPtr = Isorropia::Epetra::createBalancedCopy(graph);
}
else if (noCosts){
graphPtr = Isorropia::Epetra::createBalancedCopy(graph, params);
}
targetBlockRowMap = &(graphPtr->RowMap());
targetBlockColMap = &(graphPtr->ColMap());
}
else if (objectType == EPETRA_ROWMATRIX){
if (noParams && noCosts){
rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix);
}
else if (noCosts){
rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params);
}
targetRowMap = &(rowMatrixPtr->RowMatrixRowMap());
targetColMap = &(rowMatrixPtr->RowMatrixColMap());
}
else if (objectType == EPETRA_LINEARPROBLEM){
// Create a linear problem with this matrix.
LHSmap = new Epetra_Map(numCols, base, Comm);
int myRHSsize = sourceRowMap.NumMyElements();
int myLHSsize = LHSmap->NumMyElements();
int valSize = ((myRHSsize > myLHSsize) ? myRHSsize : myLHSsize);
double *vals = NULL;
if (valSize){
vals = new double [valSize];
}
if (valSize){
for (int i=0; i < valSize; i++){
// put my rank in my portion of LHS and my portion of RHS
vals[i] = localProc;
}
}
RHS = new Epetra_MultiVector(Copy, sourceRowMap, vals, 1, 1);
LHS = new Epetra_MultiVector(Copy, *LHSmap, vals, 1, 1);
if (valSize){
delete [] vals;
}
problem = new Epetra_LinearProblem(matrix.get(), LHS, RHS);
Epetra_LinearProblem lp = *problem;
if (lp.CheckInput()){
ERROREXIT((localProc==0), "Error creating a LinearProblem");
}
if (noParams && noCosts){
problemPtr = Isorropia::Epetra::createBalancedCopy(lp);
}
else if (noCosts){
problemPtr = Isorropia::Epetra::createBalancedCopy(lp, params);
}
targetRowMap = &(problemPtr->GetMatrix()->RowMatrixRowMap());
targetColMap = &(problemPtr->GetMatrix()->RowMatrixColMap());
}
// Redistribute the edge weights
// Comment this out since we don't redistribute columns
if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){
if (partitioningType == GRAPH_PARTITIONING){
Epetra_Import *importer = NULL;
if (objectType == EPETRA_CRSGRAPH){
newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *graphPtr));
targetRowMap = &(newewgts->RowMap());
targetColMap = &(newewgts->ColMap());
}
else{
newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *targetRowMap, *targetColMap, 0));
}
importer = new Epetra_Import(*targetRowMap, sourceRowMap);
newewgts->Import(*eptr, *importer, Insert);
newewgts->FillComplete(*targetColMap, *targetRowMap);
costs.setGraphEdgeWeights(newewgts);
}
}
// Redistribute the vertex weights
if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){
Epetra_Import *importer = NULL;
if (objectType == EPETRA_CRSGRAPH){
newvwgts = Teuchos::rcp(new Epetra_Vector(*targetBlockRowMap));
importer = new Epetra_Import(*targetBlockRowMap, sourceRowMap);
}
else{
newvwgts = Teuchos::rcp(new Epetra_Vector(*targetRowMap));
importer = new Epetra_Import(*targetRowMap, sourceRowMap);
}
newvwgts->Import(*vptr, *importer, Insert);
costs.setVertexWeights(newvwgts);
}
if (localProc == 0){
test_type(numPartitions, partitioningType, vertexWeightType, edgeWeightType, objectType);
}
if (verbose){
// Picture of problem before balancing
if (objectType == EPETRA_LINEARPROBLEM){
ispatest::show_matrix("Before load balancing", *problem, Comm);
}
else{
ispatest::show_matrix("Before load balancing", matrix->Graph(), Comm);
}
// Picture of problem after balancing
if (objectType == EPETRA_LINEARPROBLEM){
ispatest::show_matrix("After load balancing (x in Ax=b is not redistributed)", *problemPtr, Comm);
}
else if (objectType == EPETRA_ROWMATRIX){
ispatest::show_matrix("After load balancing", *rowMatrixPtr, Comm);
}
else if (objectType == EPETRA_CRSMATRIX){
ispatest::show_matrix("After load balancing", matrixPtr->Graph(), Comm);
}
else if (objectType == EPETRA_CRSGRAPH){
ispatest::show_matrix("After load balancing", *graphPtr, Comm);
}
}
// After partitioning, recompute the metrics
if (partitioningType == GRAPH_PARTITIONING){
if (objectType == EPETRA_LINEARPROBLEM){
rc = ispatest::compute_graph_metrics(*(problemPtr->GetMatrix()), costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else if (objectType == EPETRA_ROWMATRIX){
rc = ispatest::compute_graph_metrics(*rowMatrixPtr, costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else if (objectType == EPETRA_CRSMATRIX){
rc = ispatest::compute_graph_metrics(matrixPtr->Graph(), costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else {
rc = ispatest::compute_graph_metrics(*graphPtr, costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
}
else{
if (objectType == EPETRA_LINEARPROBLEM){
rc = ispatest::compute_hypergraph_metrics(*(problemPtr->GetMatrix()), costs,
myShare, balance2, cutn2, cutl2);
}
else if (objectType == EPETRA_ROWMATRIX){
rc = ispatest::compute_hypergraph_metrics(*rowMatrixPtr, costs,
myShare, balance2, cutn2, cutl2);
}
else if (objectType == EPETRA_CRSMATRIX){
rc = ispatest::compute_hypergraph_metrics(matrixPtr->Graph(), costs,
myShare, balance2, cutn2, cutl2);
}
else{
rc = ispatest::compute_hypergraph_metrics(*graphPtr, costs,
myShare, balance2, cutn2, cutl2);
}
}
if (rc){
ERROREXIT((localProc==0), "Error in computing partitioning metrics")
}
std::string why;
if (partitioningType == GRAPH_PARTITIONING){
fail = (cutWgt2 > cutWgt1);
why = "New weighted edge cuts are worse";
if (localProc == 0){
std::cout << "Before partitioning: Balance " << balance1 ;
std::cout << " cutn " << cutn1 ;
std::cout << " cutl " << cutl1 ;
if (contract){
std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl;
std::cout << "Before partitioning: Balance " << balance3 ;
std::cout << " cutn " << cutn3 ;
std::cout << " cutl " << cutl3 ;
std::cout << " (wrt balancing over " << numProcs << " partitions)" ;
}
std::cout << std::endl;
std::cout << " Total edge cuts: " << numCuts1;
std::cout << " Total weighted edge cuts: " << cutWgt1 << std::endl;
std::cout << "After partitioning: Balance " << balance2 ;
std::cout << " cutn " << cutn2 ;
std::cout << " cutl " << cutl2 << std::endl;
std::cout << " Total edge cuts: " << numCuts2;
std::cout << " Total weighted edge cuts: " << cutWgt2 << std::endl;
}
}
else{
fail = (cutl2 > cutl1);
why = "New cutl is worse";
if (localProc == 0){
std::cout << "Before partitioning: Balance " << balance1 ;
std::cout << " cutn " << cutn1 ;
std::cout << " cutl " << cutl1 ;
if (contract){
std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl;
std::cout << "Before partitioning: Balance " << balance3 ;
std::cout << " cutn " << cutn3 ;
std::cout << " cutl " << cutl3 ;
std::cout << " (wrt balancing over " << numProcs << " partitions)" ;
}
std::cout << std::endl;
std::cout << "After partitioning: Balance " << balance2 ;
std::cout << " cutn " << cutn2 ;
std::cout << " cutl " << cutl2 << std::endl;
}
}
if (fail){
if (localProc == 0) std::cout << "ERROR: "+why << std::endl;
}
// Check that input matrix is valid. This test constructs an "x"
// with the matrix->DomainMap() and a "y" with matrix->RangeMap()
// and then calculates y = Ax.
if (objectType == EPETRA_LINEARPROBLEM){
valid = ispatest::test_matrix_vector_multiply(*problemPtr);
}
else if (objectType == EPETRA_ROWMATRIX){
valid = ispatest::test_row_matrix_vector_multiply(*rowMatrixPtr);
}
else if (objectType == EPETRA_CRSMATRIX){
valid = ispatest::test_matrix_vector_multiply(*matrixPtr);
}
else{
valid = ispatest::test_matrix_vector_multiply(*graphPtr);
}
if (!valid){
if (localProc == 0) std::cout << "Rebalanced matrix is not a valid Epetra matrix" << std::endl;
fail = 1;
}
else{
if (localProc == 0) std::cout << "Rebalanced matrix is a valid Epetra matrix" << std::endl;
}
if (localProc == 0)
std::cout << std::endl;
#else
std::cout << "test_simple main: currently can only test "
<< "with Epetra and EpetraExt enabled." << std::endl;
rc = -1;
#endif
return fail;
}
int TestMultiLevelPreconditioner(char ProblemType[],
Teuchos::ParameterList & MLList,
Epetra_LinearProblem & Problem, double & TotalErrorResidual,
double & TotalErrorExactSol)
{
Epetra_MultiVector* lhs = Problem.GetLHS();
Epetra_MultiVector* rhs = Problem.GetRHS();
Epetra_CrsMatrix* A = dynamic_cast<Epetra_CrsMatrix*>(Problem.GetMatrix());
int PID = A->Comm().MyPID();
int numProcs = A->Comm().NumProc();
RCP<const Epetra_RowMatrix> Arcp = Teuchos::rcp(A, false);
double n1, n2,nf;
// ======================================== //
// create a rhs corresponding to lhs or 1's //
// ======================================== //
lhs->PutScalar(1.0);
A->Multiply(false,*lhs,*rhs);
lhs->PutScalar(0.0);
MLList.set("ML output", 0);
RowMatrixToMatlabFile("mat_f.dat",*A);
MultiVectorToMatrixMarketFile("lhs_f.dat",*lhs,0,0,false);
MultiVectorToMatrixMarketFile("rhs_f.dat",*rhs,0,0,false);
Epetra_Time Time(A->Comm());
/* Build the Zoltan list - Group #1 */
ParameterList Zlist1,Sublist1;
Sublist1.set("DEBUG_LEVEL","0");
Sublist1.set("NUM_GLOBAL_PARTITIONS","2");
Zlist1.set("Zoltan",Sublist1);
/* Start Isorropia's Ninja Magic - Group #1 */
RefCountPtr<Isorropia::Epetra::Partitioner> partitioner1 =
Isorropia::Epetra::create_partitioner(Arcp, Zlist1);
Isorropia::Epetra::Redistributor rd1(partitioner1);
Teuchos::RCP<Epetra_CrsMatrix> ResA1=rd1.redistribute(*A);
Teuchos::RCP<Epetra_MultiVector> ResX1=rd1.redistribute(*lhs);
Teuchos::RCP<Epetra_MultiVector> ResB1=rd1.redistribute(*rhs);
RestrictedCrsMatrixWrapper RW1;
RW1.restrict_comm(ResA1);
RestrictedMultiVectorWrapper RX1,RB1;
RX1.restrict_comm(ResX1);
RB1.restrict_comm(ResB1);
/* Build the Zoltan list - Group #2 */
ParameterList Zlist2,Sublist2;
Sublist2.set("DEBUG_LEVEL","0");
if(PID > 1) Sublist2.set("NUM_LOCAL_PARTITIONS","1");
else Sublist2.set("NUM_LOCAL_PARTITIONS","0");
Zlist2.set("Zoltan",Sublist2);
/* Start Isorropia's Ninja Magic - Group #2 */
RefCountPtr<Isorropia::Epetra::Partitioner> partitioner2 =
Isorropia::Epetra::create_partitioner(Arcp, Zlist2);
Isorropia::Epetra::Redistributor rd2(partitioner2);
Teuchos::RCP<Epetra_CrsMatrix> ResA2=rd2.redistribute(*A);
Teuchos::RCP<Epetra_MultiVector> ResX2=rd2.redistribute(*lhs);
Teuchos::RCP<Epetra_MultiVector> ResB2=rd2.redistribute(*rhs);
RestrictedCrsMatrixWrapper RW2;
RW2.restrict_comm(ResA2);
RestrictedMultiVectorWrapper RX2,RB2;
RX2.restrict_comm(ResX2);
RB2.restrict_comm(ResB2);
if(RW1.RestrictedProcIsActive()){
Teuchos::RCP<Epetra_CrsMatrix> SubA1 = RW1.RestrictedMatrix();
Teuchos::RCP<Epetra_MultiVector> SubX1 = RX1.RestrictedMultiVector();
Teuchos::RCP<Epetra_MultiVector> SubB1 = RB1.RestrictedMultiVector();
ML_Epetra::MultiLevelPreconditioner * SubPrec1 = new ML_Epetra::MultiLevelPreconditioner(*SubA1, MLList, true);
Epetra_LinearProblem Problem1(&*SubA1,&*SubX1,&*SubB1);
AztecOO solver1(Problem1);
solver1.SetPrecOperator(SubPrec1);
solver1.SetAztecOption(AZ_solver, AZ_gmres);
solver1.SetAztecOption(AZ_output, 32);
solver1.SetAztecOption(AZ_kspace, 160);
solver1.Iterate(1550, 1e-12);
delete SubPrec1;
}
else{
Teuchos::RCP<Epetra_CrsMatrix> SubA2 = RW2.RestrictedMatrix();
Teuchos::RCP<Epetra_MultiVector> SubX2 = RX2.RestrictedMultiVector();
Teuchos::RCP<Epetra_MultiVector> SubB2 = RB2.RestrictedMultiVector();
ML_Epetra::MultiLevelPreconditioner * SubPrec2 = new ML_Epetra::MultiLevelPreconditioner(*SubA2, MLList, true);
Epetra_LinearProblem Problem2(&*SubA2,&*SubX2,&*SubB2);
AztecOO solver2(Problem2);
solver2.SetPrecOperator(SubPrec2);
solver2.SetAztecOption(AZ_solver, AZ_gmres);
solver2.SetAztecOption(AZ_output, 32);
solver2.SetAztecOption(AZ_kspace, 160);
solver2.Iterate(1550, 1e-12);
delete SubPrec2;
}
/* Post-processing exports */
Epetra_MultiVector ans1(*lhs), ans2(*lhs);
rd1.redistribute_reverse(*ResX1,ans1);
rd2.redistribute_reverse(*ResX2,ans2);
/* Run on Full Problem */
A->Comm().Barrier();
ML_Epetra::MultiLevelPreconditioner * FullPrec = new ML_Epetra::MultiLevelPreconditioner(*A, MLList, true);
AztecOO solverF(Problem);
solverF.SetPrecOperator(FullPrec);
solverF.SetAztecOption(AZ_solver, AZ_gmres);
solverF.SetAztecOption(AZ_output, 32);
solverF.SetAztecOption(AZ_kspace, 160);
solverF.Iterate(1550, 1e-12);
delete FullPrec;
/* Solution Comparison */
ans1.Update(1.0,*lhs,-1.0);
ans2.Update(1.0,*lhs,-1.0);
ans1.Norm2(&n1);
ans2.Norm2(&n2);
if(!PID) {
printf("Norm Diff 1 = %6.4e\n",n1);
printf("Norm Diff 2 = %6.4e\n",n2);
}
TotalErrorExactSol += n1 + n2;
}
double Ifpack_Condest(const Ifpack_Preconditioner& IFP,
const Ifpack_CondestType CT,
const int MaxIters,
const double Tol,
Epetra_RowMatrix* Matrix)
{
double ConditionNumberEstimate = -1.0;
if (CT == Ifpack_Cheap) {
// Create a vector with all values equal to one
Epetra_Vector Ones(IFP.OperatorDomainMap());
Ones.PutScalar(1.0);
// Create the vector of results
Epetra_Vector OnesResult(IFP.OperatorRangeMap());
// Compute the effect of the solve on the vector of ones
IFPACK_CHK_ERR(IFP.ApplyInverse(Ones, OnesResult));
// Make all values non-negative
IFPACK_CHK_ERR(OnesResult.Abs(OnesResult));
// Get the maximum value across all processors
IFPACK_CHK_ERR(OnesResult.MaxValue(&ConditionNumberEstimate));
}
else if (CT == Ifpack_CG) {
#ifdef HAVE_IFPACK_AZTECOO
if (Matrix == 0)
Matrix = (Epetra_RowMatrix*)&(IFP.Matrix());
Epetra_Vector LHS(IFP.OperatorDomainMap());
LHS.PutScalar(0.0);
Epetra_Vector RHS(IFP.OperatorRangeMap());
RHS.Random();
Epetra_LinearProblem Problem;
Problem.SetOperator(Matrix);
Problem.SetLHS(&LHS);
Problem.SetRHS(&RHS);
AztecOO Solver(Problem);
Solver.SetAztecOption(AZ_output,AZ_none);
Solver.SetAztecOption(AZ_solver,AZ_cg_condnum);
Solver.Iterate(MaxIters,Tol);
const double* status = Solver.GetAztecStatus();
ConditionNumberEstimate = status[AZ_condnum];
#endif
} else if (CT == Ifpack_GMRES) {
#ifdef HAVE_IFPACK_AZTECOO
if (Matrix == 0)
Matrix = (Epetra_RowMatrix*)&(IFP.Matrix());
Epetra_Vector LHS(IFP.OperatorDomainMap());
LHS.PutScalar(0.0);
Epetra_Vector RHS(IFP.OperatorRangeMap());
RHS.Random();
Epetra_LinearProblem Problem;
Problem.SetOperator(Matrix);
Problem.SetLHS(&LHS);
Problem.SetRHS(&RHS);
AztecOO Solver(Problem);
Solver.SetAztecOption(AZ_solver,AZ_gmres_condnum);
Solver.SetAztecOption(AZ_output,AZ_none);
// the following can be problematic for large problems,
// but any restart would destroy useful information about
// the condition number.
Solver.SetAztecOption(AZ_kspace,MaxIters);
Solver.Iterate(MaxIters,Tol);
const double* status = Solver.GetAztecStatus();
ConditionNumberEstimate = status[AZ_condnum];
#endif
}
return(ConditionNumberEstimate);
}
int main(int argc, char *argv[]) {
int i;
#ifdef EPETRA_MPI
// Initialize MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int MyPID = Comm.MyPID();
// Number of dimension of the domain
int space_dim = 2;
// Size of each of the dimensions of the domain
std::vector<double> brick_dim( space_dim );
brick_dim[0] = 1.0;
brick_dim[1] = 1.0;
// Number of elements in each of the dimensions of the domain
std::vector<int> elements( space_dim );
elements[0] = 10;
elements[1] = 10;
// Create problem
Teuchos::RCP<ModalProblem> testCase = Teuchos::rcp( new ModeLaplace2DQ2(Comm, brick_dim[0], elements[0], brick_dim[1], elements[1]) );
// Get the stiffness and mass matrices
Teuchos::RCP<Epetra_CrsMatrix> K = Teuchos::rcp( const_cast<Epetra_CrsMatrix *>(testCase->getStiffness()), false );
Teuchos::RCP<Epetra_CrsMatrix> M = Teuchos::rcp( const_cast<Epetra_CrsMatrix *>(testCase->getMass()), false );
//
// *******************************************************
// Set up Amesos direct solver for inner iteration
// *******************************************************
//
// Create the shifted system K - sigma * M.
// For the buckling transformation, this shift must be nonzero.
double sigma = 1.0;
Epetra_CrsMatrix Kcopy( *K );
int addErr = EpetraExt::MatrixMatrix::Add( *M, false, -sigma, Kcopy, 1.0 );
if (addErr != 0) {
if (MyPID == 0) {
std::cout << "EpetraExt::MatrixMatrix::Add returned with error: " << addErr << std::endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
// Create Epetra linear problem class to solve "x = b"
Epetra_LinearProblem AmesosProblem;
AmesosProblem.SetOperator(&Kcopy);
// Create Amesos factory and solver for solving "(K - sigma*M)x = b" using a direct factorization
Amesos amesosFactory;
Teuchos::RCP<Amesos_BaseSolver> AmesosSolver =
Teuchos::rcp( amesosFactory.Create( "Klu", AmesosProblem ) );
// The AmesosBucklingOp class assumes that the symbolic/numeric factorizations have already
// been performed on the linear problem.
AmesosSolver->SymbolicFactorization();
AmesosSolver->NumericFactorization();
//
// ************************************
// Start the block Arnoldi iteration
// ************************************
//
// Variables used for the Block Arnoldi Method
//
int nev = 10;
int blockSize = 3;
int numBlocks = 3*nev/blockSize;
int maxRestarts = 5;
//int step = 5;
double tol = 1.0e-8;
std::string which = "LM";
int verbosity = Anasazi::Errors + Anasazi::Warnings + Anasazi::FinalSummary;
//
// Create parameter list to pass into solver
//
Teuchos::ParameterList MyPL;
MyPL.set( "Verbosity", verbosity );
MyPL.set( "Which", which );
MyPL.set( "Block Size", blockSize );
MyPL.set( "Num Blocks", numBlocks );
MyPL.set( "Maximum Restarts", maxRestarts );
MyPL.set( "Convergence Tolerance", tol );
//MyPL.set( "Step Size", step );
typedef Epetra_MultiVector MV;
typedef Epetra_Operator OP;
typedef Anasazi::MultiVecTraits<double, MV> MVT;
typedef Anasazi::OperatorTraits<double, MV, OP> OPT;
// Create an Epetra_MultiVector for an initial vector to start the solver.
// Note: This needs to have the same number of columns as the blocksize.
Teuchos::RCP<Epetra_MultiVector> ivec = Teuchos::rcp( new Epetra_MultiVector(K->Map(), blockSize) );
MVT::MvRandom( *ivec );
// Create the Epetra_Operator for the buckling transformation using the Amesos direct solver.
Teuchos::RCP<AmesosBucklingOp> BucklingOp
= Teuchos::rcp( new AmesosBucklingOp(AmesosProblem, AmesosSolver, K) );
Teuchos::RCP<Anasazi::BasicEigenproblem<double,MV,OP> > MyProblem =
Teuchos::rcp( new Anasazi::BasicEigenproblem<double,MV,OP>(BucklingOp, K, ivec) );
// Inform the eigenproblem that the matrix pencil (K,M) is symmetric
MyProblem->setHermitian(true);
// Set the number of eigenvalues requested
MyProblem->setNEV( nev );
// Inform the eigenproblem that you are finished passing it information
bool boolret = MyProblem->setProblem();
if (boolret != true) {
if (MyPID == 0) {
std::cout << "Anasazi::BasicEigenproblem::setProblem() returned with error." << std::endl;
}
#ifdef HAVE_MPI
MPI_Finalize() ;
#endif
return -1;
}
// Initialize the Block Arnoldi solver
Anasazi::BlockKrylovSchurSolMgr<double, MV, OP> MySolverMgr(MyProblem, MyPL);
// Solve the problem to the specified tolerances or length
Anasazi::ReturnType returnCode = MySolverMgr.solve();
if (returnCode != Anasazi::Converged && MyPID==0) {
std::cout << "Anasazi::EigensolverMgr::solve() returned unconverged." << std::endl;
}
// Get the eigenvalues and eigenvectors from the eigenproblem
Anasazi::Eigensolution<double,MV> sol = MyProblem->getSolution();
std::vector<Anasazi::Value<double> > evals = sol.Evals;
Teuchos::RCP<MV> evecs = sol.Evecs;
int numev = sol.numVecs;
if (numev > 0) {
// Undo buckling transformation; computed eigenvalues are real
std::vector<double> compEvals(numev);
for (i=0; i<numev; ++i) {
compEvals[i] = sigma*evals[i].realpart/(evals[i].realpart-1.0);
}
// Remember, eigenvectors are constructed K-orthogonal to preserve symmetry,
// so numerator of the Rayleigh quotient is 1.0.
Teuchos::SerialDenseMatrix<int,double> dmatr(numev,numev);
Epetra_MultiVector tempvec(M->Map(), MVT::GetNumberVecs( *evecs ));
OPT::Apply( *M, *evecs, tempvec );
MVT::MvTransMv( 1.0, tempvec, *evecs, dmatr );
if (MyPID==0) {
double rq_eval = 0.0;
std::cout.setf(std::ios_base::right, std::ios_base::adjustfield);
std::cout<<"Actual Eigenvalues (obtained by Rayleigh quotient) : "<<std::endl;
std::cout<<"------------------------------------------------------"<<std::endl;
std::cout<<std::setw(16)<<"Real Part"
<<std::setw(16)<<"Rayleigh Error"<<std::endl;
std::cout<<"------------------------------------------------------"<<std::endl;
for (i=0; i<numev; i++) {
rq_eval = 1.0 / dmatr(i,i);
std::cout<<std::setw(16)<<rq_eval
<<std::setw(16)<<Teuchos::ScalarTraits<double>::magnitude(rq_eval-compEvals[i])
<<std::endl;
}
std::cout<<"------------------------------------------------------"<<std::endl;
}
}
#ifdef EPETRA_MPI
MPI_Finalize();
#endif
return 0;
}
int main(int argc, char** argv) {
#if defined(HAVE_MPI) && defined(HAVE_EPETRA)
int p, numProcs = 1;
int localProc = 0;
//first, set up our MPI environment...
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &localProc);
MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
int local_n = 600;
//Create a Epetra_LinearProblem object.
Epetra_LinearProblem* linprob = 0;
try {
linprob = create_epetra_problem(numProcs, localProc, local_n);
}
catch(std::exception& exc) {
std::cout << "linsys example: create_epetra_problem threw exception '"
<< exc.what() << "' on proc " << localProc << std::endl;
MPI_Finalize();
return(-1);
}
//We'll need a Teuchos::ParameterList object to pass to the
//Isorropia::Epetra::Partitioner class.
Teuchos::ParameterList paramlist;
// If Zoltan is available, the Zoltan package will be used for
// the partitioning operation. By default, Isorropia selects Zoltan's
// Hypergraph partitioner. If a method other than Hypergraph is
// desired, it can be specified by first creating a parameter sublist
// named "Zoltan", and then setting appropriate Zoltan parameters in
// that sublist. A sublist is created like this:
// Teuchos::ParameterList& sublist = paramlist.sublist("Zoltan");
//
// If Zoltan is not available, a simple linear partitioner will be
// used to partition such that the number of nonzeros is equal (or
// close to equal) on each processor.
Epetra_RowMatrix* rowmatrix = linprob->GetMatrix();
Teuchos::RCP<const Epetra_RowMatrix> rowmat =
Teuchos::rcp(rowmatrix, false);
//Now create the partitioner
Teuchos::RCP<Isorropia::Epetra::Partitioner> partitioner =
Teuchos::rcp(new Isorropia::Epetra::Partitioner(rowmat, paramlist));
//Next create a Redistributor object and use it to create balanced
//copies of the objects in linprob.
Isorropia::Epetra::Redistributor rd(partitioner);
Teuchos::RCP<Epetra_CrsMatrix> bal_matrix;
Teuchos::RCP<Epetra_MultiVector> bal_x;
Teuchos::RCP<Epetra_MultiVector> bal_b;
//Use a try-catch block because Isorropia will throw an exception
//if it encounters an error.
if (localProc == 0) {
std::cout << " calling Isorropia::Epetra::Redistributor::redistribute..."
<< std::endl;
}
try {
bal_matrix = rd.redistribute(*linprob->GetMatrix());
bal_x = rd.redistribute(*linprob->GetLHS());
bal_b = rd.redistribute(*linprob->GetRHS());
}
catch(std::exception& exc) {
std::cout << "linsys example: Isorropia::Epetra::Redistributor threw "
<< "exception '" << exc.what() << "' on proc "
<< localProc << std::endl;
MPI_Finalize();
return(-1);
}
Epetra_LinearProblem balanced_problem(bal_matrix.get(),
bal_x.get(), bal_b.get());
// Results
double goalWeight = 1.0 / (double)numProcs;
double bal0, bal1, cutn0, cutn1, cutl0, cutl1;
Isorropia::Epetra::CostDescriber default_costs;
// Balance and cut quality before partitioning
ispatest::compute_hypergraph_metrics(*(linprob->GetMatrix()), default_costs, goalWeight,
bal0, cutn0, cutl0);
// Balance and cut quality after partitioning
ispatest::compute_hypergraph_metrics(*bal_matrix, default_costs, goalWeight,
bal1, cutn1, cutl1);
if (localProc == 0){
std::cout << "Before partitioning: ";
std::cout << "Balance " << bal0 << " cutN " << cutn0 << " cutL " << cutl0;
std::cout << std::endl;
std::cout << "After partitioning: ";
std::cout << "Balance " << bal1 << " cutN " << cutn1 << " cutL " << cutl1;
std::cout << std::endl;
}
//Finally, delete the pointer objects that we asked to be created.
delete linprob->GetMatrix();
delete linprob->GetLHS();
delete linprob->GetRHS();
delete linprob;
if (localProc == 0) {
std::cout << std::endl;
}
MPI_Finalize();
#else
std::cout << "part_redist: must have both MPI and EPETRA. Make sure Trilinos "
<< "is configured with --enable-mpi and --enable-epetra." << std::endl;
#endif
return(0);
}
