int GMGSolver::solve() {
int rank = Teuchos::GlobalMPISession::getRank();
// in place of doing the scaling ourselves, for the moment I've switched
// over to using Aztec's built-in scaling. This appears to be functionally identical.
bool useAztecToScaleDiagonally = true;
AztecOO solver(problem());
Epetra_CrsMatrix *A = dynamic_cast<Epetra_CrsMatrix *>( problem().GetMatrix() );
if (A == NULL) {
cout << "Error: GMGSolver requires an Epetra_CrsMatrix.\n";
TEUCHOS_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Error: GMGSolver requires an Epetra_CrsMatrix.\n");
}
// EpetraExt::RowMatrixToMatlabFile("/tmp/A_pre_scaling.dat",*A);
// Epetra_MultiVector *b = problem().GetRHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/b_pre_scaling.dat",*b);
// Epetra_MultiVector *x = problem().GetLHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/x_initial_guess.dat",*x);
const Epetra_Map* map = &A->RowMatrixRowMap();
Epetra_Vector diagA(*map);
A->ExtractDiagonalCopy(diagA);
// EpetraExt::MultiVectorToMatlabFile("/tmp/diagA.dat",diagA);
//
Epetra_Vector scale_vector(*map);
Epetra_Vector diagA_sqrt_inv(*map);
Epetra_Vector diagA_inv(*map);
if (_diagonalScaling && !useAztecToScaleDiagonally) {
int length = scale_vector.MyLength();
for (int i=0; i<length; i++) scale_vector[i] = 1.0 / sqrt(fabs(diagA[i]));
problem().LeftScale(scale_vector);
problem().RightScale(scale_vector);
}
Teuchos::RCP<Epetra_MultiVector> diagA_ptr = Teuchos::rcp( &diagA, false );
_gmgOperator.setStiffnessDiagonal(diagA_ptr);
_gmgOperator.setApplyDiagonalSmoothing(_diagonalSmoothing);
_gmgOperator.setFineSolverUsesDiagonalScaling(_diagonalScaling);
_gmgOperator.computeCoarseStiffnessMatrix(A);
if (_diagonalScaling && useAztecToScaleDiagonally) {
solver.SetAztecOption(AZ_scaling, AZ_sym_diag);
} else {
solver.SetAztecOption(AZ_scaling, AZ_none);
}
if (_useCG) {
if (_computeCondest) {
solver.SetAztecOption(AZ_solver, AZ_cg_condnum);
} else {
solver.SetAztecOption(AZ_solver, AZ_cg);
}
} else {
solver.SetAztecOption(AZ_kspace, 200); // default is 30
if (_computeCondest) {
solver.SetAztecOption(AZ_solver, AZ_gmres_condnum);
} else {
solver.SetAztecOption(AZ_solver, AZ_gmres);
}
}
solver.SetPrecOperator(&_gmgOperator);
// solver.SetAztecOption(AZ_precond, AZ_none);
solver.SetAztecOption(AZ_precond, AZ_user_precond);
solver.SetAztecOption(AZ_conv, _azConvergenceOption);
// solver.SetAztecOption(AZ_output, AZ_last);
solver.SetAztecOption(AZ_output, _azOutput);
int solveResult = solver.Iterate(_maxIters,_tol);
const double* status = solver.GetAztecStatus();
int remainingIters = _maxIters;
int whyTerminated = status[AZ_why];
int maxRestarts = 1;
int numRestarts = 0;
while ((whyTerminated==AZ_loss) && (numRestarts < maxRestarts)) {
remainingIters -= status[AZ_its];
if (rank==0) cout << "Aztec warned that the recursive residual indicates convergence even though the true residual is too large. Restarting with the new solution as initial guess, with maxIters = " << remainingIters << endl;
solveResult = solver.Iterate(remainingIters,_tol);
whyTerminated = status[AZ_why];
numRestarts++;
}
remainingIters -= status[AZ_its];
_iterationCount = _maxIters - remainingIters;
_condest = solver.Condest(); // will be -1 if running without condest
if (rank==0) {
switch (whyTerminated) {
case AZ_normal:
// cout << "whyTerminated: AZ_normal " << endl;
break;
case AZ_param:
cout << "whyTerminated: AZ_param " << endl;
break;
case AZ_breakdown:
cout << "whyTerminated: AZ_breakdown " << endl;
break;
case AZ_loss:
cout << "whyTerminated: AZ_loss " << endl;
break;
case AZ_ill_cond:
cout << "whyTerminated: AZ_ill_cond " << endl;
break;
case AZ_maxits:
cout << "whyTerminated: AZ_maxits " << endl;
break;
default:
break;
}
}
// Epetra_MultiVector *x = problem().GetLHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/x.dat",*x);
if (_diagonalScaling && !useAztecToScaleDiagonally) {
// reverse the scaling here
scale_vector.Reciprocal(scale_vector);
problem().LeftScale(scale_vector);
problem().RightScale(scale_vector);
}
double norminf = A->NormInf();
double normone = A->NormOne();
int numIters = solver.NumIters();
if (_printToConsole) {
cout << "\n Inf-norm of stiffness matrix after scaling = " << norminf;
cout << "\n One-norm of stiffness matrix after scaling = " << normone << endl << endl;
cout << "Num iterations: " << numIters << endl;
}
_gmgOperator.setStiffnessDiagonal(Teuchos::rcp((Epetra_MultiVector*) NULL ));
return solveResult;
}
int test_azoo_scaling(Epetra_CrsMatrix& A,
Epetra_Vector& x,
Epetra_Vector& b,
bool verbose)
{
Epetra_Vector vec1(x);
Epetra_Vector vec2(x);
Epetra_Vector diag(x);
Epetra_Vector vec3(x);
Epetra_Vector vec4(x);
Epetra_Vector rhs(x);
Epetra_Vector soln_none(x);
Epetra_Vector soln_jacobi(x);
Epetra_Vector soln_rowsum(x);
Epetra_Vector soln_symdiag(x);
vec1.PutScalar(1.0);
A.Multiply(false, vec1, vec2);
A.ExtractDiagonalCopy(diag);
double* diag_vals = NULL;
diag.ExtractView(&diag_vals);
int* options = new int[AZ_OPTIONS_SIZE];
double* params = new double[AZ_PARAMS_SIZE];
AZ_defaults(options, params);
options[AZ_output] = verbose ? 1 : AZ_none;
options[AZ_scaling] = AZ_Jacobi;
AztecOO::MatrixData mdata(&A);
AZ_MATRIX* Amat = AZ_matrix_create(vec1.Map().NumMyElements());
AZ_set_MATFREE(Amat, (void*)(&mdata), Epetra_Aztec_matvec);
AZ_SCALING* scaling = AZ_scaling_create();
double* xvals = NULL, *bvals = NULL;
x.ExtractView(&xvals);
b.ExtractView(&bvals);
int err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
cout << "AztecOO_scale_epetra returned err="<<err<<endl;
}
return(err);
}
A.Multiply(false, vec1, vec3);
vec4.Multiply(1.0, diag, vec3, 0.0);
double vec2nrm, vec4nrm;
vec2.Norm2(&vec2nrm);
vec4.Norm2(&vec4nrm);
if (fabs(vec2nrm - vec4nrm) > 1.e-6) {
return(-1);
}
//now call the scaling function again, just to allow for
//testing memory-leak issues.
err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
cout << "AztecOO_scale_epetra returned err="<<err<<endl;
}
return(err);
}
AztecOO_scale_epetra(AZ_DESTROY_SCALING_DATA, Amat, options,
bvals, xvals, NULL, scaling);
x.PutScalar(1.0);
Epetra_CrsMatrix* Atmp = create_and_fill_crs_matrix(A.RowMap());
Atmp->Multiply(false, x, rhs);
x.PutScalar(0.0);
AztecOO azoo(&A, &x, &b);
azoo.SetAztecOption(AZ_scaling, AZ_Jacobi);
if (verbose) {
azoo.SetAztecOption(AZ_output, 1);
}
else {
azoo.SetAztecOption(AZ_output, AZ_none);
}
azoo.Iterate(100, 1.e-6);
delete Atmp;
Epetra_CrsMatrix* Atmp1 = create_and_fill_crs_matrix(A.RowMap());
x.PutScalar(1.0);
Atmp1->Multiply(false, x, rhs);
soln_rowsum.PutScalar(0.0);
AztecOO azoo1(Atmp1, &soln_rowsum, &rhs);
azoo1.SetAztecOption(AZ_scaling, AZ_row_sum);
azoo1.Iterate(100, 1.e-8);
delete Atmp1;
Epetra_CrsMatrix* Atmp2 = create_and_fill_crs_matrix(A.RowMap());
x.PutScalar(1.0);
Atmp2->Multiply(false, x, rhs);
soln_symdiag.PutScalar(0.0);
AztecOO azoo2(Atmp2, &soln_symdiag, &rhs);
azoo2.SetAztecOption(AZ_scaling, AZ_sym_diag);
azoo2.Iterate(100, 1.e-8);
delete Atmp2;
Epetra_CrsMatrix* Atmp3 = create_and_fill_crs_matrix(A.RowMap());
x.PutScalar(1.0);
Atmp3->Multiply(false, x, rhs);
soln_none.PutScalar(0.0);
AztecOO azoo3(Atmp3, &soln_none, &rhs);
azoo3.SetAztecOption(AZ_scaling, AZ_none);
azoo3.Iterate(100, 1.e-8);
delete Atmp3;
Epetra_CrsMatrix* Atmp4 = create_and_fill_crs_matrix(A.RowMap());
x.PutScalar(1.0);
Atmp4->Multiply(false, x, rhs);
soln_jacobi.PutScalar(0.0);
AztecOO azoo4(Atmp4, &soln_jacobi, &rhs);
azoo4.SetAztecOption(AZ_scaling, AZ_Jacobi);
azoo4.Iterate(100, 1.e-8);
delete Atmp4;
//at this point, soln_none, soln_jacobi, soln_rowsum and soln_symdiag
//should be the same or at least close to the same, since the
//matrix used in the solution has well-behaved coefficients.
//form vec1 = soln_none - soln_rowsum
vec1.PutScalar(0.0);
vec1.Update(1.0, soln_none, 0.0);
vec1.Update(-1.0, soln_rowsum, 1.0);
double norm_check1= 0.0;
vec1.Norm2(&norm_check1);
//form vec2 = soln_none - soln_symdiag
vec2.PutScalar(0.0);
vec2.Update(1.0, soln_none, 0.0);
vec2.Update(-1.0, soln_symdiag, 1.0);
double norm_check2= 0.0;
vec2.Norm2(&norm_check2);
//form vec3 = soln_none - soln_jacobi
vec3.PutScalar(0.0);
vec3.Update(1.0, soln_none, 0.0);
vec3.Update(-1.0, soln_jacobi, 1.0);
double norm_check3= 0.0;
vec3.Norm2(&norm_check3);
if (std::abs(norm_check1) > 1.e-6) {
if (verbose) {
cerr << "AZ_row_sum scaling produced bad soln"
<< endl;
}
return(-1);
}
if (std::abs(norm_check2) > 1.e-6) {
if (verbose) {
cerr << "AZ_sym_diag scaling produced bad soln"
<< endl;
}
return(-1);
}
if (std::abs(norm_check3) > 1.e-6) {
if (verbose) {
cerr << "AZ_Jacobi scaling produced bad soln"
<< endl;
}
return(-1);
}
options[AZ_pre_calc] = AZ_reuse;
err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err == 0) {
if (verbose) {
cerr << "AztecOO_scale_epetra failed to return err when"
<< " asked to reuse non-existent scaling data."<<endl;
}
return(-1);
}
options[AZ_keep_info] = 1;
options[AZ_pre_calc] = AZ_calc;
err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
cerr << "AztecOO_scale_epetra returned err=="<<err<<endl;
}
return(err);
}
options[AZ_keep_info] = 0;
options[AZ_pre_calc] = AZ_reuse;
err = AztecOO_scale_epetra(AZ_SCALE_MAT_RHS_SOL, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
cerr << "AztecOO_scale_epetra returned err=="<<err
<<" when asked to reuse scaling data"<<endl;
}
return(err);
}
options[AZ_pre_calc] = AZ_calc;
err = AztecOO_scale_epetra(AZ_DESTROY_SCALING_DATA, Amat,
options, bvals, xvals, NULL, scaling);
if (err != 0) {
if (verbose) {
std::cerr << "AztecOO_scale_epetra returned err=="<<err
<< " when asked to destroy scaling data."<<std::endl;
}
return(err);
}
AZ_matrix_destroy(&Amat);
delete [] options;
delete [] params;
AZ_scaling_destroy(&scaling);
AZ_manage_memory(0, AZ_CLEAR_ALL, 0, 0, 0);
return(0);
}
