bool probing_test(Epetra_CrsMatrix & in_mat, bool build_list){
const Epetra_CrsGraph & graph=in_mat.Graph();
Teuchos::ParameterList main, zoltan;
if(build_list){
// zoltan.set("DISTANCE","2");
main.set("ZOLTAN",zoltan);
}
Isorropia::Epetra::Prober prober(Teuchos::rcp<const Epetra_CrsGraph>(&graph,false),main);
Epetra_CrsMatrix out_mat(Copy,graph);
int rv=prober.probe(in_mat,out_mat);
if(rv!=0) {printf("ERROR: probing failed\n");return false;}
#ifdef HAVE_EPETRAEXT
EpetraExt::MatrixMatrix::Add(in_mat,false,1,out_mat,-1);
double nrm=out_mat.NormInf()/in_mat.NormInf();
if(!in_mat.Comm().MyPID())
printf("diff norm = %22.16e\n",nrm);
if(nrm < 1e-12) return true;
else return false;
#endif
return true;
}
int test_azoo_conv_anorm(Epetra_CrsMatrix& A,
Epetra_Vector& x,
Epetra_Vector& b,
bool verbose)
{
if (verbose) {
cout << "testing AztecOO with AZ_conv = AZ_Anorm" << endl;
}
Epetra_Vector soln_Anorm(x), soln_none(x), vec1(x), rhs(x);
//We'll put large numbers in a vector and use that to generate an rhs
//which has norm much larger than the infinity-norm of the matrix.
vec1.PutScalar(1000.0);
soln_Anorm.PutScalar(0.0);
soln_none.PutScalar(0.0);
A.Multiply(false, vec1, rhs);
AztecOO azoo(&A, &soln_Anorm, &rhs);
azoo.SetAztecOption(AZ_conv, AZ_Anorm);
azoo.SetAztecOption(AZ_solver, AZ_cg);
//azoo.SetAztecOption(AZ_scaling, AZ_sym_diag);
azoo.Iterate(30, 1.e-5);
AztecOO azoo1(&A, &soln_none, &rhs);
azoo1.SetAztecOption(AZ_conv, AZ_rhs);
azoo1.SetAztecOption(AZ_solver, AZ_cg);
azoo1.Iterate(30, 1.e-5);
double rhsnorm = 0.0;
rhs.Norm2(&rhsnorm);
double anorm = A.NormInf();
double rnrm_anorm = resid2norm(A, soln_Anorm, rhs);
double rnrm_rhs = resid2norm(A, soln_none, rhs);
//we expect the ratio rnrm_anorm/rnrm_rhs to roughly equal
//the ratio anorm/rhsnorm, since rnrm_anorm is the residual norm
//obtained for the solve that used Anorm scaling to determine
//convergence, and rnrm_rhs is the residual norm obtained for
//the solve that used rhs scaling.
double ratio1 = rnrm_anorm/rnrm_rhs;
double ratio2 = anorm/rhsnorm;
cout << "ratio1: " << ratio1 << ", ratio2: " << ratio2 << endl;
if (std::abs(ratio1 - ratio2) > 1.e-1) {
if (verbose) {
cout << "anorm: " << anorm << ", rhsnorm: " << rhsnorm
<< "rnrm_anorm: " << rnrm_anorm << ", rnrm_rhs: " << rnrm_rhs<<endl;
}
return(-1);
}
return(0);
}
int runTests(Epetra_Map & map, Epetra_CrsMatrix & A, Epetra_Vector & x, Epetra_Vector & b, Epetra_Vector & xexact, bool verbose) {
int ierr = 0;
// Create MultiVectors and put x, b, xexact in both columns of X, B, and Xexact, respectively.
Epetra_MultiVector X( map, 2, false );
Epetra_MultiVector B( map, 2, false );
Epetra_MultiVector Xexact( map, 2, false );
for (int i=0; i<X.NumVectors(); ++i) {
*X(i) = x;
*B(i) = b;
*Xexact(i) = xexact;
}
double residual;
std::vector<double> residualmv(2);
residual = A.NormInf(); double rAInf = residual;
if (verbose) std::cout << "Inf Norm of A = " << residual << std::endl;
residual = A.NormOne(); double rAOne = residual;
if (verbose) std::cout << "One Norm of A = " << residual << std::endl;
xexact.Norm2(&residual); double rxx = residual;
Xexact.Norm2(&residualmv[0]); std::vector<double> rXX( residualmv );
if (verbose) std::cout << "Norm of xexact = " << residual << std::endl;
if (verbose) std::cout << "Norm of Xexact = (" << residualmv[0] << ", " <<residualmv[1] <<")"<< std::endl;
Epetra_Vector tmp1(map);
Epetra_MultiVector tmp1mv(map,2,false);
A.Multiply(false, xexact, tmp1);
A.Multiply(false, Xexact, tmp1mv);
tmp1.Norm2(&residual); double rAx = residual;
tmp1mv.Norm2(&residualmv[0]); std::vector<double> rAX( residualmv );
if (verbose) std::cout << "Norm of Ax = " << residual << std::endl;
if (verbose) std::cout << "Norm of AX = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
b.Norm2(&residual); double rb = residual;
B.Norm2(&residualmv[0]); std::vector<double> rB( residualmv );
if (verbose) std::cout << "Norm of b (should equal norm of Ax) = " << residual << std::endl;
if (verbose) std::cout << "Norm of B (should equal norm of AX) = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
tmp1.Update(1.0, b, -1.0);
tmp1mv.Update(1.0, B, -1.0);
tmp1.Norm2(&residual);
tmp1mv.Norm2(&residualmv[0]);
if (verbose) std::cout << "Norm of difference between compute Ax and Ax from file = " << residual << std::endl;
if (verbose) std::cout << "Norm of difference between compute AX and AX from file = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
map.Comm().Barrier();
EPETRA_CHK_ERR(EpetraExt::BlockMapToMatrixMarketFile("Test_map.mm", map, "Official EpetraExt test map",
"This is the official EpetraExt test map generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::RowMatrixToMatrixMarketFile("Test_A.mm", A, "Official EpetraExt test matrix",
"This is the official EpetraExt test matrix generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_x.mm", x, "Official EpetraExt test initial guess",
"This is the official EpetraExt test initial guess generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvX.mm", X, "Official EpetraExt test initial guess",
"This is the official EpetraExt test initial guess generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_xexact.mm", xexact, "Official EpetraExt test exact solution",
"This is the official EpetraExt test exact solution generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvXexact.mm", Xexact, "Official EpetraExt test exact solution",
"This is the official EpetraExt test exact solution generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_b.mm", b, "Official EpetraExt test right hand side",
"This is the official EpetraExt test right hand side generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvB.mm", B, "Official EpetraExt test right hand side",
"This is the official EpetraExt test right hand side generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatlabFile("Test_mvB.mat", B));
EPETRA_CHK_ERR(EpetraExt::RowMatrixToMatlabFile("Test_A.dat", A));
Epetra_Map * map1;
Epetra_CrsMatrix * A1;
Epetra_CrsMatrix * A2;
Epetra_CrsMatrix * A3;
Epetra_Vector * x1;
Epetra_Vector * b1;
Epetra_Vector * xexact1;
Epetra_MultiVector * X1;
Epetra_MultiVector * B1;
Epetra_MultiVector * Xexact1;
EpetraExt::MatrixMarketFileToMap("Test_map.mm", map.Comm(), map1);
if (map.SameAs(*map1)) {
if (verbose) std::cout << "Maps are equal. In/Out works." << std::endl;
}
else {
if (verbose) std::cout << "Maps are not equal. In/Out fails." << std::endl;
ierr += 1;
}
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToCrsMatrix("Test_A.mm", *map1, A1));
// If map is zero-based, then we can compare to the convenient reading versions
if (map1->IndexBase()==0) EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToCrsMatrix("Test_A.mm", map1->Comm(), A2));
if (map1->IndexBase()==0) EPETRA_CHK_ERR(EpetraExt::MatlabFileToCrsMatrix("Test_A.dat", map1->Comm(), A3));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_x.mm", *map1, x1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_xexact.mm", *map1, xexact1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_b.mm", *map1, b1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvX.mm", *map1, X1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvXexact.mm", *map1, Xexact1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvB.mm", *map1, B1));
residual = A1->NormInf(); double rA1Inf = residual;
if (verbose) std::cout << "Inf Norm of A1 = " << residual << std::endl;
ierr += checkValues(rA1Inf,rAInf,"Inf Norm of A", verbose);
residual = A1->NormOne(); double rA1One = residual;
if (verbose) std::cout << "One Norm of A1 = " << residual << std::endl;
ierr += checkValues(rA1One,rAOne,"One Norm of A", verbose);
xexact1->Norm2(&residual); double rxx1 = residual;
if (verbose) std::cout << "Norm of xexact1 = " << residual << std::endl;
ierr += checkValues(rxx1,rxx,"Norm of xexact", verbose);
Xexact1->Norm2(&residualmv[0]); std::vector<double> rXX1(residualmv);
if (verbose) std::cout << "Norm of Xexact1 = (" << residualmv[0] <<", " <<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rXX1[0],rXX[0],"Norm of Xexact", verbose);
ierr += checkValues(rXX1[1],rXX[1],"Norm of Xexact", verbose);
Epetra_Vector tmp11(*map1);
A1->Multiply(false, *xexact1, tmp11);
Epetra_MultiVector tmp11mv(*map1,2,false);
A1->Multiply(false, *Xexact1, tmp11mv);
tmp11.Norm2(&residual); double rAx1 = residual;
if (verbose) std::cout << "Norm of A1*x1 = " << residual << std::endl;
ierr += checkValues(rAx1,rAx,"Norm of A1*x", verbose);
tmp11mv.Norm2(&residualmv[0]); std::vector<double> rAX1(residualmv);
if (verbose) std::cout << "Norm of A1*X1 = (" << residualmv[0] <<", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rAX1[0],rAX[0],"Norm of A1*X", verbose);
ierr += checkValues(rAX1[1],rAX[1],"Norm of A1*X", verbose);
if (map1->IndexBase()==0) {
Epetra_Vector tmp12(*map1);
A2->Multiply(false, *xexact1, tmp12);
tmp12.Norm2(&residual); double rAx2 = residual;
if (verbose) std::cout << "Norm of A2*x1 = " << residual << std::endl;
ierr += checkValues(rAx2,rAx,"Norm of A2*x", verbose);
Epetra_Vector tmp13(*map1);
A3->Multiply(false, *xexact1, tmp13);
tmp13.Norm2(&residual); double rAx3 = residual;
if (verbose) std::cout << "Norm of A3*x1 = " << residual << std::endl;
ierr += checkValues(rAx3,rAx,"Norm of A3*x", verbose);
}
b1->Norm2(&residual); double rb1 = residual;
if (verbose) std::cout << "Norm of b1 (should equal norm of Ax) = " << residual << std::endl;
ierr += checkValues(rb1,rb,"Norm of b", verbose);
B1->Norm2(&residualmv[0]); std::vector<double> rB1(residualmv);
if (verbose) std::cout << "Norm of B1 (should equal norm of AX) = (" << residualmv[0] <<", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rB1[0],rB[0],"Norm of B", verbose);
ierr += checkValues(rB1[1],rB[1],"Norm of B", verbose);
tmp11.Update(1.0, *b1, -1.0);
tmp11.Norm2(&residual);
if (verbose) std::cout << "Norm of difference between computed A1x1 and A1x1 from file = " << residual << std::endl;
ierr += checkValues(residual,0.0,"Norm of difference between computed A1x1 and A1x1 from file", verbose);
tmp11mv.Update(1.0, *B1, -1.0);
tmp11mv.Norm2(&residualmv[0]);
if (verbose) std::cout << "Norm of difference between computed A1X1 and A1X1 from file = (" << residualmv[0] << ", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(residualmv[0],0.0,"Norm of difference between computed A1X1 and A1X1 from file", verbose);
ierr += checkValues(residualmv[1],0.0,"Norm of difference between computed A1X1 and A1X1 from file", verbose);
if (map1->IndexBase()==0) {delete A2; delete A3;}
delete A1;
delete x1;
delete b1;
delete xexact1;
delete X1;
delete B1;
delete Xexact1;
delete map1;
return(ierr);
}
bool CrsMatrixInfo( const Epetra_CrsMatrix & A,
ostream & os )
{
int MyPID = A.Comm().MyPID();
// take care that matrix is already trasformed
bool IndicesAreGlobal = A.IndicesAreGlobal();
if( IndicesAreGlobal == true ) {
if( MyPID == 0 ) {
os << "WARNING : matrix must be transformed to local\n";
os << " before calling CrsMatrixInfo\n";
os << " Now returning...\n";
}
return false;
}
int NumGlobalRows = A.NumGlobalRows();
int NumGlobalNonzeros = A.NumGlobalNonzeros();
int NumGlobalCols = A.NumGlobalCols();
double NormInf = A.NormInf();
double NormOne = A.NormOne();
int NumGlobalDiagonals = A.NumGlobalDiagonals();
int GlobalMaxNumEntries = A.GlobalMaxNumEntries();
int IndexBase = A.IndexBase();
bool StorageOptimized = A.StorageOptimized();
bool LowerTriangular = A.LowerTriangular();
bool UpperTriangular = A.UpperTriangular();
bool NoDiagonal = A.NoDiagonal();
// these variables identifies quantities I have to compute,
// since not provided by Epetra_CrsMatrix
double MyFrobeniusNorm( 0.0 ), FrobeniusNorm( 0.0 );
double MyMinElement( DBL_MAX ), MinElement( DBL_MAX );
double MyMaxElement( DBL_MIN ), MaxElement( DBL_MIN );
double MyMinAbsElement( DBL_MAX ), MinAbsElement( DBL_MAX );
double MyMaxAbsElement( 0.0 ), MaxAbsElement( 0.0 );
int NumMyRows = A.NumMyRows();
int * NzPerRow = new int[NumMyRows];
int Row; // iterator on rows
int Col; // iterator on cols
int MaxNumEntries = A.MaxNumEntries();
double * Values = new double[MaxNumEntries];
int * Indices = new int[MaxNumEntries];
double Element, AbsElement; // generic nonzero element and its abs value
int NumEntries;
double * Diagonal = new double [NumMyRows];
// SumOffDiagonal is the sum of absolute values for off-diagonals
double * SumOffDiagonal = new double [NumMyRows];
for( Row=0 ; Row<NumMyRows ; ++Row ) {
SumOffDiagonal[Row] = 0.0;
}
int * IsDiagonallyDominant = new int [NumMyRows];
int GlobalRow;
// cycle over all matrix elements
for( Row=0 ; Row<NumMyRows ; ++Row ) {
GlobalRow = A.GRID(Row);
NzPerRow[Row] = A.NumMyEntries(Row);
A.ExtractMyRowCopy(Row,NzPerRow[Row],NumEntries,Values,Indices);
for( Col=0 ; Col<NumEntries ; ++Col ) {
Element = Values[Col];
AbsElement = abs(Element);
if( Element<MyMinElement ) MyMinElement = Element;
if( Element>MyMaxElement ) MyMaxElement = Element;
if( AbsElement<MyMinAbsElement ) MyMinAbsElement = AbsElement;
if( AbsElement>MyMaxAbsElement ) MyMaxAbsElement = AbsElement;
if( Indices[Col] == Row ) Diagonal[Row] = Element;
else
SumOffDiagonal[Row] += abs(Element);
MyFrobeniusNorm += pow(Element,2);
}
}
// analise storage per row
int MyMinNzPerRow( NumMyRows ), MinNzPerRow( NumMyRows );
int MyMaxNzPerRow( 0 ), MaxNzPerRow( 0 );
for( Row=0 ; Row<NumMyRows ; ++Row ) {
if( NzPerRow[Row]<MyMinNzPerRow ) MyMinNzPerRow=NzPerRow[Row];
if( NzPerRow[Row]>MyMaxNzPerRow ) MyMaxNzPerRow=NzPerRow[Row];
}
// a test to see if matrix is diagonally-dominant
int MyDiagonalDominance( 0 ), DiagonalDominance( 0 );
int MyWeakDiagonalDominance( 0 ), WeakDiagonalDominance( 0 );
for( Row=0 ; Row<NumMyRows ; ++Row ) {
if( abs(Diagonal[Row])>SumOffDiagonal[Row] )
++MyDiagonalDominance;
else if( abs(Diagonal[Row])==SumOffDiagonal[Row] )
++MyWeakDiagonalDominance;
}
// reduction operations
A.Comm().SumAll(&MyFrobeniusNorm, &FrobeniusNorm, 1);
A.Comm().MinAll(&MyMinElement, &MinElement, 1);
A.Comm().MaxAll(&MyMaxElement, &MaxElement, 1);
A.Comm().MinAll(&MyMinAbsElement, &MinAbsElement, 1);
A.Comm().MaxAll(&MyMaxAbsElement, &MaxAbsElement, 1);
A.Comm().MinAll(&MyMinNzPerRow, &MinNzPerRow, 1);
A.Comm().MaxAll(&MyMaxNzPerRow, &MaxNzPerRow, 1);
A.Comm().SumAll(&MyDiagonalDominance, &DiagonalDominance, 1);
A.Comm().SumAll(&MyWeakDiagonalDominance, &WeakDiagonalDominance, 1);
// free memory
delete Values;
delete Indices;
delete Diagonal;
delete SumOffDiagonal;
delete IsDiagonallyDominant;
delete NzPerRow;
// simply no output for MyPID>0, only proc 0 write on os
if( MyPID != 0 ) return true;
os << "*** general Information about the matrix\n";
os << "Number of Global Rows = " << NumGlobalRows << endl;
os << "Number of Global Cols = " << NumGlobalCols << endl;
os << "is the matrix square = " <<
((NumGlobalRows==NumGlobalCols)?"yes":"no") << endl;
os << "||A||_\\infty = " << NormInf << endl;
os << "||A||_1 = " << NormOne << endl;
os << "||A||_F = " << sqrt(FrobeniusNorm) << endl;
os << "Number of nonzero diagonal entries = "
<< NumGlobalDiagonals
<< "( " << 1.0* NumGlobalDiagonals/NumGlobalRows*100
<< " %)\n";
os << "Nonzero per row : min = " << MinNzPerRow
<< " average = " << 1.0*NumGlobalNonzeros/NumGlobalRows
<< " max = " << MaxNzPerRow << endl;
os << "Maximum number of nonzero elements/row = "
<< GlobalMaxNumEntries << endl;
os << "min( a_{i,j} ) = " << MinElement << endl;
os << "max( a_{i,j} ) = " << MaxElement << endl;
os << "min( abs(a_{i,j}) ) = " << MinAbsElement << endl;
os << "max( abs(a_{i,j}) ) = " << MaxAbsElement << endl;
os << "Number of diagonal dominant rows = " << DiagonalDominance
<< " (" << 100.0*DiagonalDominance/NumGlobalRows << " % of total)\n";
os << "Number of weakly diagonal dominant rows = "
<< WeakDiagonalDominance
<< " (" << 100.0*WeakDiagonalDominance/NumGlobalRows << " % of total)\n";
os << "*** Information about the Trilinos storage\n";
os << "Base Index = " << IndexBase << endl;
os << "is storage optimized = "
<< ((StorageOptimized==true)?"yes":"no") << endl;
os << "are indices global = "
<< ((IndicesAreGlobal==true)?"yes":"no") << endl;
os << "is matrix lower triangular = "
<< ((LowerTriangular==true)?"yes":"no") << endl;
os << "is matrix upper triangular = "
<< ((UpperTriangular==true)?"yes":"no") << endl;
os << "are there diagonal entries = "
<< ((NoDiagonal==false)?"yes":"no") << endl;
return true;
}
// run tests with "Local" permutation strategy and nDofsPerNode = 3
bool runPermutationTest2(const std::string input_filename, const std::string expected_filename, const Teuchos::RCP<const Teuchos::Comm<int> >& comm) {
#ifndef HAVE_MUELU_INST_COMPLEX_INT_INT
Teuchos::RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
out->setOutputToRootOnly(0);
Epetra_CrsMatrix * ptrA = NULL;
Epetra_CrsMatrix * ptrExpected = NULL;
int ret = EpetraExt::MatlabFileToCrsMatrix ( input_filename.c_str(),
*Xpetra::toEpetra(comm),
ptrA
);
if(ret!=0)
std::cout << "failed to read matrix from file" << std::endl;
if(expected_filename.size() > 0)
{
int ret2 = EpetraExt::MatlabFileToCrsMatrix (expected_filename.c_str(),
*Xpetra::toEpetra(comm),
ptrExpected
);
if(ret2!=0)
std::cout << "failed to read matrix from file" << std::endl;
}
Teuchos::RCP<Epetra_CrsMatrix> epA = Teuchos::rcp(ptrA);
Teuchos::RCP<Epetra_CrsMatrix> epExpected = Teuchos::rcp(ptrExpected);
// Epetra_CrsMatrix -> Xpetra::Matrix
Teuchos::RCP<CrsMatrix> exA = Teuchos::rcp(new Xpetra::EpetraCrsMatrix(epA));
Teuchos::RCP<CrsMatrixWrap> crsOp = Teuchos::rcp(new CrsMatrixWrap(exA));
Teuchos::RCP<Matrix> A = Teuchos::rcp_dynamic_cast<Matrix>(crsOp);
A->SetFixedBlockSize(3);
Teuchos::RCP<Level> Finest = Teuchos::rcp(new Level());
Finest->SetLevelID(0); // must be level 0 for NullspaceFactory
Finest->Set("A", A);
// permute full matrix
Teuchos::RCP<PermutationFactory> PermFact = Teuchos::rcp(new MueLu::PermutationFactory<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps>());
PermFact->SetParameter("PermutationStrategy",Teuchos::ParameterEntry(std::string("Local")));
PermFact->SetParameter("PermutationRowMapName",Teuchos::ParameterEntry(std::string("")));
PermFact->SetFactory("PermutationRowMapFactory", Teuchos::null);
// setup main factory manager
Teuchos::RCP<FactoryManager> M = Teuchos::rcp(new FactoryManager());
M->SetFactory("permQT", PermFact);
M->SetFactory("A", MueLu::NoFactory::getRCP()); // this is the input matrix
MueLu::SetFactoryManager SFMFinest(Finest, M);
// prepare building process for permutation operators
Finest->Request("A", PermFact.get());
Finest->Request("permA", PermFact.get());
Finest->Request("permP", PermFact.get());
Finest->Request("permQT", PermFact.get());
Finest->Request("permScaling", PermFact.get());
Finest->Request("#RowPermutations", PermFact.get());
Finest->Request("#ColPermutations", PermFact.get());
Finest->Request("#WideRangeRowPermutations", PermFact.get());
Finest->Request("#WideRangeColPermutations", PermFact.get());
// build permutation operators
PermFact->Build(*Finest);
//std::cout << "P" << *GetEpetraMatrix("permP", Finest, PermFact) << std::endl;
//std::cout << "Q^T" << *GetEpetraMatrix("permQT", Finest, PermFact) << std::endl;
//std::cout << "permA" << *GetEpetraMatrix("A", Finest, PermFact) << std::endl;
Teuchos::RCP<const Epetra_CrsMatrix> epResult = GetEpetraMatrix("A", Finest, PermFact);
//std::cout << *epResult << std::endl;
if(epExpected != Teuchos::null) {
Epetra_CrsMatrix* comparison = NULL;
EpetraExt::MatrixMatrix::Add(*epResult, false, -1.0, *epExpected, false, 1.0, comparison);
comparison->FillComplete();
//std::cout << *comparison << std::endl;
double norm = comparison->NormInf();
delete comparison;
comparison = NULL;
if(norm < 1.0e-14) {
*out << "** PASSED **: " << input_filename << std::endl;
return true;
}
else {
*out << "-- FAILED --: " << input_filename << std::endl;
return false;
}
}
#endif
*out << "-- FAILED --: " << input_filename << " no result file found" << std::endl;
return false; // no result for comparison available
}
int GMGSolver::solve() {
int rank = Teuchos::GlobalMPISession::getRank();
// in place of doing the scaling ourselves, for the moment I've switched
// over to using Aztec's built-in scaling. This appears to be functionally identical.
bool useAztecToScaleDiagonally = true;
AztecOO solver(problem());
Epetra_CrsMatrix *A = dynamic_cast<Epetra_CrsMatrix *>( problem().GetMatrix() );
if (A == NULL) {
cout << "Error: GMGSolver requires an Epetra_CrsMatrix.\n";
TEUCHOS_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Error: GMGSolver requires an Epetra_CrsMatrix.\n");
}
// EpetraExt::RowMatrixToMatlabFile("/tmp/A_pre_scaling.dat",*A);
// Epetra_MultiVector *b = problem().GetRHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/b_pre_scaling.dat",*b);
// Epetra_MultiVector *x = problem().GetLHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/x_initial_guess.dat",*x);
const Epetra_Map* map = &A->RowMatrixRowMap();
Epetra_Vector diagA(*map);
A->ExtractDiagonalCopy(diagA);
// EpetraExt::MultiVectorToMatlabFile("/tmp/diagA.dat",diagA);
//
Epetra_Vector scale_vector(*map);
Epetra_Vector diagA_sqrt_inv(*map);
Epetra_Vector diagA_inv(*map);
if (_diagonalScaling && !useAztecToScaleDiagonally) {
int length = scale_vector.MyLength();
for (int i=0; i<length; i++) scale_vector[i] = 1.0 / sqrt(fabs(diagA[i]));
problem().LeftScale(scale_vector);
problem().RightScale(scale_vector);
}
Teuchos::RCP<Epetra_MultiVector> diagA_ptr = Teuchos::rcp( &diagA, false );
_gmgOperator.setStiffnessDiagonal(diagA_ptr);
_gmgOperator.setApplyDiagonalSmoothing(_diagonalSmoothing);
_gmgOperator.setFineSolverUsesDiagonalScaling(_diagonalScaling);
_gmgOperator.computeCoarseStiffnessMatrix(A);
if (_diagonalScaling && useAztecToScaleDiagonally) {
solver.SetAztecOption(AZ_scaling, AZ_sym_diag);
} else {
solver.SetAztecOption(AZ_scaling, AZ_none);
}
if (_useCG) {
if (_computeCondest) {
solver.SetAztecOption(AZ_solver, AZ_cg_condnum);
} else {
solver.SetAztecOption(AZ_solver, AZ_cg);
}
} else {
solver.SetAztecOption(AZ_kspace, 200); // default is 30
if (_computeCondest) {
solver.SetAztecOption(AZ_solver, AZ_gmres_condnum);
} else {
solver.SetAztecOption(AZ_solver, AZ_gmres);
}
}
solver.SetPrecOperator(&_gmgOperator);
// solver.SetAztecOption(AZ_precond, AZ_none);
solver.SetAztecOption(AZ_precond, AZ_user_precond);
solver.SetAztecOption(AZ_conv, _azConvergenceOption);
// solver.SetAztecOption(AZ_output, AZ_last);
solver.SetAztecOption(AZ_output, _azOutput);
int solveResult = solver.Iterate(_maxIters,_tol);
const double* status = solver.GetAztecStatus();
int remainingIters = _maxIters;
int whyTerminated = status[AZ_why];
int maxRestarts = 1;
int numRestarts = 0;
while ((whyTerminated==AZ_loss) && (numRestarts < maxRestarts)) {
remainingIters -= status[AZ_its];
if (rank==0) cout << "Aztec warned that the recursive residual indicates convergence even though the true residual is too large. Restarting with the new solution as initial guess, with maxIters = " << remainingIters << endl;
solveResult = solver.Iterate(remainingIters,_tol);
whyTerminated = status[AZ_why];
numRestarts++;
}
remainingIters -= status[AZ_its];
_iterationCount = _maxIters - remainingIters;
_condest = solver.Condest(); // will be -1 if running without condest
if (rank==0) {
switch (whyTerminated) {
case AZ_normal:
// cout << "whyTerminated: AZ_normal " << endl;
break;
case AZ_param:
cout << "whyTerminated: AZ_param " << endl;
break;
case AZ_breakdown:
cout << "whyTerminated: AZ_breakdown " << endl;
break;
case AZ_loss:
cout << "whyTerminated: AZ_loss " << endl;
break;
case AZ_ill_cond:
cout << "whyTerminated: AZ_ill_cond " << endl;
break;
case AZ_maxits:
cout << "whyTerminated: AZ_maxits " << endl;
break;
default:
break;
}
}
// Epetra_MultiVector *x = problem().GetLHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/x.dat",*x);
if (_diagonalScaling && !useAztecToScaleDiagonally) {
// reverse the scaling here
scale_vector.Reciprocal(scale_vector);
problem().LeftScale(scale_vector);
problem().RightScale(scale_vector);
}
double norminf = A->NormInf();
double normone = A->NormOne();
int numIters = solver.NumIters();
if (_printToConsole) {
cout << "\n Inf-norm of stiffness matrix after scaling = " << norminf;
cout << "\n One-norm of stiffness matrix after scaling = " << normone << endl << endl;
cout << "Num iterations: " << numIters << endl;
}
_gmgOperator.setStiffnessDiagonal(Teuchos::rcp((Epetra_MultiVector*) NULL ));
return solveResult;
}
int main(int argc, char *argv[]) {
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
cout << Comm << endl;
int MyPID = Comm.MyPID();
bool verbose = false;
bool verbose1 = true;
if (MyPID==0) verbose = true;
if(argc < 2 && verbose) {
cerr << "Usage: " << argv[0]
<< " HB_filename [level_fill [level_overlap [absolute_threshold [ relative_threshold]]]]" << endl
<< "where:" << endl
<< "HB_filename - filename and path of a Harwell-Boeing data set" << endl
<< "level_fill - The amount of fill to use for ILU(k) preconditioner (default 0)" << endl
<< "level_overlap - The amount of overlap used for overlapping Schwarz subdomains (default 0)" << endl
<< "absolute_threshold - The minimum value to place on the diagonal prior to factorization (default 0.0)" << endl
<< "relative_threshold - The relative amount to perturb the diagonal prior to factorization (default 1.0)" << endl << endl
<< "To specify a non-default value for one of these parameters, you must specify all" << endl
<< " preceding values but not any subsequent parameters. Example:" << endl
<< "ifpackHpcSerialMsr.exe mymatrix.hpc 1 - loads mymatrix.hpc, uses level fill of one, all other values are defaults" << endl
<< endl;
return(1);
}
// Uncomment the next three lines to debug in mpi mode
//int tmp;
//if (MyPID==0) cin >> tmp;
//Comm.Barrier();
Epetra_Map * readMap;
Epetra_CrsMatrix * readA;
Epetra_Vector * readx;
Epetra_Vector * readb;
Epetra_Vector * readxexact;
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra(argv[1], Comm, readMap, readA, readx, readb, readxexact);
// Create uniform distributed map
Epetra_Map map(readMap->NumGlobalElements(), 0, Comm);
// Create Exporter to distribute read-in matrix and vectors
Epetra_Export exporter(*readMap, map);
Epetra_CrsMatrix A(Copy, map, 0);
Epetra_Vector x(map);
Epetra_Vector b(map);
Epetra_Vector xexact(map);
Epetra_Time FillTimer(Comm);
x.Export(*readx, exporter, Add);
b.Export(*readb, exporter, Add);
xexact.Export(*readxexact, exporter, Add);
Comm.Barrier();
double vectorRedistributeTime = FillTimer.ElapsedTime();
A.Export(*readA, exporter, Add);
Comm.Barrier();
double matrixRedistributeTime = FillTimer.ElapsedTime() - vectorRedistributeTime;
assert(A.FillComplete()==0);
Comm.Barrier();
double fillCompleteTime = FillTimer.ElapsedTime() - matrixRedistributeTime;
if (Comm.MyPID()==0) {
cout << "\n\n****************************************************" << endl;
cout << "\n Vector redistribute time (sec) = " << vectorRedistributeTime<< endl;
cout << " Matrix redistribute time (sec) = " << matrixRedistributeTime << endl;
cout << " Transform to Local time (sec) = " << fillCompleteTime << endl<< endl;
}
Epetra_Vector tmp1(*readMap);
Epetra_Vector tmp2(map);
readA->Multiply(false, *readxexact, tmp1);
A.Multiply(false, xexact, tmp2);
double residual;
tmp1.Norm2(&residual);
if (verbose) cout << "Norm of Ax from file = " << residual << endl;
tmp2.Norm2(&residual);
if (verbose) cout << "Norm of Ax after redistribution = " << residual << endl << endl << endl;
//cout << "A from file = " << *readA << endl << endl << endl;
//cout << "A after dist = " << A << endl << endl << endl;
delete readA;
delete readx;
delete readb;
delete readxexact;
delete readMap;
Comm.Barrier();
bool smallProblem = false;
if (A.RowMap().NumGlobalElements()<100) smallProblem = true;
if (smallProblem)
cout << "Original Matrix = " << endl << A << endl;
x.PutScalar(0.0);
Epetra_LinearProblem FullProblem(&A, &x, &b);
double normb, norma;
b.NormInf(&normb);
norma = A.NormInf();
if (verbose)
cout << "Inf norm of Original Matrix = " << norma << endl
<< "Inf norm of Original RHS = " << normb << endl;
Epetra_Time ReductionTimer(Comm);
Epetra_CrsSingletonFilter SingletonFilter;
Comm.Barrier();
double reduceInitTime = ReductionTimer.ElapsedTime();
SingletonFilter.Analyze(&A);
Comm.Barrier();
double reduceAnalyzeTime = ReductionTimer.ElapsedTime() - reduceInitTime;
if (SingletonFilter.SingletonsDetected())
cout << "Singletons found" << endl;
else {
cout << "Singletons not found" << endl;
exit(1);
}
SingletonFilter.ConstructReducedProblem(&FullProblem);
Comm.Barrier();
double reduceConstructTime = ReductionTimer.ElapsedTime() - reduceInitTime;
double totalReduceTime = ReductionTimer.ElapsedTime();
if (verbose)
cout << "\n\n****************************************************" << endl
<< " Reduction init time (sec) = " << reduceInitTime<< endl
<< " Reduction Analyze time (sec) = " << reduceAnalyzeTime << endl
<< " Construct Reduced Problem time (sec) = " << reduceConstructTime << endl
<< " Reduction Total time (sec) = " << totalReduceTime << endl<< endl;
Statistics(SingletonFilter);
Epetra_LinearProblem * ReducedProblem = SingletonFilter.ReducedProblem();
Epetra_CrsMatrix * Ap = dynamic_cast<Epetra_CrsMatrix *>(ReducedProblem->GetMatrix());
Epetra_Vector * bp = (*ReducedProblem->GetRHS())(0);
Epetra_Vector * xp = (*ReducedProblem->GetLHS())(0);
if (smallProblem)
cout << " Reduced Matrix = " << endl << *Ap << endl
<< " LHS before sol = " << endl << *xp << endl
<< " RHS = " << endl << *bp << endl;
// Construct ILU preconditioner
double elapsed_time, total_flops, MFLOPs;
Epetra_Time timer(Comm);
int LevelFill = 0;
if (argc > 2) LevelFill = atoi(argv[2]);
if (verbose) cout << "Using Level Fill = " << LevelFill << endl;
int Overlap = 0;
if (argc > 3) Overlap = atoi(argv[3]);
if (verbose) cout << "Using Level Overlap = " << Overlap << endl;
double Athresh = 0.0;
if (argc > 4) Athresh = atof(argv[4]);
if (verbose) cout << "Using Absolute Threshold Value of = " << Athresh << endl;
double Rthresh = 1.0;
if (argc > 5) Rthresh = atof(argv[5]);
if (verbose) cout << "Using Relative Threshold Value of = " << Rthresh << endl;
Ifpack_IlukGraph * IlukGraph = 0;
Ifpack_CrsRiluk * ILUK = 0;
if (LevelFill>-1) {
elapsed_time = timer.ElapsedTime();
IlukGraph = new Ifpack_IlukGraph(Ap->Graph(), LevelFill, Overlap);
assert(IlukGraph->ConstructFilledGraph()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
if (verbose) cout << "Time to construct ILUK graph = " << elapsed_time << endl;
Epetra_Flops fact_counter;
elapsed_time = timer.ElapsedTime();
ILUK = new Ifpack_CrsRiluk(*IlukGraph);
ILUK->SetFlopCounter(fact_counter);
ILUK->SetAbsoluteThreshold(Athresh);
ILUK->SetRelativeThreshold(Rthresh);
//assert(ILUK->InitValues()==0);
int initerr = ILUK->InitValues(*Ap);
if (initerr!=0) {
cout << endl << Comm << endl << " InitValues error = " << initerr;
if (initerr==1) cout << " Zero diagonal found, warning error only";
cout << endl << endl;
}
assert(ILUK->Factor()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = ILUK->Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute preconditioner values = "
<< elapsed_time << endl
<< "MFLOPS for Factorization = " << MFLOPs << endl;
//cout << *ILUK << endl;
double Condest;
ILUK->Condest(false, Condest);
if (verbose) cout << "Condition number estimate for this preconditioner = " << Condest << endl;
}
int Maxiter = 100;
double Tolerance = 1.0E-8;
Epetra_Flops counter;
Ap->SetFlopCounter(counter);
xp->SetFlopCounter(*Ap);
bp->SetFlopCounter(*Ap);
if (ILUK!=0) ILUK->SetFlopCounter(*Ap);
elapsed_time = timer.ElapsedTime();
double normreducedb, normreduceda;
bp->NormInf(&normreducedb);
normreduceda = Ap->NormInf();
if (verbose)
cout << "Inf norm of Reduced Matrix = " << normreduceda << endl
<< "Inf norm of Reduced RHS = " << normreducedb << endl;
BiCGSTAB(*Ap, *xp, *bp, ILUK, Maxiter, Tolerance, &residual, verbose);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = counter.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute solution = "
<< elapsed_time << endl
<< "Number of operations in solve = " << total_flops << endl
<< "MFLOPS for Solve = " << MFLOPs<< endl << endl;
SingletonFilter.ComputeFullSolution();
if (smallProblem)
cout << " Reduced LHS after sol = " << endl << *xp << endl
<< " Full LHS after sol = " << endl << x << endl
<< " Full Exact LHS = " << endl << xexact << endl;
Epetra_Vector resid(x);
resid.Update(1.0, x, -1.0, xexact, 0.0); // resid = xcomp - xexact
resid.Norm2(&residual);
double normx, normxexact;
x.Norm2(&normx);
xexact.Norm2(&normxexact);
if (verbose)
cout << "2-norm of computed solution = " << normx << endl
<< "2-norm of exact solution = " << normxexact << endl
<< "2-norm of difference between computed and exact solution = " << residual << endl;
if (verbose1 && residual>1.0e-5) {
if (verbose)
cout << "Difference between computed and exact solution appears large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
Epetra_Vector bdiff(b);
assert(A.Multiply(false, resid, bdiff)==0);
assert(bdiff.Norm2(&residual)==0);
if (verbose)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
if (verbose)
cout << "********************************************************" << endl
<< " Solving again with 2*Ax=2*b" << endl
<< "********************************************************" << endl;
A.Scale(1.0); // A = 2*A
b.Scale(1.0); // b = 2*b
x.PutScalar(0.0);
b.NormInf(&normb);
norma = A.NormInf();
if (verbose)
cout << "Inf norm of Original Matrix = " << norma << endl
<< "Inf norm of Original RHS = " << normb << endl;
double updateReducedProblemTime = ReductionTimer.ElapsedTime();
SingletonFilter.UpdateReducedProblem(&FullProblem);
Comm.Barrier();
updateReducedProblemTime = ReductionTimer.ElapsedTime() - updateReducedProblemTime;
if (verbose)
cout << "\n\n****************************************************" << endl
<< " Update Reduced Problem time (sec) = " << updateReducedProblemTime<< endl
<< "****************************************************" << endl;
Statistics(SingletonFilter);
if (LevelFill>-1) {
Epetra_Flops fact_counter;
elapsed_time = timer.ElapsedTime();
int initerr = ILUK->InitValues(*Ap);
if (initerr!=0) {
cout << endl << Comm << endl << " InitValues error = " << initerr;
if (initerr==1) cout << " Zero diagonal found, warning error only";
cout << endl << endl;
}
assert(ILUK->Factor()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = ILUK->Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute preconditioner values = "
<< elapsed_time << endl
<< "MFLOPS for Factorization = " << MFLOPs << endl;
double Condest;
ILUK->Condest(false, Condest);
if (verbose) cout << "Condition number estimate for this preconditioner = " << Condest << endl;
}
bp->NormInf(&normreducedb);
normreduceda = Ap->NormInf();
if (verbose)
cout << "Inf norm of Reduced Matrix = " << normreduceda << endl
<< "Inf norm of Reduced RHS = " << normreducedb << endl;
BiCGSTAB(*Ap, *xp, *bp, ILUK, Maxiter, Tolerance, &residual, verbose);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = counter.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute solution = "
<< elapsed_time << endl
<< "Number of operations in solve = " << total_flops << endl
<< "MFLOPS for Solve = " << MFLOPs<< endl << endl;
SingletonFilter.ComputeFullSolution();
if (smallProblem)
cout << " Reduced LHS after sol = " << endl << *xp << endl
<< " Full LHS after sol = " << endl << x << endl
<< " Full Exact LHS = " << endl << xexact << endl;
resid.Update(1.0, x, -1.0, xexact, 0.0); // resid = xcomp - xexact
resid.Norm2(&residual);
x.Norm2(&normx);
xexact.Norm2(&normxexact);
if (verbose)
cout << "2-norm of computed solution = " << normx << endl
<< "2-norm of exact solution = " << normxexact << endl
<< "2-norm of difference between computed and exact solution = " << residual << endl;
if (verbose1 && residual>1.0e-5) {
if (verbose)
cout << "Difference between computed and exact solution appears large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
Epetra_Vector bdiff(b);
assert(A.Multiply(false, resid, bdiff)==0);
assert(bdiff.Norm2(&residual)==0);
if (verbose)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
if (ILUK!=0) delete ILUK;
if (IlukGraph!=0) delete IlukGraph;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return 0 ;
}
