int runTests(Epetra_Map & map, Epetra_CrsMatrix & A, Epetra_Vector & x, Epetra_Vector & b, Epetra_Vector & xexact, bool verbose) {
int ierr = 0;
// Create MultiVectors and put x, b, xexact in both columns of X, B, and Xexact, respectively.
Epetra_MultiVector X( map, 2, false );
Epetra_MultiVector B( map, 2, false );
Epetra_MultiVector Xexact( map, 2, false );
for (int i=0; i<X.NumVectors(); ++i) {
*X(i) = x;
*B(i) = b;
*Xexact(i) = xexact;
}
double residual;
std::vector<double> residualmv(2);
residual = A.NormInf(); double rAInf = residual;
if (verbose) std::cout << "Inf Norm of A = " << residual << std::endl;
residual = A.NormOne(); double rAOne = residual;
if (verbose) std::cout << "One Norm of A = " << residual << std::endl;
xexact.Norm2(&residual); double rxx = residual;
Xexact.Norm2(&residualmv[0]); std::vector<double> rXX( residualmv );
if (verbose) std::cout << "Norm of xexact = " << residual << std::endl;
if (verbose) std::cout << "Norm of Xexact = (" << residualmv[0] << ", " <<residualmv[1] <<")"<< std::endl;
Epetra_Vector tmp1(map);
Epetra_MultiVector tmp1mv(map,2,false);
A.Multiply(false, xexact, tmp1);
A.Multiply(false, Xexact, tmp1mv);
tmp1.Norm2(&residual); double rAx = residual;
tmp1mv.Norm2(&residualmv[0]); std::vector<double> rAX( residualmv );
if (verbose) std::cout << "Norm of Ax = " << residual << std::endl;
if (verbose) std::cout << "Norm of AX = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
b.Norm2(&residual); double rb = residual;
B.Norm2(&residualmv[0]); std::vector<double> rB( residualmv );
if (verbose) std::cout << "Norm of b (should equal norm of Ax) = " << residual << std::endl;
if (verbose) std::cout << "Norm of B (should equal norm of AX) = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
tmp1.Update(1.0, b, -1.0);
tmp1mv.Update(1.0, B, -1.0);
tmp1.Norm2(&residual);
tmp1mv.Norm2(&residualmv[0]);
if (verbose) std::cout << "Norm of difference between compute Ax and Ax from file = " << residual << std::endl;
if (verbose) std::cout << "Norm of difference between compute AX and AX from file = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
map.Comm().Barrier();
EPETRA_CHK_ERR(EpetraExt::BlockMapToMatrixMarketFile("Test_map.mm", map, "Official EpetraExt test map",
"This is the official EpetraExt test map generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::RowMatrixToMatrixMarketFile("Test_A.mm", A, "Official EpetraExt test matrix",
"This is the official EpetraExt test matrix generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_x.mm", x, "Official EpetraExt test initial guess",
"This is the official EpetraExt test initial guess generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvX.mm", X, "Official EpetraExt test initial guess",
"This is the official EpetraExt test initial guess generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_xexact.mm", xexact, "Official EpetraExt test exact solution",
"This is the official EpetraExt test exact solution generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvXexact.mm", Xexact, "Official EpetraExt test exact solution",
"This is the official EpetraExt test exact solution generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_b.mm", b, "Official EpetraExt test right hand side",
"This is the official EpetraExt test right hand side generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvB.mm", B, "Official EpetraExt test right hand side",
"This is the official EpetraExt test right hand side generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatlabFile("Test_mvB.mat", B));
EPETRA_CHK_ERR(EpetraExt::RowMatrixToMatlabFile("Test_A.dat", A));
Epetra_Map * map1;
Epetra_CrsMatrix * A1;
Epetra_CrsMatrix * A2;
Epetra_CrsMatrix * A3;
Epetra_Vector * x1;
Epetra_Vector * b1;
Epetra_Vector * xexact1;
Epetra_MultiVector * X1;
Epetra_MultiVector * B1;
Epetra_MultiVector * Xexact1;
EpetraExt::MatrixMarketFileToMap("Test_map.mm", map.Comm(), map1);
if (map.SameAs(*map1)) {
if (verbose) std::cout << "Maps are equal. In/Out works." << std::endl;
}
else {
if (verbose) std::cout << "Maps are not equal. In/Out fails." << std::endl;
ierr += 1;
}
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToCrsMatrix("Test_A.mm", *map1, A1));
// If map is zero-based, then we can compare to the convenient reading versions
if (map1->IndexBase()==0) EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToCrsMatrix("Test_A.mm", map1->Comm(), A2));
if (map1->IndexBase()==0) EPETRA_CHK_ERR(EpetraExt::MatlabFileToCrsMatrix("Test_A.dat", map1->Comm(), A3));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_x.mm", *map1, x1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_xexact.mm", *map1, xexact1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_b.mm", *map1, b1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvX.mm", *map1, X1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvXexact.mm", *map1, Xexact1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvB.mm", *map1, B1));
residual = A1->NormInf(); double rA1Inf = residual;
if (verbose) std::cout << "Inf Norm of A1 = " << residual << std::endl;
ierr += checkValues(rA1Inf,rAInf,"Inf Norm of A", verbose);
residual = A1->NormOne(); double rA1One = residual;
if (verbose) std::cout << "One Norm of A1 = " << residual << std::endl;
ierr += checkValues(rA1One,rAOne,"One Norm of A", verbose);
xexact1->Norm2(&residual); double rxx1 = residual;
if (verbose) std::cout << "Norm of xexact1 = " << residual << std::endl;
ierr += checkValues(rxx1,rxx,"Norm of xexact", verbose);
Xexact1->Norm2(&residualmv[0]); std::vector<double> rXX1(residualmv);
if (verbose) std::cout << "Norm of Xexact1 = (" << residualmv[0] <<", " <<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rXX1[0],rXX[0],"Norm of Xexact", verbose);
ierr += checkValues(rXX1[1],rXX[1],"Norm of Xexact", verbose);
Epetra_Vector tmp11(*map1);
A1->Multiply(false, *xexact1, tmp11);
Epetra_MultiVector tmp11mv(*map1,2,false);
A1->Multiply(false, *Xexact1, tmp11mv);
tmp11.Norm2(&residual); double rAx1 = residual;
if (verbose) std::cout << "Norm of A1*x1 = " << residual << std::endl;
ierr += checkValues(rAx1,rAx,"Norm of A1*x", verbose);
tmp11mv.Norm2(&residualmv[0]); std::vector<double> rAX1(residualmv);
if (verbose) std::cout << "Norm of A1*X1 = (" << residualmv[0] <<", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rAX1[0],rAX[0],"Norm of A1*X", verbose);
ierr += checkValues(rAX1[1],rAX[1],"Norm of A1*X", verbose);
if (map1->IndexBase()==0) {
Epetra_Vector tmp12(*map1);
A2->Multiply(false, *xexact1, tmp12);
tmp12.Norm2(&residual); double rAx2 = residual;
if (verbose) std::cout << "Norm of A2*x1 = " << residual << std::endl;
ierr += checkValues(rAx2,rAx,"Norm of A2*x", verbose);
Epetra_Vector tmp13(*map1);
A3->Multiply(false, *xexact1, tmp13);
tmp13.Norm2(&residual); double rAx3 = residual;
if (verbose) std::cout << "Norm of A3*x1 = " << residual << std::endl;
ierr += checkValues(rAx3,rAx,"Norm of A3*x", verbose);
}
b1->Norm2(&residual); double rb1 = residual;
if (verbose) std::cout << "Norm of b1 (should equal norm of Ax) = " << residual << std::endl;
ierr += checkValues(rb1,rb,"Norm of b", verbose);
B1->Norm2(&residualmv[0]); std::vector<double> rB1(residualmv);
if (verbose) std::cout << "Norm of B1 (should equal norm of AX) = (" << residualmv[0] <<", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rB1[0],rB[0],"Norm of B", verbose);
ierr += checkValues(rB1[1],rB[1],"Norm of B", verbose);
tmp11.Update(1.0, *b1, -1.0);
tmp11.Norm2(&residual);
if (verbose) std::cout << "Norm of difference between computed A1x1 and A1x1 from file = " << residual << std::endl;
ierr += checkValues(residual,0.0,"Norm of difference between computed A1x1 and A1x1 from file", verbose);
tmp11mv.Update(1.0, *B1, -1.0);
tmp11mv.Norm2(&residualmv[0]);
if (verbose) std::cout << "Norm of difference between computed A1X1 and A1X1 from file = (" << residualmv[0] << ", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(residualmv[0],0.0,"Norm of difference between computed A1X1 and A1X1 from file", verbose);
ierr += checkValues(residualmv[1],0.0,"Norm of difference between computed A1X1 and A1X1 from file", verbose);
if (map1->IndexBase()==0) {delete A2; delete A3;}
delete A1;
delete x1;
delete b1;
delete xexact1;
delete X1;
delete B1;
delete Xexact1;
delete map1;
return(ierr);
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
Teuchos::GlobalMPISession mpiSession(&argc, &argv, 0);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int nProcs, myPID ;
Teuchos::ParameterList pLUList ; // ParaLU parameters
Teuchos::ParameterList isoList ; // Isorropia parameters
Teuchos::ParameterList shyLUList ; // shyLU parameters
Teuchos::ParameterList ifpackList ; // shyLU parameters
string ipFileName = "ShyLU.xml"; // TODO : Accept as i/p
nProcs = mpiSession.getNProc();
myPID = Comm.MyPID();
if (myPID == 0)
{
cout <<"Parallel execution: nProcs="<< nProcs << endl;
}
// =================== Read input xml file =============================
Teuchos::updateParametersFromXmlFile(ipFileName, &pLUList);
isoList = pLUList.sublist("Isorropia Input");
shyLUList = pLUList.sublist("ShyLU Input");
shyLUList.set("Outer Solver Library", "AztecOO");
// Get matrix market file name
string MMFileName = Teuchos::getParameter<string>(pLUList, "mm_file");
string prec_type = Teuchos::getParameter<string>(pLUList, "preconditioner");
int maxiters = Teuchos::getParameter<int>(pLUList, "Outer Solver MaxIters");
double tol = Teuchos::getParameter<double>(pLUList, "Outer Solver Tolerance");
string rhsFileName = pLUList.get<string>("rhs_file", "");
if (myPID == 0)
{
cout << "Input :" << endl;
cout << "ParaLU params " << endl;
pLUList.print(std::cout, 2, true, true);
cout << "Matrix market file name: " << MMFileName << endl;
}
// ==================== Read input Matrix ==============================
Epetra_CrsMatrix *A;
Epetra_MultiVector *b1;
int err = EpetraExt::MatrixMarketFileToCrsMatrix(MMFileName.c_str(), Comm,
A);
//EpetraExt::MatlabFileToCrsMatrix(MMFileName.c_str(), Comm, A);
//assert(err != 0);
//cout <<"Done reading the matrix"<< endl;
int n = A->NumGlobalRows();
//cout <<"n="<< n << endl;
// Create input vectors
Epetra_Map vecMap(n, 0, Comm);
if (rhsFileName != "")
{
err = EpetraExt::MatrixMarketFileToMultiVector(rhsFileName.c_str(),
vecMap, b1);
}
else
{
b1 = new Epetra_MultiVector(vecMap, 1, false);
b1->PutScalar(1.0);
}
Epetra_MultiVector x(vecMap, 1);
//cout << "Created the vectors" << endl;
// Partition the matrix with hypergraph partitioning and redisstribute
Isorropia::Epetra::Partitioner *partitioner = new
Isorropia::Epetra::Partitioner(A, isoList, false);
partitioner->partition();
Isorropia::Epetra::Redistributor rd(partitioner);
Epetra_CrsMatrix *newA;
Epetra_MultiVector *newX, *newB;
rd.redistribute(*A, newA);
delete A;
A = newA;
rd.redistribute(x, newX);
rd.redistribute(*b1, newB);
Epetra_LinearProblem problem(A, newX, newB);
AztecOO solver(problem);
ifpackList ;
Ifpack_Preconditioner *prec;
ML_Epetra::MultiLevelPreconditioner *MLprec;
if (prec_type.compare("ShyLU") == 0)
{
prec = new Ifpack_ShyLU(A);
prec->SetParameters(shyLUList);
prec->Initialize();
prec->Compute();
//(dynamic_cast<Ifpack_ShyLU *>(prec))->JustTryIt();
//cout << " Going to set it in solver" << endl ;
solver.SetPrecOperator(prec);
//cout << " Done setting the solver" << endl ;
}
else if (prec_type.compare("ILU") == 0)
{
ifpackList.set( "fact: level-of-fill", 1 );
prec = new Ifpack_ILU(A);
prec->SetParameters(ifpackList);
prec->Initialize();
prec->Compute();
solver.SetPrecOperator(prec);
}
else if (prec_type.compare("ILUT") == 0)
{
ifpackList.set( "fact: ilut level-of-fill", 2 );
ifpackList.set( "fact: drop tolerance", 1e-8);
prec = new Ifpack_ILUT(A);
prec->SetParameters(ifpackList);
prec->Initialize();
prec->Compute();
solver.SetPrecOperator(prec);
}
else if (prec_type.compare("ML") == 0)
{
Teuchos::ParameterList mlList; // TODO : Take it from i/p
MLprec = new ML_Epetra::MultiLevelPreconditioner(*A, mlList, true);
solver.SetPrecOperator(MLprec);
}
solver.SetAztecOption(AZ_solver, AZ_gmres);
solver.SetMatrixName(333);
//solver.SetAztecOption(AZ_output, 1);
//solver.SetAztecOption(AZ_conv, AZ_Anorm);
//cout << "Going to iterate for the global problem" << endl;
solver.Iterate(maxiters, tol);
// compute ||Ax - b||
double Norm;
Epetra_MultiVector Ax(vecMap, 1);
Epetra_MultiVector *newAx;
rd.redistribute(Ax, newAx);
A->Multiply(false, *newX, *newAx);
newAx->Update(1.0, *newB, -1.0);
newAx->Norm2(&Norm);
double ANorm = A->NormOne();
cout << "|Ax-b |/|A| = " << Norm/ANorm << endl;
delete newAx;
if (prec_type.compare("ML") == 0)
{
delete MLprec;
}
else
{
delete prec;
}
delete b1;
delete newX;
delete newB;
delete A;
delete partitioner;
}
bool CrsMatrixInfo( const Epetra_CrsMatrix & A,
ostream & os )
{
int MyPID = A.Comm().MyPID();
// take care that matrix is already trasformed
bool IndicesAreGlobal = A.IndicesAreGlobal();
if( IndicesAreGlobal == true ) {
if( MyPID == 0 ) {
os << "WARNING : matrix must be transformed to local\n";
os << " before calling CrsMatrixInfo\n";
os << " Now returning...\n";
}
return false;
}
int NumGlobalRows = A.NumGlobalRows();
int NumGlobalNonzeros = A.NumGlobalNonzeros();
int NumGlobalCols = A.NumGlobalCols();
double NormInf = A.NormInf();
double NormOne = A.NormOne();
int NumGlobalDiagonals = A.NumGlobalDiagonals();
int GlobalMaxNumEntries = A.GlobalMaxNumEntries();
int IndexBase = A.IndexBase();
bool StorageOptimized = A.StorageOptimized();
bool LowerTriangular = A.LowerTriangular();
bool UpperTriangular = A.UpperTriangular();
bool NoDiagonal = A.NoDiagonal();
// these variables identifies quantities I have to compute,
// since not provided by Epetra_CrsMatrix
double MyFrobeniusNorm( 0.0 ), FrobeniusNorm( 0.0 );
double MyMinElement( DBL_MAX ), MinElement( DBL_MAX );
double MyMaxElement( DBL_MIN ), MaxElement( DBL_MIN );
double MyMinAbsElement( DBL_MAX ), MinAbsElement( DBL_MAX );
double MyMaxAbsElement( 0.0 ), MaxAbsElement( 0.0 );
int NumMyRows = A.NumMyRows();
int * NzPerRow = new int[NumMyRows];
int Row; // iterator on rows
int Col; // iterator on cols
int MaxNumEntries = A.MaxNumEntries();
double * Values = new double[MaxNumEntries];
int * Indices = new int[MaxNumEntries];
double Element, AbsElement; // generic nonzero element and its abs value
int NumEntries;
double * Diagonal = new double [NumMyRows];
// SumOffDiagonal is the sum of absolute values for off-diagonals
double * SumOffDiagonal = new double [NumMyRows];
for( Row=0 ; Row<NumMyRows ; ++Row ) {
SumOffDiagonal[Row] = 0.0;
}
int * IsDiagonallyDominant = new int [NumMyRows];
int GlobalRow;
// cycle over all matrix elements
for( Row=0 ; Row<NumMyRows ; ++Row ) {
GlobalRow = A.GRID(Row);
NzPerRow[Row] = A.NumMyEntries(Row);
A.ExtractMyRowCopy(Row,NzPerRow[Row],NumEntries,Values,Indices);
for( Col=0 ; Col<NumEntries ; ++Col ) {
Element = Values[Col];
AbsElement = abs(Element);
if( Element<MyMinElement ) MyMinElement = Element;
if( Element>MyMaxElement ) MyMaxElement = Element;
if( AbsElement<MyMinAbsElement ) MyMinAbsElement = AbsElement;
if( AbsElement>MyMaxAbsElement ) MyMaxAbsElement = AbsElement;
if( Indices[Col] == Row ) Diagonal[Row] = Element;
else
SumOffDiagonal[Row] += abs(Element);
MyFrobeniusNorm += pow(Element,2);
}
}
// analise storage per row
int MyMinNzPerRow( NumMyRows ), MinNzPerRow( NumMyRows );
int MyMaxNzPerRow( 0 ), MaxNzPerRow( 0 );
for( Row=0 ; Row<NumMyRows ; ++Row ) {
if( NzPerRow[Row]<MyMinNzPerRow ) MyMinNzPerRow=NzPerRow[Row];
if( NzPerRow[Row]>MyMaxNzPerRow ) MyMaxNzPerRow=NzPerRow[Row];
}
// a test to see if matrix is diagonally-dominant
int MyDiagonalDominance( 0 ), DiagonalDominance( 0 );
int MyWeakDiagonalDominance( 0 ), WeakDiagonalDominance( 0 );
for( Row=0 ; Row<NumMyRows ; ++Row ) {
if( abs(Diagonal[Row])>SumOffDiagonal[Row] )
++MyDiagonalDominance;
else if( abs(Diagonal[Row])==SumOffDiagonal[Row] )
++MyWeakDiagonalDominance;
}
// reduction operations
A.Comm().SumAll(&MyFrobeniusNorm, &FrobeniusNorm, 1);
A.Comm().MinAll(&MyMinElement, &MinElement, 1);
A.Comm().MaxAll(&MyMaxElement, &MaxElement, 1);
A.Comm().MinAll(&MyMinAbsElement, &MinAbsElement, 1);
A.Comm().MaxAll(&MyMaxAbsElement, &MaxAbsElement, 1);
A.Comm().MinAll(&MyMinNzPerRow, &MinNzPerRow, 1);
A.Comm().MaxAll(&MyMaxNzPerRow, &MaxNzPerRow, 1);
A.Comm().SumAll(&MyDiagonalDominance, &DiagonalDominance, 1);
A.Comm().SumAll(&MyWeakDiagonalDominance, &WeakDiagonalDominance, 1);
// free memory
delete Values;
delete Indices;
delete Diagonal;
delete SumOffDiagonal;
delete IsDiagonallyDominant;
delete NzPerRow;
// simply no output for MyPID>0, only proc 0 write on os
if( MyPID != 0 ) return true;
os << "*** general Information about the matrix\n";
os << "Number of Global Rows = " << NumGlobalRows << endl;
os << "Number of Global Cols = " << NumGlobalCols << endl;
os << "is the matrix square = " <<
((NumGlobalRows==NumGlobalCols)?"yes":"no") << endl;
os << "||A||_\\infty = " << NormInf << endl;
os << "||A||_1 = " << NormOne << endl;
os << "||A||_F = " << sqrt(FrobeniusNorm) << endl;
os << "Number of nonzero diagonal entries = "
<< NumGlobalDiagonals
<< "( " << 1.0* NumGlobalDiagonals/NumGlobalRows*100
<< " %)\n";
os << "Nonzero per row : min = " << MinNzPerRow
<< " average = " << 1.0*NumGlobalNonzeros/NumGlobalRows
<< " max = " << MaxNzPerRow << endl;
os << "Maximum number of nonzero elements/row = "
<< GlobalMaxNumEntries << endl;
os << "min( a_{i,j} ) = " << MinElement << endl;
os << "max( a_{i,j} ) = " << MaxElement << endl;
os << "min( abs(a_{i,j}) ) = " << MinAbsElement << endl;
os << "max( abs(a_{i,j}) ) = " << MaxAbsElement << endl;
os << "Number of diagonal dominant rows = " << DiagonalDominance
<< " (" << 100.0*DiagonalDominance/NumGlobalRows << " % of total)\n";
os << "Number of weakly diagonal dominant rows = "
<< WeakDiagonalDominance
<< " (" << 100.0*WeakDiagonalDominance/NumGlobalRows << " % of total)\n";
os << "*** Information about the Trilinos storage\n";
os << "Base Index = " << IndexBase << endl;
os << "is storage optimized = "
<< ((StorageOptimized==true)?"yes":"no") << endl;
os << "are indices global = "
<< ((IndicesAreGlobal==true)?"yes":"no") << endl;
os << "is matrix lower triangular = "
<< ((LowerTriangular==true)?"yes":"no") << endl;
os << "is matrix upper triangular = "
<< ((UpperTriangular==true)?"yes":"no") << endl;
os << "are there diagonal entries = "
<< ((NoDiagonal==false)?"yes":"no") << endl;
return true;
}
int TestOneMatrix( std::string HBname, std::string MMname, std::string TRIname, Epetra_Comm &Comm, bool verbose ) {
if ( Comm.MyPID() != 0 ) verbose = false ;
Epetra_Map * readMap = 0;
Epetra_CrsMatrix * HbA = 0;
Epetra_Vector * Hbx = 0;
Epetra_Vector * Hbb = 0;
Epetra_Vector * Hbxexact = 0;
Epetra_CrsMatrix * TriplesA = 0;
Epetra_Vector * Triplesx = 0;
Epetra_Vector * Triplesb = 0;
Epetra_Vector * Triplesxexact = 0;
Epetra_CrsMatrix * MatrixMarketA = 0;
Epetra_Vector * MatrixMarketx = 0;
Epetra_Vector * MatrixMarketb = 0;
Epetra_Vector * MatrixMarketxexact = 0;
int TRI_Size = TRIname.size() ;
std::string LastFiveBytes = TRIname.substr( EPETRA_MAX(0,TRI_Size-5), TRI_Size );
if ( LastFiveBytes == ".TimD" ) {
// Call routine to read in a file with a Tim Davis header and zero-based indexing
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra64( &TRIname[0], false, Comm,
readMap, TriplesA, Triplesx,
Triplesb, Triplesxexact, false, true, true ) );
delete readMap;
} else {
if ( LastFiveBytes == ".triU" ) {
// Call routine to read in unsymmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra64( &TRIname[0], false, Comm,
readMap, TriplesA, Triplesx,
Triplesb, Triplesxexact, false, false ) );
delete readMap;
} else {
if ( LastFiveBytes == ".triS" ) {
// Call routine to read in symmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra64( &TRIname[0], true, Comm,
readMap, TriplesA, Triplesx,
Triplesb, Triplesxexact, false, false ) );
delete readMap;
} else {
assert( false ) ;
}
}
}
EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra64( &MMname[0], Comm, readMap,
MatrixMarketA, MatrixMarketx,
MatrixMarketb, MatrixMarketxexact) );
delete readMap;
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra64( &HBname[0], Comm, readMap, HbA, Hbx,
Hbb, Hbxexact) ;
#if 0
std::cout << " HbA " ;
HbA->Print( std::cout ) ;
std::cout << std::endl ;
std::cout << " MatrixMarketA " ;
MatrixMarketA->Print( std::cout ) ;
std::cout << std::endl ;
std::cout << " TriplesA " ;
TriplesA->Print( std::cout ) ;
std::cout << std::endl ;
#endif
int TripleErr = 0 ;
int MMerr = 0 ;
for ( int i = 0 ; i < 10 ; i++ )
{
double resid_Hb_Triples;
double resid_Hb_Matrix_Market;
double norm_A ;
Hbx->Random();
//
// Set the output vectors to different values:
//
Triplesb->PutScalar(1.1);
Hbb->PutScalar(1.2);
MatrixMarketb->PutScalar(1.3);
HbA->Multiply( false, *Hbx, *Hbb );
norm_A = HbA->NormOne( ) ;
TriplesA->Multiply( false, *Hbx, *Triplesb );
Triplesb->Update( 1.0, *Hbb, -1.0 ) ;
MatrixMarketA->Multiply( false, *Hbx, *MatrixMarketb );
MatrixMarketb->Update( 1.0, *Hbb, -1.0 ) ;
Triplesb->Norm1( &resid_Hb_Triples ) ;
MatrixMarketb->Norm1( &resid_Hb_Matrix_Market ) ;
TripleErr += ( resid_Hb_Triples > 1e-11 * norm_A ) ;
MMerr += ( resid_Hb_Matrix_Market > 1e-11 * norm_A ) ;
if ( verbose && resid_Hb_Triples > 1e-11 * norm_A )
std::cout << " resid_Hb_Triples = " << resid_Hb_Triples
<< " norm_A = " << norm_A << std::endl ;
if ( verbose && resid_Hb_Matrix_Market > 1e-11 * norm_A )
std::cout << " resid_Hb_Matrix_Market = " << resid_Hb_Matrix_Market
<< " norm_A = " << norm_A << std::endl ;
}
if ( verbose ) {
if ( TripleErr ) std::cout << " Error in reading " << HBname << " or " << TRIname << std::endl ;
if ( MMerr ) std::cout << " Error in reading " << HBname << " or " << MMname << std::endl ;
}
delete HbA;
delete Hbx;
delete Hbb;
delete Hbxexact;
delete TriplesA;
delete Triplesx;
delete Triplesb;
delete Triplesxexact;
delete MatrixMarketA;
delete MatrixMarketx;
delete MatrixMarketb;
delete MatrixMarketxexact;
delete readMap;
return TripleErr+MMerr ;
}
int GMGSolver::solve() {
int rank = Teuchos::GlobalMPISession::getRank();
// in place of doing the scaling ourselves, for the moment I've switched
// over to using Aztec's built-in scaling. This appears to be functionally identical.
bool useAztecToScaleDiagonally = true;
AztecOO solver(problem());
Epetra_CrsMatrix *A = dynamic_cast<Epetra_CrsMatrix *>( problem().GetMatrix() );
if (A == NULL) {
cout << "Error: GMGSolver requires an Epetra_CrsMatrix.\n";
TEUCHOS_TEST_FOR_EXCEPTION(true, std::invalid_argument, "Error: GMGSolver requires an Epetra_CrsMatrix.\n");
}
// EpetraExt::RowMatrixToMatlabFile("/tmp/A_pre_scaling.dat",*A);
// Epetra_MultiVector *b = problem().GetRHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/b_pre_scaling.dat",*b);
// Epetra_MultiVector *x = problem().GetLHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/x_initial_guess.dat",*x);
const Epetra_Map* map = &A->RowMatrixRowMap();
Epetra_Vector diagA(*map);
A->ExtractDiagonalCopy(diagA);
// EpetraExt::MultiVectorToMatlabFile("/tmp/diagA.dat",diagA);
//
Epetra_Vector scale_vector(*map);
Epetra_Vector diagA_sqrt_inv(*map);
Epetra_Vector diagA_inv(*map);
if (_diagonalScaling && !useAztecToScaleDiagonally) {
int length = scale_vector.MyLength();
for (int i=0; i<length; i++) scale_vector[i] = 1.0 / sqrt(fabs(diagA[i]));
problem().LeftScale(scale_vector);
problem().RightScale(scale_vector);
}
Teuchos::RCP<Epetra_MultiVector> diagA_ptr = Teuchos::rcp( &diagA, false );
_gmgOperator.setStiffnessDiagonal(diagA_ptr);
_gmgOperator.setApplyDiagonalSmoothing(_diagonalSmoothing);
_gmgOperator.setFineSolverUsesDiagonalScaling(_diagonalScaling);
_gmgOperator.computeCoarseStiffnessMatrix(A);
if (_diagonalScaling && useAztecToScaleDiagonally) {
solver.SetAztecOption(AZ_scaling, AZ_sym_diag);
} else {
solver.SetAztecOption(AZ_scaling, AZ_none);
}
if (_useCG) {
if (_computeCondest) {
solver.SetAztecOption(AZ_solver, AZ_cg_condnum);
} else {
solver.SetAztecOption(AZ_solver, AZ_cg);
}
} else {
solver.SetAztecOption(AZ_kspace, 200); // default is 30
if (_computeCondest) {
solver.SetAztecOption(AZ_solver, AZ_gmres_condnum);
} else {
solver.SetAztecOption(AZ_solver, AZ_gmres);
}
}
solver.SetPrecOperator(&_gmgOperator);
// solver.SetAztecOption(AZ_precond, AZ_none);
solver.SetAztecOption(AZ_precond, AZ_user_precond);
solver.SetAztecOption(AZ_conv, _azConvergenceOption);
// solver.SetAztecOption(AZ_output, AZ_last);
solver.SetAztecOption(AZ_output, _azOutput);
int solveResult = solver.Iterate(_maxIters,_tol);
const double* status = solver.GetAztecStatus();
int remainingIters = _maxIters;
int whyTerminated = status[AZ_why];
int maxRestarts = 1;
int numRestarts = 0;
while ((whyTerminated==AZ_loss) && (numRestarts < maxRestarts)) {
remainingIters -= status[AZ_its];
if (rank==0) cout << "Aztec warned that the recursive residual indicates convergence even though the true residual is too large. Restarting with the new solution as initial guess, with maxIters = " << remainingIters << endl;
solveResult = solver.Iterate(remainingIters,_tol);
whyTerminated = status[AZ_why];
numRestarts++;
}
remainingIters -= status[AZ_its];
_iterationCount = _maxIters - remainingIters;
_condest = solver.Condest(); // will be -1 if running without condest
if (rank==0) {
switch (whyTerminated) {
case AZ_normal:
// cout << "whyTerminated: AZ_normal " << endl;
break;
case AZ_param:
cout << "whyTerminated: AZ_param " << endl;
break;
case AZ_breakdown:
cout << "whyTerminated: AZ_breakdown " << endl;
break;
case AZ_loss:
cout << "whyTerminated: AZ_loss " << endl;
break;
case AZ_ill_cond:
cout << "whyTerminated: AZ_ill_cond " << endl;
break;
case AZ_maxits:
cout << "whyTerminated: AZ_maxits " << endl;
break;
default:
break;
}
}
// Epetra_MultiVector *x = problem().GetLHS();
// EpetraExt::MultiVectorToMatlabFile("/tmp/x.dat",*x);
if (_diagonalScaling && !useAztecToScaleDiagonally) {
// reverse the scaling here
scale_vector.Reciprocal(scale_vector);
problem().LeftScale(scale_vector);
problem().RightScale(scale_vector);
}
double norminf = A->NormInf();
double normone = A->NormOne();
int numIters = solver.NumIters();
if (_printToConsole) {
cout << "\n Inf-norm of stiffness matrix after scaling = " << norminf;
cout << "\n One-norm of stiffness matrix after scaling = " << normone << endl << endl;
cout << "Num iterations: " << numIters << endl;
}
_gmgOperator.setStiffnessDiagonal(Teuchos::rcp((Epetra_MultiVector*) NULL ));
return solveResult;
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
Teuchos::GlobalMPISession mpiSession(&argc, &argv, 0);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
int nProcs, myPID ;
Teuchos::ParameterList pLUList ; // ParaLU parameters
Teuchos::ParameterList isoList ; // Isorropia parameters
string ipFileName = "ShyLU.xml"; // TODO : Accept as i/p
nProcs = mpiSession.getNProc();
myPID = Comm.MyPID();
if (myPID == 0)
{
cout <<"Parallel execution: nProcs="<< nProcs << endl;
}
// =================== Read input xml file =============================
Teuchos::updateParametersFromXmlFile(ipFileName, &pLUList);
isoList = pLUList.sublist("Isorropia Input");
// Get matrix market file name
string MMFileName = Teuchos::getParameter<string>(pLUList, "mm_file");
string prec_type = Teuchos::getParameter<string>(pLUList, "preconditioner");
if (myPID == 0)
{
cout << "Input :" << endl;
cout << "ParaLU params " << endl;
pLUList.print(std::cout, 2, true, true);
cout << "Matrix market file name: " << MMFileName << endl;
}
// ==================== Read input Matrix ==============================
Epetra_CrsMatrix *A;
int err = EpetraExt::MatrixMarketFileToCrsMatrix(MMFileName.c_str(), Comm, A);
//EpetraExt::MatlabFileToCrsMatrix(MMFileName.c_str(), Comm, A);
//assert(err != 0);
cout <<"Done reading the matrix"<< endl;
int n = A->NumGlobalRows();
cout <<"n="<< n << endl;
// Create input vectors
Epetra_Map vecMap(n, 0, Comm);
Epetra_MultiVector x(vecMap, 1);
Epetra_MultiVector b(vecMap, 1, false);
b.PutScalar(1.0); // TODO : Accept it as input
// Partition the matrix with hypergraph partitioning and redisstribute
Isorropia::Epetra::Partitioner *partitioner = new
Isorropia::Epetra::Partitioner(A, isoList, false);
partitioner->partition();
Isorropia::Epetra::Redistributor rd(partitioner);
Epetra_CrsMatrix *newA;
Epetra_MultiVector *newX, *newB;
rd.redistribute(*A, newA);
delete A;
A = newA;
rd.redistribute(x, newX);
rd.redistribute(b, newB);
Epetra_LinearProblem problem(A, newX, newB);
Amesos Factory;
char* SolverType = "Amesos_Klu";
bool IsAvailable = Factory.Query(SolverType);
Epetra_LinearProblem *LP = new Epetra_LinearProblem();
LP->SetOperator(A);
LP->SetLHS(newX);
LP->SetRHS(newB);
Amesos_BaseSolver *Solver = Factory.Create(SolverType, *LP);
Solver->SymbolicFactorization();
Teuchos::Time ftime("setup time");
ftime.start();
Solver->NumericFactorization();
cout << "Numeric Factorization" << endl;
Solver->Solve();
cout << "Solve done" << endl;
ftime.stop();
cout << "Time to setup" << ftime.totalElapsedTime() << endl;
// compute ||Ax - b||
double Norm;
Epetra_MultiVector Ax(vecMap, 1);
Epetra_MultiVector *newAx;
rd.redistribute(Ax, newAx);
A->Multiply(false, *newX, *newAx);
newAx->Update(1.0, *newB, -1.0);
newAx->Norm2(&Norm);
double ANorm = A->NormOne();
cout << "|Ax-b |/|A| = " << Norm/ANorm << endl;
delete newAx;
delete newX;
delete newB;
delete A;
delete partitioner;
}
