int SchwarzSolver::solve()
{
// compute some statistics for the original problem
double condest = -1;
Epetra_LinearProblem problem(_stiffnessMatrix.get(), _lhs.get(), _rhs.get());
AztecOO solverForConditionEstimate(problem);
solverForConditionEstimate.SetAztecOption(AZ_solver, AZ_cg_condnum);
solverForConditionEstimate.ConstructPreconditioner(condest);
Epetra_RowMatrix *A = problem.GetMatrix();
double norminf = A->NormInf();
double normone = A->NormOne();
if (_printToConsole)
{
cout << "\n Inf-norm of stiffness matrix before scaling = " << norminf;
cout << "\n One-norm of stiffness matrix before scaling = " << normone << endl << endl;
cout << "Condition number estimate: " << condest << endl;
}
AztecOO solver(problem);
int otherRows = A->NumGlobalRows() - A->NumMyRows();
int overlapLevel = std::min(otherRows,_overlapLevel);
solver.SetAztecOption(AZ_precond, AZ_dom_decomp); // additive schwarz
solver.SetAztecOption(AZ_overlap, overlapLevel); // level of overlap for schwarz
solver.SetAztecOption(AZ_type_overlap, AZ_symmetric);
solver.SetAztecOption(AZ_solver, AZ_cg_condnum); // more expensive than AZ_cg, but allows estimate of condition #
solver.SetAztecOption(AZ_subdomain_solve, AZ_ilut); // TODO: look these up (copied from example)
solver.SetAztecParam(AZ_ilut_fill, 1.0); // TODO: look these up (copied from example)
solver.SetAztecParam(AZ_drop, 0.0); // TODO: look these up (copied from example)
int solveResult = solver.Iterate(_maxIters,_tol);
// int solveResult = solver.AdaptiveIterate(_maxIters,1,_tol); // an experiment (was Iterate())
norminf = A->NormInf();
normone = A->NormOne();
condest = solver.Condest();
int numIters = solver.NumIters();
if (_printToConsole)
{
cout << "\n Inf-norm of stiffness matrix after scaling = " << norminf;
cout << "\n One-norm of stiffness matrix after scaling = " << normone << endl << endl;
cout << "Condition number estimate: " << condest << endl;
cout << "Num iterations: " << numIters << endl;
}
return solveResult;
}
int check(Epetra_RowMatrix& A, Epetra_RowMatrix & B, bool verbose) {
int ierr = 0;
EPETRA_TEST_ERR(!A.Comm().NumProc()==B.Comm().NumProc(),ierr);
EPETRA_TEST_ERR(!A.Comm().MyPID()==B.Comm().MyPID(),ierr);
EPETRA_TEST_ERR(!A.Filled()==B.Filled(),ierr);
EPETRA_TEST_ERR(!A.HasNormInf()==B.HasNormInf(),ierr);
EPETRA_TEST_ERR(!A.LowerTriangular()==B.LowerTriangular(),ierr);
EPETRA_TEST_ERR(!A.Map().SameAs(B.Map()),ierr);
EPETRA_TEST_ERR(!A.MaxNumEntries()==B.MaxNumEntries(),ierr);
EPETRA_TEST_ERR(!A.NumGlobalCols64()==B.NumGlobalCols64(),ierr);
EPETRA_TEST_ERR(!A.NumGlobalDiagonals64()==B.NumGlobalDiagonals64(),ierr);
EPETRA_TEST_ERR(!A.NumGlobalNonzeros64()==B.NumGlobalNonzeros64(),ierr);
EPETRA_TEST_ERR(!A.NumGlobalRows64()==B.NumGlobalRows64(),ierr);
EPETRA_TEST_ERR(!A.NumMyCols()==B.NumMyCols(),ierr);
EPETRA_TEST_ERR(!A.NumMyDiagonals()==B.NumMyDiagonals(),ierr);
EPETRA_TEST_ERR(!A.NumMyNonzeros()==B.NumMyNonzeros(),ierr);
for (int i=0; i<A.NumMyRows(); i++) {
int nA, nB;
A.NumMyRowEntries(i,nA); B.NumMyRowEntries(i,nB);
EPETRA_TEST_ERR(!nA==nB,ierr);
}
EPETRA_TEST_ERR(!A.NumMyRows()==B.NumMyRows(),ierr);
EPETRA_TEST_ERR(!A.OperatorDomainMap().SameAs(B.OperatorDomainMap()),ierr);
EPETRA_TEST_ERR(!A.OperatorRangeMap().SameAs(B.OperatorRangeMap()),ierr);
EPETRA_TEST_ERR(!A.RowMatrixColMap().SameAs(B.RowMatrixColMap()),ierr);
EPETRA_TEST_ERR(!A.RowMatrixRowMap().SameAs(B.RowMatrixRowMap()),ierr);
EPETRA_TEST_ERR(!A.UpperTriangular()==B.UpperTriangular(),ierr);
EPETRA_TEST_ERR(!A.UseTranspose()==B.UseTranspose(),ierr);
int NumVectors = 5;
{ // No transpose case
Epetra_MultiVector X(A.OperatorDomainMap(), NumVectors);
Epetra_MultiVector YA1(A.OperatorRangeMap(), NumVectors);
Epetra_MultiVector YA2(YA1);
Epetra_MultiVector YB1(YA1);
Epetra_MultiVector YB2(YA1);
X.Random();
bool transA = false;
A.SetUseTranspose(transA);
B.SetUseTranspose(transA);
A.Apply(X,YA1);
A.Multiply(transA, X, YA2);
EPETRA_TEST_ERR(checkMultiVectors(YA1,YA2,"A Multiply and A Apply", verbose),ierr);
B.Apply(X,YB1);
EPETRA_TEST_ERR(checkMultiVectors(YA1,YB1,"A Multiply and B Multiply", verbose),ierr);
B.Multiply(transA, X, YB2);
EPETRA_TEST_ERR(checkMultiVectors(YA1,YB2,"A Multiply and B Apply", verbose), ierr);
}
{// transpose case
Epetra_MultiVector X(A.OperatorRangeMap(), NumVectors);
Epetra_MultiVector YA1(A.OperatorDomainMap(), NumVectors);
Epetra_MultiVector YA2(YA1);
Epetra_MultiVector YB1(YA1);
Epetra_MultiVector YB2(YA1);
X.Random();
bool transA = true;
A.SetUseTranspose(transA);
B.SetUseTranspose(transA);
A.Apply(X,YA1);
A.Multiply(transA, X, YA2);
EPETRA_TEST_ERR(checkMultiVectors(YA1,YA2, "A Multiply and A Apply (transpose)", verbose),ierr);
B.Apply(X,YB1);
EPETRA_TEST_ERR(checkMultiVectors(YA1,YB1, "A Multiply and B Multiply (transpose)", verbose),ierr);
B.Multiply(transA, X,YB2);
EPETRA_TEST_ERR(checkMultiVectors(YA1,YB2, "A Multiply and B Apply (transpose)", verbose),ierr);
}
Epetra_Vector diagA(A.RowMatrixRowMap());
EPETRA_TEST_ERR(A.ExtractDiagonalCopy(diagA),ierr);
Epetra_Vector diagB(B.RowMatrixRowMap());
EPETRA_TEST_ERR(B.ExtractDiagonalCopy(diagB),ierr);
EPETRA_TEST_ERR(checkMultiVectors(diagA,diagB, "ExtractDiagonalCopy", verbose),ierr);
Epetra_Vector rowA(A.RowMatrixRowMap());
EPETRA_TEST_ERR(A.InvRowSums(rowA),ierr);
Epetra_Vector rowB(B.RowMatrixRowMap());
EPETRA_TEST_ERR(B.InvRowSums(rowB),ierr)
EPETRA_TEST_ERR(checkMultiVectors(rowA,rowB, "InvRowSums", verbose),ierr);
Epetra_Vector colA(A.RowMatrixColMap());
EPETRA_TEST_ERR(A.InvColSums(colA),ierr);
Epetra_Vector colB(B.RowMatrixColMap());
EPETRA_TEST_ERR(B.InvColSums(colB),ierr);
EPETRA_TEST_ERR(checkMultiVectors(colA,colB, "InvColSums", verbose),ierr);
EPETRA_TEST_ERR(checkValues(A.NormInf(), B.NormInf(), "NormInf before scaling", verbose), ierr);
EPETRA_TEST_ERR(checkValues(A.NormOne(), B.NormOne(), "NormOne before scaling", verbose),ierr);
EPETRA_TEST_ERR(A.RightScale(colA),ierr);
EPETRA_TEST_ERR(B.RightScale(colB),ierr);
EPETRA_TEST_ERR(A.LeftScale(rowA),ierr);
EPETRA_TEST_ERR(B.LeftScale(rowB),ierr);
EPETRA_TEST_ERR(checkValues(A.NormInf(), B.NormInf(), "NormInf after scaling", verbose), ierr);
EPETRA_TEST_ERR(checkValues(A.NormOne(), B.NormOne(), "NormOne after scaling", verbose),ierr);
vector<double> valuesA(A.MaxNumEntries());
vector<int> indicesA(A.MaxNumEntries());
vector<double> valuesB(B.MaxNumEntries());
vector<int> indicesB(B.MaxNumEntries());
return(0);
for (int i=0; i<A.NumMyRows(); i++) {
int nA, nB;
EPETRA_TEST_ERR(A.ExtractMyRowCopy(i, A.MaxNumEntries(), nA, &valuesA[0], &indicesA[0]),ierr);
EPETRA_TEST_ERR(B.ExtractMyRowCopy(i, B.MaxNumEntries(), nB, &valuesB[0], &indicesB[0]),ierr);
EPETRA_TEST_ERR(!nA==nB,ierr);
for (int j=0; j<nA; j++) {
double curVal = valuesA[j];
int curIndex = indicesA[j];
bool notfound = true;
int jj = 0;
while (notfound && jj< nB) {
if (!checkValues(curVal, valuesB[jj])) notfound = false;
jj++;
}
EPETRA_TEST_ERR(notfound, ierr);
vector<int>::iterator p = find(indicesB.begin(),indicesB.end(),curIndex); // find curIndex in indicesB
EPETRA_TEST_ERR(p==indicesB.end(), ierr);
}
}
if (verbose) cout << "RowMatrix Methods check OK" << endl;
return (ierr);
}
//
// Amesos_TestMultiSolver.cpp reads in a matrix in Harwell-Boeing format,
// calls one of the sparse direct solvers, using blocked right hand sides
// and computes the error and residual.
//
// TestSolver ignores the Harwell-Boeing right hand sides, creating
// random right hand sides instead.
//
// Amesos_TestMultiSolver can test either A x = b or A^T x = b.
// This can be a bit confusing because sparse direct solvers
// use compressed column storage - the transpose of Trilinos'
// sparse row storage.
//
// Matrices:
// readA - Serial. As read from the file.
// transposeA - Serial. The transpose of readA.
// serialA - if (transpose) then transposeA else readA
// distributedA - readA distributed to all processes
// passA - if ( distributed ) then distributedA else serialA
//
//
int Amesos_TestMultiSolver( Epetra_Comm &Comm, char *matrix_file, int numsolves,
SparseSolverType SparseSolver, bool transpose,
int special, AMESOS_MatrixType matrix_type ) {
int iam = Comm.MyPID() ;
// int hatever;
// if ( iam == 0 ) std::cin >> hatever ;
Comm.Barrier();
Epetra_Map * readMap;
Epetra_CrsMatrix * readA;
Epetra_Vector * readx;
Epetra_Vector * readb;
Epetra_Vector * readxexact;
std::string FileName = matrix_file ;
int FN_Size = FileName.size() ;
std::string LastFiveBytes = FileName.substr( EPETRA_MAX(0,FN_Size-5), FN_Size );
std::string LastFourBytes = FileName.substr( EPETRA_MAX(0,FN_Size-4), FN_Size );
bool NonContiguousMap = false;
if ( LastFiveBytes == ".triU" ) {
NonContiguousMap = true;
// Call routine to read in unsymmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, false, Comm, readMap, readA, readx,
readb, readxexact, NonContiguousMap ) );
} else {
if ( LastFiveBytes == ".triS" ) {
NonContiguousMap = true;
// Call routine to read in symmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, true, Comm,
readMap, readA, readx,
readb, readxexact, NonContiguousMap ) );
} else {
if ( LastFourBytes == ".mtx" ) {
EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra( matrix_file, Comm, readMap,
readA, readx, readb, readxexact) );
} else {
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra( matrix_file, Comm, readMap, readA, readx,
readb, readxexact) ;
}
}
}
Epetra_CrsMatrix transposeA(Copy, *readMap, 0);
Epetra_CrsMatrix *serialA ;
if ( transpose ) {
assert( CrsMatrixTranspose( readA, &transposeA ) == 0 );
serialA = &transposeA ;
} else {
serialA = readA ;
}
// Create uniform distributed map
Epetra_Map map(readMap->NumGlobalElements(), 0, Comm);
Epetra_Map* map_;
if( NonContiguousMap ) {
//
// map gives us NumMyElements and MyFirstElement;
//
int NumGlobalElements = readMap->NumGlobalElements();
int NumMyElements = map.NumMyElements();
int MyFirstElement = map.MinMyGID();
std::vector<int> MapMap_( NumGlobalElements );
readMap->MyGlobalElements( &MapMap_[0] ) ;
Comm.Broadcast( &MapMap_[0], NumGlobalElements, 0 ) ;
map_ = new Epetra_Map( NumGlobalElements, NumMyElements, &MapMap_[MyFirstElement], 0, Comm);
} else {
map_ = new Epetra_Map( map ) ;
}
// Create Exporter to distribute read-in matrix and vectors
Epetra_Export exporter(*readMap, *map_);
Epetra_CrsMatrix A(Copy, *map_, 0);
Epetra_RowMatrix * passA = 0;
Epetra_MultiVector * passx = 0;
Epetra_MultiVector * passb = 0;
Epetra_MultiVector * passxexact = 0;
Epetra_MultiVector * passresid = 0;
Epetra_MultiVector * passtmp = 0;
Epetra_MultiVector x(*map_,numsolves);
Epetra_MultiVector b(*map_,numsolves);
Epetra_MultiVector xexact(*map_,numsolves);
Epetra_MultiVector resid(*map_,numsolves);
Epetra_MultiVector tmp(*map_,numsolves);
Epetra_MultiVector serialx(*readMap,numsolves);
Epetra_MultiVector serialb(*readMap,numsolves);
Epetra_MultiVector serialxexact(*readMap,numsolves);
Epetra_MultiVector serialresid(*readMap,numsolves);
Epetra_MultiVector serialtmp(*readMap,numsolves);
bool distribute_matrix = ( matrix_type == AMESOS_Distributed ) ;
if ( distribute_matrix ) {
//
// Initialize x, b and xexact to the values read in from the file
//
A.Export(*serialA, exporter, Add);
Comm.Barrier();
assert(A.FillComplete()==0);
Comm.Barrier();
passA = &A;
passx = &x;
passb = &b;
passxexact = &xexact;
passresid = &resid;
passtmp = &tmp;
} else {
passA = serialA;
passx = &serialx;
passb = &serialb;
passxexact = &serialxexact;
passresid = &serialresid;
passtmp = &serialtmp;
}
passxexact->SetSeed(131) ;
passxexact->Random();
passx->SetSeed(11231) ;
passx->Random();
passb->PutScalar( 0.0 );
passA->Multiply( transpose, *passxexact, *passb ) ;
Epetra_MultiVector CopyB( *passb ) ;
double Anorm = passA->NormInf() ;
SparseDirectTimingVars::SS_Result.Set_Anorm(Anorm) ;
Epetra_LinearProblem Problem( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb );
double max_resid = 0.0;
for ( int j = 0 ; j < special+1 ; j++ ) {
Epetra_Time TotalTime( Comm ) ;
if ( false ) {
#ifdef TEST_UMFPACK
unused code
} else if ( SparseSolver == UMFPACK ) {
UmfpackOO umfpack( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb ) ;
umfpack.SetTrans( transpose ) ;
umfpack.Solve() ;
#endif
#ifdef TEST_SUPERLU
} else if ( SparseSolver == SuperLU ) {
SuperluserialOO superluserial( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb ) ;
superluserial.SetPermc( SuperLU_permc ) ;
superluserial.SetTrans( transpose ) ;
superluserial.SetUseDGSSV( special == 0 ) ;
superluserial.Solve() ;
#endif
#ifdef HAVE_AMESOS_SLUD
} else if ( SparseSolver == SuperLUdist ) {
SuperludistOO superludist( Problem ) ;
superludist.SetTrans( transpose ) ;
EPETRA_CHK_ERR( superludist.Solve( true ) ) ;
#endif
#ifdef HAVE_AMESOS_SLUD2
} else if ( SparseSolver == SuperLUdist2 ) {
Superludist2_OO superludist2( Problem ) ;
superludist2.SetTrans( transpose ) ;
EPETRA_CHK_ERR( superludist2.Solve( true ) ) ;
#endif
#ifdef TEST_SPOOLES
} else if ( SparseSolver == SPOOLES ) {
SpoolesOO spooles( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb ) ;
spooles.SetTrans( transpose ) ;
spooles.Solve() ;
#endif
#ifdef HAVE_AMESOS_DSCPACK
} else if ( SparseSolver == DSCPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Dscpack dscpack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( dscpack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( dscpack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_UMFPACK
} else if ( SparseSolver == UMFPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Umfpack umfpack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( umfpack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( umfpack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( umfpack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_KLU
} else if ( SparseSolver == KLU ) {
Teuchos::ParameterList ParamList ;
Amesos_Klu klu( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( klu.SetParameters( ParamList ) );
EPETRA_CHK_ERR( klu.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( klu.SymbolicFactorization( ) );
EPETRA_CHK_ERR( klu.NumericFactorization( ) );
EPETRA_CHK_ERR( klu.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARAKLETE
} else if ( SparseSolver == PARAKLETE ) {
Teuchos::ParameterList ParamList ;
Amesos_Paraklete paraklete( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( paraklete.SetParameters( ParamList ) );
EPETRA_CHK_ERR( paraklete.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( paraklete.SymbolicFactorization( ) );
EPETRA_CHK_ERR( paraklete.NumericFactorization( ) );
EPETRA_CHK_ERR( paraklete.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SLUS
} else if ( SparseSolver == SuperLU ) {
Epetra_SLU superluserial( &Problem ) ;
EPETRA_CHK_ERR( superluserial.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( superluserial.SymbolicFactorization( ) );
EPETRA_CHK_ERR( superluserial.NumericFactorization( ) );
EPETRA_CHK_ERR( superluserial.Solve( ) );
#endif
#ifdef HAVE_AMESOS_LAPACK
} else if ( SparseSolver == LAPACK ) {
Teuchos::ParameterList ParamList ;
ParamList.set( "MaxProcs", -3 );
Amesos_Lapack lapack( Problem ) ;
EPETRA_CHK_ERR( lapack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( lapack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( lapack.NumericFactorization( ) );
EPETRA_CHK_ERR( lapack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_TAUCS
} else if ( SparseSolver == TAUCS ) {
Teuchos::ParameterList ParamList ;
Amesos_Taucs taucs( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( taucs.SetParameters( ParamList ) );
EPETRA_CHK_ERR( taucs.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( taucs.SymbolicFactorization( ) );
EPETRA_CHK_ERR( taucs.NumericFactorization( ) );
EPETRA_CHK_ERR( taucs.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARDISO
} else if ( SparseSolver == PARDISO ) {
Teuchos::ParameterList ParamList ;
Amesos_Pardiso pardiso( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( pardiso.SetParameters( ParamList ) );
EPETRA_CHK_ERR( pardiso.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( pardiso.SymbolicFactorization( ) );
EPETRA_CHK_ERR( pardiso.NumericFactorization( ) );
EPETRA_CHK_ERR( pardiso.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARKLETE
} else if ( SparseSolver == PARKLETE ) {
Teuchos::ParameterList ParamList ;
Amesos_Parklete parklete( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( parklete.SetParameters( ParamList ) );
EPETRA_CHK_ERR( parklete.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( parklete.SymbolicFactorization( ) );
EPETRA_CHK_ERR( parklete.NumericFactorization( ) );
EPETRA_CHK_ERR( parklete.Solve( ) );
#endif
#ifdef HAVE_AMESOS_MUMPS
} else if ( SparseSolver == MUMPS ) {
Teuchos::ParameterList ParamList ;
Amesos_Mumps mumps( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( mumps.SetParameters( ParamList ) );
EPETRA_CHK_ERR( mumps.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( mumps.SymbolicFactorization( ) );
EPETRA_CHK_ERR( mumps.NumericFactorization( ) );
EPETRA_CHK_ERR( mumps.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SCALAPACK
} else if ( SparseSolver == SCALAPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Scalapack scalapack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( scalapack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( scalapack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( scalapack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( scalapack.NumericFactorization( ) );
EPETRA_CHK_ERR( scalapack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SUPERLUDIST
} else if ( SparseSolver == SUPERLUDIST ) {
Teuchos::ParameterList ParamList ;
Amesos_Superludist superludist( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( superludist.SetParameters( ParamList ) );
EPETRA_CHK_ERR( superludist.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( superludist.SymbolicFactorization( ) );
EPETRA_CHK_ERR( superludist.NumericFactorization( ) );
EPETRA_CHK_ERR( superludist.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SUPERLU
} else if ( SparseSolver == SUPERLU ) {
Teuchos::ParameterList ParamList ;
Amesos_Superlu superlu( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( superlu.SetParameters( ParamList ) );
EPETRA_CHK_ERR( superlu.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( superlu.SymbolicFactorization( ) );
EPETRA_CHK_ERR( superlu.NumericFactorization( ) );
EPETRA_CHK_ERR( superlu.Solve( ) );
#endif
#ifdef TEST_SPOOLESSERIAL
} else if ( SparseSolver == SPOOLESSERIAL ) {
SpoolesserialOO spoolesserial( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb ) ;
spoolesserial.Solve() ;
#endif
} else {
SparseDirectTimingVars::log_file << "Solver not implemented yet" << std::endl ;
std::cerr << "\n\n#################### Requested solver not available (Or not tested with blocked RHS) on this platform #####################\n" << std::endl ;
}
SparseDirectTimingVars::SS_Result.Set_Total_Time( TotalTime.ElapsedTime() );
// SparseDirectTimingVars::SS_Result.Set_First_Time( 0.0 );
// SparseDirectTimingVars::SS_Result.Set_Middle_Time( 0.0 );
// SparseDirectTimingVars::SS_Result.Set_Last_Time( 0.0 );
//
// Compute the error = norm(xcomp - xexact )
//
std::vector <double> error(numsolves) ;
double max_error = 0.0;
passresid->Update(1.0, *passx, -1.0, *passxexact, 0.0);
passresid->Norm2(&error[0]);
for ( int i = 0 ; i< numsolves; i++ )
if ( error[i] > max_error ) max_error = error[i] ;
SparseDirectTimingVars::SS_Result.Set_Error(max_error) ;
// passxexact->Norm2(&error[0] ) ;
// passx->Norm2(&error ) ;
//
// Compute the residual = norm(Ax - b)
//
std::vector <double> residual(numsolves) ;
passtmp->PutScalar(0.0);
passA->Multiply( transpose, *passx, *passtmp);
passresid->Update(1.0, *passtmp, -1.0, *passb, 0.0);
// passresid->Update(1.0, *passtmp, -1.0, CopyB, 0.0);
passresid->Norm2(&residual[0]);
for ( int i = 0 ; i< numsolves; i++ )
if ( residual[i] > max_resid ) max_resid = residual[i] ;
SparseDirectTimingVars::SS_Result.Set_Residual(max_resid) ;
std::vector <double> bnorm(numsolves);
passb->Norm2( &bnorm[0] ) ;
SparseDirectTimingVars::SS_Result.Set_Bnorm(bnorm[0]) ;
std::vector <double> xnorm(numsolves);
passx->Norm2( &xnorm[0] ) ;
SparseDirectTimingVars::SS_Result.Set_Xnorm(xnorm[0]) ;
if ( false && iam == 0 ) {
std::cout << " Amesos_TestMutliSolver.cpp " << std::endl ;
for ( int i = 0 ; i< numsolves && i < 10 ; i++ ) {
std::cout << "i=" << i
<< " error = " << error[i]
<< " xnorm = " << xnorm[i]
<< " residual = " << residual[i]
<< " bnorm = " << bnorm[i]
<< std::endl ;
}
std::cout << std::endl << " max_resid = " << max_resid ;
std::cout << " max_error = " << max_error << std::endl ;
std::cout << " Get_residual() again = " << SparseDirectTimingVars::SS_Result.Get_Residual() << std::endl ;
}
}
delete readA;
delete readx;
delete readb;
delete readxexact;
delete readMap;
delete map_;
Comm.Barrier();
return 0 ;
}
void AZOO_iterate(double * xsolve, double * b,
int * options, double * params,
double * status, int *proc_config,
AZ_MATRIX * Amat,
AZ_PRECOND *precond, struct AZ_SCALING *scaling)
{
(void)precond;
(void)scaling;
bool verbose = (options[AZ_output]!=AZ_none); // Print info unless all output is turned off
Epetra_Comm * comm;
Epetra_BlockMap * map;
Epetra_RowMatrix * A;
Epetra_Vector * px;
Epetra_Vector * pb;
int * global_indices;
int ierr = Aztec2Petra(proc_config, Amat, xsolve, b, comm, map, A, px, pb, &global_indices);
if (ierr!=0) {
cerr << "Error detected in Aztec2Petra. Value = " << ierr << endl;
exit(1);
}
Epetra_LinearProblem problem(A, px, pb);
Epetra_Vector * leftScaleVec = 0;
Epetra_Vector * rightScaleVec = 0;
bool doRowScaling = false;
bool doColScaling = false;
if ((options[AZ_scaling]==AZ_Jacobi) || options[AZ_scaling]==AZ_BJacobi) {
doRowScaling = true;
leftScaleVec = new Epetra_Vector(*map);
A->ExtractDiagonalCopy(*leftScaleVec); // Extract diagonal of matrix
leftScaleVec->Reciprocal(*leftScaleVec); // invert it
}
else if (options[AZ_scaling]==AZ_row_sum) {
doRowScaling = true;
leftScaleVec = new Epetra_Vector(*map);
A->InvRowSums(*leftScaleVec);
}
else if (options[AZ_scaling]==AZ_sym_diag) {
doRowScaling = true;
doColScaling = true;
leftScaleVec = new Epetra_Vector(*map);
A->ExtractDiagonalCopy(*leftScaleVec); // Extract diagonal of matrix
int length = leftScaleVec->MyLength();
for (int i=0; i<length; i++) (*leftScaleVec)[i] = sqrt(fabs((*leftScaleVec)[i])); // Take its sqrt
rightScaleVec = leftScaleVec; // symmetric, so left and right the same
leftScaleVec->Reciprocal(*leftScaleVec); // invert it
}
else if (options[AZ_scaling]==AZ_sym_row_sum) {
doRowScaling = true;
doColScaling = true;
leftScaleVec = new Epetra_Vector(*map);
A->InvRowSums(*leftScaleVec);
int length = leftScaleVec->MyLength();
for (int i=0; i<length; i++) (*leftScaleVec)[i] = sqrt(fabs((*leftScaleVec)[i])); // Take its sqrt
rightScaleVec = leftScaleVec; // symmetric, so left and right the same
}
if ((doRowScaling || doColScaling) && verbose) {
double norminf = A->NormInf();
double normone = A->NormOne();
if (comm->MyPID()==0)
cout << "\n Inf-norm of A before scaling = " << norminf
<< "\n One-norm of A before scaling = " << normone<< endl << endl;
}
if (doRowScaling) problem.LeftScale(*leftScaleVec);
if (doColScaling) problem.RightScale(*rightScaleVec);
if ((doRowScaling || doColScaling) && verbose) {
double norminf = A->NormInf();
double normone = A->NormOne();
if (comm->MyPID()==0)
cout << "\n Inf-norm of A after scaling = " << norminf
<< "\n One-norm of A after scaling = " << normone << endl << endl;
}
AztecOO solver(problem);
solver.SetAllAztecParams(params); // set all AztecOO params with user-provided params
solver.SetAllAztecOptions(options); // set all AztecOO options with user-provided options
solver.CheckInput();
solver.SetAztecOption(AZ_scaling, AZ_none); // Always must have scaling off
solver.Iterate(options[AZ_max_iter], params[AZ_tol]);
solver.GetAllAztecStatus(status);
if (doColScaling) {
rightScaleVec->Reciprocal(*rightScaleVec);
problem.RightScale(*rightScaleVec);
}
if (doRowScaling) {
leftScaleVec->Reciprocal(*leftScaleVec);
problem.LeftScale(*leftScaleVec);
}
if ((rightScaleVec!=0) && (rightScaleVec!=leftScaleVec)) delete rightScaleVec;
if (leftScaleVec!=0) delete leftScaleVec;
delete pb; // These are all objects created here and we have to delete them
delete px;
delete A;
delete map;
delete comm;
if (global_indices!=0) AZ_free((void *) global_indices); // Note: we used a special version of free here
return;
}
int Ifpack_Analyze(const Epetra_RowMatrix& A, const bool Cheap,
const int NumPDEEqns)
{
int NumMyRows = A.NumMyRows();
long long NumGlobalRows = A.NumGlobalRows64();
long long NumGlobalCols = A.NumGlobalCols64();
long long MyBandwidth = 0, GlobalBandwidth;
long long MyLowerNonzeros = 0, MyUpperNonzeros = 0;
long long GlobalLowerNonzeros, GlobalUpperNonzeros;
long long MyDiagonallyDominant = 0, GlobalDiagonallyDominant;
long long MyWeaklyDiagonallyDominant = 0, GlobalWeaklyDiagonallyDominant;
double MyMin, MyAvg, MyMax;
double GlobalMin, GlobalAvg, GlobalMax;
long long GlobalStorage;
bool verbose = (A.Comm().MyPID() == 0);
GlobalStorage = sizeof(int*) * NumGlobalRows +
sizeof(int) * A.NumGlobalNonzeros64() +
sizeof(double) * A.NumGlobalNonzeros64();
if (verbose) {
print();
Ifpack_PrintLine();
print<const char*>("Label", A.Label());
print<long long>("Global rows", NumGlobalRows);
print<long long>("Global columns", NumGlobalCols);
print<long long>("Stored nonzeros", A.NumGlobalNonzeros64());
print<long long>("Nonzeros / row", A.NumGlobalNonzeros64() / NumGlobalRows);
print<double>("Estimated storage (Mbytes)", 1.0e-6 * GlobalStorage);
}
long long NumMyActualNonzeros = 0, NumGlobalActualNonzeros;
long long NumMyEmptyRows = 0, NumGlobalEmptyRows;
long long NumMyDirichletRows = 0, NumGlobalDirichletRows;
std::vector<int> colInd(A.MaxNumEntries());
std::vector<double> colVal(A.MaxNumEntries());
Epetra_Vector Diag(A.RowMatrixRowMap());
Epetra_Vector RowSum(A.RowMatrixRowMap());
Diag.PutScalar(0.0);
RowSum.PutScalar(0.0);
for (int i = 0 ; i < NumMyRows ; ++i) {
long long GRID = A.RowMatrixRowMap().GID64(i);
int Nnz;
IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz,
&colVal[0],&colInd[0]));
if (Nnz == 0)
NumMyEmptyRows++;
if (Nnz == 1)
NumMyDirichletRows++;
for (int j = 0 ; j < Nnz ; ++j) {
double v = colVal[j];
if (v < 0) v = -v;
if (colVal[j] != 0.0)
NumMyActualNonzeros++;
long long GCID = A.RowMatrixColMap().GID64(colInd[j]);
if (GCID != GRID)
RowSum[i] += v;
else
Diag[i] = v;
if (GCID < GRID)
MyLowerNonzeros++;
else if (GCID > GRID)
MyUpperNonzeros++;
long long b = GCID - GRID;
if (b < 0) b = -b;
if (b > MyBandwidth)
MyBandwidth = b;
}
if (Diag[i] > RowSum[i])
MyDiagonallyDominant++;
if (Diag[i] >= RowSum[i])
MyWeaklyDiagonallyDominant++;
RowSum[i] += Diag[i];
}
// ======================== //
// summing up global values //
// ======================== //
A.Comm().SumAll(&MyDiagonallyDominant,&GlobalDiagonallyDominant,1);
A.Comm().SumAll(&MyWeaklyDiagonallyDominant,&GlobalWeaklyDiagonallyDominant,1);
A.Comm().SumAll(&NumMyActualNonzeros, &NumGlobalActualNonzeros, 1);
A.Comm().SumAll(&NumMyEmptyRows, &NumGlobalEmptyRows, 1);
A.Comm().SumAll(&NumMyDirichletRows, &NumGlobalDirichletRows, 1);
A.Comm().SumAll(&MyBandwidth, &GlobalBandwidth, 1);
A.Comm().SumAll(&MyLowerNonzeros, &GlobalLowerNonzeros, 1);
A.Comm().SumAll(&MyUpperNonzeros, &GlobalUpperNonzeros, 1);
A.Comm().SumAll(&MyDiagonallyDominant, &GlobalDiagonallyDominant, 1);
A.Comm().SumAll(&MyWeaklyDiagonallyDominant, &GlobalWeaklyDiagonallyDominant, 1);
double NormOne = A.NormOne();
double NormInf = A.NormInf();
double NormF = Ifpack_FrobeniusNorm(A);
if (verbose) {
print();
print<long long>("Actual nonzeros", NumGlobalActualNonzeros);
print<long long>("Nonzeros in strict lower part", GlobalLowerNonzeros);
print<long long>("Nonzeros in strict upper part", GlobalUpperNonzeros);
print();
print<long long>("Empty rows", NumGlobalEmptyRows,
100.0 * NumGlobalEmptyRows / NumGlobalRows);
print<long long>("Dirichlet rows", NumGlobalDirichletRows,
100.0 * NumGlobalDirichletRows / NumGlobalRows);
print<long long>("Diagonally dominant rows", GlobalDiagonallyDominant,
100.0 * GlobalDiagonallyDominant / NumGlobalRows);
print<long long>("Weakly diag. dominant rows",
GlobalWeaklyDiagonallyDominant,
100.0 * GlobalWeaklyDiagonallyDominant / NumGlobalRows);
print();
print<long long>("Maximum bandwidth", GlobalBandwidth);
print();
print("", "one-norm", "inf-norm", "Frobenius", false);
print("", "========", "========", "=========", false);
print();
print<double>("A", NormOne, NormInf, NormF);
}
if (Cheap == false) {
// create A + A^T and A - A^T
Epetra_FECrsMatrix AplusAT(Copy, A.RowMatrixRowMap(), 0);
Epetra_FECrsMatrix AminusAT(Copy, A.RowMatrixRowMap(), 0);
#ifndef EPETRA_NO_32BIT_GLOBAL_INDICES
if(A.RowMatrixRowMap().GlobalIndicesInt()) {
for (int i = 0 ; i < NumMyRows ; ++i) {
int GRID = A.RowMatrixRowMap().GID(i);
assert (GRID != -1);
int Nnz;
IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz,
&colVal[0],&colInd[0]));
for (int j = 0 ; j < Nnz ; ++j) {
int GCID = A.RowMatrixColMap().GID(colInd[j]);
assert (GCID != -1);
double plus_val = colVal[j];
double minus_val = -colVal[j];
if (AplusAT.SumIntoGlobalValues(1,&GRID,1,&GCID,&plus_val) != 0) {
IFPACK_CHK_ERR(AplusAT.InsertGlobalValues(1,&GRID,1,&GCID,&plus_val));
}
if (AplusAT.SumIntoGlobalValues(1,&GCID,1,&GRID,&plus_val) != 0) {
IFPACK_CHK_ERR(AplusAT.InsertGlobalValues(1,&GCID,1,&GRID,&plus_val));
}
if (AminusAT.SumIntoGlobalValues(1,&GRID,1,&GCID,&plus_val) != 0) {
IFPACK_CHK_ERR(AminusAT.InsertGlobalValues(1,&GRID,1,&GCID,&plus_val));
}
if (AminusAT.SumIntoGlobalValues(1,&GCID,1,&GRID,&minus_val) != 0) {
IFPACK_CHK_ERR(AminusAT.InsertGlobalValues(1,&GCID,1,&GRID,&minus_val));
}
}
}
}
else
#endif
#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
if(A.RowMatrixRowMap().GlobalIndicesLongLong()) {
for (int i = 0 ; i < NumMyRows ; ++i) {
long long GRID = A.RowMatrixRowMap().GID64(i);
assert (GRID != -1);
int Nnz;
IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz,
&colVal[0],&colInd[0]));
for (int j = 0 ; j < Nnz ; ++j) {
long long GCID = A.RowMatrixColMap().GID64(colInd[j]);
assert (GCID != -1);
double plus_val = colVal[j];
double minus_val = -colVal[j];
if (AplusAT.SumIntoGlobalValues(1,&GRID,1,&GCID,&plus_val) != 0) {
IFPACK_CHK_ERR(AplusAT.InsertGlobalValues(1,&GRID,1,&GCID,&plus_val));
}
if (AplusAT.SumIntoGlobalValues(1,&GCID,1,&GRID,&plus_val) != 0) {
IFPACK_CHK_ERR(AplusAT.InsertGlobalValues(1,&GCID,1,&GRID,&plus_val));
}
if (AminusAT.SumIntoGlobalValues(1,&GRID,1,&GCID,&plus_val) != 0) {
IFPACK_CHK_ERR(AminusAT.InsertGlobalValues(1,&GRID,1,&GCID,&plus_val));
}
if (AminusAT.SumIntoGlobalValues(1,&GCID,1,&GRID,&minus_val) != 0) {
IFPACK_CHK_ERR(AminusAT.InsertGlobalValues(1,&GCID,1,&GRID,&minus_val));
}
}
}
}
else
#endif
throw "Ifpack_Analyze: GlobalIndices type unknown";
AplusAT.FillComplete();
AminusAT.FillComplete();
AplusAT.Scale(0.5);
AminusAT.Scale(0.5);
NormOne = AplusAT.NormOne();
NormInf = AplusAT.NormInf();
NormF = Ifpack_FrobeniusNorm(AplusAT);
if (verbose) {
print<double>("A + A^T", NormOne, NormInf, NormF);
}
NormOne = AminusAT.NormOne();
NormInf = AminusAT.NormInf();
NormF = Ifpack_FrobeniusNorm(AminusAT);
if (verbose) {
print<double>("A - A^T", NormOne, NormInf, NormF);
}
}
if (verbose) {
print();
print<const char*>("", "min", "avg", "max", false);
print<const char*>("", "===", "===", "===", false);
}
MyMax = -DBL_MAX;
MyMin = DBL_MAX;
MyAvg = 0.0;
for (int i = 0 ; i < NumMyRows ; ++i) {
int Nnz;
IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz,
&colVal[0],&colInd[0]));
for (int j = 0 ; j < Nnz ; ++j) {
MyAvg += colVal[j];
if (colVal[j] > MyMax) MyMax = colVal[j];
if (colVal[j] < MyMin) MyMin = colVal[j];
}
}
A.Comm().MaxAll(&MyMax, &GlobalMax, 1);
A.Comm().MinAll(&MyMin, &GlobalMin, 1);
A.Comm().SumAll(&MyAvg, &GlobalAvg, 1);
GlobalAvg /= A.NumGlobalNonzeros64();
if (verbose) {
print();
print<double>(" A(i,j)", GlobalMin, GlobalAvg, GlobalMax);
}
MyMax = 0.0;
MyMin = DBL_MAX;
MyAvg = 0.0;
for (int i = 0 ; i < NumMyRows ; ++i) {
int Nnz;
IFPACK_CHK_ERR(A.ExtractMyRowCopy(i,A.MaxNumEntries(),Nnz,
&colVal[0],&colInd[0]));
for (int j = 0 ; j < Nnz ; ++j) {
double v = colVal[j];
if (v < 0) v = -v;
MyAvg += v;
if (colVal[j] > MyMax) MyMax = v;
if (colVal[j] < MyMin) MyMin = v;
}
}
A.Comm().MaxAll(&MyMax, &GlobalMax, 1);
A.Comm().MinAll(&MyMin, &GlobalMin, 1);
A.Comm().SumAll(&MyAvg, &GlobalAvg, 1);
GlobalAvg /= A.NumGlobalNonzeros64();
if (verbose) {
print<double>("|A(i,j)|", GlobalMin, GlobalAvg, GlobalMax);
}
// ================= //
// diagonal elements //
// ================= //
Diag.MinValue(&GlobalMin);
Diag.MaxValue(&GlobalMax);
Diag.MeanValue(&GlobalAvg);
if (verbose) {
print();
print<double>(" A(k,k)", GlobalMin, GlobalAvg, GlobalMax);
}
Diag.Abs(Diag);
Diag.MinValue(&GlobalMin);
Diag.MaxValue(&GlobalMax);
Diag.MeanValue(&GlobalAvg);
if (verbose) {
print<double>("|A(k,k)|", GlobalMin, GlobalAvg, GlobalMax);
}
// ============================================== //
// cycle over all equations for diagonal elements //
// ============================================== //
if (NumPDEEqns > 1 ) {
if (verbose) print();
for (int ie = 0 ; ie < NumPDEEqns ; ie++) {
MyMin = DBL_MAX;
MyMax = -DBL_MAX;
MyAvg = 0.0;
for (int i = ie ; i < Diag.MyLength() ; i += NumPDEEqns) {
double d = Diag[i];
MyAvg += d;
if (d < MyMin)
MyMin = d;
if (d > MyMax)
MyMax = d;
}
A.Comm().MinAll(&MyMin, &GlobalMin, 1);
A.Comm().MaxAll(&MyMax, &GlobalMax, 1);
A.Comm().SumAll(&MyAvg, &GlobalAvg, 1);
// does not really work fine if the number of global
// elements is not a multiple of NumPDEEqns
GlobalAvg /= (Diag.GlobalLength64() / NumPDEEqns);
if (verbose) {
char str[80];
sprintf(str, " A(k,k), eq %d", ie);
print<double>(str, GlobalMin, GlobalAvg, GlobalMax);
}
}
}
// ======== //
// row sums //
// ======== //
RowSum.MinValue(&GlobalMin);
RowSum.MaxValue(&GlobalMax);
RowSum.MeanValue(&GlobalAvg);
if (verbose) {
print();
print<double>(" sum_j A(k,j)", GlobalMin, GlobalAvg, GlobalMax);
}
// ===================================== //
// cycle over all equations for row sums //
// ===================================== //
if (NumPDEEqns > 1 ) {
if (verbose) print();
for (int ie = 0 ; ie < NumPDEEqns ; ie++) {
MyMin = DBL_MAX;
MyMax = -DBL_MAX;
MyAvg = 0.0;
for (int i = ie ; i < Diag.MyLength() ; i += NumPDEEqns) {
double d = RowSum[i];
MyAvg += d;
if (d < MyMin)
MyMin = d;
if (d > MyMax)
MyMax = d;
}
A.Comm().MinAll(&MyMin, &GlobalMin, 1);
A.Comm().MaxAll(&MyMax, &GlobalMax, 1);
A.Comm().SumAll(&MyAvg, &GlobalAvg, 1);
// does not really work fine if the number of global
// elements is not a multiple of NumPDEEqns
GlobalAvg /= (Diag.GlobalLength64() / NumPDEEqns);
if (verbose) {
char str[80];
sprintf(str, " sum_j A(k,j), eq %d", ie);
print<double>(str, GlobalMin, GlobalAvg, GlobalMax);
}
}
}
if (verbose)
Ifpack_PrintLine();
return(0);
}
int Amesos_TestSolver( Epetra_Comm &Comm, char *matrix_file,
SparseSolverType SparseSolver,
bool transpose,
int special, AMESOS_MatrixType matrix_type ) {
Epetra_Map * readMap;
Epetra_CrsMatrix * readA;
Epetra_Vector * readx;
Epetra_Vector * readb;
Epetra_Vector * readxexact;
std::string FileName = matrix_file ;
int FN_Size = FileName.size() ;
std::string LastFiveBytes = FileName.substr( EPETRA_MAX(0,FN_Size-5), FN_Size );
std::string LastFourBytes = FileName.substr( EPETRA_MAX(0,FN_Size-4), FN_Size );
bool NonContiguousMap = false;
if ( LastFiveBytes == ".triU" ) {
// Call routine to read in unsymmetric Triplet matrix
NonContiguousMap = true;
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, false, Comm, readMap, readA, readx,
readb, readxexact, NonContiguousMap ) );
} else {
if ( LastFiveBytes == ".triS" ) {
NonContiguousMap = true;
// Call routine to read in symmetric Triplet matrix
EPETRA_CHK_ERR( Trilinos_Util_ReadTriples2Epetra( matrix_file, true, Comm, readMap, readA, readx,
readb, readxexact, NonContiguousMap ) );
} else {
if ( LastFourBytes == ".mtx" ) {
EPETRA_CHK_ERR( Trilinos_Util_ReadMatrixMarket2Epetra( matrix_file, Comm, readMap,
readA, readx, readb, readxexact) );
} else {
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra( matrix_file, Comm, readMap, readA, readx,
readb, readxexact) ;
}
}
}
Epetra_CrsMatrix transposeA(Copy, *readMap, 0);
Epetra_CrsMatrix *serialA ;
if ( transpose ) {
assert( CrsMatrixTranspose( readA, &transposeA ) == 0 );
serialA = &transposeA ;
} else {
serialA = readA ;
}
Epetra_RowMatrix * passA = 0;
Epetra_Vector * passx = 0;
Epetra_Vector * passb = 0;
Epetra_Vector * passxexact = 0;
Epetra_Vector * passresid = 0;
Epetra_Vector * passtmp = 0;
// Create uniform distributed map
Epetra_Map map(readMap->NumGlobalElements(), 0, Comm);
Epetra_Map* map_;
if( NonContiguousMap ) {
//
// map gives us NumMyElements and MyFirstElement;
//
int NumGlobalElements = readMap->NumGlobalElements();
int NumMyElements = map.NumMyElements();
int MyFirstElement = map.MinMyGID();
std::vector<int> MapMap_( NumGlobalElements );
readMap->MyGlobalElements( &MapMap_[0] ) ;
Comm.Broadcast( &MapMap_[0], NumGlobalElements, 0 ) ;
map_ = new Epetra_Map( NumGlobalElements, NumMyElements, &MapMap_[MyFirstElement], 0, Comm);
} else {
map_ = new Epetra_Map( map ) ;
}
Epetra_CrsMatrix A(Copy, *map_, 0);
const Epetra_Map &OriginalMap = serialA->RowMatrixRowMap() ;
assert( OriginalMap.SameAs(*readMap) );
Epetra_Export exporter(OriginalMap, *map_);
Epetra_Export exporter2(OriginalMap, *map_);
Epetra_Export MatrixExporter(OriginalMap, *map_);
Epetra_CrsMatrix AwithDiag(Copy, *map_, 0);
Epetra_Vector x(*map_);
Epetra_Vector b(*map_);
Epetra_Vector xexact(*map_);
Epetra_Vector resid(*map_);
Epetra_Vector readresid(*readMap);
Epetra_Vector tmp(*map_);
Epetra_Vector readtmp(*readMap);
// Epetra_Vector xcomp(*map_); // X as computed by the solver
bool distribute_matrix = ( matrix_type == AMESOS_Distributed ) ;
if ( distribute_matrix ) {
// Create Exporter to distribute read-in matrix and vectors
//
// Initialize x, b and xexact to the values read in from the file
//
x.Export(*readx, exporter, Add);
b.Export(*readb, exporter, Add);
xexact.Export(*readxexact, exporter, Add);
Comm.Barrier();
A.Export(*serialA, exporter, Add);
assert(A.FillComplete()==0);
Comm.Barrier();
passA = &A;
passx = &x;
passb = &b;
passxexact = &xexact;
passresid = &resid;
passtmp = &tmp;
} else {
passA = serialA;
passx = readx;
passb = readb;
passxexact = readxexact;
passresid = &readresid;
passtmp = &readtmp;
}
Epetra_MultiVector CopyB( *passb ) ;
double Anorm = passA->NormInf() ;
SparseDirectTimingVars::SS_Result.Set_Anorm(Anorm) ;
Epetra_LinearProblem Problem( (Epetra_RowMatrix *) passA,
(Epetra_MultiVector *) passx,
(Epetra_MultiVector *) passb );
for ( int i = 0; i < 1+special ; i++ ) {
Epetra_Time TotalTime( Comm ) ;
if ( false ) {
// TEST_UMFPACK is never set by configure
#ifdef HAVE_AMESOS_SUPERLUDIST
} else if ( SparseSolver == SUPERLUDIST ) {
Teuchos::ParameterList ParamList ;
ParamList.set( "MaxProcs", -3 );
Amesos_Superludist A_Superludist( Problem ) ;
//ParamList.set( "Redistribute", true );
//ParamList.set( "AddZeroToDiag", true );
Teuchos::ParameterList& SuperludistParams = ParamList.sublist("Superludist") ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_Superludist.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_Superludist.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_Superludist.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_Superludist.NumericFactorization( ) );
EPETRA_CHK_ERR( A_Superludist.Solve( ) );
#endif
#ifdef HAVE_AMESOS_DSCPACK
} else if ( SparseSolver == DSCPACK ) {
Teuchos::ParameterList ParamList ;
ParamList.set( "MaxProcs", -3 );
Amesos_Dscpack A_dscpack( Problem ) ;
EPETRA_CHK_ERR( A_dscpack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_dscpack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_dscpack.NumericFactorization( ) );
EPETRA_CHK_ERR( A_dscpack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SCALAPACK
} else if ( SparseSolver == SCALAPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Scalapack A_scalapack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_scalapack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_scalapack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_scalapack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_scalapack.NumericFactorization( ) );
EPETRA_CHK_ERR( A_scalapack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_TAUCS
} else if ( SparseSolver == TAUCS ) {
Teuchos::ParameterList ParamList ;
Amesos_Taucs A_taucs( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_taucs.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_taucs.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_taucs.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_taucs.NumericFactorization( ) );
EPETRA_CHK_ERR( A_taucs.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARDISO
} else if ( SparseSolver == PARDISO ) {
Teuchos::ParameterList ParamList ;
Amesos_Pardiso A_pardiso( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_pardiso.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_pardiso.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_pardiso.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_pardiso.NumericFactorization( ) );
EPETRA_CHK_ERR( A_pardiso.Solve( ) );
#endif
#ifdef HAVE_AMESOS_PARAKLETE
} else if ( SparseSolver == PARAKLETE ) {
Teuchos::ParameterList ParamList ;
Amesos_Paraklete A_paraklete( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_paraklete.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_paraklete.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_paraklete.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_paraklete.NumericFactorization( ) );
EPETRA_CHK_ERR( A_paraklete.Solve( ) );
#endif
#ifdef HAVE_AMESOS_MUMPS
} else if ( SparseSolver == MUMPS ) {
Teuchos::ParameterList ParamList ;
Amesos_Mumps A_mumps( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_mumps.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_mumps.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_mumps.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_mumps.NumericFactorization( ) );
EPETRA_CHK_ERR( A_mumps.Solve( ) );
#endif
#ifdef HAVE_AMESOS_SUPERLU
} else if ( SparseSolver == SUPERLU ) {
Teuchos::ParameterList ParamList ;
Amesos_Superlu A_superlu( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_superlu.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_superlu.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_superlu.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_superlu.NumericFactorization( ) );
EPETRA_CHK_ERR( A_superlu.Solve( ) );
#endif
#ifdef HAVE_AMESOS_LAPACK
} else if ( SparseSolver == LAPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Lapack A_lapack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_lapack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_lapack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_lapack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_lapack.NumericFactorization( ) );
EPETRA_CHK_ERR( A_lapack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_UMFPACK
} else if ( SparseSolver == UMFPACK ) {
Teuchos::ParameterList ParamList ;
Amesos_Umfpack A_umfpack( Problem ) ;
ParamList.set( "MaxProcs", -3 );
EPETRA_CHK_ERR( A_umfpack.SetParameters( ParamList ) );
EPETRA_CHK_ERR( A_umfpack.SetUseTranspose( transpose ) );
EPETRA_CHK_ERR( A_umfpack.SymbolicFactorization( ) );
EPETRA_CHK_ERR( A_umfpack.NumericFactorization( ) );
EPETRA_CHK_ERR( A_umfpack.Solve( ) );
#endif
#ifdef HAVE_AMESOS_KLU
} else if ( SparseSolver == KLU ) {
using namespace Teuchos;
Amesos_Time AT;
int setupTimePtr = -1, symTimePtr = -1, numTimePtr = -1, refacTimePtr = -1, solveTimePtr = -1;
AT.CreateTimer(Comm, 2);
AT.ResetTimer(0);
Teuchos::ParameterList ParamList ;
// ParamList.set("OutputLevel",2);
Amesos_Klu A_klu( Problem );
ParamList.set( "MaxProcs", -3 );
ParamList.set( "TrustMe", false );
// ParamList.set( "Refactorize", true );
EPETRA_CHK_ERR( A_klu.SetParameters( ParamList ) ) ;
EPETRA_CHK_ERR( A_klu.SetUseTranspose( transpose ) );
setupTimePtr = AT.AddTime("Setup", setupTimePtr, 0);
EPETRA_CHK_ERR( A_klu.SymbolicFactorization( ) );
symTimePtr = AT.AddTime("Symbolic", symTimePtr, 0);
EPETRA_CHK_ERR( A_klu.NumericFactorization( ) );
numTimePtr = AT.AddTime("Numeric", numTimePtr, 0);
EPETRA_CHK_ERR( A_klu.NumericFactorization( ) );
refacTimePtr = AT.AddTime("Refactor", refacTimePtr, 0);
// for ( int i=0; i<100000 ; i++ )
EPETRA_CHK_ERR( A_klu.Solve( ) );
solveTimePtr = AT.AddTime("Solve", solveTimePtr, 0);
double SetupTime = AT.GetTime(setupTimePtr);
double SymbolicTime = AT.GetTime(symTimePtr);
double NumericTime = AT.GetTime(numTimePtr);
double RefactorTime = AT.GetTime(refacTimePtr);
double SolveTime = AT.GetTime(solveTimePtr);
std::cout << __FILE__ << "::" << __LINE__ << " SetupTime = " << SetupTime << std::endl ;
std::cout << __FILE__ << "::" << __LINE__ << " SymbolicTime = " << SymbolicTime - SetupTime << std::endl ;
std::cout << __FILE__ << "::" << __LINE__ << " NumericTime = " << NumericTime - SymbolicTime<< std::endl ;
std::cout << __FILE__ << "::" << __LINE__ << " RefactorTime = " << RefactorTime - NumericTime << std::endl ;
std::cout << __FILE__ << "::" << __LINE__ << " SolveTime = " << SolveTime - RefactorTime << std::endl ;
#endif
} else {
SparseDirectTimingVars::log_file << "Solver not implemented yet" << std::endl ;
std::cerr << "\n\n#################### Requested solver not available on this platform ##################### ATS\n" << std::endl ;
std::cout << " SparseSolver = " << SparseSolver << std::endl ;
std::cerr << " SparseSolver = " << SparseSolver << std::endl ;
}
SparseDirectTimingVars::SS_Result.Set_Total_Time( TotalTime.ElapsedTime() );
} // end for (int i=0; i<special; i++ )
//
// Compute the error = norm(xcomp - xexact )
//
double error;
passresid->Update(1.0, *passx, -1.0, *passxexact, 0.0);
passresid->Norm2(&error);
SparseDirectTimingVars::SS_Result.Set_Error(error) ;
// passxexact->Norm2(&error ) ;
// passx->Norm2(&error ) ;
//
// Compute the residual = norm(Ax - b)
//
double residual ;
passA->Multiply( transpose, *passx, *passtmp);
passresid->Update(1.0, *passtmp, -1.0, *passb, 0.0);
// passresid->Update(1.0, *passtmp, -1.0, CopyB, 0.0);
passresid->Norm2(&residual);
SparseDirectTimingVars::SS_Result.Set_Residual(residual) ;
double bnorm;
passb->Norm2( &bnorm ) ;
SparseDirectTimingVars::SS_Result.Set_Bnorm(bnorm) ;
double xnorm;
passx->Norm2( &xnorm ) ;
SparseDirectTimingVars::SS_Result.Set_Xnorm(xnorm) ;
delete readA;
delete readx;
delete readb;
delete readxexact;
delete readMap;
delete map_;
Comm.Barrier();
return 0;
}
