// ===========================================================================
Epetra_Import& Amesos_Mumps::SerialImporter()
{
if (SerialImporter_ == null) {
SerialImporter_ = rcp(new Epetra_Import(SerialMap(),Matrix().OperatorDomainMap()));
assert (SerialImporter_ != null);
}
return(*SerialImporter_);
}
//=============================================================================
int Amesos_Umfpack::Solve()
{
// if necessary, perform numeric factorization.
// This may call SymbolicFactorization() as well.
if (!IsNumericFactorizationOK_)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
Epetra_MultiVector *SerialB, *SerialX;
// Extract Serial versions of X and B
//
double *SerialXvalues ;
double *SerialBvalues ;
Epetra_MultiVector* SerialXextract = 0;
Epetra_MultiVector* SerialBextract = 0;
// Copy B to the serial version of B
//
ResetTimer(0);
if (IsLocal_ == 1) {
SerialB = vecB ;
SerialX = vecX ;
} else {
assert (IsLocal_ == 0);
SerialXextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract->Import(*vecB,Importer(),Insert);
SerialB = SerialBextract;
SerialX = SerialXextract;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call UMFPACK to perform the solve
// Note: UMFPACK uses a Compressed Column Storage instead of compressed row storage,
// Hence to compute A X = B, we ask UMFPACK to perform A^T X = B and vice versa
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
ResetTimer(0);
int SerialBlda, SerialXlda ;
int UmfpackRequest = UseTranspose()?UMFPACK_A:UMFPACK_At ;
int status = 0;
if ( MyPID_ == 0 ) {
int ierr;
ierr = SerialB->ExtractView(&SerialBvalues, &SerialBlda);
assert (ierr == 0);
ierr = SerialX->ExtractView(&SerialXvalues, &SerialXlda);
assert (ierr == 0);
assert( SerialBlda == NumGlobalElements_ ) ;
assert( SerialXlda == NumGlobalElements_ ) ;
for ( int j =0 ; j < NumVectors; j++ ) {
double *Control = (double *) NULL, *Info = (double *) NULL ;
status = umfpack_di_solve (UmfpackRequest, &Ap[0],
&Ai[0], &Aval[0],
&SerialXvalues[j*SerialXlda],
&SerialBvalues[j*SerialBlda],
Numeric, Control, Info) ;
}
}
if (status) AMESOS_CHK_ERR(status);
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if ( IsLocal_ == 0 ) {
vecX->Export(*SerialX, Importer(), Insert ) ;
if (SerialBextract) delete SerialBextract ;
if (SerialXextract) delete SerialXextract ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
{
Epetra_RowMatrix* Matrix =
dynamic_cast<Epetra_RowMatrix*>(Problem_->GetOperator());
ComputeTrueResidual(*Matrix, *vecX, *vecB, UseTranspose(), "Amesos_Umfpack");
}
if (ComputeVectorNorms_) {
ComputeVectorNorms(*vecX, *vecB, "Amesos_Umfpack");
}
NumSolve_++;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); // Amesos overhead
return(0);
}
//=============================================================================
// If FirstTime is true, then build SerialMap and ImportToSerial,
// otherwise simply re-ship the matrix, so that the numerical values
// are updated.
int Amesos_Umfpack::ConvertToSerial(const bool FirstTime)
{
ResetTimer(0);
ResetTimer(1);
const Epetra_Map &OriginalMap = Matrix()->RowMatrixRowMap() ;
NumGlobalElements_ = Matrix()->NumGlobalRows();
numentries_ = Matrix()->NumGlobalNonzeros();
assert (NumGlobalElements_ == Matrix()->NumGlobalCols());
int NumMyElements_ = 0 ;
if (MyPID_ == 0) NumMyElements_ = NumGlobalElements_;
IsLocal_ = ( OriginalMap.NumMyElements() ==
OriginalMap.NumGlobalElements() )?1:0;
// if ( AddZeroToDiag_ ) IsLocal_ = 0 ; // bug # Umfpack does not support AddZeroToDiag_
Comm().Broadcast( &IsLocal_, 1, 0 ) ;
// Convert Original Matrix to Serial (if it is not already)
//
if (IsLocal_== 1) {
SerialMatrix_ = Matrix();
}
else
{
if (FirstTime)
{
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_,NumMyElements_,
0,Comm()));
if (SerialMap_.get() == 0)
AMESOS_CHK_ERR(-1);
ImportToSerial_ = rcp(new Epetra_Import (SerialMap(),OriginalMap));
if (ImportToSerial_.get() == 0)
AMESOS_CHK_ERR(-1);
}
SerialCrsMatrixA_ = rcp(new Epetra_CrsMatrix(Copy,SerialMap(),0));
if (SerialCrsMatrixA_.get() == 0)
AMESOS_CHK_ERR(-1);
SerialCrsMatrix().Import(*Matrix(), Importer(),Insert);
#if 0
I was not able to make this work - 11 Feb 2006
if (AddZeroToDiag_ ) {
int OriginalTracebackMode = SerialCrsMatrix().GetTracebackMode() ;
SerialCrsMatrix().SetTracebackMode( EPETRA_MIN( OriginalTracebackMode, 0) ) ; // ExportToSerial is called both by PerformSymbolicFactorization() and PerformNumericFactorization(). When called by the latter, the call to insertglobalvalues is both unnecessary and illegal. Fortunately, Epetra allows us to ignore the error message by setting the traceback mode to 0.
//
// Add 0.0 to each diagonal entry to avoid empty diagonal entries;
//
double zero = 0.0;
for ( int i = 0 ; i < SerialMap_->NumGlobalElements(); i++ )
if ( SerialCrsMatrix().LRID(i) >= 0 )
SerialCrsMatrix().InsertGlobalValues( i, 1, &zero, &i ) ;
SerialCrsMatrix().SetTracebackMode( OriginalTracebackMode ) ;
}
#endif
SerialCrsMatrix().FillComplete();
SerialMatrix_ = &SerialCrsMatrix();
assert( numentries_ == SerialMatrix_->NumGlobalNonzeros()); // This should be set to an assignment if AddToZeroDiag is non -zero
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
return(0);
}
int checkResults( bool trans,
Epetra_LinearProblemRedistor * redistor,
Epetra_LinearProblem * A,
Epetra_LinearProblem * R,
bool verbose) {
int m = A->GetRHS()->MyLength();
int n = A->GetLHS()->MyLength();
assert( m == n ) ;
Epetra_MultiVector *x = A->GetLHS() ;
Epetra_MultiVector x1( *x ) ;
// Epetra_MultiVector Difference( x1 ) ;
Epetra_MultiVector *b = A->GetRHS();
Epetra_RowMatrix *matrixA = A->GetMatrix();
assert( matrixA != 0 ) ;
int iam = matrixA->Comm().MyPID();
// Epetra_Time timer(A->Comm());
// double start = timer.ElapsedTime();
matrixA->Multiply(trans, *b, x1) ; // x = Ab
int M,N,nz;
int *ptr, *ind;
double *val, *rhs, *lhs;
int Nrhs, ldrhs, ldlhs;
redistor->ExtractHbData( M, N, nz, ptr, ind,
val, Nrhs, rhs, ldrhs,
lhs, ldlhs);
assert( M == N ) ;
if ( verbose ) {
cout << " iam = " << iam
<< " m = " << m << " n = " << n << " M = " << M << endl ;
cout << " iam = " << iam << " ptr = " << ptr[0] << " " << ptr[1] << " " << ptr[2] << " " << ptr[3] << " " << ptr[4] << " " << ptr[5] << endl ;
cout << " iam = " << iam << " ind = " << ind[0] << " " << ind[1] << " " << ind[2] << " " << ind[3] << " " << ind[4] << " " << ind[5] << endl ;
cout << " iam = " << iam << " val = " << val[0] << " " << val[1] << " " << val[2] << " " << val[3] << " " << val[4] << " " << val[5] << endl ;
}
// Create a serial map in case we end up needing it
// If it is created inside the else block below it would have to
// be with a call to new().
int NumMyElements_ = 0 ;
if (matrixA->Comm().MyPID()==0) NumMyElements_ = n;
Epetra_Map SerialMap( n, NumMyElements_, 0, matrixA->Comm() );
// These are unnecessary and useless
// Epetra_Vector serial_A_rhs( SerialMap ) ;
// Epetra_Vector serial_A_lhs( SerialMap ) ;
// Epetra_Export exporter( matrixA->BlockRowMap(), SerialMap) ;
//
// In each process, we will compute Rb putting the answer into LHS
//
for ( int k = 0 ; k < Nrhs; k ++ ) {
for ( int i = 0 ; i < M ; i ++ ) {
lhs[ i + k * ldlhs ] = 0.0;
}
for ( int i = 0 ; i < M ; i++ ) {
for ( int l = ptr[i]; l < ptr[i+1]; l++ ) {
int j = ind[l] ;
if ( verbose && N < 40 ) {
cout << " i = " << i << " j = " << j ;
cout << " l = " << l << " val[l] = " << val[l] ;
cout << " rhs = " << rhs[ j + k * ldrhs ] << endl ;
}
lhs[ i + k * ldrhs ] += val[l] * rhs[ j + k * ldrhs ] ;
}
}
if ( verbose && N < 40 ) {
cout << " lhs = " ;
for ( int j = 0 ; j < N ; j++ ) cout << " " << lhs[j] ;
cout << endl ;
cout << " rhs = " ;
for ( int j = 0 ; j < N ; j++ ) cout << " " << rhs[j] ;
cout << endl ;
}
const Epetra_Comm &comm = matrixA->Comm() ;
#ifdef HAVE_COMM_ASSERT_EQUAL
//
// Here we double check to make sure that lhs and rhs are
// replicated.
//
for ( int j = 0 ; j < N ; j++ ) {
assert( Comm_assert_equal( &comm, lhs[ j + k * ldrhs ] ) ) ;
assert( Comm_assert_equal( &comm, rhs[ j + k * ldrhs ] ) ) ;
}
#endif
}
//
// Now we have to redistribue them back
//
redistor->UpdateOriginalLHS( A->GetLHS() ) ;
//
// Now we want to compare x and x1 which have been computed as follows:
// x = Rb
// x1 = Ab
//
double Norm_x1, Norm_diff ;
EPETRA_CHK_ERR( x1.Norm2( &Norm_x1 ) ) ;
// cout << " x1 = " << x1 << endl ;
// cout << " *x = " << *x << endl ;
x1.Update( -1.0, *x, 1.0 ) ;
EPETRA_CHK_ERR( x1.Norm2( &Norm_diff ) ) ;
// cout << " diff, i.e. updated x1 = " << x1 << endl ;
int ierr = 0;
if ( verbose ) {
cout << " Norm_diff = " << Norm_diff << endl ;
cout << " Norm_x1 = " << Norm_x1 << endl ;
}
if ( Norm_diff / Norm_x1 > n * error_tolerance ) ierr++ ;
if (ierr!=0 && verbose) cerr << "Status: Test failed" << endl;
else if (verbose) cerr << "Status: Test passed" << endl;
return(ierr);
}
int Amesos_Mumps::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
int NumVectors = vecX->NumVectors();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
if (Comm().NumProc() == 1)
{
// do not import any data
for (int j = 0 ; j < NumVectors; j++)
{
ResetTimer();
MDS.job = 3; // Request solve
for (int i = 0 ; i < Matrix().NumMyRows() ; ++i)
(*vecX)[j][i] = (*vecB)[j][i];
MDS.rhs = (*vecX)[j];
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
}
else
{
Epetra_MultiVector SerialVector(SerialMap(),NumVectors);
ResetTimer();
AMESOS_CHK_ERR(SerialVector.Import(*vecB,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
for (int j = 0 ; j < NumVectors; j++)
{
if (Comm().MyPID() == 0)
MDS.rhs = SerialVector[j];
// solve the linear system and take time
MDS.job = 3;
ResetTimer();
if (Comm().MyPID() < MaxProcs_)
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
// ship solution back and take timing
ResetTimer();
AMESOS_CHK_ERR(vecX->Export(SerialVector,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
}
if (ComputeTrueResidual_)
ComputeTrueResidual(Matrix(), *vecX, *vecB, UseTranspose(), "Amesos_Mumps");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Mumps");
NumSolve_++;
return(0) ;
}
