//=============================================================================
int Amesos_Dscpack::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(0);
ResetTimer(1);
Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
// MS // some checks on matrix size
if (RowMatrixA->NumGlobalRows() != RowMatrixA->NumGlobalCols())
AMESOS_CHK_ERR(-1);
// Convert vector b to a vector in the form that DSCPACK needs it
//
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong with input
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-2);
double *dscmapXvalues ;
int dscmapXlda ;
Epetra_MultiVector dscmapX(DscRowMap(),NumVectors) ;
int ierr;
AMESOS_CHK_ERR(dscmapX.ExtractView(&dscmapXvalues,&dscmapXlda));
assert (dscmapXlda == NumLocalCols);
double *dscmapBvalues ;
int dscmapBlda ;
Epetra_MultiVector dscmapB(DscRowMap(), NumVectors ) ;
ierr = dscmapB.ExtractView( &dscmapBvalues, &dscmapBlda );
AMESOS_CHK_ERR(ierr);
assert( dscmapBlda == NumLocalCols ) ;
AMESOS_CHK_ERR(dscmapB.Import(*vecB, Importer(), Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
ResetTimer(0);
// MS // now solve the problem
std::vector<double> ValuesInNewOrder( NumLocalCols ) ;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
if ( MyDscRank >= 0 ) {
for ( int j =0 ; j < NumVectors; j++ ) {
for ( int i = 0; i < NumLocalCols; i++ ) {
ValuesInNewOrder[i] = dscmapBvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapBlda ] ;
}
AMESOS_CHK_ERR( DSC_InputRhsLocalVec ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
AMESOS_CHK_ERR( DSC_Solve ( PrivateDscpackData_->MyDSCObject_ ) ) ;
AMESOS_CHK_ERR( DSC_GetLocalSolution ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
for ( int i = 0; i < NumLocalCols; i++ ) {
dscmapXvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapXlda ] = ValuesInNewOrder[i];
}
}
}
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
ResetTimer(0);
ResetTimer(1);
vecX->Export( dscmapX, Importer(), Insert ) ;
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
ComputeTrueResidual(*(GetProblem()->GetMatrix()), *vecX, *vecB,
false, "Amesos_Dscpack");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Dscpack");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
NumSolve_++;
return(0) ;
}
//=============================================================================
int Amesos_Umfpack::Solve()
{
// if necessary, perform numeric factorization.
// This may call SymbolicFactorization() as well.
if (!IsNumericFactorizationOK_)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
Epetra_MultiVector *SerialB, *SerialX;
// Extract Serial versions of X and B
//
double *SerialXvalues ;
double *SerialBvalues ;
Epetra_MultiVector* SerialXextract = 0;
Epetra_MultiVector* SerialBextract = 0;
// Copy B to the serial version of B
//
ResetTimer(0);
if (IsLocal_ == 1) {
SerialB = vecB ;
SerialX = vecX ;
} else {
assert (IsLocal_ == 0);
SerialXextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract->Import(*vecB,Importer(),Insert);
SerialB = SerialBextract;
SerialX = SerialXextract;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call UMFPACK to perform the solve
// Note: UMFPACK uses a Compressed Column Storage instead of compressed row storage,
// Hence to compute A X = B, we ask UMFPACK to perform A^T X = B and vice versa
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
ResetTimer(0);
int SerialBlda, SerialXlda ;
int UmfpackRequest = UseTranspose()?UMFPACK_A:UMFPACK_At ;
int status = 0;
if ( MyPID_ == 0 ) {
int ierr;
ierr = SerialB->ExtractView(&SerialBvalues, &SerialBlda);
assert (ierr == 0);
ierr = SerialX->ExtractView(&SerialXvalues, &SerialXlda);
assert (ierr == 0);
assert( SerialBlda == NumGlobalElements_ ) ;
assert( SerialXlda == NumGlobalElements_ ) ;
for ( int j =0 ; j < NumVectors; j++ ) {
double *Control = (double *) NULL, *Info = (double *) NULL ;
status = umfpack_di_solve (UmfpackRequest, &Ap[0],
&Ai[0], &Aval[0],
&SerialXvalues[j*SerialXlda],
&SerialBvalues[j*SerialBlda],
Numeric, Control, Info) ;
}
}
if (status) AMESOS_CHK_ERR(status);
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if ( IsLocal_ == 0 ) {
vecX->Export(*SerialX, Importer(), Insert ) ;
if (SerialBextract) delete SerialBextract ;
if (SerialXextract) delete SerialXextract ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
{
Epetra_RowMatrix* Matrix =
dynamic_cast<Epetra_RowMatrix*>(Problem_->GetOperator());
ComputeTrueResidual(*Matrix, *vecX, *vecB, UseTranspose(), "Amesos_Umfpack");
}
if (ComputeVectorNorms_) {
ComputeVectorNorms(*vecX, *vecB, "Amesos_Umfpack");
}
NumSolve_++;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); // Amesos overhead
return(0);
}
int Amesos_Mumps::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
int NumVectors = vecX->NumVectors();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
if (Comm().NumProc() == 1)
{
// do not import any data
for (int j = 0 ; j < NumVectors; j++)
{
ResetTimer();
MDS.job = 3; // Request solve
for (int i = 0 ; i < Matrix().NumMyRows() ; ++i)
(*vecX)[j][i] = (*vecB)[j][i];
MDS.rhs = (*vecX)[j];
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
}
else
{
Epetra_MultiVector SerialVector(SerialMap(),NumVectors);
ResetTimer();
AMESOS_CHK_ERR(SerialVector.Import(*vecB,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
for (int j = 0 ; j < NumVectors; j++)
{
if (Comm().MyPID() == 0)
MDS.rhs = SerialVector[j];
// solve the linear system and take time
MDS.job = 3;
ResetTimer();
if (Comm().MyPID() < MaxProcs_)
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
// ship solution back and take timing
ResetTimer();
AMESOS_CHK_ERR(vecX->Export(SerialVector,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
}
if (ComputeTrueResidual_)
ComputeTrueResidual(Matrix(), *vecX, *vecB, UseTranspose(), "Amesos_Mumps");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Mumps");
NumSolve_++;
return(0) ;
}
//=============================================================================
int Amesos_Klu::Solve()
{
Epetra_MultiVector* vecX = 0 ;
Epetra_MultiVector* vecB = 0 ;
#ifdef HAVE_AMESOS_EPETRAEXT
Teuchos::RCP<Epetra_MultiVector> vecX_rcp;
Teuchos::RCP<Epetra_MultiVector> vecB_rcp;
#endif
#ifdef Bug_8212
// This demonstrates Bug #2812 - Valgrind does not catch this
// memory leak
lose_this_ = (int *) malloc( 300 ) ;
#ifdef Bug_8212_B
// This demonstrates Bug #2812 - Valgrind does catch this
// use of unitialized data - but only in TestOptions/TestOptions.exe
// not in Test_Basic/amesos_test.exe
//
if ( lose_this_[0] == 12834 ) {
std::cout << __FILE__ << "::" << __LINE__ << " very unlikely to happen " << std::endl ;
}
#endif
#endif
if ( !TrustMe_ ) {
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false);
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false);
Epetra_MultiVector* OrigVecX ;
Epetra_MultiVector* OrigVecB ;
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
//
// Reindex the LHS and RHS
//
OrigVecX = Problem_->GetLHS() ;
OrigVecB = Problem_->GetRHS() ;
if ( Reindex_ ) {
#ifdef HAVE_AMESOS_EPETRAEXT
vecX_rcp = StdIndexDomain_->StandardizeIndex( *OrigVecX ) ;
vecB_rcp = StdIndexRange_->StandardizeIndex( *OrigVecB ) ;
vecX = &*vecX_rcp;
vecB = &*vecB_rcp;
#else
AMESOS_CHK_ERR( -13 ) ; // Amesos_Klu can't handle non-standard indexing without EpetraExt
#endif
} else {
vecX = OrigVecX ;
vecB = OrigVecB ;
}
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong in input
// Extract Serial versions of X and B
ResetTimer(0);
// Copy B to the serial version of B
//
if (UseDataInPlace_ == 1) {
#ifdef HAVE_AMESOS_EPETRAEXT
if(vecX_rcp==Teuchos::null)
SerialX_ = Teuchos::rcp(vecX,false);
else
SerialX_ = vecX_rcp;
if(vecB_rcp==Teuchos::null)
SerialB_ = Teuchos::rcp(vecB,false);
else
SerialB_ = vecB_rcp;
#else
SerialB_ = Teuchos::rcp(vecB,false);
SerialX_ = Teuchos::rcp(vecX,false);
#endif
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
} else {
assert (UseDataInPlace_ == 0);
if( NumVectors_ != Problem_->GetRHS()->NumVectors() ) {
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
}
if (NumVectors_ != vecB->NumVectors())
AMESOS_CHK_ERR(-1); // internal error
//ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecB->Map() ) );
//if ( SerialBextract_->Import(*vecB,*ImportRangeToSerial_,Insert) )
Epetra_Import *UseImport;
if(!UseTranspose_) UseImport=&*ImportRangeToSerial_;
else UseImport=&*ImportDomainToSerial_;
if ( SerialBextract_->Import(*vecB,*UseImport,Insert) )
AMESOS_CHK_ERR( -1 ) ; // internal error
SerialB_ = Teuchos::rcp(&*SerialBextract_,false) ;
SerialX_ = Teuchos::rcp(&*SerialXextract_,false) ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call KLU to perform the solve
ResetTimer(0);
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
if (SerialXlda_ != NumGlobalElements_)
AMESOS_CHK_ERR(-1);
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
else
{
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false) ;
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false) ;
NumVectors_ = SerialX_->NumVectors();
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
}
}
if ( MyPID_ == 0) {
if ( NumVectors_ == 1 ) {
for ( int i = 0 ; i < NumGlobalElements_ ; i++ )
SerialXBvalues_[i] = SerialBvalues_[i] ;
} else {
SerialX_->Scale(1.0, *SerialB_ ) ; // X = B (Klu overwrites B with X)
}
if (UseTranspose()) {
amesos_klu_solve( &*PrivateKluData_->Symbolic_, &*PrivateKluData_->Numeric_,
SerialXlda_, NumVectors_, &SerialXBvalues_[0], &*PrivateKluData_->common_ );
} else {
amesos_klu_tsolve( &*PrivateKluData_->Symbolic_, &*PrivateKluData_->Numeric_,
SerialXlda_, NumVectors_, &SerialXBvalues_[0], &*PrivateKluData_->common_ );
}
}
if ( !TrustMe_ ) {
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if (UseDataInPlace_ == 0) {
Epetra_Import *UseImport;
if(!UseTranspose_) UseImport=&*ImportDomainToSerial_;
else UseImport=&*ImportRangeToSerial_;
// ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecX->Map() ) );
vecX->Export( *SerialX_, *UseImport, Insert ) ;
} // otherwise we are already in place.
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
#if 0
//
// ComputeTrueResidual causes TestOptions to fail on my linux box
// Bug #1417
if (ComputeTrueResidual_)
ComputeTrueResidual(*SerialMatrix_, *vecX, *vecB, UseTranspose(), "Amesos_Klu");
#endif
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Klu");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
++NumSolve_;
return(0) ;
}
//=============================================================================
int Amesos_Paraklete::Solve()
{
Epetra_MultiVector* vecX=0;
Epetra_MultiVector* vecB=0;
if ( !TrustMe_ ) {
SerialB_ = Problem_->GetRHS() ;
SerialX_ = Problem_->GetLHS() ;
Epetra_MultiVector* OrigVecX ;
Epetra_MultiVector* OrigVecB ;
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
//
// Reindex the LHS and RHS
//
OrigVecX = Problem_->GetLHS() ;
OrigVecB = Problem_->GetRHS() ;
if ( Reindex_ ) {
#ifdef HAVE_AMESOS_EPETRAEXT
vecX = StdIndexDomain_->StandardizeIndex( OrigVecX ) ;
vecB = StdIndexRange_->StandardizeIndex( OrigVecB ) ;
#else
AMESOS_CHK_ERR( -13 ) ; // Amesos_Paraklete can't handle non-standard indexing without EpetraExt
#endif
} else {
vecX = OrigVecX ;
vecB = OrigVecB ;
}
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong in input
// Extract Serial versions of X and B
ResetTimer(0);
// Copy B to the serial version of B
//
if (false && UseDataInPlace_ == 1) {
SerialB_ = vecB;
SerialX_ = vecX;
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
} else {
assert (UseDataInPlace_ == 0);
if( NumVectors_ != Problem_->GetRHS()->NumVectors() ) {
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
}
if (NumVectors_ != vecB->NumVectors())
AMESOS_CHK_ERR(-1); // internal error
ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecB->Map() ) );
if ( SerialBextract_->Import(*vecB,*ImportRangeToSerial_,Insert) )
AMESOS_CHK_ERR( -1 ) ; // internal error
SerialB_ = &*SerialBextract_ ;
SerialX_ = &*SerialXextract_ ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call PARAKLETE to perform the solve
ResetTimer(0);
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
if (SerialXlda_ != NumGlobalElements_)
AMESOS_CHK_ERR(-1);
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
if ( MyPID_ == 0) {
if ( NumVectors_ == 1 ) {
for ( int i = 0 ; i < NumGlobalElements_ ; i++ )
SerialXBvalues_[i] = SerialBvalues_[i] ;
} else {
SerialX_->Scale(1.0, *SerialB_ ) ; // X = B (Klu overwrites B with X)
}
}
if ( IamInGroup_ )
for (int i = 0; i < NumVectors_ ; i++ ) {
amesos_paraklete_solve( &*PrivateParakleteData_->LUnumeric_, &*PrivateParakleteData_->LUsymbolic_,
&SerialXBvalues_[i*SerialXlda_], &*PrivateParakleteData_->common_ );
}
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if (UseDataInPlace_ == 0) {
ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecX->Map() ) );
vecX->Export( *SerialX_, *ImportDomainToSerial_, Insert ) ;
} // otherwise we are already in place.
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
#if 0
//
// ComputeTrueResidual causes TestOptions to fail on my linux box // Bug #1417
if (ComputeTrueResidual_)
ComputeTrueResidual(*SerialMatrix_, *vecX, *vecB, UseTranspose(), "Amesos_Paraklete");
#endif
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Paraklete");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
++NumSolve_;
return(0) ;
}