//=============================================================================
void Amesos_Mumps::SetICNTLandCNTL()
{
std::map<int,int>::iterator i_iter;
for (i_iter = ICNTL.begin() ; i_iter != ICNTL.end() ; ++i_iter)
{
int pos = i_iter->first;
int val = i_iter->second;
if (pos < 1 || pos > 40) continue;
MDS.ICNTL(pos) = val;
}
std::map<int,double>::iterator d_iter;
for (d_iter = CNTL.begin() ; d_iter != CNTL.end() ; ++d_iter)
{
int pos = d_iter->first;
double val = d_iter->second;
if (pos < 1 || pos > 5) continue;
MDS.CNTL(pos) = val;
}
// fix some options
if (Comm().NumProc() != 1) MDS.ICNTL(18)= 3;
else MDS.ICNTL(18)= 0;
if (UseTranspose()) MDS.ICNTL(9) = 0;
else MDS.ICNTL(9) = 1;
MDS.ICNTL(5) = 0;
#if 0
if (IsComputeSchurComplementOK_) MDS.ICNTL(19) = 1;
else MDS.ICNTL(19) = 0;
// something to do if the Schur complement is required.
if (IsComputeSchurComplementOK_ && Comm().MyPID() == 0)
{
MDS.size_schur = NumSchurComplementRows_;
MDS.listvar_schur = SchurComplementRows_;
MDS.schur = new double[NumSchurComplementRows_*NumSchurComplementRows_];
}
#endif
}
//==============================================================================
int Ifpack_Chebyshev::
Apply(const Epetra_MultiVector& X, Epetra_MultiVector& Y) const
{
if (IsComputed() == false)
IFPACK_CHK_ERR(-3);
if (X.NumVectors() != Y.NumVectors())
IFPACK_CHK_ERR(-2);
if (IsRowMatrix_)
{
IFPACK_CHK_ERR(Matrix_->Multiply(UseTranspose(),X,Y));
}
else
{
IFPACK_CHK_ERR(Operator_->Apply(X,Y));
}
return(0);
}
//=============================================================================
int Amesos_Umfpack::Solve()
{
// if necessary, perform numeric factorization.
// This may call SymbolicFactorization() as well.
if (!IsNumericFactorizationOK_)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
Epetra_MultiVector *SerialB, *SerialX;
// Extract Serial versions of X and B
//
double *SerialXvalues ;
double *SerialBvalues ;
Epetra_MultiVector* SerialXextract = 0;
Epetra_MultiVector* SerialBextract = 0;
// Copy B to the serial version of B
//
ResetTimer(0);
if (IsLocal_ == 1) {
SerialB = vecB ;
SerialX = vecX ;
} else {
assert (IsLocal_ == 0);
SerialXextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract->Import(*vecB,Importer(),Insert);
SerialB = SerialBextract;
SerialX = SerialXextract;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call UMFPACK to perform the solve
// Note: UMFPACK uses a Compressed Column Storage instead of compressed row storage,
// Hence to compute A X = B, we ask UMFPACK to perform A^T X = B and vice versa
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
ResetTimer(0);
int SerialBlda, SerialXlda ;
int UmfpackRequest = UseTranspose()?UMFPACK_A:UMFPACK_At ;
int status = 0;
if ( MyPID_ == 0 ) {
int ierr;
ierr = SerialB->ExtractView(&SerialBvalues, &SerialBlda);
assert (ierr == 0);
ierr = SerialX->ExtractView(&SerialXvalues, &SerialXlda);
assert (ierr == 0);
assert( SerialBlda == NumGlobalElements_ ) ;
assert( SerialXlda == NumGlobalElements_ ) ;
for ( int j =0 ; j < NumVectors; j++ ) {
double *Control = (double *) NULL, *Info = (double *) NULL ;
status = umfpack_di_solve (UmfpackRequest, &Ap[0],
&Ai[0], &Aval[0],
&SerialXvalues[j*SerialXlda],
&SerialBvalues[j*SerialBlda],
Numeric, Control, Info) ;
}
}
if (status) AMESOS_CHK_ERR(status);
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if ( IsLocal_ == 0 ) {
vecX->Export(*SerialX, Importer(), Insert ) ;
if (SerialBextract) delete SerialBextract ;
if (SerialXextract) delete SerialXextract ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
{
Epetra_RowMatrix* Matrix =
dynamic_cast<Epetra_RowMatrix*>(Problem_->GetOperator());
ComputeTrueResidual(*Matrix, *vecX, *vecB, UseTranspose(), "Amesos_Umfpack");
}
if (ComputeVectorNorms_) {
ComputeVectorNorms(*vecX, *vecB, "Amesos_Umfpack");
}
NumSolve_++;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); // Amesos overhead
return(0);
}
int Amesos_Scalapack::Solve() {
if( debug_ == 1 ) std::cout << "Entering `Solve()'" << std::endl;
NumSolve_++;
Epetra_MultiVector *vecX = Problem_->GetLHS() ;
Epetra_MultiVector *vecB = Problem_->GetRHS() ;
//
// Compute the number of right hands sides
// (and check that X and B have the same shape)
//
int nrhs;
if ( vecX == 0 ) {
nrhs = 0 ;
EPETRA_CHK_ERR( vecB != 0 ) ;
} else {
nrhs = vecX->NumVectors() ;
EPETRA_CHK_ERR( vecB->NumVectors() != nrhs ) ;
}
Epetra_MultiVector *ScalapackB =0;
Epetra_MultiVector *ScalapackX =0;
//
// Extract Scalapack versions of X and B
//
double *ScalapackXvalues ;
Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator());
Time_->ResetStartTime(); // track time to broadcast vectors
//
// Copy B to the scalapack version of B
//
const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap();
Epetra_MultiVector *ScalapackXextract = new Epetra_MultiVector( *VectorMap_, nrhs ) ;
Epetra_MultiVector *ScalapackBextract = new Epetra_MultiVector( *VectorMap_, nrhs ) ;
Epetra_Import ImportToScalapack( *VectorMap_, OriginalMap );
ScalapackBextract->Import( *vecB, ImportToScalapack, Insert ) ;
ScalapackB = ScalapackBextract ;
ScalapackX = ScalapackXextract ;
VecTime_ += Time_->ElapsedTime();
//
// Call SCALAPACKs PDGETRS to perform the solve
//
int DescX[10];
ScalapackX->Scale(1.0, *ScalapackB) ;
int ScalapackXlda ;
Time_->ResetStartTime(); // tract time to solve
//
// Setup DescX
//
if( nrhs > nb_ ) {
EPETRA_CHK_ERR( -2 );
}
int Ierr[1] ;
Ierr[0] = 0 ;
const int zero = 0 ;
const int one = 1 ;
if ( iam_ < nprow_ * npcol_ ) {
assert( ScalapackX->ExtractView( &ScalapackXvalues, &ScalapackXlda ) == 0 ) ;
if ( false ) std::cout << "Amesos_Scalapack.cpp: " << __LINE__ << " ScalapackXlda = " << ScalapackXlda
<< " lda_ = " << lda_
<< " nprow_ = " << nprow_
<< " npcol_ = " << npcol_
<< " myprow_ = " << myprow_
<< " mypcol_ = " << mypcol_
<< " iam_ = " << iam_ << std::endl ;
if ( TwoD_distribution_ ) assert( mypcol_ >0 || EPETRA_MAX(ScalapackXlda,1) == lda_ ) ;
DESCINIT_F77(DescX,
&NumGlobalElements_,
&nrhs,
&nb_,
&nb_,
&zero,
&zero,
&ictxt_,
&lda_,
Ierr ) ;
assert( Ierr[0] == 0 ) ;
//
// For the 1D data distribution, we factor the transposed
// matrix, hence we must invert the sense of the transposition
//
char trans = 'N';
if ( TwoD_distribution_ ) {
if ( UseTranspose() ) trans = 'T' ;
} else {
if ( ! UseTranspose() ) trans = 'T' ;
}
if ( nprow_ * npcol_ == 1 ) {
DGETRS_F77(&trans,
&NumGlobalElements_,
&nrhs,
&DenseA_[0],
&lda_,
&Ipiv_[0],
ScalapackXvalues,
&lda_,
Ierr ) ;
} else {
PDGETRS_F77(&trans,
&NumGlobalElements_,
&nrhs,
&DenseA_[0],
&one,
&one,
DescA_,
&Ipiv_[0],
ScalapackXvalues,
&one,
&one,
DescX,
Ierr ) ;
}
}
SolTime_ += Time_->ElapsedTime();
Time_->ResetStartTime(); // track time to broadcast vectors
//
// Copy X back to the original vector
//
Epetra_Import ImportFromScalapack( OriginalMap, *VectorMap_ );
vecX->Import( *ScalapackX, ImportFromScalapack, Insert ) ;
delete ScalapackBextract ;
delete ScalapackXextract ;
VecTime_ += Time_->ElapsedTime();
// All processes should return the same error code
if ( nprow_ * npcol_ < Comm().NumProc() )
Comm().Broadcast( Ierr, 1, 0 ) ;
// MS // compute vector norms
if( ComputeVectorNorms_ == true || verbose_ == 2 ) {
double NormLHS, NormRHS;
for( int i=0 ; i<nrhs ; ++i ) {
assert((*vecX)(i)->Norm2(&NormLHS)==0);
assert((*vecB)(i)->Norm2(&NormRHS)==0);
if( verbose_ && Comm().MyPID() == 0 ) {
std::cout << "Amesos_Scalapack : vector " << i << ", ||x|| = " << NormLHS
<< ", ||b|| = " << NormRHS << std::endl;
}
}
}
// MS // compute true residual
if( ComputeTrueResidual_ == true || verbose_ == 2 ) {
double Norm;
Epetra_MultiVector Ax(vecB->Map(),nrhs);
for( int i=0 ; i<nrhs ; ++i ) {
(Problem_->GetMatrix()->Multiply(UseTranspose(), *((*vecX)(i)), Ax));
(Ax.Update(1.0, *((*vecB)(i)), -1.0));
(Ax.Norm2(&Norm));
if( verbose_ && Comm().MyPID() == 0 ) {
std::cout << "Amesos_Scalapack : vector " << i << ", ||Ax - b|| = " << Norm << std::endl;
}
}
}
return Ierr[0];
}
//=============================================================================
int Amesos_Klu::Solve()
{
Epetra_MultiVector* vecX = 0 ;
Epetra_MultiVector* vecB = 0 ;
#ifdef HAVE_AMESOS_EPETRAEXT
Teuchos::RCP<Epetra_MultiVector> vecX_rcp;
Teuchos::RCP<Epetra_MultiVector> vecB_rcp;
#endif
#ifdef Bug_8212
// This demonstrates Bug #2812 - Valgrind does not catch this
// memory leak
lose_this_ = (int *) malloc( 300 ) ;
#ifdef Bug_8212_B
// This demonstrates Bug #2812 - Valgrind does catch this
// use of unitialized data - but only in TestOptions/TestOptions.exe
// not in Test_Basic/amesos_test.exe
//
if ( lose_this_[0] == 12834 ) {
std::cout << __FILE__ << "::" << __LINE__ << " very unlikely to happen " << std::endl ;
}
#endif
#endif
if ( !TrustMe_ ) {
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false);
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false);
Epetra_MultiVector* OrigVecX ;
Epetra_MultiVector* OrigVecB ;
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
//
// Reindex the LHS and RHS
//
OrigVecX = Problem_->GetLHS() ;
OrigVecB = Problem_->GetRHS() ;
if ( Reindex_ ) {
#ifdef HAVE_AMESOS_EPETRAEXT
vecX_rcp = StdIndexDomain_->StandardizeIndex( *OrigVecX ) ;
vecB_rcp = StdIndexRange_->StandardizeIndex( *OrigVecB ) ;
vecX = &*vecX_rcp;
vecB = &*vecB_rcp;
#else
AMESOS_CHK_ERR( -13 ) ; // Amesos_Klu can't handle non-standard indexing without EpetraExt
#endif
} else {
vecX = OrigVecX ;
vecB = OrigVecB ;
}
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong in input
// Extract Serial versions of X and B
ResetTimer(0);
// Copy B to the serial version of B
//
if (UseDataInPlace_ == 1) {
#ifdef HAVE_AMESOS_EPETRAEXT
if(vecX_rcp==Teuchos::null)
SerialX_ = Teuchos::rcp(vecX,false);
else
SerialX_ = vecX_rcp;
if(vecB_rcp==Teuchos::null)
SerialB_ = Teuchos::rcp(vecB,false);
else
SerialB_ = vecB_rcp;
#else
SerialB_ = Teuchos::rcp(vecB,false);
SerialX_ = Teuchos::rcp(vecX,false);
#endif
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
} else {
assert (UseDataInPlace_ == 0);
if( NumVectors_ != Problem_->GetRHS()->NumVectors() ) {
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
}
if (NumVectors_ != vecB->NumVectors())
AMESOS_CHK_ERR(-1); // internal error
//ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecB->Map() ) );
//if ( SerialBextract_->Import(*vecB,*ImportRangeToSerial_,Insert) )
Epetra_Import *UseImport;
if(!UseTranspose_) UseImport=&*ImportRangeToSerial_;
else UseImport=&*ImportDomainToSerial_;
if ( SerialBextract_->Import(*vecB,*UseImport,Insert) )
AMESOS_CHK_ERR( -1 ) ; // internal error
SerialB_ = Teuchos::rcp(&*SerialBextract_,false) ;
SerialX_ = Teuchos::rcp(&*SerialXextract_,false) ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call KLU to perform the solve
ResetTimer(0);
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
if (SerialXlda_ != NumGlobalElements_)
AMESOS_CHK_ERR(-1);
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
else
{
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false) ;
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false) ;
NumVectors_ = SerialX_->NumVectors();
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
}
}
if ( MyPID_ == 0) {
if ( NumVectors_ == 1 ) {
for ( int i = 0 ; i < NumGlobalElements_ ; i++ )
SerialXBvalues_[i] = SerialBvalues_[i] ;
} else {
SerialX_->Scale(1.0, *SerialB_ ) ; // X = B (Klu overwrites B with X)
}
if (UseTranspose()) {
amesos_klu_solve( &*PrivateKluData_->Symbolic_, &*PrivateKluData_->Numeric_,
SerialXlda_, NumVectors_, &SerialXBvalues_[0], &*PrivateKluData_->common_ );
} else {
amesos_klu_tsolve( &*PrivateKluData_->Symbolic_, &*PrivateKluData_->Numeric_,
SerialXlda_, NumVectors_, &SerialXBvalues_[0], &*PrivateKluData_->common_ );
}
}
if ( !TrustMe_ ) {
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if (UseDataInPlace_ == 0) {
Epetra_Import *UseImport;
if(!UseTranspose_) UseImport=&*ImportDomainToSerial_;
else UseImport=&*ImportRangeToSerial_;
// ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecX->Map() ) );
vecX->Export( *SerialX_, *UseImport, Insert ) ;
} // otherwise we are already in place.
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
#if 0
//
// ComputeTrueResidual causes TestOptions to fail on my linux box
// Bug #1417
if (ComputeTrueResidual_)
ComputeTrueResidual(*SerialMatrix_, *vecX, *vecB, UseTranspose(), "Amesos_Klu");
#endif
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Klu");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
++NumSolve_;
return(0) ;
}
int Amesos_Klu::CreateLocalMatrixAndExporters()
{
ResetTimer(0);
RowMatrixA_ = Problem_->GetMatrix(); // MS, 18-Apr-06 //
if (RowMatrixA_ == 0) AMESOS_CHK_ERR(-1);
const Epetra_Map &OriginalMatrixMap = RowMatrixA_->RowMatrixRowMap() ;
const Epetra_Map &OriginalDomainMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorRangeMap():
GetProblem()->GetOperator()->OperatorDomainMap();
const Epetra_Map &OriginalRangeMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorDomainMap():
GetProblem()->GetOperator()->OperatorRangeMap();
NumGlobalElements_ = RowMatrixA_->NumGlobalRows();
numentries_ = RowMatrixA_->NumGlobalNonzeros();
int indexBase = OriginalMatrixMap.IndexBase();
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
//
// Create a serial matrix
//
assert( NumGlobalElements_ == OriginalMatrixMap.NumGlobalElements() ) ;
int NumMyElements_ = 0 ;
if (MyPID_==0) NumMyElements_ = NumGlobalElements_;
//
// UseDataInPlace_ is set to 1 (true) only if everything is perfectly
// normal. Anything out of the ordinary reverts to the more expensive
// path.
//
UseDataInPlace_ = ( OriginalMatrixMap.NumMyElements() ==
OriginalMatrixMap.NumGlobalElements() )?1:0;
if ( ! OriginalRangeMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( ! OriginalDomainMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( AddZeroToDiag_ ) UseDataInPlace_ = 0 ;
Comm().Broadcast( &UseDataInPlace_, 1, 0 ) ;
//
// Reindex matrix if necessary (and possible - i.e. CrsMatrix)
//
// For now, since I don't know how to determine if we need to reindex the matrix,
// we allow the user to control re-indexing through the "Reindex" parameter.
//
CrsMatrixA_ = dynamic_cast<Epetra_CrsMatrix *>(Problem_->GetOperator());
if ( Reindex_ ) {
if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR(-7);
#ifdef HAVE_AMESOS_EPETRAEXT
const Epetra_Map& OriginalMap = CrsMatrixA_->RowMap();
StdIndex_ = rcp( new Amesos_StandardIndex( OriginalMap ) );
//const Epetra_Map& OriginalColMap = CrsMatrixA_->RowMap();
StdIndexDomain_ = rcp( new Amesos_StandardIndex( OriginalDomainMap ) );
StdIndexRange_ = rcp( new Amesos_StandardIndex( OriginalRangeMap ) );
StdIndexMatrix_ = StdIndex_->StandardizeIndex( CrsMatrixA_ );
#else
std::cerr << "Amesos_Klu requires EpetraExt to reindex matrices." << std::endl
<< " Please rebuild with the EpetraExt library by adding --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR(-8);
#endif
} else {
StdIndexMatrix_ = RowMatrixA_ ;
}
//
// At this point, StdIndexMatrix_ points to a matrix with
// standard indexing.
//
//
// Convert Original Matrix to Serial (if it is not already)
//
if (UseDataInPlace_ == 1) {
SerialMatrix_ = StdIndexMatrix_;
} else {
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_, NumMyElements_, indexBase, Comm()));
if ( Problem_->GetRHS() )
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
else
NumVectors_ = 1 ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
ImportToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->RowMatrixRowMap() ) );
if (ImportToSerial_.get() == 0) AMESOS_CHK_ERR(-9);
// Build the vector data import/export once and only once
#define CHRIS
#ifdef CHRIS
if(StdIndexMatrix_->RowMatrixRowMap().SameAs(StdIndexMatrix_->OperatorRangeMap()))
ImportRangeToSerial_=ImportToSerial_;
else
ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->OperatorRangeMap() ) );
if(StdIndexMatrix_->RowMatrixRowMap().SameAs(StdIndexMatrix_->OperatorDomainMap()))
ImportDomainToSerial_=ImportToSerial_;
else
ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->OperatorDomainMap() ) );
#endif
SerialCrsMatrixA_ = rcp( new Epetra_CrsMatrix(Copy, *SerialMap_, 0) ) ;
SerialMatrix_ = &*SerialCrsMatrixA_ ;
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
return(0);
}
//==============================================================================
int Ifpack_ReorderFilter::
Apply(const Epetra_MultiVector& X, Epetra_MultiVector& Y) const
{
int ierr = Multiply(UseTranspose(),X,Y);
IFPACK_RETURN(ierr);
}
//==============================================================================
int Ifpack_DropFilter::
Apply(const Epetra_MultiVector& X, Epetra_MultiVector& Y) const
{
IFPACK_RETURN(Multiply(UseTranspose(),X,Y));
}
int Amesos_Mumps::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
int NumVectors = vecX->NumVectors();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
if (Comm().NumProc() == 1)
{
// do not import any data
for (int j = 0 ; j < NumVectors; j++)
{
ResetTimer();
MDS.job = 3; // Request solve
for (int i = 0 ; i < Matrix().NumMyRows() ; ++i)
(*vecX)[j][i] = (*vecB)[j][i];
MDS.rhs = (*vecX)[j];
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
}
else
{
Epetra_MultiVector SerialVector(SerialMap(),NumVectors);
ResetTimer();
AMESOS_CHK_ERR(SerialVector.Import(*vecB,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
for (int j = 0 ; j < NumVectors; j++)
{
if (Comm().MyPID() == 0)
MDS.rhs = SerialVector[j];
// solve the linear system and take time
MDS.job = 3;
ResetTimer();
if (Comm().MyPID() < MaxProcs_)
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
// ship solution back and take timing
ResetTimer();
AMESOS_CHK_ERR(vecX->Export(SerialVector,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
}
if (ComputeTrueResidual_)
ComputeTrueResidual(Matrix(), *vecX, *vecB, UseTranspose(), "Amesos_Mumps");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Mumps");
NumSolve_++;
return(0) ;
}
//=============================================================================
int Amesos_Paraklete::Solve()
{
Epetra_MultiVector* vecX=0;
Epetra_MultiVector* vecB=0;
if ( !TrustMe_ ) {
SerialB_ = Problem_->GetRHS() ;
SerialX_ = Problem_->GetLHS() ;
Epetra_MultiVector* OrigVecX ;
Epetra_MultiVector* OrigVecB ;
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
//
// Reindex the LHS and RHS
//
OrigVecX = Problem_->GetLHS() ;
OrigVecB = Problem_->GetRHS() ;
if ( Reindex_ ) {
#ifdef HAVE_AMESOS_EPETRAEXT
vecX = StdIndexDomain_->StandardizeIndex( OrigVecX ) ;
vecB = StdIndexRange_->StandardizeIndex( OrigVecB ) ;
#else
AMESOS_CHK_ERR( -13 ) ; // Amesos_Paraklete can't handle non-standard indexing without EpetraExt
#endif
} else {
vecX = OrigVecX ;
vecB = OrigVecB ;
}
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong in input
// Extract Serial versions of X and B
ResetTimer(0);
// Copy B to the serial version of B
//
if (false && UseDataInPlace_ == 1) {
SerialB_ = vecB;
SerialX_ = vecX;
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
} else {
assert (UseDataInPlace_ == 0);
if( NumVectors_ != Problem_->GetRHS()->NumVectors() ) {
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
}
if (NumVectors_ != vecB->NumVectors())
AMESOS_CHK_ERR(-1); // internal error
ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecB->Map() ) );
if ( SerialBextract_->Import(*vecB,*ImportRangeToSerial_,Insert) )
AMESOS_CHK_ERR( -1 ) ; // internal error
SerialB_ = &*SerialBextract_ ;
SerialX_ = &*SerialXextract_ ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call PARAKLETE to perform the solve
ResetTimer(0);
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
if (SerialXlda_ != NumGlobalElements_)
AMESOS_CHK_ERR(-1);
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
if ( MyPID_ == 0) {
if ( NumVectors_ == 1 ) {
for ( int i = 0 ; i < NumGlobalElements_ ; i++ )
SerialXBvalues_[i] = SerialBvalues_[i] ;
} else {
SerialX_->Scale(1.0, *SerialB_ ) ; // X = B (Klu overwrites B with X)
}
}
if ( IamInGroup_ )
for (int i = 0; i < NumVectors_ ; i++ ) {
amesos_paraklete_solve( &*PrivateParakleteData_->LUnumeric_, &*PrivateParakleteData_->LUsymbolic_,
&SerialXBvalues_[i*SerialXlda_], &*PrivateParakleteData_->common_ );
}
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if (UseDataInPlace_ == 0) {
ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecX->Map() ) );
vecX->Export( *SerialX_, *ImportDomainToSerial_, Insert ) ;
} // otherwise we are already in place.
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
#if 0
//
// ComputeTrueResidual causes TestOptions to fail on my linux box // Bug #1417
if (ComputeTrueResidual_)
ComputeTrueResidual(*SerialMatrix_, *vecX, *vecB, UseTranspose(), "Amesos_Paraklete");
#endif
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Paraklete");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
++NumSolve_;
return(0) ;
}
int Amesos_Paraklete::CreateLocalMatrixAndExporters()
{
ResetTimer(0);
RowMatrixA_ = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator());
if (RowMatrixA_ == 0) AMESOS_CHK_ERR(-1);
const Epetra_Map &OriginalMatrixMap = RowMatrixA_->RowMatrixRowMap() ;
const Epetra_Map &OriginalDomainMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorRangeMap():
GetProblem()->GetOperator()->OperatorDomainMap();
const Epetra_Map &OriginalRangeMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorDomainMap():
GetProblem()->GetOperator()->OperatorRangeMap();
NumGlobalElements_ = RowMatrixA_->NumGlobalRows();
numentries_ = RowMatrixA_->NumGlobalNonzeros();
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
//
// Create a serial matrix
//
assert( NumGlobalElements_ == OriginalMatrixMap.NumGlobalElements() ) ;
int NumMyElements_ = 0 ;
if (MyPID_==0) NumMyElements_ = NumGlobalElements_;
//
// UseDataInPlace_ is set to 1 (true) only if everything is perfectly
// normal. Anything out of the ordinary reverts to the more expensive
// path.
//
UseDataInPlace_ = ( OriginalMatrixMap.NumMyElements() ==
OriginalMatrixMap.NumGlobalElements() )?1:0;
if ( ! OriginalRangeMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( ! OriginalDomainMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( AddZeroToDiag_ ) UseDataInPlace_ = 0 ;
Comm().Broadcast( &UseDataInPlace_, 1, 0 ) ;
UseDataInPlace_ = 0 ; // bug - remove this someday.
//
// Reindex matrix if necessary (and possible - i.e. CrsMatrix)
//
// For now, since I don't know how to determine if we need to reindex the matrix,
// we allow the user to control re-indexing through the "Reindex" parameter.
//
CrsMatrixA_ = dynamic_cast<Epetra_CrsMatrix *>(Problem_->GetOperator());
Epetra_CrsMatrix* CcsMatrixA = 0 ;
//
// CcsMatrixA points to a matrix in Compressed Column Format
// i.e. the format needed by Paraklete. If we are solving
// A^T x = b, CcsMatrixA = CrsMatrixA_, otherwise we must
// transpose the matrix.
//
if (UseTranspose()) {
CcsMatrixA = CrsMatrixA_ ;
} else {
if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR( -7 ); // Amesos_Paraklete only supports CrsMatrix objects in the non-transpose case
#ifdef HAVE_AMESOS_EPETRAEXT
bool MakeDataContiguous = true;
transposer_ = rcp ( new EpetraExt::RowMatrix_Transpose( MakeDataContiguous ));
int OriginalTracebackMode = CrsMatrixA_->GetTracebackMode();
CrsMatrixA_->SetTracebackMode( EPETRA_MIN( OriginalTracebackMode, 0) );
CcsMatrixA = &(dynamic_cast<Epetra_CrsMatrix&>(((*transposer_)(*CrsMatrixA_))));
CrsMatrixA_->SetTracebackMode( OriginalTracebackMode );
#else
std::cerr << "Amesos_Paraklete requires the EpetraExt library to solve non-transposed problems. " << std::endl ;
std::cerr << " To rebuild Amesos with EpetraExt, add --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR( -3 );
#endif
}
if ( Reindex_ ) {
if ( CcsMatrixA == 0 ) AMESOS_CHK_ERR(-4);
#ifdef HAVE_AMESOS_EPETRAEXT
const Epetra_Map& OriginalMap = CcsMatrixA->RowMap();
StdIndex_ = rcp( new Amesos_StandardIndex( OriginalMap ) );
//const Epetra_Map& OriginalColMap = CcsMatrixA->RowMap();
StdIndexDomain_ = rcp( new Amesos_StandardIndex( OriginalDomainMap ) );
StdIndexRange_ = rcp( new Amesos_StandardIndex( OriginalRangeMap ) );
StdIndexMatrix_ = StdIndex_->StandardizeIndex( CcsMatrixA );
#else
std::cerr << "Amesos_Paraklete requires EpetraExt to reindex matrices." << std::endl
<< " Please rebuild with the EpetraExt library by adding --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR(-4);
#endif
} else {
if ( UseTranspose() ){
StdIndexMatrix_ = RowMatrixA_ ;
} else {
StdIndexMatrix_ = CcsMatrixA ;
}
}
//
// At this point, StdIndexMatrix_ points to a matrix with
// standard indexing.
//
//
// Convert Original Matrix to Serial (if it is not already)
//
if (UseDataInPlace_ == 1) {
SerialMatrix_ = StdIndexMatrix_;
} else {
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_, NumMyElements_, 0, Comm()));
if ( Problem_->GetRHS() )
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
else
NumVectors_ = 1 ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
ImportToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->RowMatrixRowMap() ) );
if (ImportToSerial_.get() == 0) AMESOS_CHK_ERR(-5);
SerialCrsMatrixA_ = rcp( new Epetra_CrsMatrix(Copy, *SerialMap_, 0) ) ;
SerialMatrix_ = &*SerialCrsMatrixA_ ;
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
return(0);
}
virtual int Apply(const Tpetra_MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node>& X,
Tpetra_MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node>& Y) const
{
IFPACK2_RETURN(Multiply(UseTranspose(),X,Y));
}
//==============================================================================
int Ifpack2_ReorderFilter::
Apply(const Tpetra_MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node>& X, Tpetra_MultiVector<Scalar,LocalOrdinal,GlobalOrdinal,Node>& Y) const
{
IFPACK2_RETURN(Multiply(UseTranspose(),X,Y));
}