int Amesos_Scalapack::Solve() {
if( debug_ == 1 ) std::cout << "Entering `Solve()'" << std::endl;
NumSolve_++;
Epetra_MultiVector *vecX = Problem_->GetLHS() ;
Epetra_MultiVector *vecB = Problem_->GetRHS() ;
//
// Compute the number of right hands sides
// (and check that X and B have the same shape)
//
int nrhs;
if ( vecX == 0 ) {
nrhs = 0 ;
EPETRA_CHK_ERR( vecB != 0 ) ;
} else {
nrhs = vecX->NumVectors() ;
EPETRA_CHK_ERR( vecB->NumVectors() != nrhs ) ;
}
Epetra_MultiVector *ScalapackB =0;
Epetra_MultiVector *ScalapackX =0;
//
// Extract Scalapack versions of X and B
//
double *ScalapackXvalues ;
Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator());
Time_->ResetStartTime(); // track time to broadcast vectors
//
// Copy B to the scalapack version of B
//
const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap();
Epetra_MultiVector *ScalapackXextract = new Epetra_MultiVector( *VectorMap_, nrhs ) ;
Epetra_MultiVector *ScalapackBextract = new Epetra_MultiVector( *VectorMap_, nrhs ) ;
Epetra_Import ImportToScalapack( *VectorMap_, OriginalMap );
ScalapackBextract->Import( *vecB, ImportToScalapack, Insert ) ;
ScalapackB = ScalapackBextract ;
ScalapackX = ScalapackXextract ;
VecTime_ += Time_->ElapsedTime();
//
// Call SCALAPACKs PDGETRS to perform the solve
//
int DescX[10];
ScalapackX->Scale(1.0, *ScalapackB) ;
int ScalapackXlda ;
Time_->ResetStartTime(); // tract time to solve
//
// Setup DescX
//
if( nrhs > nb_ ) {
EPETRA_CHK_ERR( -2 );
}
int Ierr[1] ;
Ierr[0] = 0 ;
const int zero = 0 ;
const int one = 1 ;
if ( iam_ < nprow_ * npcol_ ) {
assert( ScalapackX->ExtractView( &ScalapackXvalues, &ScalapackXlda ) == 0 ) ;
if ( false ) std::cout << "Amesos_Scalapack.cpp: " << __LINE__ << " ScalapackXlda = " << ScalapackXlda
<< " lda_ = " << lda_
<< " nprow_ = " << nprow_
<< " npcol_ = " << npcol_
<< " myprow_ = " << myprow_
<< " mypcol_ = " << mypcol_
<< " iam_ = " << iam_ << std::endl ;
if ( TwoD_distribution_ ) assert( mypcol_ >0 || EPETRA_MAX(ScalapackXlda,1) == lda_ ) ;
DESCINIT_F77(DescX,
&NumGlobalElements_,
&nrhs,
&nb_,
&nb_,
&zero,
&zero,
&ictxt_,
&lda_,
Ierr ) ;
assert( Ierr[0] == 0 ) ;
//
// For the 1D data distribution, we factor the transposed
// matrix, hence we must invert the sense of the transposition
//
char trans = 'N';
if ( TwoD_distribution_ ) {
if ( UseTranspose() ) trans = 'T' ;
} else {
if ( ! UseTranspose() ) trans = 'T' ;
}
if ( nprow_ * npcol_ == 1 ) {
DGETRS_F77(&trans,
&NumGlobalElements_,
&nrhs,
&DenseA_[0],
&lda_,
&Ipiv_[0],
ScalapackXvalues,
&lda_,
Ierr ) ;
} else {
PDGETRS_F77(&trans,
&NumGlobalElements_,
&nrhs,
&DenseA_[0],
&one,
&one,
DescA_,
&Ipiv_[0],
ScalapackXvalues,
&one,
&one,
DescX,
Ierr ) ;
}
}
SolTime_ += Time_->ElapsedTime();
Time_->ResetStartTime(); // track time to broadcast vectors
//
// Copy X back to the original vector
//
Epetra_Import ImportFromScalapack( OriginalMap, *VectorMap_ );
vecX->Import( *ScalapackX, ImportFromScalapack, Insert ) ;
delete ScalapackBextract ;
delete ScalapackXextract ;
VecTime_ += Time_->ElapsedTime();
// All processes should return the same error code
if ( nprow_ * npcol_ < Comm().NumProc() )
Comm().Broadcast( Ierr, 1, 0 ) ;
// MS // compute vector norms
if( ComputeVectorNorms_ == true || verbose_ == 2 ) {
double NormLHS, NormRHS;
for( int i=0 ; i<nrhs ; ++i ) {
assert((*vecX)(i)->Norm2(&NormLHS)==0);
assert((*vecB)(i)->Norm2(&NormRHS)==0);
if( verbose_ && Comm().MyPID() == 0 ) {
std::cout << "Amesos_Scalapack : vector " << i << ", ||x|| = " << NormLHS
<< ", ||b|| = " << NormRHS << std::endl;
}
}
}
// MS // compute true residual
if( ComputeTrueResidual_ == true || verbose_ == 2 ) {
double Norm;
Epetra_MultiVector Ax(vecB->Map(),nrhs);
for( int i=0 ; i<nrhs ; ++i ) {
(Problem_->GetMatrix()->Multiply(UseTranspose(), *((*vecX)(i)), Ax));
(Ax.Update(1.0, *((*vecB)(i)), -1.0));
(Ax.Norm2(&Norm));
if( verbose_ && Comm().MyPID() == 0 ) {
std::cout << "Amesos_Scalapack : vector " << i << ", ||Ax - b|| = " << Norm << std::endl;
}
}
}
return Ierr[0];
}
//=============================================================================
void Epetra_LAPACK::GETRS( const char TRANS, const int N, const int NRHS, const double * A, const int LDA,
const int * IPIV, double * X, const int LDX, int * INFO) const {
DGETRS_F77(CHAR_MACRO(TRANS), &N, &NRHS, A, &LDA, IPIV, X, &LDX, INFO);
}
