//=============================================================================
int Amesos_Lapack::SolveDistributed(Epetra_MultiVector& X,
const Epetra_MultiVector& B)
{
ResetTimer();
int NumVectors = X.NumVectors();
// vector that contains RHS, overwritten by solution,
// with all elements on on process 0.
Epetra_MultiVector SerialVector(SerialMap(),NumVectors);
// import off-process data
AMESOS_CHK_ERR(SerialVector.Export(B,RhsExporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
ResetTimer();
if (MyPID_ == 0) {
Epetra_SerialDenseMatrix DenseX(static_cast<int>(NumGlobalRows64()),NumVectors);
Epetra_SerialDenseMatrix DenseB(static_cast<int>(NumGlobalRows64()),NumVectors);
for (int i = 0 ; i < NumGlobalRows64() ; ++i)
for (int j = 0 ; j < NumVectors ; ++j)
DenseB(i,j) = SerialVector[j][i];
DenseSolver_.SetVectors(DenseX,DenseB);
DenseSolver_.SolveWithTranspose(UseTranspose());
AMESOS_CHK_ERR(DenseSolver_.Solve());
for (int i = 0 ; i < NumGlobalRows64() ; ++i)
for (int j = 0 ; j < NumVectors ; ++j)
SerialVector[j][i] = DenseX(i,j);
}
SolveTime_ = AddTime("Total solve time", SolveTime_);
ResetTimer();
AMESOS_CHK_ERR(X.Import(SerialVector,SolutionImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
++NumSolve_;
return(0) ;
}
//=============================================================================
int Amesos_Umfpack::Solve()
{
// if necessary, perform numeric factorization.
// This may call SymbolicFactorization() as well.
if (!IsNumericFactorizationOK_)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
Epetra_MultiVector *SerialB, *SerialX;
// Extract Serial versions of X and B
//
double *SerialXvalues ;
double *SerialBvalues ;
Epetra_MultiVector* SerialXextract = 0;
Epetra_MultiVector* SerialBextract = 0;
// Copy B to the serial version of B
//
ResetTimer(0);
if (IsLocal_ == 1) {
SerialB = vecB ;
SerialX = vecX ;
} else {
assert (IsLocal_ == 0);
SerialXextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract->Import(*vecB,Importer(),Insert);
SerialB = SerialBextract;
SerialX = SerialXextract;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call UMFPACK to perform the solve
// Note: UMFPACK uses a Compressed Column Storage instead of compressed row storage,
// Hence to compute A X = B, we ask UMFPACK to perform A^T X = B and vice versa
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
ResetTimer(0);
int SerialBlda, SerialXlda ;
int UmfpackRequest = UseTranspose()?UMFPACK_A:UMFPACK_At ;
int status = 0;
if ( MyPID_ == 0 ) {
int ierr;
ierr = SerialB->ExtractView(&SerialBvalues, &SerialBlda);
assert (ierr == 0);
ierr = SerialX->ExtractView(&SerialXvalues, &SerialXlda);
assert (ierr == 0);
assert( SerialBlda == NumGlobalElements_ ) ;
assert( SerialXlda == NumGlobalElements_ ) ;
for ( int j =0 ; j < NumVectors; j++ ) {
double *Control = (double *) NULL, *Info = (double *) NULL ;
status = umfpack_di_solve (UmfpackRequest, &Ap[0],
&Ai[0], &Aval[0],
&SerialXvalues[j*SerialXlda],
&SerialBvalues[j*SerialBlda],
Numeric, Control, Info) ;
}
}
if (status) AMESOS_CHK_ERR(status);
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if ( IsLocal_ == 0 ) {
vecX->Export(*SerialX, Importer(), Insert ) ;
if (SerialBextract) delete SerialBextract ;
if (SerialXextract) delete SerialXextract ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
{
Epetra_RowMatrix* Matrix =
dynamic_cast<Epetra_RowMatrix*>(Problem_->GetOperator());
ComputeTrueResidual(*Matrix, *vecX, *vecB, UseTranspose(), "Amesos_Umfpack");
}
if (ComputeVectorNorms_) {
ComputeVectorNorms(*vecX, *vecB, "Amesos_Umfpack");
}
NumSolve_++;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); // Amesos overhead
return(0);
}
void
Albany::SolutionResponseFunction::
cullSolution(const Epetra_MultiVector& x, Epetra_MultiVector& x_culled) const
{
x_culled.Import(x, *importer, Insert);
}
int Amesos_Scalapack::Solve() {
if( debug_ == 1 ) std::cout << "Entering `Solve()'" << std::endl;
NumSolve_++;
Epetra_MultiVector *vecX = Problem_->GetLHS() ;
Epetra_MultiVector *vecB = Problem_->GetRHS() ;
//
// Compute the number of right hands sides
// (and check that X and B have the same shape)
//
int nrhs;
if ( vecX == 0 ) {
nrhs = 0 ;
EPETRA_CHK_ERR( vecB != 0 ) ;
} else {
nrhs = vecX->NumVectors() ;
EPETRA_CHK_ERR( vecB->NumVectors() != nrhs ) ;
}
Epetra_MultiVector *ScalapackB =0;
Epetra_MultiVector *ScalapackX =0;
//
// Extract Scalapack versions of X and B
//
double *ScalapackXvalues ;
Epetra_RowMatrix *RowMatrixA = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator());
Time_->ResetStartTime(); // track time to broadcast vectors
//
// Copy B to the scalapack version of B
//
const Epetra_Map &OriginalMap = RowMatrixA->RowMatrixRowMap();
Epetra_MultiVector *ScalapackXextract = new Epetra_MultiVector( *VectorMap_, nrhs ) ;
Epetra_MultiVector *ScalapackBextract = new Epetra_MultiVector( *VectorMap_, nrhs ) ;
Epetra_Import ImportToScalapack( *VectorMap_, OriginalMap );
ScalapackBextract->Import( *vecB, ImportToScalapack, Insert ) ;
ScalapackB = ScalapackBextract ;
ScalapackX = ScalapackXextract ;
VecTime_ += Time_->ElapsedTime();
//
// Call SCALAPACKs PDGETRS to perform the solve
//
int DescX[10];
ScalapackX->Scale(1.0, *ScalapackB) ;
int ScalapackXlda ;
Time_->ResetStartTime(); // tract time to solve
//
// Setup DescX
//
if( nrhs > nb_ ) {
EPETRA_CHK_ERR( -2 );
}
int Ierr[1] ;
Ierr[0] = 0 ;
const int zero = 0 ;
const int one = 1 ;
if ( iam_ < nprow_ * npcol_ ) {
assert( ScalapackX->ExtractView( &ScalapackXvalues, &ScalapackXlda ) == 0 ) ;
if ( false ) std::cout << "Amesos_Scalapack.cpp: " << __LINE__ << " ScalapackXlda = " << ScalapackXlda
<< " lda_ = " << lda_
<< " nprow_ = " << nprow_
<< " npcol_ = " << npcol_
<< " myprow_ = " << myprow_
<< " mypcol_ = " << mypcol_
<< " iam_ = " << iam_ << std::endl ;
if ( TwoD_distribution_ ) assert( mypcol_ >0 || EPETRA_MAX(ScalapackXlda,1) == lda_ ) ;
DESCINIT_F77(DescX,
&NumGlobalElements_,
&nrhs,
&nb_,
&nb_,
&zero,
&zero,
&ictxt_,
&lda_,
Ierr ) ;
assert( Ierr[0] == 0 ) ;
//
// For the 1D data distribution, we factor the transposed
// matrix, hence we must invert the sense of the transposition
//
char trans = 'N';
if ( TwoD_distribution_ ) {
if ( UseTranspose() ) trans = 'T' ;
} else {
if ( ! UseTranspose() ) trans = 'T' ;
}
if ( nprow_ * npcol_ == 1 ) {
DGETRS_F77(&trans,
&NumGlobalElements_,
&nrhs,
&DenseA_[0],
&lda_,
&Ipiv_[0],
ScalapackXvalues,
&lda_,
Ierr ) ;
} else {
PDGETRS_F77(&trans,
&NumGlobalElements_,
&nrhs,
&DenseA_[0],
&one,
&one,
DescA_,
&Ipiv_[0],
ScalapackXvalues,
&one,
&one,
DescX,
Ierr ) ;
}
}
SolTime_ += Time_->ElapsedTime();
Time_->ResetStartTime(); // track time to broadcast vectors
//
// Copy X back to the original vector
//
Epetra_Import ImportFromScalapack( OriginalMap, *VectorMap_ );
vecX->Import( *ScalapackX, ImportFromScalapack, Insert ) ;
delete ScalapackBextract ;
delete ScalapackXextract ;
VecTime_ += Time_->ElapsedTime();
// All processes should return the same error code
if ( nprow_ * npcol_ < Comm().NumProc() )
Comm().Broadcast( Ierr, 1, 0 ) ;
// MS // compute vector norms
if( ComputeVectorNorms_ == true || verbose_ == 2 ) {
double NormLHS, NormRHS;
for( int i=0 ; i<nrhs ; ++i ) {
assert((*vecX)(i)->Norm2(&NormLHS)==0);
assert((*vecB)(i)->Norm2(&NormRHS)==0);
if( verbose_ && Comm().MyPID() == 0 ) {
std::cout << "Amesos_Scalapack : vector " << i << ", ||x|| = " << NormLHS
<< ", ||b|| = " << NormRHS << std::endl;
}
}
}
// MS // compute true residual
if( ComputeTrueResidual_ == true || verbose_ == 2 ) {
double Norm;
Epetra_MultiVector Ax(vecB->Map(),nrhs);
for( int i=0 ; i<nrhs ; ++i ) {
(Problem_->GetMatrix()->Multiply(UseTranspose(), *((*vecX)(i)), Ax));
(Ax.Update(1.0, *((*vecB)(i)), -1.0));
(Ax.Norm2(&Norm));
if( verbose_ && Comm().MyPID() == 0 ) {
std::cout << "Amesos_Scalapack : vector " << i << ", ||Ax - b|| = " << Norm << std::endl;
}
}
}
return Ierr[0];
}