//=============================================================================
int Amesos_Dscpack::NumericFactorization()
{
IsNumericFactorizationOK_ = false;
if (!IsSymbolicFactorizationOK_)
AMESOS_CHK_ERR(SymbolicFactorization());
AMESOS_CHK_ERR(PerformNumericFactorization());
IsNumericFactorizationOK_ = true;
NumNumericFact_++;
return(0);
}
int Amesos_Mumps::NumericFactorization()
{
IsNumericFactorizationOK_ = false;
if (IsSymbolicFactorizationOK_ == false)
AMESOS_CHK_ERR(SymbolicFactorization());
RedistrMatrix(true);
AMESOS_CHK_ERR(ConvertToTriplet(true));
if (Comm().NumProc() != 1)
{
if (Comm().MyPID() < MaxProcs_)
MDS.a_loc = &Val[0];
}
else
MDS.a = &Val[0];
// Request numeric factorization
MDS.job = 2;
// Perform numeric factorization
ResetTimer();
if (Comm().MyPID() < MaxProcs_) {
dmumps_c(&(MDS));
}
NumFactTime_ = AddTime("Total numeric factorization time", NumFactTime_);
int IntWrong = CheckError()?1:0 ;
int AnyWrong;
Comm().SumAll( &IntWrong, &AnyWrong, 1 ) ;
bool Wrong = AnyWrong > 0 ;
if ( Wrong ) {
AMESOS_CHK_ERR( NumericallySingularMatrixError ) ;
}
IsNumericFactorizationOK_ = true;
NumNumericFact_++;
return(0);
}
//=============================================================================
int Amesos_Umfpack::ConvertToUmfpackCRS()
{
ResetTimer(0);
ResetTimer(1);
// Convert matrix to the form that Umfpack expects (Ap, Ai, Aval),
// only on processor 0. The matrix has already been assembled in
// SerialMatrix_; if only one processor is used, then SerialMatrix_
// points to the problem's matrix.
if (MyPID_ == 0)
{
Ap.resize( NumGlobalElements_+1 );
Ai.resize( EPETRA_MAX( NumGlobalElements_, numentries_) ) ;
Aval.resize( EPETRA_MAX( NumGlobalElements_, numentries_) ) ;
int NumEntries = SerialMatrix_->MaxNumEntries();
int NumEntriesThisRow;
int Ai_index = 0 ;
int MyRow;
for (MyRow = 0 ; MyRow < NumGlobalElements_; MyRow++)
{
int ierr;
Ap[MyRow] = Ai_index ;
ierr = SerialMatrix_->ExtractMyRowCopy(MyRow, NumEntries,
NumEntriesThisRow,
&Aval[Ai_index], &Ai[Ai_index]);
if (ierr)
AMESOS_CHK_ERR(-1);
#if 1
// MS // added on 15-Mar-05 and KSS restored 8-Feb-06
if (AddToDiag_ != 0.0) {
for (int i = 0 ; i < NumEntriesThisRow ; ++i) {
if (Ai[Ai_index+i] == MyRow) {
Aval[Ai_index+i] += AddToDiag_;
break;
}
}
}
#endif
Ai_index += NumEntriesThisRow;
}
Ap[MyRow] = Ai_index ;
}
MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_, 0);
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
return 0;
}
//=============================================================================
int Amesos_Umfpack::SymbolicFactorization()
{
// MS // NOTE: If you change this method, also change
// MS // NumericFactorization(), because it performs part of the actions
// MS // of this method. This is to avoid to ship the matrix twice
// MS // (once for the symbolic factorization, and once for the numeric
// MS // factorization) when it is not necessary.
IsSymbolicFactorizationOK_ = false;
IsNumericFactorizationOK_ = false;
CreateTimer(Comm(), 2);
// MS // Initialize two timers:
// MS // timer 1: this will track all time spent in Amesos most
// MS // important functions, *including* UMFPACK functions
// MS // timer 2: this will track all time spent in this function
// MS // that is not due to UMFPACK calls, and therefore it
// MS // tracks down how much Amesos costs. The timer starts
// MS // and ends in *each* method, unless the method does not
// MS // perform any real operation. If a method calls another
// MS // method, the timer will be stopped before the called
// MS // method, then restared.
// MS // All the time of this timer goes into "overhead"
MyPID_ = Comm().MyPID();
NumProcs_ = Comm().NumProc();
AMESOS_CHK_ERR(ConvertToSerial(true));
AMESOS_CHK_ERR(ConvertToUmfpackCRS());
AMESOS_CHK_ERR(PerformSymbolicFactorization());
IsSymbolicFactorizationOK_ = true;
NumSymbolicFact_++;
return 0;
}
//=============================================================================
int Amesos_Paraklete::ExportToSerial()
{
if ( AddZeroToDiag_==0 && numentries_ != RowMatrixA_->NumGlobalNonzeros()) {
std::cerr << " The number of non zero entries in the matrix has changed since the last call to SymbolicFactorization(). " ;
AMESOS_CHK_ERR( -2 );
}
if ( UseDataInPlace_ == 0 ) {
assert ( RowMatrixA_ != 0 ) ;
if ( AddZeroToDiag_==0 && numentries_ != RowMatrixA_->NumGlobalNonzeros()) {
std::cerr << " The number of non zero entries in the matrix has changed since the last call to SymbolicFactorization(). " ;
AMESOS_CHK_ERR( -2 );
}
#ifdef HAVE_AMESOS_EPETRAEXT
if ( transposer_.get() != 0 ) {
// int OriginalTracebackMode = CrsMatrixA_->GetTracebackMode();
// CrsMatrixA_->SetTracebackMode( EPETRA_MIN( OriginalTracebackMode, 0) );
transposer_->fwd() ;
// CrsMatrixA_->SetTracebackMode( OriginalTracebackMode );
}
#endif
assert ( ImportToSerial_.get() != 0 ) ;
AMESOS_CHK_ERR(SerialCrsMatrixA_->Import(*StdIndexMatrix_,
*ImportToSerial_, Insert ));
if (AddZeroToDiag_ ) {
int OriginalTracebackMode = SerialCrsMatrixA_->GetTracebackMode() ;
SerialCrsMatrixA_->SetTracebackMode( EPETRA_MIN( OriginalTracebackMode, 0) ) ; // ExportToSerial is called both by PerformSymbolicFactorization() and PerformNumericFactorization(). When called by the latter, the call to insertglobalvalues is both unnecessary and illegal. Fortunately, Epetra allows us to ignore the error message by setting the traceback mode to 0.
//
// Add 0.0 to each diagonal entry to avoid empty diagonal entries;
//
double zero = 0.0;
for ( int i = 0 ; i < SerialMap_->NumGlobalElements(); i++ )
if ( SerialCrsMatrixA_->LRID(i) >= 0 )
SerialCrsMatrixA_->InsertGlobalValues( i, 1, &zero, &i ) ;
SerialCrsMatrixA_->SetTracebackMode( OriginalTracebackMode ) ;
}
AMESOS_CHK_ERR(SerialCrsMatrixA_->FillComplete());
//AMESOS_CHK_ERR(SerialCrsMatrixA_->OptimizeStorage());
if( !AddZeroToDiag_ && numentries_ != SerialMatrix_->NumGlobalNonzeros()) {
std::cerr << " Amesos_Paraklete cannot handle this matrix. " ;
if ( Reindex_ ) {
std::cerr << "Unknown error" << std::endl ;
AMESOS_CHK_ERR( -5 );
} else {
std::cerr << " Try setting the Reindex parameter to true. " << std::endl ;
#ifndef HAVE_AMESOS_EPETRAEXT
std::cerr << " You will need to rebuild the Amesos library with the EpetraExt library to use the reindex feature" << std::endl ;
std::cerr << " To rebuild Amesos with EpetraExt, add --enable-epetraext to your configure invocation" << std::endl ;
#endif
AMESOS_CHK_ERR( -3 );
}
}
}
return 0;
}
//=============================================================================
int Amesos_Dscpack::SymbolicFactorization()
{
IsSymbolicFactorizationOK_ = false;
IsNumericFactorizationOK_ = false;
CreateTimer(Comm(), 2);
AMESOS_CHK_ERR(PerformSymbolicFactorization());
IsSymbolicFactorizationOK_ = true;
NumSymbolicFact_++;
return(0);
}
//=============================================================================
int Amesos_Klu::PerformSymbolicFactorization()
{
ResetTimer(0);
int symbolic_ok = 0;
if (MyPID_ == 0) {
PrivateKluData_->common_ = rcp(new klu_common());
amesos_klu_defaults(&*PrivateKluData_->common_) ;
PrivateKluData_->Symbolic_ =
rcpWithDealloc(
amesos_klu_analyze (NumGlobalElements_, &Ap[0], Ai, &*PrivateKluData_->common_ )
,deallocFunctorDeleteWithCommon<klu_symbolic>(PrivateKluData_->common_,amesos_klu_free_symbolic)
,true
);
symbolic_ok = (PrivateKluData_->Symbolic_.get() != NULL)
&& PrivateKluData_->common_->status == KLU_OK ;
}
// Communicate the state of the symbolic factorization with everyone.
Comm().Broadcast(&symbolic_ok, 1, 0);
if ( !symbolic_ok ) {
if (MyPID_ == 0) {
AMESOS_CHK_ERR( StructurallySingularMatrixError );
} else
return( StructurallySingularMatrixError );
}
SymFactTime_ = AddTime("Total symbolic factorization time", SymFactTime_, 0);
return 0;
}
//=============================================================================
int Amesos_Umfpack::NumericFactorization()
{
IsNumericFactorizationOK_ = false;
if (IsSymbolicFactorizationOK_ == false)
{
// Call here what is needed, to avoid double shipping of the matrix
CreateTimer(Comm(), 2);
MyPID_ = Comm().MyPID();
NumProcs_ = Comm().NumProc();
AMESOS_CHK_ERR(ConvertToSerial(true));
AMESOS_CHK_ERR(ConvertToUmfpackCRS());
AMESOS_CHK_ERR(PerformSymbolicFactorization());
IsSymbolicFactorizationOK_ = true;
NumSymbolicFact_++;
AMESOS_CHK_ERR(PerformNumericFactorization());
}
else
{
// need to reshuffle and reconvert because entry values may have changed
AMESOS_CHK_ERR(ConvertToSerial(false));
AMESOS_CHK_ERR(ConvertToUmfpackCRS());
AMESOS_CHK_ERR(PerformNumericFactorization());
}
NumNumericFact_++;
IsNumericFactorizationOK_ = true;
return 0;
}
//=============================================================================
int Amesos_Klu::NumericFactorization()
{
if ( !TrustMe_ )
{
IsNumericFactorizationOK_ = false;
if (IsSymbolicFactorizationOK_ == false)
AMESOS_CHK_ERR(SymbolicFactorization());
ResetTimer(1); // "overhead" time
Epetra_CrsMatrix *CrsMatrixA = dynamic_cast<Epetra_CrsMatrix *>(RowMatrixA_);
if ( CrsMatrixA == 0 ) // hack to get around Bug #1502
AMESOS_CHK_ERR( CreateLocalMatrixAndExporters() ) ;
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
if ( AddZeroToDiag_ == 0 )
assert( numentries_ == RowMatrixA_->NumGlobalNonzeros() );
AMESOS_CHK_ERR( ExportToSerial() );
if ( CrsMatrixA == 0 ) { // continuation of hack to avoid bug #1502
AMESOS_CHK_ERR( ConvertToKluCRS(true) );
} else {
AMESOS_CHK_ERR( ConvertToKluCRS(false) );
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
// this time is all for KLU
AMESOS_CHK_ERR( PerformNumericFactorization() );
NumNumericFact_++;
IsNumericFactorizationOK_ = true;
return 0;
}
//=============================================================================
int Amesos_Dscpack::PerformSymbolicFactorization()
{
ResetTimer(0);
ResetTimer(1);
MyPID_ = Comm().MyPID();
NumProcs_ = Comm().NumProc();
Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ;
const Epetra_MpiComm& comm1 = dynamic_cast<const Epetra_MpiComm &> (Comm());
int numrows = RowMatrixA->NumGlobalRows();
int numentries = RowMatrixA->NumGlobalNonzeros();
Teuchos::RCP<Epetra_CrsGraph> Graph;
Epetra_CrsMatrix* CastCrsMatrixA =
dynamic_cast<Epetra_CrsMatrix*>(RowMatrixA);
if (CastCrsMatrixA)
{
Graph = Teuchos::rcp(const_cast<Epetra_CrsGraph*>(&(CastCrsMatrixA->Graph())), false);
}
else
{
int MaxNumEntries = RowMatrixA->MaxNumEntries();
Graph = Teuchos::rcp(new Epetra_CrsGraph(Copy, OriginalMap, MaxNumEntries));
std::vector<int> Indices(MaxNumEntries);
std::vector<double> Values(MaxNumEntries);
for (int i = 0 ; i < RowMatrixA->NumMyRows() ; ++i)
{
int NumEntries;
RowMatrixA->ExtractMyRowCopy(i, MaxNumEntries, NumEntries,
&Values[0], &Indices[0]);
for (int j = 0 ; j < NumEntries ; ++j)
Indices[j] = RowMatrixA->RowMatrixColMap().GID(Indices[j]);
int GlobalRow = RowMatrixA->RowMatrixRowMap().GID(i);
Graph->InsertGlobalIndices(GlobalRow, NumEntries, &Indices[0]);
}
Graph->FillComplete();
}
//
// Create a replicated map and graph
//
std::vector<int> AllIDs( numrows ) ;
for ( int i = 0; i < numrows ; i++ ) AllIDs[i] = i ;
Epetra_Map ReplicatedMap( -1, numrows, &AllIDs[0], 0, Comm());
Epetra_Import ReplicatedImporter(ReplicatedMap, OriginalMap);
Epetra_CrsGraph ReplicatedGraph( Copy, ReplicatedMap, 0 );
AMESOS_CHK_ERR(ReplicatedGraph.Import(*Graph, ReplicatedImporter, Insert));
AMESOS_CHK_ERR(ReplicatedGraph.FillComplete());
//
// Convert the matrix to Ap, Ai
//
std::vector <int> Replicates(numrows);
std::vector <int> Ap(numrows + 1);
std::vector <int> Ai(EPETRA_MAX(numrows, numentries));
for( int i = 0 ; i < numrows; i++ ) Replicates[i] = 1;
int NumEntriesPerRow ;
int *ColIndices = 0 ;
int Ai_index = 0 ;
for ( int MyRow = 0; MyRow <numrows; MyRow++ ) {
AMESOS_CHK_ERR( ReplicatedGraph.ExtractMyRowView( MyRow, NumEntriesPerRow, ColIndices ) );
Ap[MyRow] = Ai_index ;
for ( int j = 0; j < NumEntriesPerRow; j++ ) {
Ai[Ai_index] = ColIndices[j] ;
Ai_index++;
}
}
assert( Ai_index == numentries ) ;
Ap[ numrows ] = Ai_index ;
MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_, 0);
ResetTimer(0);
//
// Call Dscpack Symbolic Factorization
//
int OrderCode = 2;
std::vector<double> MyANonZ;
NumLocalNonz = 0 ;
GlobalStructNewColNum = 0 ;
GlobalStructNewNum = 0 ;
GlobalStructOwner = 0 ;
LocalStructOldNum = 0 ;
NumGlobalCols = 0 ;
// MS // Have to define the maximum number of processes to be used
// MS // This is only a suggestion as Dscpack uses a number of processes that is a power of 2
int NumGlobalNonzeros = GetProblem()->GetMatrix()->NumGlobalNonzeros();
int NumRows = GetProblem()->GetMatrix()->NumGlobalRows();
// optimal value for MaxProcs == -1
int OptNumProcs1 = 1+EPETRA_MAX( NumRows/10000, NumGlobalNonzeros/1000000 );
OptNumProcs1 = EPETRA_MIN(NumProcs_,OptNumProcs1 );
// optimal value for MaxProcs == -2
int OptNumProcs2 = (int)sqrt(1.0 * NumProcs_);
if( OptNumProcs2 < 1 ) OptNumProcs2 = 1;
// fix the value of MaxProcs
switch (MaxProcs_)
{
case -1:
MaxProcs_ = EPETRA_MIN(OptNumProcs1, NumProcs_);
break;
case -2:
MaxProcs_ = EPETRA_MIN(OptNumProcs2, NumProcs_);
break;
case -3:
MaxProcs_ = NumProcs_;
break;
}
#if 0
if (MyDscRank>=0 && A_and_LU_built) {
DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject);
}
#endif
// if ( ! A_and_LU_built ) {
// DSC_End( PrivateDscpackData_->MyDSCObject ) ;
// PrivateDscpackData_->MyDSCObject = DSC_Begin() ;
// }
// MS // here I continue with the old code...
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
DscNumProcs = 1 ;
int DscMax = DSC_Analyze( numrows, &Ap[0], &Ai[0], &Replicates[0] );
while ( DscNumProcs * 2 <=EPETRA_MIN( MaxProcs_, DscMax ) ) DscNumProcs *= 2 ;
MyDscRank = -1;
DSC_Open0( PrivateDscpackData_->MyDSCObject_, DscNumProcs, &MyDscRank, comm1.Comm()) ;
NumLocalCols = 0 ; // This is for those processes not in the Dsc grid
if ( MyDscRank >= 0 ) {
assert( MyPID_ == MyDscRank ) ;
AMESOS_CHK_ERR( DSC_Order ( PrivateDscpackData_->MyDSCObject_, OrderCode, numrows, &Ap[0], &Ai[0],
&Replicates[0], &NumGlobalCols, &NumLocalStructs,
&NumLocalCols, &NumLocalNonz,
&GlobalStructNewColNum, &GlobalStructNewNum,
&GlobalStructOwner, &LocalStructOldNum ) ) ;
assert( NumGlobalCols == numrows ) ;
assert( NumLocalCols == NumLocalStructs ) ;
}
if ( MyDscRank >= 0 ) {
int MaxSingleBlock;
const int Limit = 5000000 ; // Memory Limit set to 5 Terabytes
AMESOS_CHK_ERR( DSC_SFactor ( PrivateDscpackData_->MyDSCObject_, &TotalMemory_,
&MaxSingleBlock, Limit, DSC_LBLAS3, DSC_DBLAS2 ) ) ;
}
// A_and_LU_built = true; // If you uncomment this, TestOptions fails
SymFactTime_ = AddTime("Total symbolic factorization time", SymFactTime_, 0);
return(0);
}
//=============================================================================
int Amesos_Umfpack::Solve()
{
// if necessary, perform numeric factorization.
// This may call SymbolicFactorization() as well.
if (!IsNumericFactorizationOK_)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
Epetra_MultiVector *SerialB, *SerialX;
// Extract Serial versions of X and B
//
double *SerialXvalues ;
double *SerialBvalues ;
Epetra_MultiVector* SerialXextract = 0;
Epetra_MultiVector* SerialBextract = 0;
// Copy B to the serial version of B
//
ResetTimer(0);
if (IsLocal_ == 1) {
SerialB = vecB ;
SerialX = vecX ;
} else {
assert (IsLocal_ == 0);
SerialXextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract = new Epetra_MultiVector(SerialMap(),NumVectors);
SerialBextract->Import(*vecB,Importer(),Insert);
SerialB = SerialBextract;
SerialX = SerialXextract;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call UMFPACK to perform the solve
// Note: UMFPACK uses a Compressed Column Storage instead of compressed row storage,
// Hence to compute A X = B, we ask UMFPACK to perform A^T X = B and vice versa
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
ResetTimer(0);
int SerialBlda, SerialXlda ;
int UmfpackRequest = UseTranspose()?UMFPACK_A:UMFPACK_At ;
int status = 0;
if ( MyPID_ == 0 ) {
int ierr;
ierr = SerialB->ExtractView(&SerialBvalues, &SerialBlda);
assert (ierr == 0);
ierr = SerialX->ExtractView(&SerialXvalues, &SerialXlda);
assert (ierr == 0);
assert( SerialBlda == NumGlobalElements_ ) ;
assert( SerialXlda == NumGlobalElements_ ) ;
for ( int j =0 ; j < NumVectors; j++ ) {
double *Control = (double *) NULL, *Info = (double *) NULL ;
status = umfpack_di_solve (UmfpackRequest, &Ap[0],
&Ai[0], &Aval[0],
&SerialXvalues[j*SerialXlda],
&SerialBvalues[j*SerialBlda],
Numeric, Control, Info) ;
}
}
if (status) AMESOS_CHK_ERR(status);
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if ( IsLocal_ == 0 ) {
vecX->Export(*SerialX, Importer(), Insert ) ;
if (SerialBextract) delete SerialBextract ;
if (SerialXextract) delete SerialXextract ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
{
Epetra_RowMatrix* Matrix =
dynamic_cast<Epetra_RowMatrix*>(Problem_->GetOperator());
ComputeTrueResidual(*Matrix, *vecX, *vecB, UseTranspose(), "Amesos_Umfpack");
}
if (ComputeVectorNorms_) {
ComputeVectorNorms(*vecX, *vecB, "Amesos_Umfpack");
}
NumSolve_++;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1); // Amesos overhead
return(0);
}
//=============================================================================
//
// See also pre and post conditions in Amesos_Paraklete.h
// Preconditions:
// firsttime specifies that this is the first time that
// ConertToParakleteCrs has been called, i.e. in symbolic factorization.
// No data allocation should happen unless firsttime=true.
// SerialMatrix_ points to the matrix to be factored and solved
// NumGlobalElements_ has been set to the dimension of the matrix
// numentries_ has been set to the number of non-zeros in the matrix
// (i.e. CreateLocalMatrixAndExporters() has been callded)
//
// Postconditions:
// pk_A_ contains the matrix as Paraklete needs it
//
//
int Amesos_Paraklete::ConvertToParakleteCRS(bool firsttime)
{
ResetTimer(0);
//
// Convert matrix to the form that Klu expects (Ap, Ai, VecAval)
//
if (MyPID_==0) {
assert( NumGlobalElements_ == SerialMatrix_->NumGlobalRows());
assert( NumGlobalElements_ == SerialMatrix_->NumGlobalCols());
if ( ! AddZeroToDiag_ ) {
assert( numentries_ == SerialMatrix_->NumGlobalNonzeros()) ;
} else {
numentries_ = SerialMatrix_->NumGlobalNonzeros() ;
}
Epetra_CrsMatrix *CrsMatrix = dynamic_cast<Epetra_CrsMatrix *>(SerialMatrix_);
bool StorageOptimized = ( CrsMatrix != 0 && CrsMatrix->StorageOptimized() );
if ( AddToDiag_ != 0.0 ) StorageOptimized = false ;
if ( firsttime ) {
Ap.resize( NumGlobalElements_+1 );
Ai.resize( EPETRA_MAX( NumGlobalElements_, numentries_) ) ;
if ( ! StorageOptimized ) {
VecAval.resize( EPETRA_MAX( NumGlobalElements_, numentries_) ) ;
Aval = &VecAval[0];
}
}
double *RowValues;
int *ColIndices;
int NumEntriesThisRow;
if( StorageOptimized ) {
if ( firsttime ) {
Ap[0] = 0;
for ( int MyRow = 0; MyRow <NumGlobalElements_; MyRow++ ) {
if( CrsMatrix->
ExtractMyRowView( MyRow, NumEntriesThisRow, RowValues,
ColIndices ) != 0 )
AMESOS_CHK_ERR( -6 );
for ( int j=0; j < NumEntriesThisRow; j++ ) {
Ai[Ap[MyRow]+j] = ColIndices[j];
}
if ( MyRow == 0 ) {
Aval = RowValues ;
}
Ap[MyRow+1] = Ap[MyRow] + NumEntriesThisRow ;
}
}
} else {
int Ai_index = 0 ;
int MyRow;
int MaxNumEntries_ = SerialMatrix_->MaxNumEntries();
if ( firsttime && CrsMatrix == 0 ) {
ColIndicesV_.resize(MaxNumEntries_);
RowValuesV_.resize(MaxNumEntries_);
}
for ( MyRow = 0; MyRow <NumGlobalElements_; MyRow++ ) {
if ( CrsMatrix != 0 ) {
if( CrsMatrix->
ExtractMyRowView( MyRow, NumEntriesThisRow, RowValues,
ColIndices ) != 0 )
AMESOS_CHK_ERR( -6 );
} else {
if( SerialMatrix_->
ExtractMyRowCopy( MyRow, MaxNumEntries_,
NumEntriesThisRow, &RowValuesV_[0],
&ColIndicesV_[0] ) != 0 )
AMESOS_CHK_ERR( -6 );
RowValues = &RowValuesV_[0];
ColIndices = &ColIndicesV_[0];
}
if ( firsttime ) {
Ap[MyRow] = Ai_index ;
for ( int j = 0; j < NumEntriesThisRow; j++ ) {
Ai[Ai_index] = ColIndices[j] ;
// VecAval[Ai_index] = RowValues[j] ; // We have to do this because of the hacks to get around bug #1502
if (ColIndices[j] == MyRow) {
VecAval[Ai_index] += AddToDiag_;
}
Ai_index++;
}
} else {
for ( int j = 0; j < NumEntriesThisRow; j++ ) {
VecAval[Ai_index] = RowValues[j] ;
if (ColIndices[j] == MyRow) {
VecAval[Ai_index] += AddToDiag_;
}
Ai_index++;
}
}
}
Ap[MyRow] = Ai_index ;
}
PrivateParakleteData_->pk_A_.nrow = NumGlobalElements_ ;
cholmod_sparse& pk_A = PrivateParakleteData_->pk_A_ ;
pk_A.nrow = NumGlobalElements_ ;
pk_A.ncol = NumGlobalElements_ ;
pk_A.nzmax = numentries_ ;
pk_A.p = &Ap[0] ;
pk_A.i = &Ai[0] ;
pk_A.nz = 0;
if ( firsttime ) {
pk_A.x = 0 ;
pk_A.xtype = CHOLMOD_PATTERN ;
}
else
{
pk_A.x = Aval ;
pk_A.xtype = CHOLMOD_REAL ;
}
pk_A.z = 0 ;
pk_A.stype = 0 ; // symmetric
pk_A.itype = CHOLMOD_LONG ;
pk_A.dtype = CHOLMOD_DOUBLE ;
pk_A.sorted = 0 ;
pk_A.packed = 1 ;
}
MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_, 0);
return 0;
}
//=============================================================================
int Amesos_Paraklete::SymbolicFactorization()
{
MyPID_ = Comm().MyPID();
NumProcs_ = Comm().NumProc();
IsSymbolicFactorizationOK_ = false;
IsNumericFactorizationOK_ = false;
#ifdef HAVE_AMESOS_EPETRAEXT
transposer_ = static_cast<Teuchos::ENull>( 0 );
#endif
CreateTimer(Comm(), 2);
ResetTimer(1);
// "overhead" time for the following method is considered here
AMESOS_CHK_ERR( CreateLocalMatrixAndExporters() ) ;
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
SetMaxProcesses(MaxProcesses_, *RowMatrixA_);
//
// Perform checks in SymbolicFactorization(), but none in
// NumericFactorization() or Solve()
//
assert( ! TrustMe_ ) ;
if ( TrustMe_ ) {
if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR(10 );
if( UseDataInPlace_ != 1 ) AMESOS_CHK_ERR( 10 ) ;
if( Reindex_ ) AMESOS_CHK_ERR( 10 ) ;
if( ! Problem_->GetLHS() ) AMESOS_CHK_ERR( 10 ) ;
if( ! Problem_->GetRHS() ) AMESOS_CHK_ERR( 10 ) ;
if( ! Problem_->GetLHS()->NumVectors() ) AMESOS_CHK_ERR( 10 ) ;
if( ! Problem_->GetRHS()->NumVectors() ) AMESOS_CHK_ERR( 10 ) ;
SerialB_ = Problem_->GetRHS() ;
SerialX_ = Problem_->GetLHS() ;
NumVectors_ = SerialX_->NumVectors();
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
}
}
PrivateParakleteData_->common_ = rcp(new paraklete_common());
const Epetra_MpiComm* MpiComm = dynamic_cast<const Epetra_MpiComm*>(&Comm());
assert (MpiComm != 0);
MPI_Comm PK_Comm;
//
// Create an MPI group with MaxProcesses_ processes
//
if ( MaxProcesses_ != Comm().NumProc()) {
if(ParakleteComm_) {
MPI_Comm_free(&ParakleteComm_);
ParakleteComm_ = 0 ;
}
std::vector<int> ProcsInGroup(MaxProcesses_);
IamInGroup_ = false;
for (int i = 0 ; i < MaxProcesses_ ; ++i) {
ProcsInGroup[i] = i;
if ( Comm().MyPID() == i ) IamInGroup_ = true;
}
MPI_Group OrigGroup, ParakleteGroup;
MPI_Comm_group(MpiComm->GetMpiComm(), &OrigGroup);
MPI_Group_incl(OrigGroup, MaxProcesses_, &ProcsInGroup[0], &ParakleteGroup);
MPI_Comm_create(MpiComm->GetMpiComm(), ParakleteGroup, &ParakleteComm_);
PK_Comm = ParakleteComm_ ;
} else {
IamInGroup_ = true;
PK_Comm = MpiComm->GetMpiComm() ;
}
paraklete_common& pk_common = *PrivateParakleteData_->common_ ;
cholmod_common *cm = &(pk_common.cm) ;
amesos_cholmod_l_start (cm) ;
PK_DEBUG_INIT ("pk", cm) ;
pk_common.nproc = MaxProcesses_ ;
pk_common.myid = Comm().MyPID() ;
//pk_common.mpicomm = PK_Comm ;
cm->print = 1 ;
cm->precise = TRUE ;
cm->error_handler = my_handler ;
pk_common.tol_diag = 0.001 ;
pk_common.tol_offdiag = 0.1 ;
pk_common.growth = 2. ;
// "overhead" time for the following two methods is considered here
AMESOS_CHK_ERR( ExportToSerial() );
AMESOS_CHK_ERR( ConvertToParakleteCRS(true) );
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
// All this time if PARAKLETE time
AMESOS_CHK_ERR( PerformSymbolicFactorization() );
NumSymbolicFact_++;
IsSymbolicFactorizationOK_ = true;
return 0;
}
//=============================================================================
// If FirstTime is true, then build SerialMap and ImportToSerial,
// otherwise simply re-ship the matrix, so that the numerical values
// are updated.
int Amesos_Umfpack::ConvertToSerial(const bool FirstTime)
{
ResetTimer(0);
ResetTimer(1);
const Epetra_Map &OriginalMap = Matrix()->RowMatrixRowMap() ;
NumGlobalElements_ = Matrix()->NumGlobalRows();
numentries_ = Matrix()->NumGlobalNonzeros();
assert (NumGlobalElements_ == Matrix()->NumGlobalCols());
int NumMyElements_ = 0 ;
if (MyPID_ == 0) NumMyElements_ = NumGlobalElements_;
IsLocal_ = ( OriginalMap.NumMyElements() ==
OriginalMap.NumGlobalElements() )?1:0;
// if ( AddZeroToDiag_ ) IsLocal_ = 0 ; // bug # Umfpack does not support AddZeroToDiag_
Comm().Broadcast( &IsLocal_, 1, 0 ) ;
// Convert Original Matrix to Serial (if it is not already)
//
if (IsLocal_== 1) {
SerialMatrix_ = Matrix();
}
else
{
if (FirstTime)
{
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_,NumMyElements_,
0,Comm()));
if (SerialMap_.get() == 0)
AMESOS_CHK_ERR(-1);
ImportToSerial_ = rcp(new Epetra_Import (SerialMap(),OriginalMap));
if (ImportToSerial_.get() == 0)
AMESOS_CHK_ERR(-1);
}
SerialCrsMatrixA_ = rcp(new Epetra_CrsMatrix(Copy,SerialMap(),0));
if (SerialCrsMatrixA_.get() == 0)
AMESOS_CHK_ERR(-1);
SerialCrsMatrix().Import(*Matrix(), Importer(),Insert);
#if 0
I was not able to make this work - 11 Feb 2006
if (AddZeroToDiag_ ) {
int OriginalTracebackMode = SerialCrsMatrix().GetTracebackMode() ;
SerialCrsMatrix().SetTracebackMode( EPETRA_MIN( OriginalTracebackMode, 0) ) ; // ExportToSerial is called both by PerformSymbolicFactorization() and PerformNumericFactorization(). When called by the latter, the call to insertglobalvalues is both unnecessary and illegal. Fortunately, Epetra allows us to ignore the error message by setting the traceback mode to 0.
//
// Add 0.0 to each diagonal entry to avoid empty diagonal entries;
//
double zero = 0.0;
for ( int i = 0 ; i < SerialMap_->NumGlobalElements(); i++ )
if ( SerialCrsMatrix().LRID(i) >= 0 )
SerialCrsMatrix().InsertGlobalValues( i, 1, &zero, &i ) ;
SerialCrsMatrix().SetTracebackMode( OriginalTracebackMode ) ;
}
#endif
SerialCrsMatrix().FillComplete();
SerialMatrix_ = &SerialCrsMatrix();
assert( numentries_ == SerialMatrix_->NumGlobalNonzeros()); // This should be set to an assignment if AddToZeroDiag is non -zero
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
return(0);
}
//=============================================================================
int Amesos_Mumps::ConvertToTriplet(const bool OnlyValues)
{
Epetra_RowMatrix* ptr;
if (Comm().NumProc() == 1)
ptr = &Matrix();
else {
ptr = &RedistrMatrix(true);
}
ResetTimer();
#ifdef EXTRA_DEBUG_INFO
Epetra_CrsMatrix* Eptr = dynamic_cast<Epetra_CrsMatrix*>( ptr );
if ( ptr->NumGlobalNonzeros() < 300 ) SetICNTL(4,3 ); // Enable more debug info for small matrices
if ( ptr->NumGlobalNonzeros() < 42 && Eptr ) {
std::cout << " Matrix = " << std::endl ;
Eptr->Print( std::cout ) ;
} else {
assert( Eptr );
}
#endif
Row.resize(ptr->NumMyNonzeros());
Col.resize(ptr->NumMyNonzeros());
Val.resize(ptr->NumMyNonzeros());
int MaxNumEntries = ptr->MaxNumEntries();
std::vector<int> Indices;
std::vector<double> Values;
Indices.resize(MaxNumEntries);
Values.resize(MaxNumEntries);
int count = 0;
for (int i = 0; i < ptr->NumMyRows() ; ++i) {
int GlobalRow = ptr->RowMatrixRowMap().GID(i);
int NumEntries = 0;
int ierr;
ierr = ptr->ExtractMyRowCopy(i, MaxNumEntries,
NumEntries, &Values[0],
&Indices[0]);
AMESOS_CHK_ERR(ierr);
for (int j = 0 ; j < NumEntries ; ++j) {
if (OnlyValues == false) {
Row[count] = GlobalRow + 1;
Col[count] = ptr->RowMatrixColMap().GID(Indices[j]) + 1;
}
// MS // Added on 15-Mar-05.
if (AddToDiag_ && Indices[j] == i)
Values[j] += AddToDiag_;
Val[count] = Values[j];
count++;
}
}
MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_);
assert (count <= ptr->NumMyNonzeros());
return(0);
}
int Amesos_Paraklete::CreateLocalMatrixAndExporters()
{
ResetTimer(0);
RowMatrixA_ = dynamic_cast<Epetra_RowMatrix *>(Problem_->GetOperator());
if (RowMatrixA_ == 0) AMESOS_CHK_ERR(-1);
const Epetra_Map &OriginalMatrixMap = RowMatrixA_->RowMatrixRowMap() ;
const Epetra_Map &OriginalDomainMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorRangeMap():
GetProblem()->GetOperator()->OperatorDomainMap();
const Epetra_Map &OriginalRangeMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorDomainMap():
GetProblem()->GetOperator()->OperatorRangeMap();
NumGlobalElements_ = RowMatrixA_->NumGlobalRows();
numentries_ = RowMatrixA_->NumGlobalNonzeros();
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
//
// Create a serial matrix
//
assert( NumGlobalElements_ == OriginalMatrixMap.NumGlobalElements() ) ;
int NumMyElements_ = 0 ;
if (MyPID_==0) NumMyElements_ = NumGlobalElements_;
//
// UseDataInPlace_ is set to 1 (true) only if everything is perfectly
// normal. Anything out of the ordinary reverts to the more expensive
// path.
//
UseDataInPlace_ = ( OriginalMatrixMap.NumMyElements() ==
OriginalMatrixMap.NumGlobalElements() )?1:0;
if ( ! OriginalRangeMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( ! OriginalDomainMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( AddZeroToDiag_ ) UseDataInPlace_ = 0 ;
Comm().Broadcast( &UseDataInPlace_, 1, 0 ) ;
UseDataInPlace_ = 0 ; // bug - remove this someday.
//
// Reindex matrix if necessary (and possible - i.e. CrsMatrix)
//
// For now, since I don't know how to determine if we need to reindex the matrix,
// we allow the user to control re-indexing through the "Reindex" parameter.
//
CrsMatrixA_ = dynamic_cast<Epetra_CrsMatrix *>(Problem_->GetOperator());
Epetra_CrsMatrix* CcsMatrixA = 0 ;
//
// CcsMatrixA points to a matrix in Compressed Column Format
// i.e. the format needed by Paraklete. If we are solving
// A^T x = b, CcsMatrixA = CrsMatrixA_, otherwise we must
// transpose the matrix.
//
if (UseTranspose()) {
CcsMatrixA = CrsMatrixA_ ;
} else {
if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR( -7 ); // Amesos_Paraklete only supports CrsMatrix objects in the non-transpose case
#ifdef HAVE_AMESOS_EPETRAEXT
bool MakeDataContiguous = true;
transposer_ = rcp ( new EpetraExt::RowMatrix_Transpose( MakeDataContiguous ));
int OriginalTracebackMode = CrsMatrixA_->GetTracebackMode();
CrsMatrixA_->SetTracebackMode( EPETRA_MIN( OriginalTracebackMode, 0) );
CcsMatrixA = &(dynamic_cast<Epetra_CrsMatrix&>(((*transposer_)(*CrsMatrixA_))));
CrsMatrixA_->SetTracebackMode( OriginalTracebackMode );
#else
std::cerr << "Amesos_Paraklete requires the EpetraExt library to solve non-transposed problems. " << std::endl ;
std::cerr << " To rebuild Amesos with EpetraExt, add --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR( -3 );
#endif
}
if ( Reindex_ ) {
if ( CcsMatrixA == 0 ) AMESOS_CHK_ERR(-4);
#ifdef HAVE_AMESOS_EPETRAEXT
const Epetra_Map& OriginalMap = CcsMatrixA->RowMap();
StdIndex_ = rcp( new Amesos_StandardIndex( OriginalMap ) );
//const Epetra_Map& OriginalColMap = CcsMatrixA->RowMap();
StdIndexDomain_ = rcp( new Amesos_StandardIndex( OriginalDomainMap ) );
StdIndexRange_ = rcp( new Amesos_StandardIndex( OriginalRangeMap ) );
StdIndexMatrix_ = StdIndex_->StandardizeIndex( CcsMatrixA );
#else
std::cerr << "Amesos_Paraklete requires EpetraExt to reindex matrices." << std::endl
<< " Please rebuild with the EpetraExt library by adding --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR(-4);
#endif
} else {
if ( UseTranspose() ){
StdIndexMatrix_ = RowMatrixA_ ;
} else {
StdIndexMatrix_ = CcsMatrixA ;
}
}
//
// At this point, StdIndexMatrix_ points to a matrix with
// standard indexing.
//
//
// Convert Original Matrix to Serial (if it is not already)
//
if (UseDataInPlace_ == 1) {
SerialMatrix_ = StdIndexMatrix_;
} else {
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_, NumMyElements_, 0, Comm()));
if ( Problem_->GetRHS() )
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
else
NumVectors_ = 1 ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
ImportToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->RowMatrixRowMap() ) );
if (ImportToSerial_.get() == 0) AMESOS_CHK_ERR(-5);
SerialCrsMatrixA_ = rcp( new Epetra_CrsMatrix(Copy, *SerialMap_, 0) ) ;
SerialMatrix_ = &*SerialCrsMatrixA_ ;
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
return(0);
}
//=============================================================================
int Amesos_Paraklete::Solve()
{
Epetra_MultiVector* vecX=0;
Epetra_MultiVector* vecB=0;
if ( !TrustMe_ ) {
SerialB_ = Problem_->GetRHS() ;
SerialX_ = Problem_->GetLHS() ;
Epetra_MultiVector* OrigVecX ;
Epetra_MultiVector* OrigVecB ;
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
//
// Reindex the LHS and RHS
//
OrigVecX = Problem_->GetLHS() ;
OrigVecB = Problem_->GetRHS() ;
if ( Reindex_ ) {
#ifdef HAVE_AMESOS_EPETRAEXT
vecX = StdIndexDomain_->StandardizeIndex( OrigVecX ) ;
vecB = StdIndexRange_->StandardizeIndex( OrigVecB ) ;
#else
AMESOS_CHK_ERR( -13 ) ; // Amesos_Paraklete can't handle non-standard indexing without EpetraExt
#endif
} else {
vecX = OrigVecX ;
vecB = OrigVecB ;
}
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong in input
// Extract Serial versions of X and B
ResetTimer(0);
// Copy B to the serial version of B
//
if (false && UseDataInPlace_ == 1) {
SerialB_ = vecB;
SerialX_ = vecX;
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
} else {
assert (UseDataInPlace_ == 0);
if( NumVectors_ != Problem_->GetRHS()->NumVectors() ) {
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
}
if (NumVectors_ != vecB->NumVectors())
AMESOS_CHK_ERR(-1); // internal error
ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecB->Map() ) );
if ( SerialBextract_->Import(*vecB,*ImportRangeToSerial_,Insert) )
AMESOS_CHK_ERR( -1 ) ; // internal error
SerialB_ = &*SerialBextract_ ;
SerialX_ = &*SerialXextract_ ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call PARAKLETE to perform the solve
ResetTimer(0);
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
if (SerialXlda_ != NumGlobalElements_)
AMESOS_CHK_ERR(-1);
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
if ( MyPID_ == 0) {
if ( NumVectors_ == 1 ) {
for ( int i = 0 ; i < NumGlobalElements_ ; i++ )
SerialXBvalues_[i] = SerialBvalues_[i] ;
} else {
SerialX_->Scale(1.0, *SerialB_ ) ; // X = B (Klu overwrites B with X)
}
}
if ( IamInGroup_ )
for (int i = 0; i < NumVectors_ ; i++ ) {
amesos_paraklete_solve( &*PrivateParakleteData_->LUnumeric_, &*PrivateParakleteData_->LUsymbolic_,
&SerialXBvalues_[i*SerialXlda_], &*PrivateParakleteData_->common_ );
}
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if (UseDataInPlace_ == 0) {
ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecX->Map() ) );
vecX->Export( *SerialX_, *ImportDomainToSerial_, Insert ) ;
} // otherwise we are already in place.
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
#if 0
//
// ComputeTrueResidual causes TestOptions to fail on my linux box // Bug #1417
if (ComputeTrueResidual_)
ComputeTrueResidual(*SerialMatrix_, *vecX, *vecB, UseTranspose(), "Amesos_Paraklete");
#endif
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Paraklete");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
++NumSolve_;
return(0) ;
}
//=============================================================================
//
// See also pre and post conditions in Amesos_Klu.h
// Preconditions:
// firsttime specifies that this is the first time that
// ConertToKluCrs has been called, i.e. in symbolic factorization.
// No data allocation should happen unless firsttime=true.
// SerialMatrix_ points to the matrix to be factored and solved
// NumGlobalElements_ has been set to the dimension of the matrix
// numentries_ has been set to the number of non-zeros in the matrix
// (i.e. CreateLocalMatrixAndExporters() has been callded)
//
// Postconditions:
// Ap, VecAi, VecAval contain the matrix as Klu needs it
//
//
int Amesos_Klu::ConvertToKluCRS(bool firsttime)
{
ResetTimer(0);
//
// Convert matrix to the form that Klu expects (Ap, VecAi, VecAval)
//
if (MyPID_==0) {
assert( NumGlobalElements_ == SerialMatrix_->NumGlobalRows());
assert( NumGlobalElements_ == SerialMatrix_->NumGlobalCols());
if ( ! AddZeroToDiag_ ) {
assert( numentries_ == SerialMatrix_->NumGlobalNonzeros()) ;
} else {
numentries_ = SerialMatrix_->NumGlobalNonzeros() ;
}
Epetra_CrsMatrix *CrsMatrix = dynamic_cast<Epetra_CrsMatrix *>(SerialMatrix_);
bool StorageOptimized = ( CrsMatrix != 0 && CrsMatrix->StorageOptimized() );
if ( AddToDiag_ != 0.0 ) StorageOptimized = false ;
if ( firsttime ) {
Ap.resize( NumGlobalElements_+1 );
if ( ! StorageOptimized ) {
VecAi.resize( EPETRA_MAX( NumGlobalElements_, numentries_) ) ;
VecAval.resize( EPETRA_MAX( NumGlobalElements_, numentries_) ) ;
Ai = &VecAi[0];
Aval = &VecAval[0];
}
}
double *RowValues;
int *ColIndices;
int NumEntriesThisRow;
if( StorageOptimized ) {
if ( firsttime ) {
Ap[0] = 0;
for ( int MyRow = 0; MyRow <NumGlobalElements_; MyRow++ ) {
if( CrsMatrix->
ExtractMyRowView( MyRow, NumEntriesThisRow, RowValues,
ColIndices ) != 0 )
AMESOS_CHK_ERR( -10 );
if ( MyRow == 0 ) {
Ai = ColIndices ;
Aval = RowValues ;
}
Ap[MyRow+1] = Ap[MyRow] + NumEntriesThisRow ;
}
}
} else {
int Ai_index = 0 ;
int MyRow;
int MaxNumEntries_ = SerialMatrix_->MaxNumEntries();
if ( firsttime && CrsMatrix == 0 ) {
ColIndicesV_.resize(MaxNumEntries_);
RowValuesV_.resize(MaxNumEntries_);
}
for ( MyRow = 0; MyRow <NumGlobalElements_; MyRow++ ) {
if ( CrsMatrix != 0 ) {
if( CrsMatrix->
ExtractMyRowView( MyRow, NumEntriesThisRow, RowValues,
ColIndices ) != 0 )
AMESOS_CHK_ERR( -11 );
} else {
if( SerialMatrix_->
ExtractMyRowCopy( MyRow, MaxNumEntries_,
NumEntriesThisRow, &RowValuesV_[0],
&ColIndicesV_[0] ) != 0 )
AMESOS_CHK_ERR( -12 );
RowValues = &RowValuesV_[0];
ColIndices = &ColIndicesV_[0];
}
if ( firsttime ) {
Ap[MyRow] = Ai_index ;
for ( int j = 0; j < NumEntriesThisRow; j++ ) {
VecAi[Ai_index] = ColIndices[j] ;
// assert( VecAi[Ai_index] == Ai[Ai_index] ) ;
VecAval[Ai_index] = RowValues[j] ; // We have to do this because of the hacks to get aroun bug #1502
if (ColIndices[j] == MyRow) {
VecAval[Ai_index] += AddToDiag_;
}
Ai_index++;
}
} else {
for ( int j = 0; j < NumEntriesThisRow; j++ ) {
VecAval[Ai_index] = RowValues[j] ;
if (ColIndices[j] == MyRow) {
VecAval[Ai_index] += AddToDiag_;
}
Ai_index++;
}
}
}
Ap[MyRow] = Ai_index ;
}
}
MtxConvTime_ = AddTime("Total matrix conversion time", MtxConvTime_, 0);
return 0;
}
int Amesos_Klu::CreateLocalMatrixAndExporters()
{
ResetTimer(0);
RowMatrixA_ = Problem_->GetMatrix(); // MS, 18-Apr-06 //
if (RowMatrixA_ == 0) AMESOS_CHK_ERR(-1);
const Epetra_Map &OriginalMatrixMap = RowMatrixA_->RowMatrixRowMap() ;
const Epetra_Map &OriginalDomainMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorRangeMap():
GetProblem()->GetOperator()->OperatorDomainMap();
const Epetra_Map &OriginalRangeMap =
UseTranspose()?GetProblem()->GetOperator()->OperatorDomainMap():
GetProblem()->GetOperator()->OperatorRangeMap();
NumGlobalElements_ = RowMatrixA_->NumGlobalRows();
numentries_ = RowMatrixA_->NumGlobalNonzeros();
int indexBase = OriginalMatrixMap.IndexBase();
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
//
// Create a serial matrix
//
assert( NumGlobalElements_ == OriginalMatrixMap.NumGlobalElements() ) ;
int NumMyElements_ = 0 ;
if (MyPID_==0) NumMyElements_ = NumGlobalElements_;
//
// UseDataInPlace_ is set to 1 (true) only if everything is perfectly
// normal. Anything out of the ordinary reverts to the more expensive
// path.
//
UseDataInPlace_ = ( OriginalMatrixMap.NumMyElements() ==
OriginalMatrixMap.NumGlobalElements() )?1:0;
if ( ! OriginalRangeMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( ! OriginalDomainMap.SameAs( OriginalMatrixMap ) ) UseDataInPlace_ = 0 ;
if ( AddZeroToDiag_ ) UseDataInPlace_ = 0 ;
Comm().Broadcast( &UseDataInPlace_, 1, 0 ) ;
//
// Reindex matrix if necessary (and possible - i.e. CrsMatrix)
//
// For now, since I don't know how to determine if we need to reindex the matrix,
// we allow the user to control re-indexing through the "Reindex" parameter.
//
CrsMatrixA_ = dynamic_cast<Epetra_CrsMatrix *>(Problem_->GetOperator());
if ( Reindex_ ) {
if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR(-7);
#ifdef HAVE_AMESOS_EPETRAEXT
const Epetra_Map& OriginalMap = CrsMatrixA_->RowMap();
StdIndex_ = rcp( new Amesos_StandardIndex( OriginalMap ) );
//const Epetra_Map& OriginalColMap = CrsMatrixA_->RowMap();
StdIndexDomain_ = rcp( new Amesos_StandardIndex( OriginalDomainMap ) );
StdIndexRange_ = rcp( new Amesos_StandardIndex( OriginalRangeMap ) );
StdIndexMatrix_ = StdIndex_->StandardizeIndex( CrsMatrixA_ );
#else
std::cerr << "Amesos_Klu requires EpetraExt to reindex matrices." << std::endl
<< " Please rebuild with the EpetraExt library by adding --enable-epetraext to your configure invocation" << std::endl ;
AMESOS_CHK_ERR(-8);
#endif
} else {
StdIndexMatrix_ = RowMatrixA_ ;
}
//
// At this point, StdIndexMatrix_ points to a matrix with
// standard indexing.
//
//
// Convert Original Matrix to Serial (if it is not already)
//
if (UseDataInPlace_ == 1) {
SerialMatrix_ = StdIndexMatrix_;
} else {
SerialMap_ = rcp(new Epetra_Map(NumGlobalElements_, NumMyElements_, indexBase, Comm()));
if ( Problem_->GetRHS() )
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
else
NumVectors_ = 1 ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
ImportToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->RowMatrixRowMap() ) );
if (ImportToSerial_.get() == 0) AMESOS_CHK_ERR(-9);
// Build the vector data import/export once and only once
#define CHRIS
#ifdef CHRIS
if(StdIndexMatrix_->RowMatrixRowMap().SameAs(StdIndexMatrix_->OperatorRangeMap()))
ImportRangeToSerial_=ImportToSerial_;
else
ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->OperatorRangeMap() ) );
if(StdIndexMatrix_->RowMatrixRowMap().SameAs(StdIndexMatrix_->OperatorDomainMap()))
ImportDomainToSerial_=ImportToSerial_;
else
ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, StdIndexMatrix_->OperatorDomainMap() ) );
#endif
SerialCrsMatrixA_ = rcp( new Epetra_CrsMatrix(Copy, *SerialMap_, 0) ) ;
SerialMatrix_ = &*SerialCrsMatrixA_ ;
}
MtxRedistTime_ = AddTime("Total matrix redistribution time", MtxRedistTime_, 0);
return(0);
}
int Amesos_Mumps::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
int NumVectors = vecX->NumVectors();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1);
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-1);
if (Comm().NumProc() == 1)
{
// do not import any data
for (int j = 0 ; j < NumVectors; j++)
{
ResetTimer();
MDS.job = 3; // Request solve
for (int i = 0 ; i < Matrix().NumMyRows() ; ++i)
(*vecX)[j][i] = (*vecB)[j][i];
MDS.rhs = (*vecX)[j];
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
}
else
{
Epetra_MultiVector SerialVector(SerialMap(),NumVectors);
ResetTimer();
AMESOS_CHK_ERR(SerialVector.Import(*vecB,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
for (int j = 0 ; j < NumVectors; j++)
{
if (Comm().MyPID() == 0)
MDS.rhs = SerialVector[j];
// solve the linear system and take time
MDS.job = 3;
ResetTimer();
if (Comm().MyPID() < MaxProcs_)
dmumps_c(&(MDS)) ; // Perform solve
static_cast<void>( CheckError( ) ); // Can hang
SolveTime_ = AddTime("Total solve time", SolveTime_);
}
// ship solution back and take timing
ResetTimer();
AMESOS_CHK_ERR(vecX->Export(SerialVector,SerialImporter(),Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_);
}
if (ComputeTrueResidual_)
ComputeTrueResidual(Matrix(), *vecX, *vecB, UseTranspose(), "Amesos_Mumps");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Mumps");
NumSolve_++;
return(0) ;
}
//=============================================================================
int Amesos_Dscpack::PerformNumericFactorization()
{
ResetTimer(0);
ResetTimer(1);
Epetra_RowMatrix* RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
const Epetra_Map& OriginalMap = RowMatrixA->RowMatrixRowMap() ;
int numrows = RowMatrixA->NumGlobalRows();
assert( numrows == RowMatrixA->NumGlobalCols() );
//
// Call Dscpack to perform Numeric Factorization
//
std::vector<double> MyANonZ;
#if 0
if ( IsNumericFactorizationOK_ ) {
DSC_ReFactorInitialize(PrivateDscpackData_->MyDSCObject);
}
#endif
DscRowMap_ = Teuchos::rcp(new Epetra_Map(numrows, NumLocalCols,
LocalStructOldNum, 0, Comm()));
if (DscRowMap_.get() == 0)
AMESOS_CHK_ERR(-1);
Importer_ = rcp(new Epetra_Import(DscRowMap(), OriginalMap));
//
// Import from the CrsMatrix
//
Epetra_CrsMatrix DscMat(Copy, DscRowMap(), 0);
AMESOS_CHK_ERR(DscMat.Import(*RowMatrixA, Importer(), Insert));
AMESOS_CHK_ERR(DscMat.FillComplete());
DscColMap_ = Teuchos::rcp(new Epetra_Map(DscMat.RowMatrixColMap()));
assert( MyDscRank >= 0 || NumLocalNonz == 0 ) ;
assert( MyDscRank >= 0 || NumLocalCols == 0 ) ;
assert( MyDscRank >= 0 || NumGlobalCols == 0 ) ;
MyANonZ.resize( NumLocalNonz ) ;
int NonZIndex = 0 ;
int max_num_entries = DscMat.MaxNumEntries() ;
std::vector<int> col_indices( max_num_entries ) ;
std::vector<double> mat_values( max_num_entries ) ;
assert( NumLocalCols == DscRowMap().NumMyElements() ) ;
std::vector<int> my_global_elements( NumLocalCols ) ;
AMESOS_CHK_ERR(DscRowMap().MyGlobalElements( &my_global_elements[0] ) ) ;
std::vector<int> GlobalStructOldColNum( NumGlobalCols ) ;
typedef std::pair<int, double> Data;
std::vector<Data> sort_array(max_num_entries);
std::vector<int> sort_indices(max_num_entries);
for ( int i = 0; i < NumLocalCols ; i++ ) {
assert( my_global_elements[i] == LocalStructOldNum[i] ) ;
int num_entries_this_row;
#ifdef USE_LOCAL
AMESOS_CHK_ERR( DscMat.ExtractMyRowCopy( i, max_num_entries, num_entries_this_row,
&mat_values[0], &col_indices[0] ) ) ;
#else
AMESOS_CHK_ERR( DscMat.ExtractGlobalRowCopy( DscMat.GRID(i), max_num_entries, num_entries_this_row,
&mat_values[0], &col_indices[0] ) ) ;
#endif
int OldRowNumber = LocalStructOldNum[i] ;
if (GlobalStructOwner[ OldRowNumber ] == -1)
AMESOS_CHK_ERR(-1);
int NewRowNumber = GlobalStructNewColNum[ my_global_elements[ i ] ] ;
//
// Sort the column elements
//
for ( int j = 0; j < num_entries_this_row; j++ ) {
#ifdef USE_LOCAL
sort_array[j].first = GlobalStructNewColNum[ DscMat.GCID( col_indices[j])] ;
sort_indices[j] = GlobalStructNewColNum[ DscMat.GCID( col_indices[j])] ;
#else
sort_array[j].first = GlobalStructNewColNum[ col_indices[j] ];
#endif
sort_array[j].second = mat_values[j] ;
}
sort(&sort_array[0], &sort_array[num_entries_this_row]);
for ( int j = 0; j < num_entries_this_row; j++ ) {
int NewColNumber = sort_array[j].first ;
if ( NewRowNumber <= NewColNumber ) MyANonZ[ NonZIndex++ ] = sort_array[j].second ;
#ifndef USE_LOCAL
assert( NonZIndex <= NumLocalNonz ); // This assert can fail on non-symmetric matrices
#endif
}
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
if ( MyDscRank >= 0 ) {
const int SchemeCode = 1;
#ifndef USE_LOCAL
assert( NonZIndex == NumLocalNonz );
#endif
AMESOS_CHK_ERR( DSC_NFactor ( PrivateDscpackData_->MyDSCObject_, SchemeCode, &MyANonZ[0],
DSC_LLT, DSC_LBLAS3, DSC_DBLAS2 ) ) ;
} // if ( MyDscRank >= 0 )
IsNumericFactorizationOK_ = true ;
NumFactTime_ = AddTime("Total numeric factorization time", NumFactTime_, 0);
return(0);
}
//=============================================================================
int Amesos_Klu::Solve()
{
Epetra_MultiVector* vecX = 0 ;
Epetra_MultiVector* vecB = 0 ;
#ifdef HAVE_AMESOS_EPETRAEXT
Teuchos::RCP<Epetra_MultiVector> vecX_rcp;
Teuchos::RCP<Epetra_MultiVector> vecB_rcp;
#endif
#ifdef Bug_8212
// This demonstrates Bug #2812 - Valgrind does not catch this
// memory leak
lose_this_ = (int *) malloc( 300 ) ;
#ifdef Bug_8212_B
// This demonstrates Bug #2812 - Valgrind does catch this
// use of unitialized data - but only in TestOptions/TestOptions.exe
// not in Test_Basic/amesos_test.exe
//
if ( lose_this_[0] == 12834 ) {
std::cout << __FILE__ << "::" << __LINE__ << " very unlikely to happen " << std::endl ;
}
#endif
#endif
if ( !TrustMe_ ) {
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false);
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false);
Epetra_MultiVector* OrigVecX ;
Epetra_MultiVector* OrigVecB ;
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(1);
//
// Reindex the LHS and RHS
//
OrigVecX = Problem_->GetLHS() ;
OrigVecB = Problem_->GetRHS() ;
if ( Reindex_ ) {
#ifdef HAVE_AMESOS_EPETRAEXT
vecX_rcp = StdIndexDomain_->StandardizeIndex( *OrigVecX ) ;
vecB_rcp = StdIndexRange_->StandardizeIndex( *OrigVecB ) ;
vecX = &*vecX_rcp;
vecB = &*vecB_rcp;
#else
AMESOS_CHK_ERR( -13 ) ; // Amesos_Klu can't handle non-standard indexing without EpetraExt
#endif
} else {
vecX = OrigVecX ;
vecB = OrigVecB ;
}
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong in input
// Extract Serial versions of X and B
ResetTimer(0);
// Copy B to the serial version of B
//
if (UseDataInPlace_ == 1) {
#ifdef HAVE_AMESOS_EPETRAEXT
if(vecX_rcp==Teuchos::null)
SerialX_ = Teuchos::rcp(vecX,false);
else
SerialX_ = vecX_rcp;
if(vecB_rcp==Teuchos::null)
SerialB_ = Teuchos::rcp(vecB,false);
else
SerialB_ = vecB_rcp;
#else
SerialB_ = Teuchos::rcp(vecB,false);
SerialX_ = Teuchos::rcp(vecX,false);
#endif
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
} else {
assert (UseDataInPlace_ == 0);
if( NumVectors_ != Problem_->GetRHS()->NumVectors() ) {
NumVectors_ = Problem_->GetRHS()->NumVectors() ;
SerialXextract_ = rcp( new Epetra_MultiVector(*SerialMap_,NumVectors_));
SerialBextract_ = rcp (new Epetra_MultiVector(*SerialMap_,NumVectors_));
}
if (NumVectors_ != vecB->NumVectors())
AMESOS_CHK_ERR(-1); // internal error
//ImportRangeToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecB->Map() ) );
//if ( SerialBextract_->Import(*vecB,*ImportRangeToSerial_,Insert) )
Epetra_Import *UseImport;
if(!UseTranspose_) UseImport=&*ImportRangeToSerial_;
else UseImport=&*ImportDomainToSerial_;
if ( SerialBextract_->Import(*vecB,*UseImport,Insert) )
AMESOS_CHK_ERR( -1 ) ; // internal error
SerialB_ = Teuchos::rcp(&*SerialBextract_,false) ;
SerialX_ = Teuchos::rcp(&*SerialXextract_,false) ;
}
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
// Call KLU to perform the solve
ResetTimer(0);
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
if (SerialXlda_ != NumGlobalElements_)
AMESOS_CHK_ERR(-1);
}
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
else
{
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false) ;
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false) ;
NumVectors_ = SerialX_->NumVectors();
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
}
}
if ( MyPID_ == 0) {
if ( NumVectors_ == 1 ) {
for ( int i = 0 ; i < NumGlobalElements_ ; i++ )
SerialXBvalues_[i] = SerialBvalues_[i] ;
} else {
SerialX_->Scale(1.0, *SerialB_ ) ; // X = B (Klu overwrites B with X)
}
if (UseTranspose()) {
amesos_klu_solve( &*PrivateKluData_->Symbolic_, &*PrivateKluData_->Numeric_,
SerialXlda_, NumVectors_, &SerialXBvalues_[0], &*PrivateKluData_->common_ );
} else {
amesos_klu_tsolve( &*PrivateKluData_->Symbolic_, &*PrivateKluData_->Numeric_,
SerialXlda_, NumVectors_, &SerialXBvalues_[0], &*PrivateKluData_->common_ );
}
}
if ( !TrustMe_ ) {
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
// Copy X back to the original vector
ResetTimer(0);
ResetTimer(1);
if (UseDataInPlace_ == 0) {
Epetra_Import *UseImport;
if(!UseTranspose_) UseImport=&*ImportDomainToSerial_;
else UseImport=&*ImportRangeToSerial_;
// ImportDomainToSerial_ = rcp(new Epetra_Import ( *SerialMap_, vecX->Map() ) );
vecX->Export( *SerialX_, *UseImport, Insert ) ;
} // otherwise we are already in place.
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
#if 0
//
// ComputeTrueResidual causes TestOptions to fail on my linux box
// Bug #1417
if (ComputeTrueResidual_)
ComputeTrueResidual(*SerialMatrix_, *vecX, *vecB, UseTranspose(), "Amesos_Klu");
#endif
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Klu");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
}
++NumSolve_;
return(0) ;
}
//=============================================================================
int Amesos_Klu::PerformNumericFactorization( )
{
// Changed this; it was "if (!TrustMe)...
// The behavior is not intuitive. Maybe we should introduce a new
// parameter, FastSolvers or something like that, that does not perform
// any AddTime, ResetTimer, GetTime.
// HKT, 1/18/2007: The "TrustMe_" flag was put back in this code due to performance degradation; Bug# 3042.
if (! TrustMe_ ) ResetTimer(0);
// This needs to be 1 just in case the pivot ordering is reused.
int numeric_ok = 1;
if (MyPID_ == 0) {
bool factor_with_pivoting = true ;
// set the default parameters
PrivateKluData_->common_->scale = ScaleMethod_ ;
const bool NumericNonZero = PrivateKluData_->Numeric_.get() != 0 ;
// see if we can "refactorize"
if ( refactorize_ && NumericNonZero ) {
// refactorize using the existing Symbolic and Numeric objects, and
// using the identical pivot ordering as the prior klu_factor.
// No partial pivoting is done.
int result = amesos_klu_refactor (&Ap[0], Ai, Aval,
&*PrivateKluData_->Symbolic_,
&*PrivateKluData_->Numeric_, &*PrivateKluData_->common_) ;
// Did it work?
const bool refactor_ok = result == 1 && PrivateKluData_->common_->status == KLU_OK ;
if ( refactor_ok ) {
amesos_klu_rcond (&*PrivateKluData_->Symbolic_,
&*PrivateKluData_->Numeric_,
&*PrivateKluData_->common_) ;
double rcond = PrivateKluData_->common_->rcond;
if ( rcond > rcond_threshold_ ) {
// factorizing without pivot worked fine. We are done.
factor_with_pivoting = false ;
}
}
}
if ( factor_with_pivoting ) {
// factor with partial pivoting:
// either this is the first time we are factoring the matrix, or the
// refactorize parameter is false, or we tried to refactorize and
// found it to be too inaccurate.
// factor the matrix using partial pivoting
PrivateKluData_->Numeric_ =
rcpWithDealloc( amesos_klu_factor(&Ap[0], Ai, Aval,
&*PrivateKluData_->Symbolic_, &*PrivateKluData_->common_),
deallocFunctorDeleteWithCommon<klu_numeric>(PrivateKluData_->common_,amesos_klu_free_numeric)
,true
);
numeric_ok = PrivateKluData_->Numeric_.get()!=NULL
&& PrivateKluData_->common_->status == KLU_OK ;
}
}
// Communicate the state of the numeric factorization with everyone.
Comm().Broadcast(&numeric_ok, 1, 0);
if ( ! numeric_ok ) {
if (MyPID_ == 0) {
AMESOS_CHK_ERR( NumericallySingularMatrixError );
} else
return( NumericallySingularMatrixError );
}
if ( !TrustMe_ ) {
NumFactTime_ = AddTime("Total numeric factorization time", NumFactTime_, 0);
}
return 0;
}
//=============================================================================
int Amesos_Mumps::SymbolicFactorization()
{
// erase data if present.
if (IsSymbolicFactorizationOK_ && MDS.job != -777)
Destroy();
IsSymbolicFactorizationOK_ = false;
IsNumericFactorizationOK_ = false;
CreateTimer(Comm());
CheckParameters();
AMESOS_CHK_ERR(ConvertToTriplet(false));
#if defined(HAVE_MPI) && defined(HAVE_AMESOS_MPI_C2F)
if (MaxProcs_ != Comm().NumProc())
{
if(MUMPSComm_)
MPI_Comm_free(&MUMPSComm_);
std::vector<int> ProcsInGroup(MaxProcs_);
for (int i = 0 ; i < MaxProcs_ ; ++i)
ProcsInGroup[i] = i;
MPI_Group OrigGroup, MumpsGroup;
MPI_Comm_group(MPI_COMM_WORLD, &OrigGroup);
MPI_Group_incl(OrigGroup, MaxProcs_, &ProcsInGroup[0], &MumpsGroup);
MPI_Comm_create(MPI_COMM_WORLD, MumpsGroup, &MUMPSComm_);
#ifdef MUMPS_4_9
MDS.comm_fortran = (MUMPS_INT) MPI_Comm_c2f( MUMPSComm_);
#else
#ifndef HAVE_AMESOS_OLD_MUMPS
MDS.comm_fortran = (DMUMPS_INT) MPI_Comm_c2f( MUMPSComm_);
#else
MDS.comm_fortran = (F_INT) MPI_Comm_c2f( MUMPSComm_);
#endif
#endif
}
else
{
const Epetra_MpiComm* MpiComm = dynamic_cast<const Epetra_MpiComm*>(&Comm());
assert (MpiComm != 0);
#ifdef MUMPS_4_9
MDS.comm_fortran = (MUMPS_INT) MPI_Comm_c2f(MpiComm->GetMpiComm());
#else
#ifndef HAVE_AMESOS_OLD_MUMPS
MDS.comm_fortran = (DMUMPS_INT) MPI_Comm_c2f(MpiComm->GetMpiComm());
#else
MDS.comm_fortran = (F_INT) MPI_Comm_c2f(MpiComm->GetMpiComm());
#endif
#endif
}
#else
// This next three lines of code were required to make Amesos_Mumps work
// with Ifpack_SubdomainFilter. They is usefull in all cases
// when using MUMPS on a subdomain.
const Epetra_MpiComm* MpiComm = dynamic_cast<const Epetra_MpiComm*>(&Comm());
assert (MpiComm != 0);
MDS.comm_fortran = (MUMPS_INT) MPI_Comm_c2f(MpiComm->GetMpiComm());
// only thing I can do, use MPI_COMM_WORLD. This will work in serial as well
// Previously, the next line was uncommented, but we don't want MUMPS to work
// on the global MPI comm, but on the comm associated with the matrix
// MDS.comm_fortran = -987654;
#endif
MDS.job = -1 ; // Initialization
MDS.par = 1 ; // Host IS involved in computations
// MDS.sym = MatrixProperty_;
MDS.sym = 0; // MatrixProperty_ is unititalized. Furthermore MUMPS
// expects only half of the matrix to be provided for
// symmetric matrices. Hence setting MDS.sym to be non-zero
// indicating that the matrix is symmetric will only work
// if we change ConvertToTriplet to pass only half of the
// matrix. Bug #2331 and Bug #2332 - low priority
RedistrMatrix(true);
if (Comm().MyPID() < MaxProcs_)
{
dmumps_c(&(MDS)); // Initialize MUMPS
static_cast<void>( CheckError( ) );
}
MDS.n = Matrix().NumGlobalRows();
// fix pointers for nonzero pattern of A. Numerical values
// will be entered in PerformNumericalFactorization()
if (Comm().NumProc() != 1)
{
MDS.nz_loc = RedistrMatrix().NumMyNonzeros();
if (Comm().MyPID() < MaxProcs_)
{
MDS.irn_loc = &Row[0];
MDS.jcn_loc = &Col[0];
}
}
else
{
if (Comm().MyPID() == 0)
{
MDS.nz = Matrix().NumMyNonzeros();
MDS.irn = &Row[0];
MDS.jcn = &Col[0];
}
}
// scaling if provided by the user
if (RowSca_ != 0)
{
MDS.rowsca = RowSca_;
MDS.colsca = ColSca_;
}
// given ordering if provided by the user
if (PermIn_ != 0)
{
MDS.perm_in = PermIn_;
}
MDS.job = 1; // Request symbolic factorization
SetICNTLandCNTL();
// Perform symbolic factorization
ResetTimer();
if (Comm().MyPID() < MaxProcs_)
dmumps_c(&(MDS));
SymFactTime_ = AddTime("Total symbolic factorization time", SymFactTime_);
int IntWrong = CheckError()?1:0 ;
int AnyWrong;
Comm().SumAll( &IntWrong, &AnyWrong, 1 ) ;
bool Wrong = AnyWrong > 0 ;
if ( Wrong ) {
AMESOS_CHK_ERR( StructurallySingularMatrixError ) ;
}
IsSymbolicFactorizationOK_ = true ;
NumSymbolicFact_++;
return 0;
}
//=============================================================================
int Amesos_Dscpack::Solve()
{
if (IsNumericFactorizationOK_ == false)
AMESOS_CHK_ERR(NumericFactorization());
ResetTimer(0);
ResetTimer(1);
Epetra_RowMatrix *RowMatrixA = Problem_->GetMatrix();
if (RowMatrixA == 0)
AMESOS_CHK_ERR(-1);
// MS // some checks on matrix size
if (RowMatrixA->NumGlobalRows() != RowMatrixA->NumGlobalCols())
AMESOS_CHK_ERR(-1);
// Convert vector b to a vector in the form that DSCPACK needs it
//
Epetra_MultiVector* vecX = Problem_->GetLHS();
Epetra_MultiVector* vecB = Problem_->GetRHS();
if ((vecX == 0) || (vecB == 0))
AMESOS_CHK_ERR(-1); // something wrong with input
int NumVectors = vecX->NumVectors();
if (NumVectors != vecB->NumVectors())
AMESOS_CHK_ERR(-2);
double *dscmapXvalues ;
int dscmapXlda ;
Epetra_MultiVector dscmapX(DscRowMap(),NumVectors) ;
int ierr;
AMESOS_CHK_ERR(dscmapX.ExtractView(&dscmapXvalues,&dscmapXlda));
assert (dscmapXlda == NumLocalCols);
double *dscmapBvalues ;
int dscmapBlda ;
Epetra_MultiVector dscmapB(DscRowMap(), NumVectors ) ;
ierr = dscmapB.ExtractView( &dscmapBvalues, &dscmapBlda );
AMESOS_CHK_ERR(ierr);
assert( dscmapBlda == NumLocalCols ) ;
AMESOS_CHK_ERR(dscmapB.Import(*vecB, Importer(), Insert));
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
ResetTimer(0);
// MS // now solve the problem
std::vector<double> ValuesInNewOrder( NumLocalCols ) ;
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
if ( MyDscRank >= 0 ) {
for ( int j =0 ; j < NumVectors; j++ ) {
for ( int i = 0; i < NumLocalCols; i++ ) {
ValuesInNewOrder[i] = dscmapBvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapBlda ] ;
}
AMESOS_CHK_ERR( DSC_InputRhsLocalVec ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
AMESOS_CHK_ERR( DSC_Solve ( PrivateDscpackData_->MyDSCObject_ ) ) ;
AMESOS_CHK_ERR( DSC_GetLocalSolution ( PrivateDscpackData_->MyDSCObject_, &ValuesInNewOrder[0], NumLocalCols ) ) ;
for ( int i = 0; i < NumLocalCols; i++ ) {
dscmapXvalues[DscColMap().LID( LocalStructOldNum[i] ) +j*dscmapXlda ] = ValuesInNewOrder[i];
}
}
}
SolveTime_ = AddTime("Total solve time", SolveTime_, 0);
ResetTimer(0);
ResetTimer(1);
vecX->Export( dscmapX, Importer(), Insert ) ;
VecRedistTime_ = AddTime("Total vector redistribution time", VecRedistTime_, 0);
if (ComputeTrueResidual_)
ComputeTrueResidual(*(GetProblem()->GetMatrix()), *vecX, *vecB,
false, "Amesos_Dscpack");
if (ComputeVectorNorms_)
ComputeVectorNorms(*vecX, *vecB, "Amesos_Dscpack");
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
NumSolve_++;
return(0) ;
}
//=============================================================================
int Amesos_Klu::SymbolicFactorization()
{
MyPID_ = Comm().MyPID();
NumProcs_ = Comm().NumProc();
IsSymbolicFactorizationOK_ = false;
IsNumericFactorizationOK_ = false;
CreateTimer(Comm(), 2);
ResetTimer(1);
// "overhead" time for the following method is considered here
AMESOS_CHK_ERR( CreateLocalMatrixAndExporters() ) ;
assert( NumGlobalElements_ == RowMatrixA_->NumGlobalCols() );
//
// Perform checks in SymbolicFactorization(), but none in
// NumericFactorization() or Solve()
//
if ( TrustMe_ ) {
if ( CrsMatrixA_ == 0 ) AMESOS_CHK_ERR( -15 );
if( UseDataInPlace_ != 1 ) AMESOS_CHK_ERR( -15 ) ;
if( Reindex_ ) AMESOS_CHK_ERR( -15 ) ;
if( ! Problem_->GetLHS() ) AMESOS_CHK_ERR( -15 ) ;
if( ! Problem_->GetRHS() ) AMESOS_CHK_ERR( -15 ) ;
if( ! Problem_->GetLHS()->NumVectors() ) AMESOS_CHK_ERR( -15 ) ;
if( ! Problem_->GetRHS()->NumVectors() ) AMESOS_CHK_ERR( -15 ) ;
SerialB_ = Teuchos::rcp(Problem_->GetRHS(),false) ;
SerialX_ = Teuchos::rcp(Problem_->GetLHS(),false) ;
NumVectors_ = SerialX_->NumVectors();
if (MyPID_ == 0) {
AMESOS_CHK_ERR(SerialX_->ExtractView(&SerialXBvalues_,&SerialXlda_ ));
AMESOS_CHK_ERR(SerialB_->ExtractView(&SerialBvalues_,&SerialXlda_ ));
}
}
// "overhead" time for the following two methods is considered here
AMESOS_CHK_ERR( ExportToSerial() );
AMESOS_CHK_ERR( ConvertToKluCRS(true) );
OverheadTime_ = AddTime("Total Amesos overhead time", OverheadTime_, 1);
// All this time if KLU time
AMESOS_CHK_ERR( PerformSymbolicFactorization() );
NumSymbolicFact_++;
IsSymbolicFactorizationOK_ = true;
return 0;
}
